def search_invalid_data(params):
log = log_utils.start_day_log(params, 'data_validator')
red_dir_list = glob.glob(path.join(params['top_level_dir'], '*'))
exclude_dirs = ['logs', 'config', 'downloads']
for red_dir in red_dir_list:
if path.isdir(red_dir) and red_dir not in exclude_dirs:
review_dataset(red_dir, log)
log_utils.close_log(log)
def count_image_data(params):
log = log_utils.start_day_log(params, 'count_images')
datasets = glob.glob(
path.join(params['top_dir'], '*_???-dom?-1m0-??-f???_?p'))
nimages = 0
log.info('# Dataset N images')
for subdir in datasets:
image_list = glob.glob(path.join(subdir, 'data', '*.fits'))
log.info(path.basename(subdir) + ' ' + str(len(image_list)))
nimages += len(image_list)
log.info('Total number of images: ' + str(nimages))
log_utils.close_log(log)
def search_archive_for_data(CONFIG_FILE):
config = config_utils.get_config(CONFIG_FILE)
log = log_utils.start_day_log(config, 'data_download')
downloaded_frames = read_frame_list(config, log)
(start_time, end_time) = set_date_range(config, log)
new_frames = fetch_new_frames(config, start_time, end_time, log)
downloaded_frames = download_new_frames(config, new_frames,
downloaded_frames, log)
framelist_utils.output_frame_list(config, downloaded_frames, log)
log_utils.close_log(log)
def prepare_data_for_reduction(CONFIG_FILE):
config = config_utils.get_config(CONFIG_FILE)
log = log_utils.start_day_log(config, 'data_preparation')
compressed_frames = check_for_new_frames(config, log)
if len(compressed_frames) > 0:
decompressed_frames = decompress_new_frames(config, log, compressed_frames)
#transform_frames(decompressed_frames, log)
sort_data.sort_data(config['data_download_dir'],config['separate_instruments'],log=log)
datasets = get_dataset_list(config, log)
for dataset_dir in datasets:
transfer_to_reduction_directory(config, log, dataset_dir)
log_utils.close_log(log)
def test_build_frame_list():
proposal = 'KEY2020B-003'
config = {'log_dir': '.', 'proposal_ids': '[KEY2020B-003, LCO2020A-002]'}
log = log_utils.start_day_log(config,'test_data_download')
default_params = { 'url': 'https://archive.lco.global',
'DATE_OBS': '2020-07-06 12:30:00',
'PROPID': 'KEY2020B-003',
'INSTRUME': 'fa12',
'OBJECT': 'TEST',
'SITEID': 'cpt',
'TELID': '1m003',
'EXPTIME': 30.0,
'FILTER': 'gp',
'REQNUM': 2168224 }
frame1 = default_params.copy()
frame1['filename'] = 'cpt1m003-fa12-20200607-0123-e91.fits'
frame2 = default_params.copy()
frame2['filename'] = 'cpt1m003-fa12-20200607-skyflat-bin2x2-B.fits'
frame3 = default_params.copy()
frame3['filename'] = 'cpt1m003-fa12-20200607-dark-bin1x1.fits'
frame4 = default_params.copy()
frame4['filename'] = 'cpt1m003-fa12-20200607-bias-bin1x1.fits'
query_results = {'results': [ frame1, frame2, frame3, frame4 ]}
new_frames = []
new_frames = framelist_utils.build_frame_list(config, query_results, proposal, new_frames, log)
assert len(new_frames) == 1
log_utils.close_log(log)
def run_for_config(config):
# set the name of the model
model_name = config['general']['name']
now = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y_%m_%d_%H_%M_%S')
model_name = now + '_' + model_name if model_name is not None else now
# where we save all the outputs
scenario = config['general']['scenario']
working_dir = os.path.join(get_base_directory(), 'sgt', scenario)
init_dir(working_dir)
saver_dir = os.path.join(working_dir, 'models', model_name)
init_dir(saver_dir)
init_log(log_file_path=os.path.join(saver_dir, 'log.txt'))
copy_config(config, os.path.join(saver_dir, 'config.yml'))
episodic_success_rates_path = os.path.join(saver_dir, 'results.txt')
test_trajectories_dir = os.path.join(working_dir, 'test_trajectories',
model_name)
init_dir(test_trajectories_dir)
# generate game
game = _get_game(config)
network = Network(config, game)
network_variables = network.get_all_variables()
# save model
latest_saver = ModelSaver(os.path.join(saver_dir, 'latest_model'),
2,
'latest',
variables=network_variables)
best_saver = ModelSaver(os.path.join(saver_dir, 'best_model'),
1,
'best',
variables=network_variables)
summaries_collector = SummariesCollector(
os.path.join(working_dir, 'tensorboard', model_name), model_name)
with tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(
gpu_options=tf.compat.v1.GPUOptions(
per_process_gpu_memory_fraction=config['general']
['gpu_usage']))) as sess:
sess.run(tf.compat.v1.global_variables_initializer())
def policy_function(starts, goals, level, is_train):
res = network.predict_policy(starts, goals, level, sess, is_train)
means = 0.5 * (np.array(starts) + np.array(goals))
distance = np.linalg.norm(res[0] - means, axis=1)
print(
f'distance from mean: mean {distance.mean()} min {distance.min()} max {distance.max()}'
)
if np.any(np.isnan(res)):
print_and_log(
'######################## Nan predictions detected...')
return res
episode_runner = EpisodeRunnerSubgoal(config, game, policy_function)
trainer = TrainerSubgoal(
model_name,
config,
working_dir,
network,
sess,
episode_runner,
summaries_collector,
curriculum_coefficient=get_initial_curriculum(config))
decrease_learn_rate_if_static_success = config['model'][
'decrease_learn_rate_if_static_success']
stop_training_after_learn_rate_decrease = config['model'][
'stop_training_after_learn_rate_decrease']
reset_best_every = config['model']['reset_best_every']
global_step = 0
best_curriculum_coefficient = None
for current_level in range(config['model']['starting_level'],
config['model']['levels'] + 1):
best_cost, best_cost_global_step = None, None
no_test_improvement, consecutive_learn_rate_decrease = 0, 0
if config['model']['init_from_lower_level'] and current_level > 1:
print_and_log('initiating level {} from previous level'.format(
current_level))
network.init_policy_from_lower_level(sess, current_level)
for cycle in range(config['general']['training_cycles_per_level']):
print_and_log('starting cycle {}, level {}'.format(
cycle, current_level))
new_global_step, success_ratio = trainer.train_policy_at_level(
current_level, global_step)
if new_global_step == global_step:
print_and_log(
'no data found in training cycle {} global step still {}'
.format(cycle, global_step))
continue
else:
global_step = new_global_step
if (cycle + 1) % config['policy']['decrease_std_every'] == 0:
network.decrease_base_std(sess, current_level)
print_and_log('new base stds {}'.format(
network.get_base_stds(sess, current_level)))
print_and_log('done training cycle {} global step {}'.format(
cycle, global_step))
# save every now and then
if cycle % config['general']['save_every_cycles'] == 0:
latest_saver.save(sess, global_step=global_step)
if cycle % config['general']['test_frequency'] == 0:
# do test
test_successes, test_cost, _, endpoints_by_path = trainer.collect_test_data(
current_level, False)
summaries_collector.write_test_success_summaries(
sess, global_step, test_successes, test_cost,
trainer.curriculum_coefficient)
with open(episodic_success_rates_path, 'a') as f:
f.write('{} {} {} {} {}'.format(
current_level, trainer.train_episodes_counter,
test_successes, test_cost, os.linesep))
# decide how to act next
print_and_log('old cost was {} at step {}'.format(
best_cost, best_cost_global_step))
print_and_log('current learn rates {}'.format(
network.get_learn_rates(sess, current_level)))
print_and_log('current base stds {}'.format(
network.get_base_stds(sess, current_level)))
if best_cost is None or test_cost < best_cost:
print_and_log('new best cost {} at step {}'.format(
test_cost, global_step))
best_cost, best_cost_global_step = test_cost, global_step
best_curriculum_coefficient = trainer.curriculum_coefficient
no_test_improvement, consecutive_learn_rate_decrease = 0, 0
best_saver.save(sess, global_step)
test_trajectories_file = os.path.join(
test_trajectories_dir,
'{}.txt'.format(global_step))
serialize_compress(endpoints_by_path,
test_trajectories_file)
else:
print_and_log(
'new model is not the best with cost {} at step {}'
.format(test_cost, global_step))
no_test_improvement += 1
print_and_log('no improvement count {} of {}'.format(
no_test_improvement,
decrease_learn_rate_if_static_success))
if reset_best_every > 0 and no_test_improvement % reset_best_every == reset_best_every - 1:
# restore the model every once in a while if did not find a better solution in a while
restore_best(sess, best_saver,
best_curriculum_coefficient, trainer)
if no_test_improvement == decrease_learn_rate_if_static_success:
# restore the best model
if config['model']['restore_on_decrease']:
restore_best(sess, best_saver,
best_curriculum_coefficient,
trainer)
# decrease learn rates
network.decrease_learn_rates(sess, current_level)
no_test_improvement = 0
consecutive_learn_rate_decrease += 1
print_and_log(
'decreasing learn rates {} of {}'.format(
consecutive_learn_rate_decrease,
stop_training_after_learn_rate_decrease))
print_and_log('new learn rates {}'.format(
network.get_learn_rates(sess, current_level)))
if consecutive_learn_rate_decrease == stop_training_after_learn_rate_decrease:
break
if trainer.curriculum_coefficient is not None:
if success_ratio > config['curriculum'][
'raise_when_train_above']:
print_and_log(
'current curriculum coefficient {}'.format(
trainer.curriculum_coefficient))
trainer.curriculum_coefficient *= config['curriculum'][
'raise_times']
print_and_log(
'curriculum coefficient raised to {}'.format(
trainer.curriculum_coefficient))
# mark in log the end of cycle
print_and_log(os.linesep)
# if we finished because we ran out of cycles, we still need to make one more test
end_of_level_test(best_cost, best_cost_global_step,
best_curriculum_coefficient, best_saver, sess,
test_trajectories_dir, trainer, current_level)
print_and_log('trained all levels - needs to stop')
close_log()
return best_cost
def run_for_config(config):
# set the name of the model
model_name = config['general']['name']
now = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y_%m_%d_%H_%M_%S')
model_name = now + '_' + model_name if model_name is not None else now
# where we save all the outputs
scenario = config['general']['scenario']
working_dir = os.path.join(get_base_directory(), 'sequential', scenario)
init_dir(working_dir)
saver_dir = os.path.join(working_dir, 'models', model_name)
init_dir(saver_dir)
init_log(log_file_path=os.path.join(saver_dir, 'log.txt'))
copy_config(config, os.path.join(saver_dir, 'config.yml'))
episodic_success_rates_path = os.path.join(saver_dir, 'results.txt')
weights_log_dir = os.path.join(saver_dir, 'weights_logs')
init_dir(weights_log_dir)
test_trajectories_dir = os.path.join(working_dir, 'test_trajectories',
model_name)
init_dir(test_trajectories_dir)
# generate game
game = _get_game(config)
network = NetworkSequential(config,
game.get_state_space_size(),
game.get_action_space_size(),
is_rollout_agent=False)
network_variables = network.get_all_variables()
# save model
latest_saver = ModelSaver(os.path.join(saver_dir, 'latest_model'),
2,
'latest',
variables=network_variables)
best_saver = ModelSaver(os.path.join(saver_dir, 'best_model'),
1,
'best',
variables=network_variables)
summaries_collector = SummariesCollector(
os.path.join(working_dir, 'tensorboard', model_name), model_name)
with tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(
gpu_options=tf.compat.v1.GPUOptions(
per_process_gpu_memory_fraction=config['general']
['gpu_usage']))) as sess:
sess.run(tf.compat.v1.global_variables_initializer())
episode_runner = EpisodeRunnerSequential(
config,
game,
curriculum_coefficient=get_initial_curriculum(config))
trainer = TrainerSequential(model_name, config, working_dir, network,
sess, episode_runner, summaries_collector)
decrease_learn_rate_if_static_success = config['model'][
'decrease_learn_rate_if_static_success']
stop_training_after_learn_rate_decrease = config['model'][
'stop_training_after_learn_rate_decrease']
reset_best_every = config['model']['reset_best_every']
global_step = 0
best_cost, best_cost_global_step, best_curriculum_coefficient = None, None, None
no_test_improvement, consecutive_learn_rate_decrease = 0, 0
for cycle in range(config['general']['training_cycles']):
print_and_log('starting cycle {}'.format(cycle))
global_step, success_ratio = trainer.train_policy(global_step)
if (cycle + 1) % config['policy']['decrease_std_every'] == 0:
network.decrease_base_std(sess)
print_and_log('new base stds {}'.format(
network.get_base_std(sess)))
print_and_log('done training cycle {} global step {}'.format(
cycle, global_step))
# save every now and then
if cycle % config['general']['save_every_cycles'] == 0:
latest_saver.save(sess, global_step=global_step)
if cycle % config['general']['test_frequency'] == 0:
# do test
test_successes, test_cost, _, endpoints_by_path = trainer.collect_data(
config['general']['test_episodes'],
is_train=False,
use_fixed_start_goal_pairs=True)
summaries_collector.write_test_success_summaries(
sess, global_step, test_successes, test_cost,
episode_runner.curriculum_coefficient)
with open(episodic_success_rates_path, 'a') as f:
f.write('{} {} {} {}'.format(
trainer.train_episodes_counter, test_successes,
test_cost, os.linesep))
# decide how to act next
print_and_log('old cost was {} at step {}'.format(
best_cost, best_cost_global_step))
print_and_log('current learn rates {}'.format(
network.get_learn_rate(sess)))
print_and_log('current base stds {}'.format(
network.get_base_std(sess)))
if best_cost is None or test_cost < best_cost:
print_and_log('new best cost {} at step {}'.format(
test_cost, global_step))
best_cost, best_cost_global_step = test_cost, global_step
best_curriculum_coefficient = episode_runner.curriculum_coefficient
no_test_improvement = 0
consecutive_learn_rate_decrease = 0
best_saver.save(sess, global_step)
test_trajectories_file = os.path.join(
test_trajectories_dir, '{}.txt'.format(global_step))
serialize_compress(endpoints_by_path,
test_trajectories_file)
else:
print_and_log(
'new model is not the best with cost {} at step {}'.
format(test_cost, global_step))
no_test_improvement += 1
print_and_log('no improvement count {} of {}'.format(
no_test_improvement,
decrease_learn_rate_if_static_success))
if reset_best_every > 0 and no_test_improvement % reset_best_every == reset_best_every - 1:
# restore the model every once in a while if did not find a better solution in a while
best_saver.restore(sess)
episode_runner.curriculum_coefficient = best_curriculum_coefficient
if no_test_improvement == decrease_learn_rate_if_static_success:
# restore the best model
if config['model']['restore_on_decrease']:
best_saver.restore(sess)
episode_runner.curriculum_coefficient = best_curriculum_coefficient
network.decrease_learn_rates(sess)
no_test_improvement = 0
consecutive_learn_rate_decrease += 1
print_and_log('decreasing learn rates {} of {}'.format(
consecutive_learn_rate_decrease,
stop_training_after_learn_rate_decrease))
print_and_log('new learn rates {}'.format(
network.get_learn_rate(sess)))
if consecutive_learn_rate_decrease == stop_training_after_learn_rate_decrease:
print_and_log('needs to stop')
best_saver.restore(sess)
break
if episode_runner.curriculum_coefficient is not None:
if success_ratio > config['curriculum'][
'raise_when_train_above']:
print_and_log('current curriculum coefficient {}'.format(
episode_runner.curriculum_coefficient))
episode_runner.curriculum_coefficient *= config[
'curriculum']['raise_times']
print_and_log('curriculum coefficient raised to {}'.format(
episode_runner.curriculum_coefficient))
# mark in log the end of cycle
print_and_log(os.linesep)
print_and_log('end of run best: {} from step: {}'.format(
best_cost, best_cost_global_step))
print_and_log('testing on a new set of start-goal pairs')
best_saver.restore(sess)
test_trajectories_file = os.path.join(test_trajectories_dir, '-1.txt')
endpoints_by_path = trainer.collect_data(
config['general']['test_episodes'],
is_train=False,
use_fixed_start_goal_pairs=True)[-1]
serialize_compress(endpoints_by_path, test_trajectories_file)
close_log()
return best_cost