learning_rate = dict( type='adaptive_change', max_no_progress_points=1000, decay=.5, init=1e-3, path_to_target_metric_storage=('default_1', 'loss') ) env.train( # gpu_memory=.3, allow_growth=True, save_path='lstm/start', # restore_path='lstm/start/checkpoints/best', learning_rate=learning_rate, batch_size=BATCH_SIZE, num_unrollings=NUM_UNROLLINGS, vocabulary=vocabulary, checkpoint_steps=100, result_types=['perplexity', 'loss', 'bpc', 'accuracy'], printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'], # stop=stop_specs, stop=40000, train_dataset_text=train_text, # train_dataset_text='abc', validation_dataset_texts=[valid_text], results_collect_interval=100, additions_to_feed_dict=add_feed, validation_additions_to_feed_dict=valid_add_feed, no_validation=False ) # log_device_placement=True)
one_batch_gen=True, train_batch_kwargs=dict(valid_size=VALID_SIZE), valid_batch_kwargs=dict(valid_size=VALID_SIZE), ) env.train( # gpu_memory=.3, with_meta_optimizer=True, restore_path=the_only_pupil_restore_path, restore_optimizer_path=os.path.join(training_path, 'checkpoints', 'final'), save_path=os.path.join(base, 'loss_best', 'test', 'pupil_training'), allow_growth=True, batch_size=BATCH_SIZE, checkpoint_steps=None, result_types=['perplexity', 'loss', 'bpc', 'accuracy'], printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'], stop=OPTIMIZER_RANGE, # stop=4000, train_dataset=dict(train='train'), validation_datasets=dict(valid='validation'), train_batch_kwargs=dict(valid_size=VALID_SIZE), valid_batch_kwargs=dict(valid_size=VALID_SIZE), results_collect_interval=1, additions_to_feed_dict=opt_inf_add_feed, validation_additions_to_feed_dict=valid_add_feed, no_validation=False, ) env.test(restore_path=os.path.join(base, 'loss_best', 'test', 'pupil_training', 'checkpoints/final'), save_path=os.path.join(base, 'loss_best', 'test', 'testing'), additions_to_feed_dict=valid_add_feed,
permute=False, summary=True, add_graph_to_summary=True ) env.train( # gpu_memory=.3, num_unrollings=NUM_UNROLLINGS, vocabulary=vocabulary, with_meta_optimizer=True, restore_path=the_only_pupil_restore_path, restore_optimizer_path=os.path.join(training_path, 'checkpoints', 'final'), save_path=os.path.join(base, 'loss_best', 'test', 'pupil_training'), allow_growth=True, batch_size=BATCH_SIZE, checkpoint_steps=None, result_types=['perplexity', 'loss', 'bpc', 'accuracy'], printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'], stop=OPTIMIZER_RANGE, # stop=4000, train_dataset_text=train_text, validation_dataset_texts=[valid_text], results_collect_interval=1, additions_to_feed_dict=opt_inf_add_feed, validation_additions_to_feed_dict=valid_add_feed, no_validation=False, ) env.test( restore_path=os.path.join(base, 'loss_best', 'test', 'pupil_training', 'checkpoints/final'), save_path=os.path.join(base, 'loss_best', 'test', 'testing'), additions_to_feed_dict=valid_add_feed,
stop_specs = dict(type='while_progress', max_no_progress_points=10, changing_parameter_name='learning_rate', path_to_target_metric_storage=('valid', 'loss')) learning_rate = dict(type='adaptive_change', max_no_progress_points=10, decay=.5, init=4., path_to_target_metric_storage=('valid', 'loss')) env.train( # gpu_memory=.3, allow_growth=True, save_path='debug_early_stop', # restore_path='lstm_sample_test/scipop3_1000_bs256_11.12/checkpoints/2000', learning_rate=learning_rate, batch_size=32, checkpoint_steps=None, result_types=['perplexity', 'loss', 'bpc', 'accuracy'], printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'], stop=stop_specs, # stop=4000, train_dataset=dict(train='train'), train_batch_kwargs=dict(data_dir=data_dir), valid_batch_kwargs=dict(data_dir=data_dir), # train_dataset_text='abc', validation_datasets=dict(valid='validation'), results_collect_interval=100, additions_to_feed_dict=add_feed, validation_additions_to_feed_dict=valid_add_feed, no_validation=False) # log_device_placement=True)
stop_specs = config['stop_specs'] if isinstance(stop_specs, dict): stop_specs['changing_parameter_name'] = "learning_rate" stop_specs['path_to_target_metric_storage'] = ["valid", "loss"] stop_specs['type'] = "while_progress" env.train( allow_growth=True, save_path=os.path.expanduser(config['save_path']), restore_path=restore_path, learning_rate=learning_rate, batch_size=BATCH_SIZE, num_unrollings=config['num_unrollings'], vocabulary=vocabulary, checkpoint_steps=None, subgraphs_to_save=dict(char_enc_dec='base'), result_types=['perplexity', 'loss', 'bpc', 'accuracy'], printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'], stop=stop_specs, train_dataset_text=train_text, validation_datasets={'valid': valid_text}, results_collect_interval=config['results_collect_interval'], no_validation=False, validation_batch_size=BATCH_SIZE, valid_batch_kwargs=dict(num_unrollings=config['num_unrollings'], ), log_launch=False, ) if config['train']: restore_path = os.path.join(os.path.expanduser(config['save_path']), 'checkpoints/all_vars/best') if config['test']:
# path_to_target_metric_storage=('valid', 'loss') # ) learning_rate = dict( type='fixed', value=lr, ) tf.set_random_seed(1) env.train( allow_growth=True, save_path=save_path, result_types=['loss', 'bpc', 'perplexity', 'accuracy'], additions_to_feed_dict=train_add_feed, # pupil_restore_paths=['debug_empty_meta_optimizer/not_learning_issue_es20_nn20/checkpoints/0'], stop=stop, vocabulary=vocabulary, num_unrollings=NUM_UNROLLINGS, results_collect_interval=freq, learning_rate=dict(type='adaptive_change', max_no_progress_points=20, decay=.5, init=lr, path_to_target_metric_storage=('valid', 'loss')), # opt_inf_results_collect_interval=1, summary=False, add_graph_to_summary=False, train_dataset_text=train_text, validation_datasets=dict(valid=valid_text), batch_size=BATCH_SIZE)
init_parameter=2., regime='autonomous_training', additional_metrics=add_metrics, going_to_limit_memory=True, optimizer='sgd') tf.set_random_seed(1) env.train( allow_growth=True, # save_path='debug_grid_search', result_types=['loss', 'bpc', 'perplexity', 'accuracy'], additions_to_feed_dict=train_add_feed, # pupil_restore_paths=['debug_empty_meta_optimizer/not_learning_issue_es20_nn20/checkpoints/0'], # stop=stop_specs, stop=1000, vocabulary=vocabulary, num_unrollings=NUM_UNROLLINGS, results_collect_interval=500, learning_rate=dict( type='exponential_decay', decay=1., init=7., period=1e+6, ), # opt_inf_results_collect_interval=1, summary=False, add_graph_to_summary=False, train_dataset_text=train_text, validation_datasets=dict(valid=valid_text), batch_size=BATCH_SIZE)
{'placeholder': 'dropout', 'value': .9}, dict( placeholder='learning_rate', value=2. ) ] valid_add_feed = [ {'placeholder': 'dropout', 'value': 1.}, ] tf.set_random_seed(1) env.train( # gpu_memory=.3, num_unrollings=NUM_UNROLLINGS, vocabulary=vocabulary, with_meta_optimizer=True, allow_growth=True, save_path='debug_empty_optimizer', batch_size=BATCH_SIZE, checkpoint_steps=None, result_types=['perplexity', 'loss', 'bpc', 'accuracy'], printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'], stop=1000, # stop=4000, train_dataset_text=train_text, validation_dataset_texts=[valid_text], results_collect_interval=100, additions_to_feed_dict=add_feed, validation_additions_to_feed_dict=valid_add_feed, no_validation=False, )
env.train( # gpu_memory=.3, allow_growth=True, # save_path='results/resrnn', # # restore_path='results/resrnn/checkpoints/all_vars/best', # restore_path='results/resrnn/checkpoints/best', save_path='results/resrnn/correlation', # restore_path='results/resrnn/back/checkpoints/best', # restore_path=dict( # char_enc_dec='results/resrnn/checkpoints/all_vars/best', # ), learning_rate=learning_rate, lr_restore_saver_name='saver', batch_size=BATCH_SIZE, num_unrollings=NUM_UNROLLINGS, vocabulary=vocabulary, checkpoint_steps=None, # subgraphs_to_save=dict(char_enc_dec='base'), # subgraphs_to_save=['char_enc_dec', 'word_enc_dec'], result_types=['perplexity', 'loss', 'bpc', 'accuracy'], printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'], stop=stop_specs, # stop=40000, train_dataset_text=train_text, # train_dataset_text='abc', validation_dataset_texts=[valid_text], results_collect_interval=100, no_validation=False, validation_batch_size=32, valid_batch_kwargs=dict( num_unrollings=100, ), add_graph_to_summary=True, summary=True, state_reset_period=10, validation_tensor_schedule=valid_tensor_schedule, additions_to_feed_dict=[{'placeholder': 'dropout', 'value': 0.99}], validation_additions_to_feed_dict=[{'placeholder': 'dropout', 'value': 0.}], )
learning_rate = dict( type='adaptive_change', max_no_progress_points=10, decay=.5, init=4., path_to_target_metric_storage=('default_1', 'loss') ) env.train( # gpu_memory=.3, allow_growth=True, save_path='lstm/start', # restore_path='lstm_sample_test/scipop3_1000_bs256_11.12/checkpoints/2000', learning_rate=learning_rate, batch_size=32, num_unrollings=10, vocabulary=vocabulary, checkpoint_steps=None, result_types=['perplexity', 'loss', 'bpc', 'accuracy'], printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'], stop=stop_specs, # stop=4000, train_dataset_text=train_text, # train_dataset_text='abc', validation_dataset_texts=[valid_text], results_collect_interval=100, additions_to_feed_dict=add_feed, validation_additions_to_feed_dict=valid_add_feed, no_validation=False ) # log_device_placement=True)
'training') env.train( allow_growth=True, # save_path='debug_grid_search', result_types=['loss', 'bpc', 'perplexity', 'accuracy'], additions_to_feed_dict=train_add_feed, # pupil_restore_paths=['debug_empty_meta_optimizer/not_learning_issue_es20_nn20/checkpoints/0'], # stop=stop_specs, save_path=training_path, restore_path=RESTORE_PATH, stop=1000, results_collect_interval=1000, summary=False, add_graph_to_summary=False, train_dataset=dict(train='train'), train_batch_kwargs=dict(valid_size=VALID_SIZE), valid_batch_kwargs=dict(valid_size=VALID_SIZE), # train_dataset_text='abc', validation_datasets=dict(valid='validation'), learning_rate=dict( init=best_conf['learning_rate/init'], decay=1., period=1e+6, type='exponential_decay', ), batch_size=BATCH_SIZE, no_validation=True, ) env.test(
additional_metrics=add_metrics, going_to_limit_memory=True, optimizer='sgd') env.train( allow_growth=True, restore_path=os.path.join(*(['..'] * ROOT_HEIGHT + ['lstm', 'start', 'checkpoints', 'start'])), save_path='after_1_step', result_types=['loss', 'bpc', 'perplexity', 'accuracy'], additions_to_feed_dict=train_add_feed, # pupil_restore_paths=['debug_empty_meta_optimizer/not_learning_issue_es20_nn20/checkpoints/0'], # stop=stop_specs, stop=0, vocabulary=vocabulary, num_unrollings=NUM_UNROLLINGS, results_collect_interval=RESULTS_COLLECT_INTERVAL, learning_rate=dict( type='exponential_decay', decay=1., init=LEARNING_RATE, period=1e+6, ), # opt_inf_results_collect_interval=1, summary=False, add_graph_to_summary=False, train_dataset_text=train_text, validation_datasets=dict(valid=valid_text), batch_size=BATCH_SIZE) env.train( allow_growth=True, restore_path='after_1_step/checkpoints/final',
dict(placeholder='learning_rate', value=4.), dict(placeholder='chi_contribution', value=.01)] valid_add_feed = [ { 'placeholder': 'dropout', 'value': 1. }, ] tf.set_random_seed(1) env.train( # gpu_memory=.3, allow_growth=True, save_path='debug_early_stop', with_meta_optimizer=True, # restore_path='lstm_sample_test/scipop3_1000_bs256_11.12/checkpoints/2000', batch_size=BATCH_SIZE, checkpoint_steps=None, result_types=['perplexity', 'loss', 'bpc', 'accuracy'], printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'], stop=1000, train_dataset=dict(train='train'), train_batch_kwargs=dict(valid_size=VALID_SIZE), valid_batch_kwargs=dict(valid_size=VALID_SIZE), # train_dataset_text='abc', validation_datasets=dict(valid='validation'), results_collect_interval=100, additions_to_feed_dict=add_feed, validation_additions_to_feed_dict=valid_add_feed, no_validation=False)
env.train( # gpu_memory=.3, allow_growth=True, save_path=save_path, # restore_path='lstm_sample_test/scipop3_1000_bs256_11.12/checkpoints/2000', learning_rate=learning_rate, batch_size=BATCH_SIZE, checkpoint_steps=None, result_types=['perplexity', 'loss', 'bpc', 'accuracy'], printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'], stop=stop, # stop=2000, train_dataset=dict( train='train' ), train_batch_kwargs=dict( valid_size=VALID_SIZE ), valid_batch_kwargs=dict( valid_size=VALID_SIZE ), # train_dataset_text='abc', validation_datasets=dict( valid='validation' ), results_collect_interval=freq, additions_to_feed_dict=add_feed, validation_additions_to_feed_dict=valid_add_feed, no_validation=False, summary=False, add_graph_to_summary=False, )