learning_rate = dict(
type='adaptive_change',
max_no_progress_points=1000,
decay=.5,
init=1e-3,
path_to_target_metric_storage=('default_1', 'loss')
)
env.train(
# gpu_memory=.3,
allow_growth=True,
save_path='lstm/start',
# restore_path='lstm/start/checkpoints/best',
learning_rate=learning_rate,
batch_size=BATCH_SIZE,
num_unrollings=NUM_UNROLLINGS,
vocabulary=vocabulary,
checkpoint_steps=100,
result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
# stop=stop_specs,
stop=40000,
train_dataset_text=train_text,
# train_dataset_text='abc',
validation_dataset_texts=[valid_text],
results_collect_interval=100,
additions_to_feed_dict=add_feed,
validation_additions_to_feed_dict=valid_add_feed,
no_validation=False
)
# log_device_placement=True)
one_batch_gen=True,
train_batch_kwargs=dict(valid_size=VALID_SIZE),
valid_batch_kwargs=dict(valid_size=VALID_SIZE),
)
env.train(
# gpu_memory=.3,
with_meta_optimizer=True,
restore_path=the_only_pupil_restore_path,
restore_optimizer_path=os.path.join(training_path, 'checkpoints', 'final'),
save_path=os.path.join(base, 'loss_best', 'test', 'pupil_training'),
allow_growth=True,
batch_size=BATCH_SIZE,
checkpoint_steps=None,
result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
stop=OPTIMIZER_RANGE,
# stop=4000,
train_dataset=dict(train='train'),
validation_datasets=dict(valid='validation'),
train_batch_kwargs=dict(valid_size=VALID_SIZE),
valid_batch_kwargs=dict(valid_size=VALID_SIZE),
results_collect_interval=1,
additions_to_feed_dict=opt_inf_add_feed,
validation_additions_to_feed_dict=valid_add_feed,
no_validation=False,
)
env.test(restore_path=os.path.join(base, 'loss_best', 'test', 'pupil_training',
'checkpoints/final'),
save_path=os.path.join(base, 'loss_best', 'test', 'testing'),
additions_to_feed_dict=valid_add_feed,
permute=False,
summary=True,
add_graph_to_summary=True
)
env.train(
# gpu_memory=.3,
num_unrollings=NUM_UNROLLINGS,
vocabulary=vocabulary,
with_meta_optimizer=True,
restore_path=the_only_pupil_restore_path,
restore_optimizer_path=os.path.join(training_path, 'checkpoints', 'final'),
save_path=os.path.join(base, 'loss_best', 'test', 'pupil_training'),
allow_growth=True,
batch_size=BATCH_SIZE,
checkpoint_steps=None,
result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
stop=OPTIMIZER_RANGE,
# stop=4000,
train_dataset_text=train_text,
validation_dataset_texts=[valid_text],
results_collect_interval=1,
additions_to_feed_dict=opt_inf_add_feed,
validation_additions_to_feed_dict=valid_add_feed,
no_validation=False,
)
env.test(
restore_path=os.path.join(base, 'loss_best', 'test', 'pupil_training', 'checkpoints/final'),
save_path=os.path.join(base, 'loss_best', 'test', 'testing'),
additions_to_feed_dict=valid_add_feed,
stop_specs = dict(type='while_progress',
max_no_progress_points=10,
changing_parameter_name='learning_rate',
path_to_target_metric_storage=('valid', 'loss'))
learning_rate = dict(type='adaptive_change',
max_no_progress_points=10,
decay=.5,
init=4.,
path_to_target_metric_storage=('valid', 'loss'))
env.train(
# gpu_memory=.3,
allow_growth=True,
save_path='debug_early_stop',
# restore_path='lstm_sample_test/scipop3_1000_bs256_11.12/checkpoints/2000',
learning_rate=learning_rate,
batch_size=32,
checkpoint_steps=None,
result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
stop=stop_specs,
# stop=4000,
train_dataset=dict(train='train'),
train_batch_kwargs=dict(data_dir=data_dir),
valid_batch_kwargs=dict(data_dir=data_dir),
# train_dataset_text='abc',
validation_datasets=dict(valid='validation'),
results_collect_interval=100,
additions_to_feed_dict=add_feed,
validation_additions_to_feed_dict=valid_add_feed,
no_validation=False)
# log_device_placement=True)
stop_specs = config['stop_specs']
if isinstance(stop_specs, dict):
stop_specs['changing_parameter_name'] = "learning_rate"
stop_specs['path_to_target_metric_storage'] = ["valid", "loss"]
stop_specs['type'] = "while_progress"
env.train(
allow_growth=True,
save_path=os.path.expanduser(config['save_path']),
restore_path=restore_path,
learning_rate=learning_rate,
batch_size=BATCH_SIZE,
num_unrollings=config['num_unrollings'],
vocabulary=vocabulary,
checkpoint_steps=None,
subgraphs_to_save=dict(char_enc_dec='base'),
result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
stop=stop_specs,
train_dataset_text=train_text,
validation_datasets={'valid': valid_text},
results_collect_interval=config['results_collect_interval'],
no_validation=False,
validation_batch_size=BATCH_SIZE,
valid_batch_kwargs=dict(num_unrollings=config['num_unrollings'], ),
log_launch=False,
)
if config['train']:
restore_path = os.path.join(os.path.expanduser(config['save_path']),
'checkpoints/all_vars/best')
if config['test']:
# path_to_target_metric_storage=('valid', 'loss')
# )
learning_rate = dict(
type='fixed',
value=lr,
)
tf.set_random_seed(1)
env.train(
allow_growth=True,
save_path=save_path,
result_types=['loss', 'bpc', 'perplexity', 'accuracy'],
additions_to_feed_dict=train_add_feed,
# pupil_restore_paths=['debug_empty_meta_optimizer/not_learning_issue_es20_nn20/checkpoints/0'],
stop=stop,
vocabulary=vocabulary,
num_unrollings=NUM_UNROLLINGS,
results_collect_interval=freq,
learning_rate=dict(type='adaptive_change',
max_no_progress_points=20,
decay=.5,
init=lr,
path_to_target_metric_storage=('valid', 'loss')),
# opt_inf_results_collect_interval=1,
summary=False,
add_graph_to_summary=False,
train_dataset_text=train_text,
validation_datasets=dict(valid=valid_text),
batch_size=BATCH_SIZE)
init_parameter=2.,
regime='autonomous_training',
additional_metrics=add_metrics,
going_to_limit_memory=True,
optimizer='sgd')
tf.set_random_seed(1)
env.train(
allow_growth=True,
# save_path='debug_grid_search',
result_types=['loss', 'bpc', 'perplexity', 'accuracy'],
additions_to_feed_dict=train_add_feed,
# pupil_restore_paths=['debug_empty_meta_optimizer/not_learning_issue_es20_nn20/checkpoints/0'],
# stop=stop_specs,
stop=1000,
vocabulary=vocabulary,
num_unrollings=NUM_UNROLLINGS,
results_collect_interval=500,
learning_rate=dict(
type='exponential_decay',
decay=1.,
init=7.,
period=1e+6,
),
# opt_inf_results_collect_interval=1,
summary=False,
add_graph_to_summary=False,
train_dataset_text=train_text,
validation_datasets=dict(valid=valid_text),
batch_size=BATCH_SIZE)
{'placeholder': 'dropout', 'value': .9},
dict(
placeholder='learning_rate',
value=2.
)
]
valid_add_feed = [
{'placeholder': 'dropout', 'value': 1.},
]
tf.set_random_seed(1)
env.train(
# gpu_memory=.3,
num_unrollings=NUM_UNROLLINGS,
vocabulary=vocabulary,
with_meta_optimizer=True,
allow_growth=True,
save_path='debug_empty_optimizer',
batch_size=BATCH_SIZE,
checkpoint_steps=None,
result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
stop=1000,
# stop=4000,
train_dataset_text=train_text,
validation_dataset_texts=[valid_text],
results_collect_interval=100,
additions_to_feed_dict=add_feed,
validation_additions_to_feed_dict=valid_add_feed,
no_validation=False,
)
env.train(
# gpu_memory=.3,
allow_growth=True,
# save_path='results/resrnn',
# # restore_path='results/resrnn/checkpoints/all_vars/best',
# restore_path='results/resrnn/checkpoints/best',
save_path='results/resrnn/correlation',
# restore_path='results/resrnn/back/checkpoints/best',
# restore_path=dict(
# char_enc_dec='results/resrnn/checkpoints/all_vars/best',
# ),
learning_rate=learning_rate,
lr_restore_saver_name='saver',
batch_size=BATCH_SIZE,
num_unrollings=NUM_UNROLLINGS,
vocabulary=vocabulary,
checkpoint_steps=None,
# subgraphs_to_save=dict(char_enc_dec='base'),
# subgraphs_to_save=['char_enc_dec', 'word_enc_dec'],
result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
stop=stop_specs,
# stop=40000,
train_dataset_text=train_text,
# train_dataset_text='abc',
validation_dataset_texts=[valid_text],
results_collect_interval=100,
no_validation=False,
validation_batch_size=32,
valid_batch_kwargs=dict(
num_unrollings=100,
),
add_graph_to_summary=True,
summary=True,
state_reset_period=10,
validation_tensor_schedule=valid_tensor_schedule,
additions_to_feed_dict=[{'placeholder': 'dropout', 'value': 0.99}],
validation_additions_to_feed_dict=[{'placeholder': 'dropout', 'value': 0.}],
)
learning_rate = dict(
type='adaptive_change',
max_no_progress_points=10,
decay=.5,
init=4.,
path_to_target_metric_storage=('default_1', 'loss')
)
env.train(
# gpu_memory=.3,
allow_growth=True,
save_path='lstm/start',
# restore_path='lstm_sample_test/scipop3_1000_bs256_11.12/checkpoints/2000',
learning_rate=learning_rate,
batch_size=32,
num_unrollings=10,
vocabulary=vocabulary,
checkpoint_steps=None,
result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
stop=stop_specs,
# stop=4000,
train_dataset_text=train_text,
# train_dataset_text='abc',
validation_dataset_texts=[valid_text],
results_collect_interval=100,
additions_to_feed_dict=add_feed,
validation_additions_to_feed_dict=valid_add_feed,
no_validation=False
)
# log_device_placement=True)
'training')
env.train(
allow_growth=True,
# save_path='debug_grid_search',
result_types=['loss', 'bpc', 'perplexity', 'accuracy'],
additions_to_feed_dict=train_add_feed,
# pupil_restore_paths=['debug_empty_meta_optimizer/not_learning_issue_es20_nn20/checkpoints/0'],
# stop=stop_specs,
save_path=training_path,
restore_path=RESTORE_PATH,
stop=1000,
results_collect_interval=1000,
summary=False,
add_graph_to_summary=False,
train_dataset=dict(train='train'),
train_batch_kwargs=dict(valid_size=VALID_SIZE),
valid_batch_kwargs=dict(valid_size=VALID_SIZE),
# train_dataset_text='abc',
validation_datasets=dict(valid='validation'),
learning_rate=dict(
init=best_conf['learning_rate/init'],
decay=1.,
period=1e+6,
type='exponential_decay',
),
batch_size=BATCH_SIZE,
no_validation=True,
)
env.test(
additional_metrics=add_metrics,
going_to_limit_memory=True,
optimizer='sgd')
env.train(
allow_growth=True,
restore_path=os.path.join(*(['..'] * ROOT_HEIGHT +
['lstm', 'start', 'checkpoints', 'start'])),
save_path='after_1_step',
result_types=['loss', 'bpc', 'perplexity', 'accuracy'],
additions_to_feed_dict=train_add_feed,
# pupil_restore_paths=['debug_empty_meta_optimizer/not_learning_issue_es20_nn20/checkpoints/0'],
# stop=stop_specs,
stop=0,
vocabulary=vocabulary,
num_unrollings=NUM_UNROLLINGS,
results_collect_interval=RESULTS_COLLECT_INTERVAL,
learning_rate=dict(
type='exponential_decay',
decay=1.,
init=LEARNING_RATE,
period=1e+6,
),
# opt_inf_results_collect_interval=1,
summary=False,
add_graph_to_summary=False,
train_dataset_text=train_text,
validation_datasets=dict(valid=valid_text),
batch_size=BATCH_SIZE)
env.train(
allow_growth=True,
restore_path='after_1_step/checkpoints/final',
dict(placeholder='learning_rate', value=4.),
dict(placeholder='chi_contribution', value=.01)]
valid_add_feed = [
{
'placeholder': 'dropout',
'value': 1.
},
]
tf.set_random_seed(1)
env.train(
# gpu_memory=.3,
allow_growth=True,
save_path='debug_early_stop',
with_meta_optimizer=True,
# restore_path='lstm_sample_test/scipop3_1000_bs256_11.12/checkpoints/2000',
batch_size=BATCH_SIZE,
checkpoint_steps=None,
result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
stop=1000,
train_dataset=dict(train='train'),
train_batch_kwargs=dict(valid_size=VALID_SIZE),
valid_batch_kwargs=dict(valid_size=VALID_SIZE),
# train_dataset_text='abc',
validation_datasets=dict(valid='validation'),
results_collect_interval=100,
additions_to_feed_dict=add_feed,
validation_additions_to_feed_dict=valid_add_feed,
no_validation=False)
env.train(
# gpu_memory=.3,
allow_growth=True,
save_path=save_path,
# restore_path='lstm_sample_test/scipop3_1000_bs256_11.12/checkpoints/2000',
learning_rate=learning_rate,
batch_size=BATCH_SIZE,
checkpoint_steps=None,
result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
printed_result_types=['perplexity', 'loss', 'bpc', 'accuracy'],
stop=stop,
# stop=2000,
train_dataset=dict(
train='train'
),
train_batch_kwargs=dict(
valid_size=VALID_SIZE
),
valid_batch_kwargs=dict(
valid_size=VALID_SIZE
),
# train_dataset_text='abc',
validation_datasets=dict(
valid='validation'
),
results_collect_interval=freq,
additions_to_feed_dict=add_feed,
validation_additions_to_feed_dict=valid_add_feed,
no_validation=False,
summary=False,
add_graph_to_summary=False,
)
