train_monitor = TrainingDataMonitoring(variables=[cost],
every_n_batches=n_batches,
prefix="train")
valid_monitor = DataStreamMonitoring(
[cost],
valid_stream,
after_epoch=True,
#before_first_epoch = False,
prefix="valid")
extensions = extensions = [
Timing(every_n_batches=n_batches), train_monitor, valid_monitor,
TrackTheBest('valid_nll', after_epoch=True),
Plot(save_dir + experiment_name + ".png", [['train_nll', 'valid_nll']],
every_n_batches=4 * n_batches,
email=True),
Checkpoint(save_dir + experiment_name + ".pkl",
use_cpickle=True,
every_n_batches=n_batches * 8,
after_epoch=True),
Checkpoint(save_dir + "best_" + experiment_name + ".pkl",
after_epoch=True,
use_cpickle=True).add_condition(
['after_epoch'],
predicate=OnLogRecord('valid_nll_best_so_far')),
Printing(every_n_batches=n_batches, after_epoch=True),
FinishAfter(after_n_epochs=2),
SaveComputationGraph(emit)
]
train_monitor = TrainingDataMonitoring(variables=monitoring_variables +
data_monitoring,
every_n_batches=n_batches,
prefix="train")
valid_monitor = DataStreamMonitoring(monitoring_variables,
valid_stream,
every_n_batches=n_batches,
prefix="valid")
extensions = [
ProgressBar(),
Timing(every_n_batches=n_batches), train_monitor, valid_monitor,
TrackTheBest('valid_nll', every_n_batches=n_batches),
Plot(save_dir + "progress/" + experiment_name + ".png",
[['train_nll', 'valid_nll'], ['valid_learning_rate']],
every_n_batches=n_batches,
email=False),
Checkpoint(save_dir + "pkl/best_" + experiment_name + ".pkl",
use_cpickle=True).add_condition(
['after_batch'],
predicate=OnLogRecord('valid_nll_best_so_far')),
Printing(every_n_batches=n_batches),
Flush(every_n_batches=n_batches, before_first_epoch=True),
LearningRateSchedule(lr,
'valid_nll',
path=save_dir + "pkl/best_" + experiment_name +
".pkl",
states=states.values(),
every_n_batches=n_batches,
before_first_epoch=True)
]
try:
result = math.isnan(log.current_row['train_total_gradient_norm'])
return result
except:
return False
extensions = extensions = [
train_monitor, valid_monitor,
Timing(every_n_batches=n_batches),
Printing(every_n_batches=n_batches),
WriteConditional(emit,
every_n_batches=n_batches,
save_name=save_dir + "samples/" + exp_name + ".png"),
FinishAfter().add_condition(["after_batch"], _is_nan),
ProgressBar(),
Checkpoint(save_dir + "pkl/" + exp_name + ".pkl", after_epoch=True),
SaveComputationGraph(emit),
Plot(save_dir + exp_name + ".png", [['train_nll', 'valid_nll']],
every_n_batches=5 * n_batches,
email=False)
]
main_loop = MainLoop(model=model,
data_stream=data_stream,
algorithm=algorithm,
extensions=extensions)
main_loop.run()
ipdb.set_trace()
valid_monitor = DataStreamMonitoring(
[cost],
valid_stream,
after_epoch = True,
#before_first_epoch = False,
prefix="valid")
extensions = extensions=[
Timing(every_n_batches = n_batches),
train_monitor,
valid_monitor,
TrackTheBest('valid_sequence_log_likelihood', after_epoch = True),
Plot(save_dir+experiment_name+".png",
[['train_sequence_log_likelihood',
'valid_sequence_log_likelihood']],
every_n_batches = 4*n_batches,
email=False),
Checkpoint(save_dir+experiment_name+".pkl",
use_cpickle = True,
every_n_batches = n_batches*8,
after_epoch = True),
Checkpoint(save_dir+"best_"+experiment_name+".pkl",
after_epoch = True,
use_cpickle = True
).add_condition(['after_epoch'],
predicate=OnLogRecord('valid_sequence_log_likelihood_best_so_far')),
Printing(every_n_batches = n_batches, after_epoch = True),
FinishAfter(after_n_epochs=2),
SaveComputationGraph(emit)
]
train_monitor = TrainingDataMonitoring(variables=[cost],
every_n_batches=n_batches,
prefix="train")
valid_monitor = DataStreamMonitoring(
[cost],
valid_stream,
after_epoch=True,
#before_first_epoch = False,
prefix="valid")
extensions = extensions = [
Timing(every_n_batches=n_batches), train_monitor, valid_monitor,
TrackTheBest('valid_sequence_log_likelihood', after_epoch=True),
Plot('Blizzard_l0.png',
[['train_sequence_log_likelihood', 'valid_sequence_log_likelihood']],
every_n_batches=4 * n_batches),
Checkpoint(save_dir + experiment_name + ".pkl",
use_cpickle=True,
every_n_batches=n_batches * 8),
Checkpoint(save_dir + "best_" + experiment_name + ".pkl",
after_epoch=True,
use_cpickle=True).add_condition(
['after_epoch'],
predicate=OnLogRecord(
'valid_sequence_log_likelihood_best_so_far')),
Printing(every_n_batches=n_batches)
]
main_loop = MainLoop(model=model,
data_stream=train_stream,