train_monitor = TrainingDataMonitoring(variables=[cost], every_n_batches=n_batches, prefix="train") valid_monitor = DataStreamMonitoring( [cost], valid_stream, after_epoch=True, #before_first_epoch = False, prefix="valid") extensions = extensions = [ Timing(every_n_batches=n_batches), train_monitor, valid_monitor, TrackTheBest('valid_nll', after_epoch=True), Plot(save_dir + experiment_name + ".png", [['train_nll', 'valid_nll']], every_n_batches=4 * n_batches, email=True), Checkpoint(save_dir + experiment_name + ".pkl", use_cpickle=True, every_n_batches=n_batches * 8, after_epoch=True), Checkpoint(save_dir + "best_" + experiment_name + ".pkl", after_epoch=True, use_cpickle=True).add_condition( ['after_epoch'], predicate=OnLogRecord('valid_nll_best_so_far')), Printing(every_n_batches=n_batches, after_epoch=True), FinishAfter(after_n_epochs=2), SaveComputationGraph(emit) ]
train_monitor = TrainingDataMonitoring(variables=monitoring_variables + data_monitoring, every_n_batches=n_batches, prefix="train") valid_monitor = DataStreamMonitoring(monitoring_variables, valid_stream, every_n_batches=n_batches, prefix="valid") extensions = [ ProgressBar(), Timing(every_n_batches=n_batches), train_monitor, valid_monitor, TrackTheBest('valid_nll', every_n_batches=n_batches), Plot(save_dir + "progress/" + experiment_name + ".png", [['train_nll', 'valid_nll'], ['valid_learning_rate']], every_n_batches=n_batches, email=False), Checkpoint(save_dir + "pkl/best_" + experiment_name + ".pkl", use_cpickle=True).add_condition( ['after_batch'], predicate=OnLogRecord('valid_nll_best_so_far')), Printing(every_n_batches=n_batches), Flush(every_n_batches=n_batches, before_first_epoch=True), LearningRateSchedule(lr, 'valid_nll', path=save_dir + "pkl/best_" + experiment_name + ".pkl", states=states.values(), every_n_batches=n_batches, before_first_epoch=True) ]
try: result = math.isnan(log.current_row['train_total_gradient_norm']) return result except: return False extensions = extensions = [ train_monitor, valid_monitor, Timing(every_n_batches=n_batches), Printing(every_n_batches=n_batches), WriteConditional(emit, every_n_batches=n_batches, save_name=save_dir + "samples/" + exp_name + ".png"), FinishAfter().add_condition(["after_batch"], _is_nan), ProgressBar(), Checkpoint(save_dir + "pkl/" + exp_name + ".pkl", after_epoch=True), SaveComputationGraph(emit), Plot(save_dir + exp_name + ".png", [['train_nll', 'valid_nll']], every_n_batches=5 * n_batches, email=False) ] main_loop = MainLoop(model=model, data_stream=data_stream, algorithm=algorithm, extensions=extensions) main_loop.run() ipdb.set_trace()
valid_monitor = DataStreamMonitoring( [cost], valid_stream, after_epoch = True, #before_first_epoch = False, prefix="valid") extensions = extensions=[ Timing(every_n_batches = n_batches), train_monitor, valid_monitor, TrackTheBest('valid_sequence_log_likelihood', after_epoch = True), Plot(save_dir+experiment_name+".png", [['train_sequence_log_likelihood', 'valid_sequence_log_likelihood']], every_n_batches = 4*n_batches, email=False), Checkpoint(save_dir+experiment_name+".pkl", use_cpickle = True, every_n_batches = n_batches*8, after_epoch = True), Checkpoint(save_dir+"best_"+experiment_name+".pkl", after_epoch = True, use_cpickle = True ).add_condition(['after_epoch'], predicate=OnLogRecord('valid_sequence_log_likelihood_best_so_far')), Printing(every_n_batches = n_batches, after_epoch = True), FinishAfter(after_n_epochs=2), SaveComputationGraph(emit) ]
train_monitor = TrainingDataMonitoring(variables=[cost], every_n_batches=n_batches, prefix="train") valid_monitor = DataStreamMonitoring( [cost], valid_stream, after_epoch=True, #before_first_epoch = False, prefix="valid") extensions = extensions = [ Timing(every_n_batches=n_batches), train_monitor, valid_monitor, TrackTheBest('valid_sequence_log_likelihood', after_epoch=True), Plot('Blizzard_l0.png', [['train_sequence_log_likelihood', 'valid_sequence_log_likelihood']], every_n_batches=4 * n_batches), Checkpoint(save_dir + experiment_name + ".pkl", use_cpickle=True, every_n_batches=n_batches * 8), Checkpoint(save_dir + "best_" + experiment_name + ".pkl", after_epoch=True, use_cpickle=True).add_condition( ['after_epoch'], predicate=OnLogRecord( 'valid_sequence_log_likelihood_best_so_far')), Printing(every_n_batches=n_batches) ] main_loop = MainLoop(model=model, data_stream=train_stream,