step_rules = [RMSProp(learning_rate=learning_rate, decay_rate=decay_rate),
StepClipping(step_clipping)]
algorithm = GradientDescent(cost=cost, parameters=cg.parameters,
step_rule=CompositeRule(step_rules))
# Extensions
gradient_norm = aggregation.mean(algorithm.total_gradient_norm)
step_norm = aggregation.mean(algorithm.total_step_norm)
monitored_vars = [cost, gradient_norm, step_norm]
dev_monitor = DataStreamMonitoring(variables=[cost], after_epoch=True,
before_first_epoch=True, data_stream=dev_stream, prefix="dev")
train_monitor = TrainingDataMonitoring(variables=monitored_vars, after_batch=True,
before_first_epoch=True, prefix='tra')
extensions = [dev_monitor, train_monitor, Timing(), Printing(after_batch=True),
FinishAfter(after_n_epochs=nepochs),
saveload.Load(load_path),
saveload.Checkpoint(last_path),
] + track_best('dev_cost', save_path)
if learning_rate_decay not in (0, 1):
extensions.append(SharedVariableModifier(step_rules[0].learning_rate,
lambda n, lr: numpy.cast[theano.config.floatX](learning_rate_decay * lr), after_epoch=True, after_batch=False))
print 'number of parameters in the model: ' + str(tensor.sum([p.size for p in cg.parameters]).eval())
# Finally build the main loop and train the model
main_loop = MainLoop(data_stream=train_stream, algorithm=algorithm,
model=Model(cost), extensions=extensions)
main_loop.run()
step_rules = [RMSProp(learning_rate=learning_rate, decay_rate=decay_rate),
StepClipping(step_clipping)]
algorithm = GradientDescent(cost=cost, parameters=cg.parameters,
step_rule=CompositeRule(step_rules))
# Extensions
gradient_norm = aggregation.mean(algorithm.total_gradient_norm)
step_norm = aggregation.mean(algorithm.total_step_norm)
monitored_vars = [cost, gradient_norm, step_norm]
dev_monitor = DataStreamMonitoring(variables=[cost], after_epoch=True,
before_first_epoch=True, data_stream=dev_stream, prefix="dev")
train_monitor = TrainingDataMonitoring(variables=monitored_vars, after_batch=True,
before_first_epoch=True, prefix='tra')
extensions = [dev_monitor, train_monitor, Timing(), Printing(after_batch=True),
FinishAfter(after_n_epochs=nepochs),
saveload.Load(load_path),
saveload.Checkpoint(last_path),
] + track_best('dev_cost', save_path)
if learning_rate_decay not in (0, 1):
extensions.append(SharedVariableModifier(step_rules[0].learning_rate,
lambda n, lr: numpy.cast[theano.config.floatX](learning_rate_decay * lr), after_epoch=True, after_batch=False))
print('number of parameters in the model: ' + str(tensor.sum([p.size for p in cg.parameters]).eval()))
# Finally build the main loop and train the model
main_loop = MainLoop(data_stream=train_stream, algorithm=algorithm,
model=Model(cost), extensions=extensions)
main_loop.run()
set_train_flag = SetTrainFlag(after_epoch=True,
before_epoch=True,
flag=train_flag)
# plot = Plot('Plotting example', channels=[['cost']], after_batch=True, open_browser=True)
extensions = [
set_train_flag,
test_monitor,
train_monitor,
Timing(),
Printing(after_epoch=True),
FinishAfter(after_n_epochs=nepochs),
saveload.Load(load_path),
saveload.Checkpoint(last_path, every_n_epochs=10000),
] + track_best('test_cost', save_path) #+ track_best('train_cost', last_path)
if learning_rate_decay not in (0, 1):
extensions.append(
SharedVariableModifier(step_rules[0].learning_rate,
lambda n, lr: np.cast[theano.config.floatX]
(learning_rate_decay * lr),
after_epoch=False,
every_n_epochs=lr_decay_every_n_epochs,
after_batch=False))
print 'number of parameters in the model: ' + str(
T.sum([p.size for p in cg.parameters]).eval())
# Finally build the main loop and train the model
main_loop = MainLoop(data_stream=train_stream,
algorithm=algorithm,
dev_monitor = DataStreamMonitoring(
variables=[cost], after_epoch=True, before_first_epoch=True, data_stream=dev_stream, prefix="dev"
)
train_monitor = TrainingDataMonitoring(
variables=monitored_vars, after_batch=True, before_first_epoch=True, prefix="tra"
)
extensions = [
dev_monitor,
train_monitor,
Timing(),
Printing(after_batch=True),
FinishAfter(after_n_epochs=nepochs),
saveload.Load(load_path),
saveload.Checkpoint(last_path),
] + track_best("dev_cost", save_path)
if learning_rate_decay not in (0, 1):
extensions.append(
SharedVariableModifier(
step_rules[0].learning_rate,
lambda n, lr: numpy.cast[theano.config.floatX](learning_rate_decay * lr),
after_epoch=True,
after_batch=False,
)
)
print "number of parameters in the model: " + str(tensor.sum([p.size for p in cg.parameters]).eval())
# Finally build the main loop and train the model
main_loop = MainLoop(data_stream=train_stream, algorithm=algorithm, model=Model(cost), extensions=extensions)
main_loop.run()
prefix="test")
train_monitor = TrainingDataMonitoring(variables=monitored_vars,
after_epoch=True,
before_first_epoch=True,
prefix='train')
# plot = Plot('Plotting example', channels=[['cost']], after_batch=True, open_browser=True)
extensions = [
test_monitor,
train_monitor,
Timing(),
Printing(after_epoch=True),
FinishAfter(after_n_epochs=nepochs),
saveload.Load(load_path),
# saveload.Checkpoint(last_path,after_epoch=True),
] + track_best('test_cost', save_path)
if learning_rate_decay not in (0, 1):
extensions.append(
SharedVariableModifier(step_rules[0].learning_rate,
lambda n, lr: np.cast[theano.config.floatX]
(learning_rate_decay * lr),
after_epoch=True,
after_batch=False))
print 'number of parameters in the model: ' + str(
tensor.sum([p.size for p in cg.parameters]).eval())
# Finally build the main loop and train the model
main_loop = MainLoop(data_stream=train_stream,
algorithm=algorithm,
model=Model(cost),
prefix="test")
train_monitor = TrainingDataMonitoring(variables=monitored_vars,
after_epoch=True,
before_first_epoch=True,
prefix='train')
# plot = Plot('Plotting example', channels=[['cost']], after_batch=True, open_browser=True)
extensions = [
test_monitor,
train_monitor,
Timing(),
Printing(after_epoch=True),
FinishAfter(after_n_epochs=nepochs),
saveload.Load(load_path),
# saveload.Checkpoint(last_path,after_epoch=True),
] + track_best('test_cost', save_path[network_mode]) #test_cost
if learning_rate_decay[network_mode] not in (0, 1):
extensions.append(
SharedVariableModifier(step_rules[0].learning_rate,
lambda n, lr: np.cast[theano.config.floatX]
(learning_rate_decay[network_mode] * lr),
after_epoch=False,
every_n_epochs=100,
after_batch=False))
print 'number of parameters in the model: ' + str(
T.sum([p.size for p in cg.parameters]).eval())
# Finally build the main loop and train the model
main_loop = MainLoop(data_stream=train_stream,
algorithm=algorithm,