def prepare_opti(cost, test):
model = Model(cost)
algorithm = GradientDescent(
cost=cost,
parameters=model.parameters,
step_rule=RMSProp(),
on_unused_sources='ignore'
)
extensions = [
FinishAfter(after_n_epochs=nb_epoch),
FinishIfNoImprovementAfter(notification_name='test_cross_entropy', epochs=patience),
TrainingDataMonitoring(
[algorithm.cost],
prefix="train",
after_epoch=True),
DataStreamMonitoring(
[algorithm.cost],
test_stream,
prefix="test"),
Printing(),
ProgressBar(),
#Checkpoint(path, after_epoch=True)
]
if resume:
print "Restoring from previous breakpoint"
extensions.extend([
Load(path)
])
return model, algorithm, extensions
def prepare_opti(cost, test, *args):
model = Model(cost)
logger.info("Model created")
algorithm = GradientDescent(cost=cost,
parameters=model.parameters,
step_rule=Adam(learning_rate=0.0015),
on_unused_sources='ignore')
to_monitor = [algorithm.cost]
if args:
to_monitor.extend(args)
extensions = [
FinishAfter(after_n_epochs=nb_epoch),
FinishIfNoImprovementAfter(notification_name='loglikelihood_nat',
epochs=patience),
TrainingDataMonitoring(to_monitor, prefix="train", after_epoch=True),
DataStreamMonitoring(to_monitor, test_stream, prefix="test"),
Printing(),
ProgressBar(),
ApplyMask(before_first_epoch=True, after_batch=True),
Checkpoint(check, every_n_epochs=save_every),
SaveModel(name=path + '/' + 'pixelcnn_{}'.format(dataset),
every_n_epochs=save_every),
GenerateSamples(every_n_epochs=save_every),
#Checkpoint(path+'/'+'exp.log', save_separately=['log'],every_n_epochs=save_every),
]
if resume:
logger.info("Restoring from previous checkpoint")
extensions = [Load(path + '/' + check)]
return model, algorithm, extensions
def prepare_opti(cost, test):
model = Model(cost)
algorithm = GradientDescent(cost=cost,
parameters=model.parameters,
step_rule=Adam(),
on_unused_sources='ignore')
extensions = [
FinishAfter(after_n_epochs=nb_epoch),
FinishIfNoImprovementAfter(notification_name='test_vae_cost',
epochs=patience),
TrainingDataMonitoring([algorithm.cost], after_epoch=True),
DataStreamMonitoring([algorithm.cost], test, prefix="test"),
Printing(),
ProgressBar(),
#SaveModel(name='vae', after_n_epochs=save_every)
]
return model, algorithm, extensions
def test_finish_if_no_improvement_after_iterations(self):
ext = FinishIfNoImprovementAfter('bananas', iterations=3)
self.check_finish_if_no_improvement_after(ext, 'bananas')
def test_finish_if_no_improvement_after_epochs_log_record_specified(self):
ext = FinishIfNoImprovementAfter('melons',
patience_log_record='blueberries',
iterations=3)
self.check_finish_if_no_improvement_after(ext, 'melons', 'blueberries')
variables=[cost],
data_stream=test_stream,
prefix='test'
)
plotting = Plot('AdniNet_{}'.format(side),
channels=[
['entropy', 'validation_entropy'],
['error', 'validation_error'],
],
after_batch=False)
# The main loop will train the network and output reports, etc
stamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d-%H:%M')
main = MainLoop(
data_stream=training_stream,
model=autoencoder,
algorithm=algo,
extensions=[
FinishAfter(after_n_epochs=max_iter),
FinishIfNoImprovementAfter(notification_name='validation_error', epochs=3),
Printing(),
validation_monitor,
training_monitor,
test_monitor,
plotting,
Checkpoint('./models/{}'.format(side, stamp))
])
main.run()
def main(job_id, params):
config = ConfigParser.ConfigParser()
config.readfp(open('./params'))
max_epoch = int(config.get('hyperparams', 'max_iter', 100))
base_lr = float(config.get('hyperparams', 'base_lr', 0.01))
train_batch = int(config.get('hyperparams', 'train_batch', 256))
valid_batch = int(config.get('hyperparams', 'valid_batch', 512))
test_batch = int(config.get('hyperparams', 'valid_batch', 512))
hidden_units = int(config.get('hyperparams', 'hidden_units', 16))
W_sd = float(config.get('hyperparams', 'W_sd', 0.01))
W_mu = float(config.get('hyperparams', 'W_mu', 0.0))
b_sd = float(config.get('hyperparams', 'b_sd', 0.01))
b_mu = float(config.get('hyperparams', 'b_mu', 0.0))
dropout_ratio = float(config.get('hyperparams', 'dropout_ratio', 0.2))
weight_decay = float(config.get('hyperparams', 'weight_decay', 0.001))
max_norm = float(config.get('hyperparams', 'max_norm', 100.0))
solver = config.get('hyperparams', 'solver_type', 'rmsprop')
data_file = config.get('hyperparams', 'data_file')
fine_tune = config.getboolean('hyperparams', 'fine_tune')
# Spearmint optimization parameters:
if params:
base_lr = float(params['base_lr'][0])
dropout_ratio = float(params['dropout_ratio'][0])
hidden_units = params['hidden_units'][0]
weight_decay = params['weight_decay'][0]
if 'adagrad' in solver:
solver_type = CompositeRule([
AdaGrad(learning_rate=base_lr),
VariableClipping(threshold=max_norm)
])
else:
solver_type = CompositeRule([
RMSProp(learning_rate=base_lr),
VariableClipping(threshold=max_norm)
])
rn_file = '/projects/francisco/repositories/NI-ML/models/deepnets/blocks/ff/models/rnet/2015-06-25-18:13'
ln_file = '/projects/francisco/repositories/NI-ML/models/deepnets/blocks/ff/models/lnet/2015-06-29-11:45'
right_dim = 10519
left_dim = 11427
train = H5PYDataset(data_file, which_set='train')
valid = H5PYDataset(data_file, which_set='valid')
test = H5PYDataset(data_file, which_set='test')
l_x = tensor.matrix('l_features')
r_x = tensor.matrix('r_features')
y = tensor.lmatrix('targets')
lnet = load(ln_file).model.get_top_bricks()[0]
rnet = load(rn_file).model.get_top_bricks()[0]
# Pre-trained layers:
# Inputs -> hidden_1 -> hidden 2
for side, net in zip(['l', 'r'], [lnet, rnet]):
for child in net.children:
child.name = side + '_' + child.name
ll1 = lnet.children[0]
lr1 = lnet.children[1]
ll2 = lnet.children[2]
lr2 = lnet.children[3]
rl1 = rnet.children[0]
rr1 = rnet.children[1]
rl2 = rnet.children[2]
rr2 = rnet.children[3]
l_h = lr2.apply(ll2.apply(lr1.apply(ll1.apply(l_x))))
r_h = rr2.apply(rl2.apply(rr1.apply(rl1.apply(r_x))))
input_dim = ll2.output_dim + rl2.output_dim
# hidden_2 -> hidden_3 -> hidden_4 -> Logistic output
output_mlp = MLP(activations=[
Rectifier(name='h3'),
Rectifier(name='h4'),
Softmax(name='output'),
],
dims=[
input_dim,
hidden_units,
hidden_units,
2,
],
weights_init=IsotropicGaussian(std=W_sd, mean=W_mu),
biases_init=IsotropicGaussian(std=W_sd, mean=W_mu))
output_mlp.initialize()
# # Concatenate the inputs from the two hidden subnets into a single variable
# # for input into the next layer.
merge = tensor.concatenate([l_h, r_h], axis=1)
#
y_hat = output_mlp.apply(merge)
# Define a cost function to optimize, and a classification error rate.
# Also apply the outputs from the net and corresponding targets:
cost = CategoricalCrossEntropy().apply(y.flatten(), y_hat)
error = MisclassificationRate().apply(y.flatten(), y_hat)
error.name = 'error'
# This is the model: before applying dropout
model = Model(cost)
# Need to define the computation graph for the cost func:
cost_graph = ComputationGraph([cost])
# This returns a list of weight vectors for each layer
W = VariableFilter(roles=[WEIGHT])(cost_graph.variables)
# Add some regularization to this model:
cost += weight_decay * l2_norm(W)
cost.name = 'entropy'
# computational graph with l2 reg
cost_graph = ComputationGraph([cost])
# Apply dropout to inputs:
inputs = VariableFilter([INPUT])(cost_graph.variables)
dropout_inputs = [
input for input in inputs if input.name.startswith('linear_')
]
dropout_graph = apply_dropout(cost_graph, [dropout_inputs[0]], 0.2)
dropout_graph = apply_dropout(dropout_graph, dropout_inputs[1:],
dropout_ratio)
dropout_cost = dropout_graph.outputs[0]
dropout_cost.name = 'dropout_entropy'
# If no fine-tuning of l-r models is wanted, find the params for only
# the joint layers:
if fine_tune:
params_to_update = dropout_graph.parameters
else:
params_to_update = VariableFilter(
[PARAMETER], bricks=output_mlp.children)(cost_graph)
# Learning Algorithm:
algo = GradientDescent(step_rule=solver_type,
params=params_to_update,
cost=dropout_cost)
# algo.step_rule.learning_rate.name = 'learning_rate'
# Data stream used for training model:
training_stream = Flatten(
DataStream.default_stream(dataset=train,
iteration_scheme=ShuffledScheme(
train.num_examples,
batch_size=train_batch)))
training_monitor = TrainingDataMonitoring([
dropout_cost,
aggregation.mean(error),
aggregation.mean(algo.total_gradient_norm)
],
after_batch=True)
# Use the 'valid' set for validation during training:
validation_stream = Flatten(
DataStream.default_stream(dataset=valid,
iteration_scheme=ShuffledScheme(
valid.num_examples,
batch_size=valid_batch)))
validation_monitor = DataStreamMonitoring(variables=[cost, error],
data_stream=validation_stream,
prefix='validation',
after_epoch=True)
test_stream = Flatten(
DataStream.default_stream(
dataset=test,
iteration_scheme=ShuffledScheme(test.num_examples,
batch_size=test_batch)))
test_monitor = DataStreamMonitoring(variables=[error],
data_stream=test_stream,
prefix='test',
after_training=True)
plotting = Plot(
'AdniNet_LeftRight',
channels=[
['dropout_entropy'],
['error', 'validation_error'],
],
)
# Checkpoint class used to save model and log:
stamp = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d-%H:%M')
checkpoint = Checkpoint('./models/{}'.format(stamp),
save_separately=['model', 'log'],
every_n_epochs=1)
# The main loop will train the network and output reports, etc
main_loop = MainLoop(data_stream=training_stream,
model=model,
algorithm=algo,
extensions=[
validation_monitor,
training_monitor,
plotting,
FinishAfter(after_n_epochs=max_epoch),
FinishIfNoImprovementAfter(
notification_name='validation_error',
epochs=1),
Printing(),
ProgressBar(),
checkpoint,
test_monitor,
])
main_loop.run()
ve = float(main_loop.log.last_epoch_row['validation_error'])
te = float(main_loop.log.last_epoch_row['error'])
spearmint_loss = ve + abs(te - ve)
print 'Spearmint Loss: {}'.format(spearmint_loss)
return spearmint_loss
def main(args):
"""Run experiment. """
lr_tag = float_tag(args.learning_rate)
x_dim, train_stream, valid_stream, test_stream = datasets.get_streams(
args.data, args.batch_size)
#------------------------------------------------------------
# Setup model
deterministic_act = Tanh
deterministic_size = 1.
if args.method == 'vae':
sizes_tag = args.layer_spec.replace(",", "-")
layer_sizes = [int(i) for i in args.layer_spec.split(",")]
layer_sizes, z_dim = layer_sizes[:-1], layer_sizes[-1]
name = "%s-%s-%s-lr%s-spl%d-%s" % \
(args.data, args.method, args.name, lr_tag, args.n_samples, sizes_tag)
if args.activation == "tanh":
hidden_act = Tanh()
elif args.activation == "logistic":
hidden_act = Logistic()
elif args.activation == "relu":
hidden_act = Rectifier()
else:
raise "Unknown hidden nonlinearity %s" % args.hidden_act
model = VAE(x_dim=x_dim,
hidden_layers=layer_sizes,
hidden_act=hidden_act,
z_dim=z_dim,
batch_norm=args.batch_normalization)
model.initialize()
elif args.method == 'dvae':
sizes_tag = args.layer_spec.replace(",", "-")
layer_sizes = [int(i) for i in args.layer_spec.split(",")]
layer_sizes, z_dim = layer_sizes[:-1], layer_sizes[-1]
name = "%s-%s-%s-lr%s-spl%d-%s" % \
(args.data, args.method, args.name, lr_tag, args.n_samples, sizes_tag)
if args.activation == "tanh":
hidden_act = Tanh()
elif args.activation == "logistic":
hidden_act = Logistic()
elif args.activation == "relu":
hidden_act = Rectifier()
else:
raise "Unknown hidden nonlinearity %s" % args.hidden_act
model = DVAE(x_dim=x_dim,
hidden_layers=layer_sizes,
hidden_act=hidden_act,
z_dim=z_dim,
batch_norm=args.batch_normalization)
model.initialize()
elif args.method == 'rws':
sizes_tag = args.layer_spec.replace(",", "-")
qbase = "" if not args.no_qbaseline else "noqb-"
name = "%s-%s-%s-%slr%s-dl%d-spl%d-%s" % \
(args.data, args.method, args.name, qbase, lr_tag, args.deterministic_layers, args.n_samples, sizes_tag)
p_layers, q_layers = create_layers(args.layer_spec, x_dim,
args.deterministic_layers,
deterministic_act,
deterministic_size)
model = ReweightedWakeSleep(
p_layers,
q_layers,
qbaseline=(not args.no_qbaseline),
)
model.initialize()
elif args.method == 'bihm-rws':
sizes_tag = args.layer_spec.replace(",", "-")
name = "%s-%s-%s-lr%s-dl%d-spl%d-%s" % \
(args.data, args.method, args.name, lr_tag, args.deterministic_layers, args.n_samples, sizes_tag)
p_layers, q_layers = create_layers(args.layer_spec, x_dim,
args.deterministic_layers,
deterministic_act,
deterministic_size)
model = BiHM(
p_layers,
q_layers,
l1reg=args.l1reg,
l2reg=args.l2reg,
)
model.initialize()
elif args.method == 'continue':
import cPickle as pickle
from os.path import basename, splitext
with open(args.model_file, 'rb') as f:
m = pickle.load(f)
if isinstance(m, MainLoop):
m = m.model
model = m.get_top_bricks()[0]
while len(model.parents) > 0:
model = model.parents[0]
assert isinstance(model, (BiHM, ReweightedWakeSleep, VAE))
mname, _, _ = basename(args.model_file).rpartition("_model.pkl")
name = "%s-cont-%s-lr%s-spl%s" % (mname, args.name, lr_tag,
args.n_samples)
else:
raise ValueError("Unknown training method '%s'" % args.method)
#------------------------------------------------------------
x = tensor.matrix('features')
#------------------------------------------------------------
# Testset monitoring
train_monitors = []
valid_monitors = []
test_monitors = []
for s in [1, 10, 100, 1000]:
log_p, log_ph = model.log_likelihood(x, s)
log_p = -log_p.mean()
log_ph = -log_ph.mean()
log_p.name = "log_p_%d" % s
log_ph.name = "log_ph_%d" % s
#train_monitors += [log_p, log_ph]
#valid_monitors += [log_p, log_ph]
test_monitors += [log_p, log_ph]
#------------------------------------------------------------
# Z estimation
#for s in [100000]:
# z2 = tensor.exp(model.estimate_log_z2(s)) / s
# z2.name = "z2_%d" % s
#
# valid_monitors += [z2]
# test_monitors += [z2]
#------------------------------------------------------------
# Gradient and training monitoring
if args.method in ['vae', 'dvae']:
log_p_bound, gradients = model.get_gradients(x, args.n_samples)
log_p_bound = -log_p_bound.mean()
log_p_bound.name = "log_p_bound"
cost = log_p_bound
train_monitors += [
log_p_bound,
named(model.kl_term.mean(), 'kl_term'),
named(model.recons_term.mean(), 'recons_term')
]
valid_monitors += [
log_p_bound,
named(model.kl_term.mean(), 'kl_term'),
named(model.recons_term.mean(), 'recons_term')
]
test_monitors += [
log_p_bound,
named(model.kl_term.mean(), 'kl_term'),
named(model.recons_term.mean(), 'recons_term')
]
else:
log_p, log_ph, gradients = model.get_gradients(x, args.n_samples)
log_p = -log_p.mean()
log_ph = -log_ph.mean()
log_p.name = "log_p"
log_ph.name = "log_ph"
cost = log_ph
train_monitors += [log_p, log_ph]
valid_monitors += [log_p, log_ph]
#------------------------------------------------------------
# Detailed monitoring
"""
n_layers = len(p_layers)
log_px, w, log_p, log_q, samples = model.log_likelihood(x, n_samples)
exp_samples = []
for l in xrange(n_layers):
e = (w.dimshuffle(0, 1, 'x')*samples[l]).sum(axis=1)
e.name = "inference_h%d" % l
e.tag.aggregation_scheme = aggregation.TakeLast(e)
exp_samples.append(e)
s1 = samples[1]
sh1 = s1.shape
s1_ = s1.reshape([sh1[0]*sh1[1], sh1[2]])
s0, _ = model.p_layers[0].sample_expected(s1_)
s0 = s0.reshape([sh1[0], sh1[1], s0.shape[1]])
s0 = (w.dimshuffle(0, 1, 'x')*s0).sum(axis=1)
s0.name = "inference_h0^"
s0.tag.aggregation_scheme = aggregation.TakeLast(s0)
exp_samples.append(s0)
# Draw P-samples
p_samples, _, _ = model.sample_p(100)
#weights = model.importance_weights(samples)
#weights = weights / weights.sum()
for i, s in enumerate(p_samples):
s.name = "psamples_h%d" % i
s.tag.aggregation_scheme = aggregation.TakeLast(s)
#
samples = model.sample(100, oversample=100)
for i, s in enumerate(samples):
s.name = "samples_h%d" % i
s.tag.aggregation_scheme = aggregation.TakeLast(s)
"""
cg = ComputationGraph([cost])
#------------------------------------------------------------
if args.step_rule == "momentum":
step_rule = Momentum(args.learning_rate, 0.95)
elif args.step_rule == "rmsprop":
step_rule = RMSProp(args.learning_rate)
elif args.step_rule == "adam":
step_rule = Adam(args.learning_rate)
else:
raise "Unknown step_rule %s" % args.step_rule
#parameters = cg.parameters[:4] + cg.parameters[5:]
parameters = cg.parameters
algorithm = GradientDescent(
cost=cost,
parameters=parameters,
gradients=gradients,
step_rule=CompositeRule([
#StepClipping(25),
step_rule,
#RemoveNotFinite(1.0),
]))
#------------------------------------------------------------
train_monitors += [
aggregation.mean(algorithm.total_gradient_norm),
aggregation.mean(algorithm.total_step_norm)
]
#------------------------------------------------------------
# Live plotting?
plotting_extensions = []
if args.live_plotting:
plotting_extensions = [
PlotManager(
name,
[
Plotter(channels=[[
"valid_%s" % cost.name, "valid_log_p"
], ["train_total_gradient_norm", "train_total_step_norm"]],
titles=[
"validation cost",
"norm of training gradient and step"
]),
DisplayImage(
[
WeightDisplay(model.p_layers[0].mlp.
linear_transformations[0].W,
n_weights=100,
image_shape=(28, 28))
]
#ImageDataStreamDisplay(test_stream, image_shape=(28,28))]
)
])
]
main_loop = MainLoop(
model=Model(cost),
data_stream=train_stream,
algorithm=algorithm,
extensions=[
Timing(),
ProgressBar(),
TrainingDataMonitoring(
train_monitors, prefix="train", after_epoch=True),
DataStreamMonitoring(
valid_monitors, data_stream=valid_stream, prefix="valid"),
DataStreamMonitoring(test_monitors,
data_stream=test_stream,
prefix="test",
after_epoch=False,
after_training=True,
every_n_epochs=10),
#SharedVariableModifier(
# algorithm.step_rule.components[0].learning_rate,
# half_lr_func,
# before_training=False,
# after_epoch=False,
# after_batch=False,
# every_n_epochs=half_lr),
TrackTheBest('valid_%s' % cost.name),
Checkpoint(name + ".pkl", save_separately=['log', 'model']),
FinishIfNoImprovementAfter('valid_%s_best_so_far' % cost.name,
epochs=args.patience),
FinishAfter(after_n_epochs=args.max_epochs),
Printing()
] + plotting_extensions)
main_loop.run()
def train_language_model(new_training_job, config, save_path, params,
fast_start, fuel_server, seed):
c = config
if seed:
fuel.config.default_seed = seed
blocks.config.config.default_seed = seed
data, lm, retrieval = initialize_data_and_model(config)
# full main loop can be saved...
main_loop_path = os.path.join(save_path, 'main_loop.tar')
# or only state (log + params) which can be useful not to pickle embeddings
state_path = os.path.join(save_path, 'training_state.tar')
stream_path = os.path.join(save_path, 'stream.pkl')
best_tar_path = os.path.join(save_path, "best_model.tar")
words = tensor.ltensor3('words')
words_mask = tensor.matrix('words_mask')
if theano.config.compute_test_value != 'off':
test_value_data = next(
data.get_stream('train', batch_size=4,
max_length=5).get_epoch_iterator())
words.tag.test_value = test_value_data[0]
words_mask.tag.test_value = test_value_data[1]
costs, updates = lm.apply(words, words_mask)
cost = rename(costs.mean(), 'mean_cost')
cg = Model(cost)
if params:
logger.debug("Load parameters from {}".format(params))
with open(params) as src:
cg.set_parameter_values(load_parameters(src))
length = rename(words.shape[1], 'length')
perplexity, = VariableFilter(name='perplexity')(cg)
perplexities = VariableFilter(name_regex='perplexity.*')(cg)
monitored_vars = [length, cost] + perplexities
if c['dict_path']:
num_definitions, = VariableFilter(name='num_definitions')(cg)
monitored_vars.extend([num_definitions])
parameters = cg.get_parameter_dict()
trained_parameters = parameters.values()
saved_parameters = parameters.values()
if c['embedding_path']:
logger.debug("Exclude word embeddings from the trained parameters")
trained_parameters = [
p for p in trained_parameters
if not p == lm.get_def_embeddings_params()
]
saved_parameters = [
p for p in saved_parameters
if not p == lm.get_def_embeddings_params()
]
if c['cache_size'] != 0:
logger.debug("Enable fake recursivity for looking up embeddings")
trained_parameters = [
p for p in trained_parameters if not p == lm.get_cache_params()
]
logger.info("Cost parameters" + "\n" + pprint.pformat([
" ".join(
(key, str(parameters[key].get_value().shape),
'trained' if parameters[key] in trained_parameters else 'frozen'))
for key in sorted(parameters.keys())
],
width=120))
rules = []
if c['grad_clip_threshold']:
rules.append(StepClipping(c['grad_clip_threshold']))
rules.append(Adam(learning_rate=c['learning_rate'], beta1=c['momentum']))
algorithm = GradientDescent(cost=cost,
parameters=trained_parameters,
step_rule=CompositeRule(rules))
if c['cache_size'] != 0:
algorithm.add_updates(updates)
train_monitored_vars = list(monitored_vars)
if c['grad_clip_threshold']:
train_monitored_vars.append(algorithm.total_gradient_norm)
word_emb_RMS, = VariableFilter(name='word_emb_RMS')(cg)
main_rnn_in_RMS, = VariableFilter(name='main_rnn_in_RMS')(cg)
train_monitored_vars.extend([word_emb_RMS, main_rnn_in_RMS])
if c['monitor_parameters']:
train_monitored_vars.extend(parameter_stats(parameters, algorithm))
# We use a completely random seed on purpose. With Fuel server
# it's currently not possible to restore the state of the training
# stream. That's why it's probably better to just have it stateless.
stream_seed = numpy.random.randint(0, 10000000) if fuel_server else None
training_stream = data.get_stream('train',
batch_size=c['batch_size'],
max_length=c['max_length'],
seed=stream_seed)
valid_stream = data.get_stream('valid',
batch_size=c['batch_size_valid'],
max_length=c['max_length'],
seed=stream_seed)
original_training_stream = training_stream
if fuel_server:
# the port will be configured by the StartFuelServer extension
training_stream = ServerDataStream(
sources=training_stream.sources,
produces_examples=training_stream.produces_examples)
validation = DataStreamMonitoring(monitored_vars,
valid_stream,
prefix="valid").set_conditions(
before_first_epoch=not fast_start,
on_resumption=True,
every_n_batches=c['mon_freq_valid'])
track_the_best = TrackTheBest(validation.record_name(perplexity),
choose_best=min).set_conditions(
on_resumption=True,
after_epoch=True,
every_n_batches=c['mon_freq_valid'])
# don't save them the entire main loop to avoid pickling everything
if c['fast_checkpoint']:
load = (LoadNoUnpickling(state_path,
load_iteration_state=True,
load_log=True).set_conditions(
before_training=not new_training_job))
cp_args = {
'save_main_loop': False,
'save_separately': ['log', 'iteration_state'],
'parameters': saved_parameters
}
checkpoint = Checkpoint(state_path,
before_training=not fast_start,
every_n_batches=c['save_freq_batches'],
after_training=not fast_start,
**cp_args)
if c['checkpoint_every_n_batches']:
intermediate_cp = IntermediateCheckpoint(
state_path,
every_n_batches=c['checkpoint_every_n_batches'],
after_training=False,
**cp_args)
else:
load = (Load(main_loop_path, load_iteration_state=True,
load_log=True).set_conditions(
before_training=not new_training_job))
cp_args = {
'save_separately': ['iteration_state'],
'parameters': saved_parameters
}
checkpoint = Checkpoint(main_loop_path,
before_training=not fast_start,
every_n_batches=c['save_freq_batches'],
after_training=not fast_start,
**cp_args)
if c['checkpoint_every_n_batches']:
intermediate_cp = IntermediateCheckpoint(
main_loop_path,
every_n_batches=c['checkpoint_every_n_batches'],
after_training=False,
**cp_args)
checkpoint = checkpoint.add_condition(
['after_batch', 'after_epoch'],
OnLogRecord(track_the_best.notification_name), (best_tar_path, ))
extensions = [
load,
StartFuelServer(original_training_stream,
stream_path,
before_training=fuel_server),
Timing(every_n_batches=c['mon_freq_train'])
]
if retrieval:
extensions.append(
RetrievalPrintStats(retrieval=retrieval,
every_n_batches=c['mon_freq_train'],
before_training=not fast_start))
extensions.extend([
TrainingDataMonitoring(train_monitored_vars,
prefix="train",
every_n_batches=c['mon_freq_train']),
validation, track_the_best, checkpoint
])
if c['checkpoint_every_n_batches']:
extensions.append(intermediate_cp)
extensions.extend([
DumpTensorflowSummaries(save_path,
every_n_batches=c['mon_freq_train'],
after_training=True),
Printing(on_resumption=True, every_n_batches=c['mon_freq_train']),
FinishIfNoImprovementAfter(track_the_best.notification_name,
iterations=50 * c['mon_freq_valid'],
every_n_batches=c['mon_freq_valid']),
FinishAfter(after_n_batches=c['n_batches'])
])
logger.info("monitored variables during training:" + "\n" +
pprint.pformat(train_monitored_vars, width=120))
logger.info("monitored variables during valid:" + "\n" +
pprint.pformat(monitored_vars, width=120))
main_loop = MainLoop(algorithm,
training_stream,
model=Model(cost),
extensions=extensions)
main_loop.run()
def main(args):
"""Run experiment. """
lr_tag = float_tag(args.learning_rate)
x_dim, train_stream, valid_stream, test_stream = datasets.get_streams(
args.data, args.batch_size)
#------------------------------------------------------------
# Setup model
deterministic_act = Tanh
deterministic_size = 1.
if args.method == 'vae':
sizes_tag = args.layer_spec.replace(",", "-")
layer_sizes = [int(i) for i in args.layer_spec.split(",")]
layer_sizes, z_dim = layer_sizes[:-1], layer_sizes[-1]
name = "%s-%s-%s-lr%s-spl%d-%s" % \
(args.data, args.method, args.name, lr_tag, args.n_samples, sizes_tag)
if args.activation == "tanh":
hidden_act = Tanh()
elif args.activation == "logistic":
hidden_act = Logistic()
elif args.activation == "relu":
hidden_act = Rectifier()
else:
raise "Unknown hidden nonlinearity %s" % args.hidden_act
model = VAE(x_dim=x_dim,
hidden_layers=layer_sizes,
hidden_act=hidden_act,
z_dim=z_dim,
batch_norm=args.batch_normalization)
model.initialize()
elif args.method == 'rws':
sizes_tag = args.layer_spec.replace(",", "-")
name = "%s-%s-%s-lr%s-dl%d-spl%d-%s" % \
(args.data, args.method, args.name, lr_tag, args.deterministic_layers, args.n_samples, sizes_tag)
p_layers, q_layers = create_layers(args.layer_spec, x_dim,
args.deterministic_layers,
deterministic_act,
deterministic_size)
model = ReweightedWakeSleep(
p_layers,
q_layers,
)
model.initialize()
elif args.method == 'bihm':
sizes_tag = args.layer_spec.replace(",", "-")
name = "%s-%s-%s-lr%s-dl%d-spl%d-%s" % \
(args.data, args.method, args.name, lr_tag, args.deterministic_layers, args.n_samples, sizes_tag)
p_layers, q_layers = create_layers(args.layer_spec, x_dim,
args.deterministic_layers,
deterministic_act,
deterministic_size)
model = BiHM(
p_layers,
q_layers,
l1reg=args.l1reg,
l2reg=args.l2reg,
)
model.initialize()
elif args.method == 'continue':
import cPickle as pickle
from os.path import basename, splitext
with open(args.model_file, 'rb') as f:
m = pickle.load(f)
if isinstance(m, MainLoop):
m = m.model
model = m.get_top_bricks()[0]
while len(model.parents) > 0:
model = model.parents[0]
assert isinstance(model, (BiHM, ReweightedWakeSleep, VAE))
mname, _, _ = basename(args.model_file).rpartition("_model.pkl")
name = "%s-cont-%s-lr%s-spl%s" % (mname, args.name, lr_tag,
args.n_samples)
else:
raise ValueError("Unknown training method '%s'" % args.method)
#------------------------------------------------------------
x = tensor.matrix('features')
#------------------------------------------------------------
# Testset monitoring
train_monitors = []
valid_monitors = []
test_monitors = []
for s in [
1,
10,
100,
1000,
]:
log_p, log_ph = model.log_likelihood(x, s)
log_p = -log_p.mean()
log_ph = -log_ph.mean()
log_p.name = "log_p_%d" % s
log_ph.name = "log_ph_%d" % s
#valid_monitors += [log_p, log_ph]
test_monitors += [log_p, log_ph]
#------------------------------------------------------------
# Z estimation
#for s in [100000]:
# z2 = tensor.exp(model.estimate_log_z2(s)) / s
# z2.name = "z2_%d" % s
#
# valid_monitors += [z2]
# test_monitors += [z2]
#------------------------------------------------------------
# Gradient and training monitoring
if args.method in ['vae', 'dvae']:
log_p_bound = model.log_likelihood_bound(x, args.n_samples)
gradients = None
log_p_bound = -log_p_bound.mean()
log_p_bound.name = "log_p_bound"
cost = log_p_bound
train_monitors += [
log_p_bound,
named(model.kl_term.mean(), 'kl_term'),
named(model.recons_term.mean(), 'recons_term')
]
valid_monitors += [
log_p_bound,
named(model.kl_term.mean(), 'kl_term'),
named(model.recons_term.mean(), 'recons_term')
]
test_monitors += [
log_p_bound,
named(model.kl_term.mean(), 'kl_term'),
named(model.recons_term.mean(), 'recons_term')
]
else:
log_p, log_ph, gradients = model.get_gradients(x, args.n_samples)
log_p_bound = named(-model.log_p_bound.mean(), "log_p_bound")
log_p = named(-log_p.mean(), "log_p")
log_ph = named(-log_ph.mean(), "log_ph")
cost = log_p
train_monitors += [log_p_bound, log_p, log_ph]
valid_monitors += [log_p_bound, log_p, log_ph]
#------------------------------------------------------------
cg = ComputationGraph([cost])
if args.step_rule == "momentum":
step_rule = Momentum(args.learning_rate, 0.95)
elif args.step_rule == "rmsprop":
step_rule = RMSProp(args.learning_rate)
elif args.step_rule == "adam":
step_rule = Adam(args.learning_rate)
else:
raise "Unknown step_rule %s" % args.step_rule
parameters = cg.parameters
algorithm = GradientDescent(cost=cost,
parameters=parameters,
gradients=gradients,
step_rule=CompositeRule([
step_rule,
]))
#------------------------------------------------------------
train_monitors += [
aggregation.mean(algorithm.total_gradient_norm),
aggregation.mean(algorithm.total_step_norm)
]
#------------------------------------------------------------
# Live plotting?
plotting_extensions = []
if args.live_plotting:
plotting_extensions = [
PlotManager(
name,
[
Plotter(channels=[[
"valid_%s" % cost.name, "valid_log_p"
], ["train_total_gradient_norm", "train_total_step_norm"]],
titles=[
"validation cost",
"norm of training gradient and step"
]),
DisplayImage(
[
WeightDisplay(model.p_layers[0].mlp.
linear_transformations[0].W,
n_weights=100,
image_shape=(28, 28))
]
#ImageDataStreamDisplay(test_stream, image_shape=(28,28))]
)
])
]
main_loop = MainLoop(
model=Model(cost),
data_stream=train_stream,
algorithm=algorithm,
extensions=[
Timing(),
ProgressBar(),
TrainingDataMonitoring(train_monitors,
prefix="train",
after_epoch=False,
after_batch=True),
DataStreamMonitoring(
valid_monitors, data_stream=valid_stream, prefix="valid"),
DataStreamMonitoring(test_monitors,
data_stream=test_stream,
prefix="test",
after_epoch=False,
after_training=True,
every_n_epochs=10),
TrackTheBest('valid_%s' % cost.name),
Checkpoint(name + ".pkl", save_separately=['log', 'model']),
FinishIfNoImprovementAfter('valid_%s_best_so_far' % cost.name,
epochs=args.patience),
FinishAfter(after_n_epochs=args.max_epochs),
Printing()
] + plotting_extensions)
main_loop.run()
def construct_main_loop(name, task_name, batch_size, max_epochs,
patience_epochs, learning_rate, hyperparameters,
**kwargs):
task = tasks.get_task(**hyperparameters)
hyperparameters["n_channels"] = task.n_channels
extensions = []
print "constructing graphs..."
graphs, outputs, updates = construct_graphs(task=task, **hyperparameters)
print "setting up main loop..."
from blocks.model import Model
model = Model(outputs["train"]["cost"])
from blocks.algorithms import GradientDescent, CompositeRule, StepClipping, Adam
algorithm = GradientDescent(cost=outputs["train"]["cost"],
parameters=graphs["train"].parameters,
step_rule=CompositeRule([
StepClipping(1e1),
Adam(learning_rate=learning_rate),
StepClipping(1e2)
]),
on_unused_sources="warn")
algorithm.add_updates(updates["train"])
extensions.extend(
construct_monitors(algorithm=algorithm,
task=task,
model=model,
graphs=graphs,
outputs=outputs,
**hyperparameters))
from blocks.extensions import FinishAfter, Printing, ProgressBar, Timing
from blocks.extensions.stopping import FinishIfNoImprovementAfter
from blocks.extensions.training import TrackTheBest
from blocks.extensions.saveload import Checkpoint
from dump import DumpBest, LightCheckpoint, PrintingTo
extensions.extend([
TrackTheBest("valid_error_rate", "best_valid_error_rate"),
FinishIfNoImprovementAfter("best_valid_error_rate",
epochs=patience_epochs),
FinishAfter(after_n_epochs=max_epochs),
DumpBest("best_valid_error_rate", name + "_best.zip"),
Checkpoint(hyperparameters["checkpoint_save_path"],
on_interrupt=False,
every_n_epochs=5,
before_training=True,
use_cpickle=True),
ProgressBar(),
Timing(),
Printing(),
PrintingTo(name + "_log")
])
from blocks.main_loop import MainLoop
main_loop = MainLoop(data_stream=task.get_stream("train"),
algorithm=algorithm,
extensions=extensions,
model=model)
# note blocks will crash and burn because it cannot deal with an
# already-initialized Algorithm, so this should be enabled only for
# debugging
if False:
with open("graph", "w") as graphfile:
algorithm.initialize()
theano.printing.debugprint(algorithm._function, file=graphfile)
from tabulate import tabulate
print "parameter sizes:"
print tabulate(
(key, "x".join(map(str,
value.get_value().shape)), value.get_value().size)
for key, value in main_loop.model.get_parameter_dict().items())
return main_loop
def train_lstm(train, test, input_dim,
hidden_dimension, columns, epochs,
save_file, execution_name, batch_size, plot):
stream_train = build_stream(train, batch_size, columns)
stream_test = build_stream(test, batch_size, columns)
# The train stream will return (TimeSequence, BatchSize, Dimensions) for
# and the train test will return (TimeSequence, BatchSize, 1)
x = T.tensor3('x')
y = T.tensor3('y')
y = y.reshape((y.shape[1], y.shape[0], y.shape[2]))
# input_dim = 6
# output_dim = 1
linear_lstm = LinearLSTM(input_dim, 1, hidden_dimension,
# print_intermediate=True,
print_attrs=['__str__', 'shape'])
y_hat = linear_lstm.apply(x)
linear_lstm.initialize()
c_test = AbsolutePercentageError().apply(y, y_hat)
c_test.name = 'mape'
c = SquaredError().apply(y, y_hat)
c.name = 'cost'
cg = ComputationGraph(c_test)
def one_perc_min(current_value, best_value):
if (1 - best_value / current_value) > 0.01:
return best_value
else:
return current_value
extensions = []
extensions.append(DataStreamMonitoring(variables=[c, c_test],
data_stream=stream_test,
prefix='test',
after_epoch=False,
every_n_epochs=100))
extensions.append(TrainingDataMonitoring(variables=[c_test],
prefix='train',
after_epoch=True))
extensions.append(FinishAfter(after_n_epochs=epochs))
# extensions.append(Printing())
# extensions.append(ProgressBar())
extensions.append(TrackTheBest('test_mape', choose_best=one_perc_min))
extensions.append(TrackTheBest('test_cost', choose_best=one_perc_min))
extensions.append(FinishIfNoImprovementAfter('test_cost_best_so_far', epochs=500))
# Save only parameters, not the whole main loop and only when best_test_cost is updated
checkpoint = Checkpoint(save_file, save_main_loop=False, after_training=False)
checkpoint.add_condition(['after_epoch'], predicate=OnLogRecord('test_cost_best_so_far'))
extensions.append(checkpoint)
if BOKEH_AVAILABLE and plot:
extensions.append(Plot(execution_name, channels=[[ # 'train_cost',
'test_cost']]))
step_rule = Adam()
algorithm = GradientDescent(cost=c_test, parameters=cg.parameters, step_rule=step_rule)
main_loop = MainLoop(algorithm, stream_train, model=Model(c_test), extensions=extensions)
main_loop.run()
test_mape = 0
if main_loop.log.status.get('best_test_mape', None) is None:
with open(save_file, 'rb') as f:
parameters = load_parameters(f)
model = main_loop.model
model.set_parameter_values(parameters)
ev = DatasetEvaluator([c_test])
test_mape = ev.evaluate(stream_test)['mape']
else:
test_mape = main_loop.log.status['best_test_mape']
return test_mape, main_loop.log.status['epochs_done']
def build_and_run(experimentconfig, modelconfig, save_to=None): #modelconfig,
""" part of this is adapted from lasagne tutorial"""
# Prepare Theano variables for inputs and targets
input_var = T.tensor4('image_features')
target_var = T.lmatrix('targets')
target_vec = T.extra_ops.to_one_hot(target_var[:,0],2)
# Create vgg model
print("Building model...")
image_size = modelconfig['image_size']
network = vgg16.build_small_model()
prediction = lasagne.utils.as_theano_expression(lasagne.layers.get_output(network["prob"],input_var))
# test_prediction = lasagne.layers.get_output(network["prob"],input_var,deterministic=True)
# Loss function -> The objective to minimize
print("Instanciation of loss function...")
# loss = lasagne.objectives.categorical_crossentropy(prediction, target_var.flatten())
loss = lasagne.objectives.squared_error(prediction,target_vec)
# test_loss = lasagne.objectives.squared_error(test_prediction,target_vec)
loss = loss.mean()
# layers = network.values()
#l1 and l2 regularization
# pondlayers = {x:0.01 for x in layers}
# l1_penality = lasagne.regularization.regularize_layer_params_weighted(pondlayers, lasagne.regularization.l2)
# l2_penality = lasagne.regularization.regularize_layer_params(layers[len(layers)/4:], lasagne.regularization.l1) * 1e-4
# reg_penalty = l1_penality + l2_penality
# reg_penalty.name = 'reg_penalty'
#loss = loss + reg_penalty
loss.name = 'loss'
error_rate = MisclassificationRate().apply(target_var.flatten(), prediction).copy(
name='error_rate')
# Load the dataset
print("Loading data...")
if 'test' in experimentconfig.keys() and experimentconfig['test'] is True:
train_stream, valid_stream, test_stream = get_stream(experimentconfig['batch_size'],image_size,test=True)
else :
train_stream, valid_stream, test_stream = get_stream(experimentconfig['batch_size'],image_size,test=False)
# Defining step rule and algorithm
if 'step_rule' in experimentconfig.keys() and not experimentconfig['step_rule'] is None :
step_rule = experimentconfig['step_rule'](learning_rate=experimentconfig['learning_rate'])
else :
step_rule=Scale(learning_rate=experimentconfig['learning_rate'])
params = map(lasagne.utils.as_theano_expression,lasagne.layers.get_all_params(network['prob'], trainable=True))
algorithm = GradientDescent(
cost=loss, gradients={var:T.grad(loss,var) for var in params},
step_rule=step_rule)
grad_norm = aggregation.mean(algorithm.total_gradient_norm)
grad_norm.name='grad_norm'
print("Initializing extensions...")
plot = Plot(save_to, channels=[['train_loss','valid_loss','train_grad_norm'],['train_error_rate','valid_error_rate']], server_url='http://hades.calculquebec.ca:5042')
checkpoint = Checkpoint('models/best_'+save_to+'.tar')
# checkpoint.add_condition(['after_n_batches=25'],
checkpoint.add_condition(['after_epoch'],
predicate=OnLogRecord('valid_error_rate_best_so_far'))
#Defining extensions
extensions = [Timing(),
FinishAfter(after_n_epochs=experimentconfig['num_epochs'],
after_n_batches=experimentconfig['num_batches']),
TrainingDataMonitoring([loss, error_rate, grad_norm, reg_penalty], prefix="train", after_epoch=True), #after_n_epochs=1
DataStreamMonitoring([loss, error_rate],valid_stream,prefix="valid", after_epoch=True), #after_n_epochs=1
#Checkpoint(save_to,after_n_epochs=5),
#ProgressBar(),
plot,
# after_batch=True),
Printing(after_epoch=True),
TrackTheBest('valid_error_rate',min), #Keep best
checkpoint, #Save best
FinishIfNoImprovementAfter('valid_error_rate_best_so_far', epochs=5)] # Early-stopping
# model = Model(ComputationGraph(network))
main_loop = MainLoop(
algorithm,
train_stream,
# model=model,
extensions=extensions)
print("Starting main loop...")
main_loop.run()
from blocks.main_loop import MainLoop
main_loop = MainLoop(data_stream=train_stream, algorithm=algorithm,
extensions=extensions, model=model)
main_loop.run()
# reinit adam
main_loop.algorithm.step_rule = Adam(adaminitlr)
extensions[4] = FinishIfNoImprovementAfter('loss_best_so_far', epochs=20)
extensions[4].main_loop = main_loop
for i in range(5):
# print_img()
new_lr = 0.2*algorithm.step_rule.learning_rate.get_value()
print '===\n(%d) Learning rate set to %e\n===' % (i, new_lr)
#
algorithm.step_rule.learning_rate.set_value(
numpy.float32(new_lr))
#main_loop.algorithm.step_rule = Adam(new_lr)
# reinit early stopping
extensions[4].last_best_iter = main_loop.log.status['iterations_done']
def train_model(new_training_job, config, save_path, params, fast_start,
fuel_server, seed):
c = config
if seed:
fuel.config.default_seed = seed
blocks.config.config.default_seed = seed
data, model = initialize_data_and_model(config, train_phase=True)
# full main loop can be saved...
main_loop_path = os.path.join(save_path, 'main_loop.tar')
# or only state (log + params) which can be useful not to pickle embeddings
state_path = os.path.join(save_path, 'training_state.tar')
stream_path = os.path.join(save_path, 'stream.pkl')
best_tar_path = os.path.join(save_path, "best_model.tar")
keys = tensor.lmatrix('keys')
n_identical_keys = tensor.lvector('n_identical_keys')
words = tensor.ltensor3('words')
words_mask = tensor.matrix('words_mask')
if theano.config.compute_test_value != 'off':
#TODO
test_value_data = next(
data.get_stream('train', batch_size=4,
max_length=5).get_epoch_iterator())
words.tag.test_value = test_value_data[0]
words_mask.tag.test_value = test_value_data[1]
if use_keys(c) and use_n_identical_keys(c):
costs = model.apply(words,
words_mask,
keys,
n_identical_keys,
train_phase=True)
elif use_keys(c):
costs = model.apply(words, words_mask, keys, train_phase=True)
else:
costs = model.apply(words, words_mask, train_phase=True)
cost = rename(costs.mean(), 'mean_cost')
cg = Model(cost)
if params:
logger.debug("Load parameters from {}".format(params))
with open(params) as src:
cg.set_parameter_values(load_parameters(src))
length = rename(words.shape[1], 'length')
perplexity, = VariableFilter(name='perplexity')(cg)
monitored_vars = [length, cost, perplexity]
if c['proximity_coef']:
proximity_term, = VariableFilter(name='proximity_term')(cg)
monitored_vars.append(proximity_term)
print "inputs of the model:", cg.inputs
parameters = cg.get_parameter_dict()
trained_parameters = parameters.values()
saved_parameters = parameters.values()
if c['embedding_path']:
if c['freeze_pretrained']:
logger.debug(
"Exclude pretrained encoder embeddings from the trained parameters"
)
to_freeze = 'main'
elif c['provide_targets']:
logger.debug(
"Exclude pretrained targets from the trained parameters")
to_freeze = 'target'
trained_parameters = [
p for p in trained_parameters
if not p == model.get_def_embeddings_params(to_freeze)
]
saved_parameters = [
p for p in saved_parameters
if not p == model.get_def_embeddings_params(to_freeze)
]
logger.info("Cost parameters" + "\n" + pprint.pformat([
" ".join(
(key, str(parameters[key].get_value().shape),
'trained' if parameters[key] in trained_parameters else 'frozen'))
for key in sorted(parameters.keys())
],
width=120))
rules = []
if c['grad_clip_threshold']:
rules.append(StepClipping(c['grad_clip_threshold']))
rules.append(Adam(learning_rate=c['learning_rate'], beta1=c['momentum']))
algorithm = GradientDescent(cost=cost,
parameters=trained_parameters,
step_rule=CompositeRule(rules))
train_monitored_vars = list(monitored_vars)
if c['grad_clip_threshold']:
train_monitored_vars.append(algorithm.total_gradient_norm)
if c['monitor_parameters']:
train_monitored_vars.extend(parameter_stats(parameters, algorithm))
# We use a completely random seed on purpose. With Fuel server
# it's currently not possible to restore the state of the training
# stream. That's why it's probably better to just have it stateless.
stream_seed = numpy.random.randint(0, 10000000) if fuel_server else None
training_stream = data.get_stream(
'train',
batch_size=c['batch_size'],
max_length=c['max_length'],
seed=stream_seed,
remove_keys=not use_keys(c),
remove_n_identical_keys=not use_n_identical_keys(c))
print "trainin_stream will contains sources:", training_stream.sources
original_training_stream = training_stream
if fuel_server:
# the port will be configured by the StartFuelServer extension
training_stream = ServerDataStream(
sources=training_stream.sources,
produces_examples=training_stream.produces_examples)
validate = c['mon_freq_valid'] > 0
if validate:
valid_stream = data.get_stream(
'valid',
batch_size=c['batch_size_valid'],
max_length=c['max_length'],
seed=stream_seed,
remove_keys=not use_keys(c),
remove_n_identical_keys=not use_n_identical_keys(c))
validation = DataStreamMonitoring(
monitored_vars, valid_stream,
prefix="valid").set_conditions(before_first_epoch=not fast_start,
on_resumption=True,
every_n_batches=c['mon_freq_valid'])
track_the_best = TrackTheBest(validation.record_name(cost),
choose_best=min).set_conditions(
on_resumption=True,
after_epoch=True,
every_n_batches=c['mon_freq_valid'])
# don't save them the entire main loop to avoid pickling everything
if c['fast_checkpoint']:
cp_path = state_path
load = (LoadNoUnpickling(cp_path,
load_iteration_state=True,
load_log=True).set_conditions(
before_training=not new_training_job))
cp_args = {
'save_main_loop': False,
'save_separately': ['log', 'iteration_state'],
'parameters': saved_parameters
}
else:
cp_path = main_loop_path
load = (Load(cp_path, load_iteration_state=True,
load_log=True).set_conditions(
before_training=not new_training_job))
cp_args = {
'save_separately': ['iteration_state'],
'parameters': saved_parameters
}
checkpoint = Checkpoint(cp_path,
before_training=not fast_start,
every_n_batches=c['save_freq_batches'],
after_training=not fast_start,
**cp_args)
if c['checkpoint_every_n_batches'] > 0 or c[
'checkpoint_every_n_epochs'] > 0:
intermediate_cp = IntermediateCheckpoint(
cp_path,
every_n_epochs=c['checkpoint_every_n_epochs'],
every_n_batches=c['checkpoint_every_n_batches'],
after_training=False,
**cp_args)
if validate:
checkpoint = checkpoint.add_condition(
['after_batch', 'after_epoch'],
OnLogRecord(track_the_best.notification_name), (best_tar_path, ))
extensions = [
load,
StartFuelServer(original_training_stream,
stream_path,
before_training=fuel_server),
Timing(every_n_batches=c['mon_freq_train'])
]
extensions.extend([
TrainingDataMonitoring(train_monitored_vars,
prefix="train",
every_n_batches=c['mon_freq_train']),
])
if validate:
extensions.extend([validation, track_the_best])
extensions.append(checkpoint)
if c['checkpoint_every_n_batches'] > 0 or c[
'checkpoint_every_n_epochs'] > 0:
extensions.append(intermediate_cp)
extensions.extend(
[Printing(on_resumption=True, every_n_batches=c['mon_freq_train'])])
if validate and c['n_valid_early'] > 0:
extensions.append(
FinishIfNoImprovementAfter(track_the_best.notification_name,
iterations=c['n_valid_early'] *
c['mon_freq_valid'],
every_n_batches=c['mon_freq_valid']))
extensions.append(FinishAfter(after_n_epochs=c['n_epochs']))
logger.info("monitored variables during training:" + "\n" +
pprint.pformat(train_monitored_vars, width=120))
logger.info("monitored variables during valid:" + "\n" +
pprint.pformat(monitored_vars, width=120))
main_loop = MainLoop(algorithm,
training_stream,
model=Model(cost),
extensions=extensions)
main_loop.run()
def test_finish_if_no_improvement_after_epochs(self):
ext = FinishIfNoImprovementAfter('mangos', epochs=3)
self.check_finish_if_no_improvement_after(ext, 'mangos', epochs=True)
def train(self,
cost,
y_hat,
train_stream,
accuracy=None,
prediction_cost=None,
regularization_cost=None,
params_to_optimize=None,
valid_stream=None,
extra_extensions=None,
model=None,
vars_to_monitor_on_train=None,
vars_to_monitor_on_valid=None,
step_rule=None,
additional_streams=None,
save_on_best=None,
use_own_validation=False,
objects_to_dump=None):
"""
Generic method for training models. It extends functionality already provided by Blocks.
:param cost: Theano var with cost function
:param y_hat: Theano var with predictions from the model
:param train_stream: Fuel stream with training data
:param accuracy: Theano var with accuracy
:param prediction_cost:
:param regularization_cost:
:param params_to_optimize:
:param valid_stream: Fuel stream with validation data
:param extra_extensions:
:param model:
:param vars_to_monitor_on_train:
:param vars_to_monitor_on_valid:
:param step_rule:
:param additional_streams:
:param save_on_best:
:param use_own_validation:
:param objects_to_dump:
:return:
"""
if not vars_to_monitor_on_valid:
vars_to_monitor_on_valid = [(cost, min)]
if accuracy:
vars_to_monitor_on_valid.append((accuracy, max))
if not save_on_best:
# use default metrics for saving the best model
save_on_best = [(cost, min)]
if accuracy:
save_on_best.append((accuracy, max))
# setup the training algorithm #######################################
# step_rule = Scale(learning_rate=0.01)
# step_rule = Adam()
model_save_suffix = ""
if self.args.append_metaparams:
model_save_suffix = "." + get_current_metaparams_str(
self.parser, self.args)
# get a list of variables that will be monitored during training
vars_to_monitor = [cost]
if accuracy:
vars_to_monitor.append(accuracy)
if prediction_cost:
vars_to_monitor.append(prediction_cost)
if regularization_cost:
vars_to_monitor.append(regularization_cost)
theano_vars_to_monitor = [
var for var, comparator in vars_to_monitor_on_valid
]
if not params_to_optimize:
# use all parameters of the model for optimization
cg = ComputationGraph(cost)
params_to_optimize = cg.parameters
self.print_parameters_info(params_to_optimize)
if not model:
if accuracy:
model = MultiOutputModel([cost, accuracy, y_hat] +
theano_vars_to_monitor)
else:
model = MultiOutputModel([cost, y_hat] +
theano_vars_to_monitor)
if not step_rule:
step_rule = AdaDelta() # learning_rate=0.02, momentum=0.9)
step_rules = [
StepClipping(self.args.gradient_clip), step_rule,
RemoveNotFinite()
]
# optionally add gradient noise
if self.args.gradient_noise:
step_rules = [
GradientNoise(self.args.gradient_noise, self.args.gn_decay)
] + step_rules
algorithm = GradientDescent(cost=cost,
parameters=params_to_optimize,
step_rule=CompositeRule(step_rules),
on_unused_sources="warn")
# this variable aggregates all extensions executed periodically during training
extensions = []
if self.args.epochs_max:
# finis training after fixed number of epochs
extensions.append(FinishAfter(after_n_epochs=self.args.epochs_max))
# training data monitoring
def create_training_data_monitoring():
if "every_n_epochs" in self.args.evaluate_every_n:
return TrainingDataMonitoring(vars_to_monitor,
prefix='train',
after_epoch=True)
else:
return TrainingDataMonitoring(vars_to_monitor,
prefix='train',
after_epoch=True,
**self.args.evaluate_every_n)
# add extensions that monitors progress of training on train set
extensions.extend([create_training_data_monitoring()])
if not self.args.disable_progress_bar:
extensions.append(ProgressBar())
def add_data_stream_monitor(data_stream, prefix):
if not use_own_validation:
extensions.append(
DataStreamMonitoring(variables=theano_vars_to_monitor,
data_stream=data_stream,
prefix=prefix,
before_epoch=False,
**self.args.evaluate_every_n))
# additional streams that should be monitored
if additional_streams:
for stream_name, stream in additional_streams:
add_data_stream_monitor(stream, stream_name)
# extra extensions need to be called before Printing extension
if extra_extensions:
extensions.extend(extra_extensions)
if valid_stream:
# add validation set monitoring
add_data_stream_monitor(valid_stream, 'valid')
# add best val monitoring
for var, comparator in vars_to_monitor_on_valid:
extensions.append(
TrackTheBest("valid_" + var.name,
choose_best=comparator,
**self.args.evaluate_every_n))
if self.args.patience_metric == 'cost':
patience_metric_name = cost.name
elif self.args.patience_metric == 'accuracy':
patience_metric_name = accuracy.name
else:
print "WARNING: Falling back to COST function for patience."
patience_metric_name = cost.name
extensions.append(
# "valid_cost_best_so_far" message will be entered to the main loop log by TrackTheBest extension
FinishIfNoImprovementAfter(
"valid_" + patience_metric_name + "_best_so_far",
epochs=self.args.epochs_patience_valid))
if not self.args.do_not_save:
# use user provided metrics for saving
valid_save_extensions = map(
lambda metric_comparator: SaveTheBest(
"valid_" + metric_comparator[0].name,
self.args.save_path + ".best." + metric_comparator[
0].name + model_save_suffix,
choose_best=metric_comparator[1],
**self.args.evaluate_every_n), save_on_best)
extensions.extend(valid_save_extensions)
extensions.extend([
Timing(**self.args.evaluate_every_n),
Printing(after_epoch=False, **self.args.evaluate_every_n),
])
if not self.args.do_not_save or self.args.save_only_best:
extensions.append(
Checkpoint(self.args.save_path + model_save_suffix,
**self.args.save_every_n))
extensions.append(FlushStreams(**self.args.evaluate_every_n))
# main loop ##########################################################
main_loop = MainLoop(data_stream=train_stream,
model=model,
algorithm=algorithm,
extensions=extensions)
sys.setrecursionlimit(1000000)
main_loop.run()
from blocks.extensions.stopping import FinishIfNoImprovementAfter
from blocks_extras.extensions.plot import Plot
from blocks.extensions.saveload import Checkpoint
import datetime
loss.name = 'loss'
extensions = [
Timing(),
TrainingDataMonitoring([loss], after_epoch=True),
Plot('FF text gen %s' % (datetime.datetime.now(), ),
channels=[['loss']],
after_batch=True),
TrackTheBest('loss'),
FinishIfNoImprovementAfter('loss_best_so_far', epochs=5),
Printing(),
PrintImageExtension(every_n_epochs=5),
Checkpoint(BASEPATH + 'model.pkl')
]
from blocks.model import Model
model = Model(generated_image_graph)
from blocks.main_loop import MainLoop
main_loop = MainLoop(data_stream=train_stream,
algorithm=algorithm,
extensions=extensions,
model=model)
def build_and_run(save_to,modelconfig,experimentconfig):
""" part of this is adapted from lasagne tutorial"""
n, num_filters, image_size, num_blockstack = modelconfig['depth'], modelconfig['num_filters'], modelconfig['image_size'], modelconfig['num_blockstack']
print("Amount of bottlenecks: %d" % n)
# Prepare Theano variables for inputs and targets
input_var = T.tensor4('image_features')
#target_value = T.ivector('targets')
target_var = T.lmatrix('targets')
target_vec = T.extra_ops.to_one_hot(target_var[:,0],2)
#target_var = T.matrix('targets')
# Create residual net model
print("Building model...")
network = build_cnn(input_var, image_size, n, num_blockstack, num_filters)
get_info(network)
prediction = lasagne.utils.as_theano_expression(lasagne.layers.get_output(network))
test_prediction = lasagne.utils.as_theano_expression(lasagne.layers.get_output(network,deterministic=True))
# Loss function -> The objective to minimize
print("Instanciation of loss function...")
#loss = CategoricalCrossEntropy().apply(target_var.flatten(), prediction)
#test_loss = CategoricalCrossEntropy().apply(target_var.flatten(), test_prediction)
# loss = lasagne.objectives.categorical_crossentropy(prediction, target_var.flatten()).mean()
# test_loss = lasagne.objectives.categorical_crossentropy(test_prediction, target_var.flatten()).mean()
loss = lasagne.objectives.squared_error(prediction,target_vec).mean()
test_loss = lasagne.objectives.squared_error(test_prediction,target_vec).mean()
# loss = tensor.nnet.binary_crossentropy(prediction, target_var).mean()
# test_loss = tensor.nnet.binary_crossentropy(test_prediction, target_var).mean()
test_loss.name = "loss"
# loss.name = 'x-ent_error'
# loss.name = 'sqr_error'
layers = lasagne.layers.get_all_layers(network)
#l1 and l2 regularization
#pondlayers = {x:0.000025 for i,x in enumerate(layers)}
#l1_penality = lasagne.regularization.regularize_layer_params_weighted(pondlayers, lasagne.regularization.l2)
#l2_penality = lasagne.regularization.regularize_layer_params(layers[len(layers)/4:], lasagne.regularization.l1) * 25e-6
#reg_penalty = l1_penality + l2_penality
#reg_penalty.name = 'reg_penalty'
#loss = loss + reg_penalty
loss.name = 'reg_loss'
error_rate = MisclassificationRate().apply(target_var.flatten(), test_prediction).copy(
name='error_rate')
# Load the dataset
print("Loading data...")
istest = 'test' in experimentconfig.keys()
if istest:
print("Using test stream")
train_stream, valid_stream, test_stream = get_stream(experimentconfig['batch_size'],image_size,test=istest)
# Defining step rule and algorithm
if 'step_rule' in experimentconfig.keys() and not experimentconfig['step_rule'] is None :
step_rule = experimentconfig['step_rule'](learning_rate=experimentconfig['learning_rate'])
else :
step_rule=Scale(learning_rate=experimentconfig['learning_rate'])
params = map(lasagne.utils.as_theano_expression,lasagne.layers.get_all_params(network, trainable=True))
print("Initializing algorithm")
algorithm = GradientDescent(
cost=loss, gradients={var:T.grad(loss,var) for var in params},#parameters=cg.parameters, #params
step_rule=step_rule)
#algorithm.add_updates(extra_updates)
grad_norm = aggregation.mean(algorithm.total_gradient_norm)
grad_norm.name = "grad_norm"
print("Initializing extensions...")
plot = Plot(save_to, channels=[['train_loss','valid_loss'],
['train_grad_norm'],
#['train_grad_norm','train_reg_penalty'],
['train_error_rate','valid_error_rate']], server_url='http://hades.calculquebec.ca:5042')
checkpoint = Checkpoint('models/best_'+save_to+'.tar')
# checkpoint.add_condition(['after_n_batches=25'],
checkpoint.add_condition(['after_epoch'],
predicate=OnLogRecord('valid_error_rate_best_so_far'))
#Defining extensions
extensions = [Timing(),
FinishAfter(after_n_epochs=experimentconfig['num_epochs'],
after_n_batches=experimentconfig['num_batches']),
TrainingDataMonitoring([test_loss, error_rate, grad_norm], # reg_penalty],
prefix="train", after_epoch=True), #after_n_epochs=1
DataStreamMonitoring([test_loss, error_rate],valid_stream,prefix="valid", after_epoch=True), #after_n_epochs=1
plot,
#Checkpoint(save_to,after_n_epochs=5),
#ProgressBar(),
# Plot(save_to, channels=[['train_loss','valid_loss'], ['train_error_rate','valid_error_rate']], server_url='http://hades.calculquebec.ca:5042'), #'grad_norm'
# after_batch=True),
Printing(after_epoch=True),
TrackTheBest('valid_error_rate',min), #Keep best
checkpoint, #Save best
FinishIfNoImprovementAfter('valid_error_rate_best_so_far', epochs=5)] # Early-stopping
# model = Model(loss)
# print("Model",model)
main_loop = MainLoop(
algorithm,
train_stream,
# model=model,
extensions=extensions)
print("Starting main loop...")
main_loop.run()
def build_and_run(label, config):
############## CREATE THE NETWORK ###############
#Define the parameters
num_epochs, num_batches, num_channels, image_shape, filter_size, num_filter, pooling_sizes, mlp_hiddens, output_size, batch_size, activation, mlp_activation = config[
'num_epochs'], config['num_batches'], config['num_channels'], config[
'image_shape'], config['filter_size'], config[
'num_filter'], config['pooling_sizes'], config[
'mlp_hiddens'], config['output_size'], config[
'batch_size'], config['activation'], config[
'mlp_activation']
# print(num_epochs, num_channels, image_shape, filter_size, num_filter, pooling_sizes, mlp_hiddens, output_size, batch_size, activation, mlp_activation)
lambda_l1 = 0.000025
lambda_l2 = 0.000025
print("Building model")
#Create the symbolics variable
x = T.tensor4('image_features')
y = T.lmatrix('targets')
#Get the parameters
conv_parameters = zip(filter_size, num_filter)
#Create the convolutions layers
conv_layers = list(
interleave([(Convolutional(filter_size=filter_size,
num_filters=num_filter,
name='conv_{}'.format(i))
for i, (filter_size,
num_filter) in enumerate(conv_parameters)),
(activation),
(MaxPooling(size, name='pool_{}'.format(i))
for i, size in enumerate(pooling_sizes))]))
# (AveragePooling(size, name='pool_{}'.format(i)) for i, size in enumerate(pooling_sizes))]))
#Create the sequence
conv_sequence = ConvolutionalSequence(conv_layers,
num_channels,
image_size=image_shape,
weights_init=Uniform(width=0.2),
biases_init=Constant(0.))
#Initialize the convnet
conv_sequence.initialize()
#Add the MLP
top_mlp_dims = [np.prod(conv_sequence.get_dim('output'))
] + mlp_hiddens + [output_size]
out = Flattener().apply(conv_sequence.apply(x))
mlp = MLP(mlp_activation,
top_mlp_dims,
weights_init=Uniform(0, 0.2),
biases_init=Constant(0.))
#Initialisze the MLP
mlp.initialize()
#Get the output
predict = mlp.apply(out)
cost = CategoricalCrossEntropy().apply(y.flatten(),
predict).copy(name='cost')
error = MisclassificationRate().apply(y.flatten(), predict)
#Little trick to plot the error rate in two different plots (We can't use two time the same data in the plot for a unknow reason)
error_rate = error.copy(name='error_rate')
error_rate2 = error.copy(name='error_rate2')
########### REGULARIZATION ##################
cg = ComputationGraph([cost])
weights = VariableFilter(roles=[WEIGHT])(cg.variables)
biases = VariableFilter(roles=[BIAS])(cg.variables)
# # l2_penalty_weights = T.sum([i*lambda_l2/len(weights) * (W ** 2).sum() for i,W in enumerate(weights)]) # Gradually increase penalty for layer
l2_penalty = T.sum([
lambda_l2 * (W**2).sum() for i, W in enumerate(weights + biases)
]) # Gradually increase penalty for layer
# # #l2_penalty_bias = T.sum([lambda_l2*(B **2).sum() for B in biases])
# # #l2_penalty = l2_penalty_weights + l2_penalty_bias
l2_penalty.name = 'l2_penalty'
l1_penalty = T.sum([lambda_l1 * T.abs_(z).sum() for z in weights + biases])
# l1_penalty_weights = T.sum([i*lambda_l1/len(weights) * T.abs_(W).sum() for i,W in enumerate(weights)]) # Gradually increase penalty for layer
# l1_penalty_biases = T.sum([lambda_l1 * T.abs_(B).sum() for B in biases])
# l1_penalty = l1_penalty_biases + l1_penalty_weights
l1_penalty.name = 'l1_penalty'
costreg = cost + l2_penalty + l1_penalty
costreg.name = 'costreg'
########### DEFINE THE ALGORITHM #############
# algorithm = GradientDescent(cost=cost, parameters=cg.parameters, step_rule=Momentum())
algorithm = GradientDescent(cost=costreg,
parameters=cg.parameters,
step_rule=Adam())
########### GET THE DATA #####################
istest = 'test' in config.keys()
train_stream, valid_stream, test_stream = get_stream(batch_size,
image_shape,
test=istest)
########### INITIALIZING EXTENSIONS ##########
checkpoint = Checkpoint('models/best_' + label + '.tar')
checkpoint.add_condition(
['after_epoch'], predicate=OnLogRecord('valid_error_rate_best_so_far'))
#Adding a live plot with the bokeh server
plot = Plot(
label,
channels=[
['train_error_rate', 'valid_error_rate'],
['valid_cost', 'valid_error_rate2'],
# ['train_costreg','train_grad_norm']], #
[
'train_costreg', 'train_total_gradient_norm',
'train_l2_penalty', 'train_l1_penalty'
]
],
server_url="http://hades.calculquebec.ca:5042")
grad_norm = aggregation.mean(algorithm.total_gradient_norm)
grad_norm.name = 'grad_norm'
extensions = [
Timing(),
FinishAfter(after_n_epochs=num_epochs, after_n_batches=num_batches),
DataStreamMonitoring([cost, error_rate, error_rate2],
valid_stream,
prefix="valid"),
TrainingDataMonitoring([
costreg, error_rate, error_rate2, grad_norm, l2_penalty, l1_penalty
],
prefix="train",
after_epoch=True),
plot,
ProgressBar(),
Printing(),
TrackTheBest('valid_error_rate', min), #Keep best
checkpoint, #Save best
FinishIfNoImprovementAfter('valid_error_rate_best_so_far', epochs=4)
] # Early-stopping
model = Model(cost)
main_loop = MainLoop(algorithm,
data_stream=train_stream,
model=model,
extensions=extensions)
main_loop.run()
host_plot = 'http://tfjgeorge.com:5006'
cost.name = 'cost'
valid_cost.name = 'valid_cost'
extensions = [
Timing(),
TrainingDataMonitoring([cost], after_epoch=True, prefix='train'),
DataStreamMonitoring(variables=[valid_cost], data_stream=valid_stream),
Plot('%s %s' % (
socket.gethostname(),
datetime.datetime.now(),
),
channels=[['train_cost', 'valid_cost']],
after_epoch=True,
server_url=host_plot),
TrackTheBest('valid_cost'),
Checkpoint('model', save_separately=["model", "log"]),
FinishIfNoImprovementAfter('valid_cost_best_so_far', epochs=5),
#FinishAfter(after_n_epochs=100),
Printing()
]
from blocks.main_loop import MainLoop
main_loop = MainLoop(model=model,
data_stream=train_stream,
algorithm=algorithm,
extensions=extensions)
main_loop.run()
def training(self,
fea2obj,
batch_size,
learning_rate=0.005,
steprule='adagrad',
wait_epochs=5,
kl_weight_init=None,
klw_ep=50,
klw_inc_rate=0,
num_epochs=None):
networkfile = self._config['net']
n_epochs = num_epochs or int(self._config['nepochs'])
reg_weight = float(self._config['loss_weight'])
reg_type = self._config['loss_reg']
numtrain = int(
self._config['num_train']) if 'num_train' in self._config else None
train_stream, num_samples_train = get_comb_stream(
fea2obj, 'train', batch_size, shuffle=True, num_examples=numtrain)
dev_stream, num_samples_dev = get_comb_stream(fea2obj,
'dev',
batch_size=None,
shuffle=False)
logger.info('sources: %s -- number of train/dev samples: %d/%d',
train_stream.sources, num_samples_train, num_samples_dev)
t2idx = fea2obj['targets'].t2idx
klw_init = kl_weight_init or float(
self._config['kld_weight']) if 'kld_weight' in self._config else 1
logger.info('kl_weight_init: %d', klw_init)
kl_weight = shared_floatx(klw_init, 'kl_weight')
entropy_weight = shared_floatx(1., 'entropy_weight')
cost, p_at_1, _, KLD, logpy_xz, pat1_recog, misclassify_rate = build_model_new(
fea2obj, len(t2idx), self._config, kl_weight, entropy_weight)
cg = ComputationGraph(cost)
weights = VariableFilter(roles=[WEIGHT])(cg.parameters)
logger.info('Model weights are: %s', weights)
if 'L2' in reg_type:
cost += reg_weight * l2_norm(weights)
logger.info('applying %s with weight: %f ', reg_type, reg_weight)
dropout = -0.1
if dropout > 0:
cg = apply_dropout(cg, weights, dropout)
cost = cg.outputs[0]
cost.name = 'cost'
logger.info('Our Algorithm is : %s, and learning_rate: %f', steprule,
learning_rate)
if 'adagrad' in steprule:
cnf_step_rule = AdaGrad(learning_rate)
elif 'adadelta' in steprule:
cnf_step_rule = AdaDelta(decay_rate=0.95)
elif 'decay' in steprule:
cnf_step_rule = RMSProp(learning_rate=learning_rate,
decay_rate=0.90)
cnf_step_rule = CompositeRule([cnf_step_rule, StepClipping(1)])
elif 'momentum' in steprule:
cnf_step_rule = Momentum(learning_rate=learning_rate, momentum=0.9)
elif 'adam' in steprule:
cnf_step_rule = Adam(learning_rate=learning_rate)
else:
logger.info('The steprule param is wrong! which is: %s', steprule)
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=cnf_step_rule,
on_unused_sources='warn')
#algorithm.add_updates(updates)
gradient_norm = aggregation.mean(algorithm.total_gradient_norm)
step_norm = aggregation.mean(algorithm.total_step_norm)
monitored_vars = [
cost, gradient_norm, step_norm, p_at_1, KLD, logpy_xz, kl_weight,
pat1_recog
]
train_monitor = TrainingDataMonitoring(variables=monitored_vars,
after_batch=True,
before_first_epoch=True,
prefix='tra')
dev_monitor = DataStreamMonitoring(variables=[
cost, p_at_1, KLD, logpy_xz, pat1_recog, misclassify_rate
],
after_epoch=True,
before_first_epoch=True,
data_stream=dev_stream,
prefix="dev")
extensions = [
dev_monitor,
train_monitor,
Timing(),
TrackTheBest('dev_cost'),
FinishIfNoImprovementAfter('dev_cost_best_so_far',
epochs=wait_epochs),
Printing(after_batch=False), #, ProgressBar()
FinishAfter(after_n_epochs=n_epochs),
saveload.Load(networkfile + '.toload.pkl'),
] + track_best('dev_cost', networkfile + '.best.pkl')
#extensions.append(SharedVariableModifier(kl_weight,
# lambda n, klw: numpy.cast[theano.config.floatX] (klw_inc_rate + klw), after_epoch=False, every_n_epochs=klw_ep, after_batch=False))
# extensions.append(SharedVariableModifier(entropy_weight,
# lambda n, crw: numpy.cast[theano.config.floatX](crw - klw_inc_rate), after_epoch=False, every_n_epochs=klw_ep, after_batch=False))
logger.info('number of parameters in the model: %d',
tensor.sum([p.size for p in cg.parameters]).eval())
logger.info('Lookup table sizes: %s',
[p.size.eval() for p in cg.parameters if 'lt' in p.name])
main_loop = MainLoop(data_stream=train_stream,
algorithm=algorithm,
model=Model(cost),
extensions=extensions)
main_loop.run()
def construct_main_loop(name, task_name, patch_shape, batch_size,
n_spatial_dims, n_patches, max_epochs, patience_epochs,
learning_rate, gradient_limiter, hyperparameters,
**kwargs):
task = tasks.get_task(**hyperparameters)
hyperparameters["n_channels"] = task.n_channels
extensions = []
# let theta noise decay as training progresses
for key in "location_std scale_std".split():
hyperparameters[key] = theano.shared(hyperparameters[key], name=key)
extensions.append(
util.ExponentialDecay(hyperparameters[key],
hyperparameters["%s_decay" % key],
after_batch=True))
print "constructing graphs..."
graphs, outputs, updates = construct_graphs(task=task, **hyperparameters)
print "setting up main loop..."
from blocks.model import Model
model = Model(outputs["train"]["cost"])
from blocks.algorithms import GradientDescent, CompositeRule, StepClipping, Adam, RMSProp
from extensions import Compressor
if gradient_limiter == "clip":
limiter = StepClipping(1.)
elif gradient_limiter == "compress":
limiter = Compressor()
else:
raise ValueError()
algorithm = GradientDescent(
cost=outputs["train"]["cost"],
parameters=graphs["train"].parameters,
step_rule=CompositeRule([limiter,
Adam(learning_rate=learning_rate)]))
algorithm.add_updates(updates["train"])
extensions.extend(
construct_monitors(algorithm=algorithm,
task=task,
model=model,
graphs=graphs,
outputs=outputs,
updates=updates,
**hyperparameters))
from blocks.extensions import FinishAfter, Printing, ProgressBar, Timing
from blocks.extensions.stopping import FinishIfNoImprovementAfter
from blocks.extensions.training import TrackTheBest
from blocks.extensions.saveload import Checkpoint
from dump import DumpBest, LightCheckpoint, PrintingTo, DumpGraph, DumpLog
extensions.extend([
TrackTheBest("valid_error_rate", "best_valid_error_rate"),
FinishIfNoImprovementAfter("best_valid_error_rate",
epochs=patience_epochs),
FinishAfter(after_n_epochs=max_epochs),
DumpBest("best_valid_error_rate", name + "_best.zip"),
Checkpoint(hyperparameters["checkpoint_save_path"],
on_interrupt=False,
every_n_epochs=10,
use_cpickle=True),
DumpLog("log.pkl", after_epoch=True),
ProgressBar(),
Timing(),
Printing(),
PrintingTo(name + "_log"),
DumpGraph(name + "_grad_graph")
])
from blocks.main_loop import MainLoop
main_loop = MainLoop(data_stream=task.get_stream("train"),
algorithm=algorithm,
extensions=extensions,
model=model)
from tabulate import tabulate
print "parameter sizes:"
print tabulate(
(key, "x".join(map(str,
value.get_value().shape)), value.get_value().size)
for key, value in main_loop.model.get_parameter_dict().items())
return main_loop
theano.function([x, x_m], y_hat_softmax),
before_first_epoch=False,
every_n_epochs=5,
prefix='testPER')
checkpoint = Checkpoint(conf.path_to_model, after_training=False)
checkpoint.add_condition(
['after_epoch'],
predicate=predicates.OnLogRecord('valid_log_p_best_so_far'))
extensions = [
val_monitor,
train_monitor,
per_val_monitor,
per_test_monitor,
Timing(),
FinishAfter(after_n_epochs=conf.max_epochs),
checkpoint,
Printing(),
TrackTheBest(record_name='val_monitor',
notification_name='valid_log_p_best_so_far'),
FinishIfNoImprovementAfter(notification_name='valid_log_p_best_so_far',
epochs=conf.epochs_early_stopping),
]
main_loop = MainLoop(
algorithm=algorithm,
data_stream=stream_train,
model=model,
extensions=extensions,
)
main_loop.run()