def __init__(self,
step_rule=None,
gradients=None,
known_grads=None,
**kwargs):
if gradients:
kwargs.setdefault("params", gradients.keys())
super(GradientDescent, self).__init__(**kwargs)
self.gradients = gradients
if not self.gradients:
logger.info("Taking the cost gradient")
self.gradients = dict(
equizip(
self.params,
tensor.grad(self.cost,
self.params,
known_grads=known_grads)))
logger.info("The cost gradient computation graph is built")
else:
if known_grads:
raise ValueError("known_grads has no effect when gradients "
"are passed in")
self.step_rule = step_rule if step_rule else Scale()
self.total_gradient_norm = named_copy(l2_norm(self.gradients.values()),
"total_gradient_norm")
self.steps, self.step_rule_updates = (self.step_rule.compute_steps(
self.gradients))
self.total_step_norm = named_copy(l2_norm(self.steps.values()),
"total_step_norm")
def __init__(self, step_rule=None, gradients=None, known_grads=None,
consider_constant=None, on_unused_sources='raise',
theano_func_kwargs=None, **kwargs):
if gradients:
kwargs.setdefault("parameters", gradients.keys())
super(GradientDescent, self).__init__(**kwargs)
self.gradients = gradients
if not self.gradients:
logger.info("Taking the cost gradient")
self.gradients = dict(
equizip(self.parameters, tensor.grad(
self.cost, self.parameters,
known_grads=known_grads,
consider_constant=consider_constant)))
logger.info("The cost gradient computation graph is built")
else:
if known_grads:
raise ValueError("known_grads has no effect when gradients "
"are passed in")
if consider_constant is not None:
raise ValueError("consider_constant has no effect when "
"gradients are passed in")
self.step_rule = step_rule if step_rule else Scale()
self.total_gradient_norm = named_copy(l2_norm(self.gradients.values()),
"total_gradient_norm")
self.steps, self.step_rule_updates = (
self.step_rule.compute_steps(self.gradients))
self.total_step_norm = named_copy(l2_norm(self.steps.values()),
"total_step_norm")
self.on_unused_sources = on_unused_sources
self.theano_func_kwargs = (theano_func_kwargs if theano_func_kwargs
is not None else dict())
def __init__(self, step_rule=None, gradients=None, known_grads=None,
**kwargs):
if gradients:
kwargs.setdefault("params", gradients.keys())
super(GradientDescent, self).__init__(**kwargs)
self.gradients = gradients
if not self.gradients:
logger.info("Taking the cost gradient")
self.gradients = dict(
equizip(self.params, tensor.grad(self.cost, self.params,
known_grads=known_grads)))
logger.info("The cost gradient computation graph is built")
else:
if known_grads:
raise ValueError("known_grads has no effect when gradients "
"are passed in")
self.step_rule = step_rule if step_rule else Scale()
self.total_gradient_norm = named_copy(l2_norm(self.gradients.values()),
"total_gradient_norm")
self.steps, self.step_rule_updates = (
self.step_rule.compute_steps(self.gradients))
self.total_step_norm = named_copy(l2_norm(self.steps.values()),
"total_step_norm")
def __init__(self,
cost,
params,
subtensor_params={},
step_rule=None,
*args,
**kwargs):
full_params = params
self.subtensor_params = subtensor_params
# For each LookupTable, we replace it by its subtensors appearing in the graph
params = [
param for param in full_params if param not in subtensor_params
]
for _, (_, _, outputs, _) in subtensor_params.iteritems():
params.extend(outputs)
super(GradientDescent, self).__init__(cost=cost,
params=params,
**kwargs)
# self.params contains the list of outputs of the lookup tables
logger.info("Taking the cost gradient")
self.gradients = dict(
equizip(self.params, tensor.grad(self.cost, self.params)))
# We combine the gradients extracted from the same parameter
for param, (subparam, canonized_indices, outputs,
indices) in subtensor_params.iteritems():
# This is necessary if we want to compute the l2 norm correctly (e.g. for StepClipping)
tmp = shared_floatx(param.get_value() * 0.)
for (output, indice) in zip(outputs, indices):
tmp = tensor.inc_subtensor(tmp[indice], self.gradients[output])
del self.gradients[output]
self.gradients[subparam] = tmp[canonized_indices]
# We remove the subtensors from the list of parameters
self.params = full_params
logger.info("The cost gradient computation graph is built")
self.step_rule = step_rule if step_rule else Scale()
self.total_gradient_norm = named_copy(l2_norm(self.gradients.values()),
"total_gradient_norm")
self.steps, self.step_rule_updates = (self.step_rule.compute_steps(
self.gradients))
self.total_step_norm = named_copy(l2_norm(self.steps.values()),
"total_step_norm")
def __init__(self, step_rule=None, gradients=None, **kwargs):
super(GradientDescent, self).__init__(**kwargs)
self.gradients = gradients
if not self.gradients:
logger.info("Taking the cost gradient")
self.gradients = dict(
zip(self.params, tensor.grad(self.cost, self.params)))
logger.info("The cost gradient computation graph is built")
self.step_rule = step_rule if step_rule else SteepestDescent()
self.total_gradient_norm = named_copy(L2_norm(self.gradients.values()),
"total_gradient_norm")
self.steps = self.step_rule.compute_steps(self.gradients)
self.total_step_norm = named_copy(L2_norm(self.steps.values()),
"total_step_norm")
def attach_aggregation_schemes(variables):
# Aggregation specification has to be factored out as a separate
# function as it has to be applied at the very last stage
# separately to training and validation observables.
result = []
for var in variables:
if var.name == 'weights_penalty':
result.append(named_copy(aggregation.mean(var, batch_size),
'weights_penalty_per_recording'))
elif var.name == 'weights_entropy':
result.append(named_copy(aggregation.mean(
var, recognizer.labels_mask.sum()), 'weights_entropy_per_label'))
else:
result.append(var)
return result
def categorical_cross_entropy(self, application_call, y, x):
"""Computationally stable cross-entropy for pre-softmax values.
Parameters
----------
y : :class:`~tensor.TensorVariable`
In the case of a matrix argument, each row represents a
probabilility distribution. In the vector case, each element
represents a distribution by specifying the position of 1 in a
1-hot vector.
x : :class:`~tensor.TensorVariable`
A matrix, each row contains unnormalized probabilities of a
distribution.
Returns
-------
cost : :class:`~tensor.TensorVariable`
A vector of cross-entropies between respective distributions
from y and x.
"""
x = self.log_probabilities(x)
application_call.add_auxiliary_variable(
named_copy(x, 'log_probabilities'))
if y.ndim == x.ndim - 1:
indices = tensor.arange(y.shape[0]) * x.shape[1] + y
cost = -x.flatten()[indices]
elif y.ndim == x.ndim:
cost = -(x * y).sum(axis=1)
else:
raise TypeError('rank mismatch between x and y')
return cost
def test_training_data_monitoring():
weights = numpy.array([-1, 1], dtype=theano.config.floatX)
features = [
numpy.array(f, dtype=theano.config.floatX)
for f in [[1, 2], [3, 4], [5, 6]]
]
targets = [(weights * f).sum() for f in features]
n_batches = 3
dataset = IterableDataset(dict(features=features, targets=targets))
x = tensor.vector('features')
y = tensor.scalar('targets')
W = shared_floatx([0, 0], name='W')
V = shared_floatx(7, name='V')
W_sum = named_copy(W.sum(), 'W_sum')
cost = ((x * W).sum() - y)**2
cost.name = 'cost'
class TrueCostExtension(TrainingExtension):
def before_batch(self, data):
self.main_loop.log.current_row['true_cost'] = ((
(W.get_value() * data["features"]).sum() - data["targets"])**2)
main_loop = MainLoop(model=None,
data_stream=dataset.get_example_stream(),
algorithm=GradientDescent(cost=cost,
parameters=[W],
step_rule=Scale(0.001)),
extensions=[
FinishAfter(after_n_epochs=1),
TrainingDataMonitoring([W_sum, cost, V],
prefix="train1",
after_batch=True),
TrainingDataMonitoring(
[aggregation.mean(W_sum), cost],
prefix="train2",
after_epoch=True),
TrueCostExtension()
])
main_loop.run()
# Check monitoring of a shared varible
assert_allclose(main_loop.log.current_row['train1_V'], 7.0)
for i in range(n_batches):
# The ground truth is written to the log before the batch is
# processed, where as the extension writes after the batch is
# processed. This is why the iteration numbers differs here.
assert_allclose(main_loop.log[i]['true_cost'],
main_loop.log[i + 1]['train1_cost'])
assert_allclose(
main_loop.log[n_batches]['train2_cost'],
sum([main_loop.log[i]['true_cost']
for i in range(n_batches)]) / n_batches)
assert_allclose(
main_loop.log[n_batches]['train2_W_sum'],
sum([
main_loop.log[i]['train1_W_sum'] for i in range(1, n_batches + 1)
]) / n_batches)
def categorical_cross_entropy(self, application_call, y, x):
"""Computationally stable cross-entropy for pre-softmax values.
Parameters
----------
y : :class:`~tensor.TensorVariable`
In the case of a matrix argument, each row represents a
probabilility distribution. In the vector case, each element
represents a distribution by specifying the position of 1 in a
1-hot vector.
x : :class:`~tensor.TensorVariable`
A matrix, each row contains unnormalized probabilities of a
distribution.
Returns
-------
cost : :class:`~tensor.TensorVariable`
A vector of cross-entropies between respective distributions
from y and x.
"""
x = self.log_probabilities(x)
application_call.add_auxiliary_variable(
named_copy(x, 'log_probabilities'))
if y.ndim == x.ndim - 1:
indices = tensor.arange(y.shape[0]) * x.shape[1] + y
cost = -x.flatten()[indices]
elif y.ndim == x.ndim:
cost = -(x * y).sum(axis=1)
else:
raise TypeError('rank mismatch between x and y')
return cost
def __init__(self, step_rule=None, gradients=None, **kwargs):
if gradients:
kwargs.setdefault("params", gradients.keys())
super(GradientDescent, self).__init__(**kwargs)
self.gradients = gradients
if not self.gradients:
logger.info("Taking the cost gradient")
self.gradients = dict(
zip(self.params, tensor.grad(self.cost, self.params)))
logger.info("The cost gradient computation graph is built")
self.step_rule = step_rule if step_rule else Scale()
self.total_gradient_norm = named_copy(l2_norm(self.gradients.values()),
"total_gradient_norm")
self.steps, self.step_rule_updates = (self.step_rule.compute_steps(
self.gradients))
self.total_step_norm = named_copy(l2_norm(self.steps.values()),
"total_step_norm")
def get_channels(self):
channels = []
for _, quantities in self.dikt.items():
if len(quantities) == 1:
channels.append(quantities[0])
else:
# name not unique; uniquefy
for i, quantity in enumerate(quantities):
channels.append(named_copy(quantity, "%s[%i]" % (quantity.name, i)))
return channels
def get_channels(self):
channels = []
for _, quantities in self.dikt.items():
if len(quantities) == 1:
channels.append(quantities[0])
else:
# name not unique; uniquefy
for i, quantity in enumerate(quantities):
channels.append(
named_copy(quantity, "%s[%i]" % (quantity.name, i)))
return channels
def test_training_data_monitoring():
weights = numpy.array([-1, 1], dtype=theano.config.floatX)
features = [numpy.array(f, dtype=theano.config.floatX)
for f in [[1, 2], [3, 4], [5, 6]]]
targets = [(weights * f).sum() for f in features]
n_batches = 3
dataset = IterableDataset(dict(features=features, targets=targets))
x = tensor.vector('features')
y = tensor.scalar('targets')
W = shared_floatx([0, 0], name='W')
V = shared_floatx(7, name='V')
W_sum = named_copy(W.sum(), 'W_sum')
cost = ((x * W).sum() - y) ** 2
cost.name = 'cost'
class TrueCostExtension(TrainingExtension):
def before_batch(self, data):
self.main_loop.log.current_row['true_cost'] = (
((W.get_value() * data["features"]).sum() -
data["targets"]) ** 2)
main_loop = MainLoop(
model=None, data_stream=dataset.get_example_stream(),
algorithm=GradientDescent(cost=cost, params=[W],
step_rule=Scale(0.001)),
extensions=[
FinishAfter(after_n_epochs=1),
TrainingDataMonitoring([W_sum, cost, V], prefix="train1",
after_batch=True),
TrainingDataMonitoring([aggregation.mean(W_sum), cost],
prefix="train2", after_epoch=True),
TrueCostExtension()])
main_loop.run()
# Check monitoring of a shared varible
assert_allclose(main_loop.log.current_row['train1_V'], 7.0)
for i in range(n_batches):
# The ground truth is written to the log before the batch is
# processed, where as the extension writes after the batch is
# processed. This is why the iteration numbers differs here.
assert_allclose(main_loop.log[i]['true_cost'],
main_loop.log[i + 1]['train1_cost'])
assert_allclose(
main_loop.log[n_batches]['train2_cost'],
sum([main_loop.log[i]['true_cost']
for i in range(n_batches)]) / n_batches)
assert_allclose(
main_loop.log[n_batches]['train2_W_sum'],
sum([main_loop.log[i]['train1_W_sum']
for i in range(1, n_batches + 1)]) / n_batches)
def __init__(self, cost, params, subtensor_params={}, step_rule=None, *args, **kwargs):
full_params = params
self.subtensor_params = subtensor_params
# For each LookupTable, we replace it by its subtensors appearing in the graph
params = [param for param in full_params if param not in subtensor_params]
for _, (_, _, outputs, _) in subtensor_params.iteritems():
params.extend(outputs)
super(GradientDescent, self).__init__(cost=cost, params=params, **kwargs)
# self.params contains the list of outputs of the lookup tables
logger.info("Taking the cost gradient")
self.gradients = dict(
equizip(self.params, tensor.grad(self.cost, self.params)))
# We combine the gradients extracted from the same parameter
for param, (subparam, canonized_indices, outputs, indices) in subtensor_params.iteritems():
# This is necessary if we want to compute the l2 norm correctly (e.g. for StepClipping)
tmp = shared_floatx(param.get_value() * 0.)
for (output, indice) in zip(outputs, indices):
tmp = tensor.inc_subtensor(tmp[indice], self.gradients[output])
del self.gradients[output]
self.gradients[subparam] = tmp[canonized_indices]
# We remove the subtensors from the list of parameters
self.params = full_params
logger.info("The cost gradient computation graph is built")
self.step_rule = step_rule if step_rule else Scale()
self.total_gradient_norm = named_copy(l2_norm(self.gradients.values()),
"total_gradient_norm")
self.steps, self.step_rule_updates = (
self.step_rule.compute_steps(self.gradients))
self.total_step_norm = named_copy(l2_norm(self.steps.values()),
"total_step_norm")
def main(name, epochs, batch_size, learning_rate, dim, mix_dim, old_model_name,
max_length, bokeh, GRU, dropout, depth, max_grad, step_method,
epsilon, sample):
#----------------------------------------------------------------------
datasource = name
def shnum(x):
""" Convert a positive float into a short tag-usable string
E.g.: 0 -> 0, 0.005 -> 53, 100 -> 1-2
"""
return '0' if x <= 0 else '%s%d' % (
("%e" % x)[0], -np.floor(np.log10(x)))
jobname = "%s-%dX%dm%dd%dr%sb%de%s" % (
datasource, depth, dim, mix_dim, int(
dropout * 10), shnum(learning_rate), batch_size, shnum(epsilon))
if max_length != 600:
jobname += '-L%d' % max_length
if GRU:
jobname += 'g'
if max_grad != 5.:
jobname += 'G%g' % max_grad
if step_method != 'adam':
jobname += step_method
if sample:
print("Sampling")
else:
print("\nRunning experiment %s" % jobname)
#----------------------------------------------------------------------
if depth > 1:
transition = LSTMstack(dim=dim,
depth=depth,
name="transition",
lstm_name="transition")
assert not GRU
elif GRU:
transition = GatedRecurrent(dim=dim, name="transition")
else:
transition = LSTM(dim=dim, name="transition")
emitter = SketchEmitter(mix_dim=mix_dim, epsilon=epsilon, name="emitter")
readout = Readout(readout_dim=emitter.get_dim('inputs'),
source_names=['states'],
emitter=emitter,
name="readout")
normal_inputs = [
name for name in transition.apply.sequences if 'mask' not in name
]
fork = Fork(normal_inputs, prototype=Linear(use_bias=True))
generator = SequenceGenerator(readout=readout,
transition=transition,
fork=fork)
# Initialization settings
generator.weights_init = OrthogonalGlorot()
generator.biases_init = Constant(0)
# Build the cost computation graph [steps,batch_size, 3]
x = T.tensor3('features', dtype=floatX)[:max_length, :, :]
x.tag.test_value = np.ones((max_length, batch_size, 3)).astype(np.float32)
cost = generator.cost(x)
cost.name = "sequence_log_likelihood"
# Give an idea of what's going on
model = Model(cost)
params = model.get_params()
logger.info("Parameters:\n" +
pprint.pformat([(key, value.get_value().shape)
for key, value in params.items()],
width=120))
model_size = 0
for v in params.itervalues():
s = v.get_value().shape
model_size += s[0] * (s[1] if len(s) > 1 else 1)
logger.info("Total number of parameters %d" % model_size)
#------------------------------------------------------------
extensions = []
if old_model_name == 'continue':
extensions.append(LoadFromDump(jobname))
elif old_model_name:
# or you can just load the weights without state using:
old_params = LoadFromDump(old_model_name).manager.load_parameters()
model.set_param_values(old_params)
else:
# Initialize parameters
for brick in model.get_top_bricks():
brick.initialize()
if sample:
assert old_model_name and old_model_name != 'continue'
Sample(generator, steps=max_length, path='.').do(None)
exit(0)
#------------------------------------------------------------
# Define the training algorithm.
cg = ComputationGraph(cost)
if dropout > 0.:
from blocks.roles import INPUT, OUTPUT
dropout_target = VariableFilter(roles=[OUTPUT],
bricks=[transition],
name_regex='states')(cg.variables)
cg = apply_dropout(cg, dropout_target, dropout)
cost = cg.outputs[0]
if step_method == 'adam':
step_rule = Adam(learning_rate)
elif step_method == 'rmsprop':
step_rule = RMSProp(learning_rate, decay_rate=0.95)
elif step_method == 'adagrad':
step_rule = AdaGrad(learning_rate)
elif step_method == 'adadelta':
step_rule = AdaDelta()
elif step_method == 'scale':
step_rule = Scale(learning_rate=0.1)
else:
raise Exception('Unknown sttep method %s' % step_method)
step_rule = CompositeRule([StepClipping(max_grad), step_rule])
algorithm = GradientDescent(cost=cost,
params=cg.parameters,
step_rule=step_rule)
#------------------------------------------------------------
observables = [cost]
# Fetch variables useful for debugging
(energies, ) = VariableFilter(applications=[generator.readout.readout],
name_regex="output")(cg.variables)
(activations, ) = VariableFilter(
applications=[generator.transition.apply],
name=generator.transition.apply.states[0])(cg.variables)
min_energy = named_copy(energies.min(), "min_energy")
max_energy = named_copy(energies.max(), "max_energy")
mean_activation = named_copy(abs(activations).mean(), "mean_activation")
observables += [min_energy, max_energy, mean_activation]
observables += [algorithm.total_step_norm, algorithm.total_gradient_norm]
for name, param in params.items():
observables.append(named_copy(param.norm(2), name + "_norm"))
observables.append(
named_copy(algorithm.gradients[param].norm(2),
name + "_grad_norm"))
#------------------------------------------------------------
datasource_fname = os.path.join(fuel.config.data_path, datasource,
datasource + '.hdf5')
train_ds = H5PYDataset(
datasource_fname, #max_length=max_length,
which_set='train',
sources=('features', ),
load_in_memory=True)
train_stream = DataStream(train_ds,
iteration_scheme=ShuffledScheme(
train_ds.num_examples, batch_size))
test_ds = H5PYDataset(
datasource_fname, #max_length=max_length,
which_set='test',
sources=('features', ),
load_in_memory=True)
test_stream = DataStream(test_ds,
iteration_scheme=SequentialScheme(
test_ds.num_examples, batch_size))
train_stream = Mapping(train_stream, _transpose)
test_stream = Mapping(test_stream, _transpose)
def stream_stats(ds, label):
itr = ds.get_epoch_iterator(as_dict=True)
batch_count = 0
examples_count = 0
for batch in itr:
batch_count += 1
examples_count += batch['features'].shape[1]
print('%s #batch %d #examples %d' %
(label, batch_count, examples_count))
stream_stats(train_stream, 'train')
stream_stats(test_stream, 'test')
extensions += [
Timing(every_n_batches=10),
TrainingDataMonitoring(observables, prefix="train",
every_n_batches=10),
DataStreamMonitoring(
[cost],
test_stream,
prefix="test",
on_resumption=True,
after_epoch=False, # by default this is True
every_n_batches=100),
# all monitored data is ready so print it...
# (next steps may take more time and we want to see the
# results as soon as possible so print as soon as you can)
Printing(every_n_batches=10),
# perform multiple dumps at different intervals
# so if one of them breaks (has nan) we can hopefully
# find a model from few batches ago in the other
Dump(jobname, every_n_batches=11),
Dump(jobname + '.test', every_n_batches=100),
Sample(generator,
steps=max_length,
path=jobname + '.test',
every_n_batches=100),
ProgressBar(),
FinishAfter(after_n_epochs=epochs)
# This shows a way to handle NaN emerging during
# training: simply finish it.
.add_condition("after_batch", _is_nan),
]
if bokeh:
extensions.append(Plot('sketch', channels=[
['cost'],
]))
# Construct the main loop and start training!
main_loop = MainLoop(model=model,
data_stream=train_stream,
algorithm=algorithm,
extensions=extensions)
main_loop.run()
def initialize_all(config, save_path, bokeh_name, params, bokeh_server, bokeh,
test_tag, use_load_ext, load_log, fast_start):
root_path, extension = os.path.splitext(save_path)
data = Data(**config['data'])
train_conf = config['training']
recognizer = create_model(config, data, test_tag)
# Separate attention_params to be handled differently
# when regularization is applied
attention = recognizer.generator.transition.attention
attention_params = Selector(attention).get_parameters().values()
logger.info(
"Initialization schemes for all bricks.\n"
"Works well only in my branch with __repr__ added to all them,\n"
"there is an issue #463 in Blocks to do that properly.")
def show_init_scheme(cur):
result = dict()
for attr in dir(cur):
if attr.endswith('_init'):
result[attr] = getattr(cur, attr)
for child in cur.children:
result[child.name] = show_init_scheme(child)
return result
logger.info(pprint.pformat(show_init_scheme(recognizer)))
prediction, prediction_mask = add_exploration(recognizer, data, train_conf)
#
# Observables:
#
primary_observables = [] # monitored each batch
secondary_observables = [] # monitored every 10 batches
validation_observables = [] # monitored on the validation set
cg = recognizer.get_cost_graph(batch=True,
prediction=prediction,
prediction_mask=prediction_mask)
labels, = VariableFilter(applications=[recognizer.cost], name='labels')(cg)
labels_mask, = VariableFilter(applications=[recognizer.cost],
name='labels_mask')(cg)
gain_matrix = VariableFilter(
theano_name=RewardRegressionEmitter.GAIN_MATRIX)(cg)
if len(gain_matrix):
gain_matrix, = gain_matrix
primary_observables.append(named_copy(gain_matrix.min(), 'min_gain'))
primary_observables.append(named_copy(gain_matrix.max(), 'max_gain'))
batch_cost = cg.outputs[0].sum()
batch_size = named_copy(recognizer.recordings.shape[1], "batch_size")
# Assumes constant batch size. `aggregation.mean` is not used because
# of Blocks #514.
cost = batch_cost / batch_size
cost.name = "sequence_total_cost"
logger.info("Cost graph is built")
# Fetch variables useful for debugging.
# It is important not to use any aggregation schemes here,
# as it's currently impossible to spread the effect of
# regularization on their variables, see Blocks #514.
cost_cg = ComputationGraph(cost)
r = recognizer
energies, = VariableFilter(applications=[r.generator.readout.readout],
name="output_0")(cost_cg)
bottom_output = VariableFilter(applications=[r.bottom.apply],
name="output")(cost_cg)[-1]
attended, = VariableFilter(applications=[r.generator.transition.apply],
name="attended")(cost_cg)
attended_mask, = VariableFilter(applications=[
r.generator.transition.apply
],
name="attended_mask")(cost_cg)
weights, = VariableFilter(applications=[r.generator.evaluate],
name="weights")(cost_cg)
max_recording_length = named_copy(r.recordings.shape[0],
"max_recording_length")
# To exclude subsampling related bugs
max_attended_mask_length = named_copy(attended_mask.shape[0],
"max_attended_mask_length")
max_attended_length = named_copy(attended.shape[0], "max_attended_length")
max_num_phonemes = named_copy(labels.shape[0], "max_num_phonemes")
min_energy = named_copy(energies.min(), "min_energy")
max_energy = named_copy(energies.max(), "max_energy")
mean_attended = named_copy(abs(attended).mean(), "mean_attended")
mean_bottom_output = named_copy(
abs(bottom_output).mean(), "mean_bottom_output")
weights_penalty = named_copy(monotonicity_penalty(weights, labels_mask),
"weights_penalty")
weights_entropy = named_copy(entropy(weights, labels_mask),
"weights_entropy")
mask_density = named_copy(labels_mask.mean(), "mask_density")
cg = ComputationGraph([
cost, weights_penalty, weights_entropy, min_energy, max_energy,
mean_attended, mean_bottom_output, batch_size, max_num_phonemes,
mask_density
])
# Regularization. It is applied explicitly to all variables
# of interest, it could not be applied to the cost only as it
# would not have effect on auxiliary variables, see Blocks #514.
reg_config = config['regularization']
regularized_cg = cg
if reg_config.get('dropout'):
logger.info('apply dropout')
regularized_cg = apply_dropout(cg, [bottom_output], 0.5)
if reg_config.get('noise'):
logger.info('apply noise')
noise_subjects = [
p for p in cg.parameters if p not in attention_params
]
regularized_cg = apply_noise(cg, noise_subjects, reg_config['noise'])
train_cost = regularized_cg.outputs[0]
if reg_config.get("penalty_coof", .0) > 0:
# big warning!!!
# here we assume that:
# regularized_weights_penalty = regularized_cg.outputs[1]
train_cost = (train_cost + reg_config.get("penalty_coof", .0) *
regularized_cg.outputs[1] / batch_size)
if reg_config.get("decay", .0) > 0:
train_cost = (
train_cost + reg_config.get("decay", .0) *
l2_norm(VariableFilter(roles=[WEIGHT])(cg.parameters))**2)
train_cost = named_copy(train_cost, 'train_cost')
gradients = None
if reg_config.get('adaptive_noise'):
logger.info('apply adaptive noise')
if ((reg_config.get("penalty_coof", .0) > 0)
or (reg_config.get("decay", .0) > 0)):
logger.error('using adaptive noise with alignment weight panalty '
'or weight decay is probably stupid')
train_cost, regularized_cg, gradients, noise_brick = apply_adaptive_noise(
cg,
cg.outputs[0],
variables=cg.parameters,
num_examples=data.get_dataset('train').num_examples,
parameters=SpeechModel(
regularized_cg.outputs[0]).get_parameter_dict().values(),
**reg_config.get('adaptive_noise'))
train_cost.name = 'train_cost'
adapt_noise_cg = ComputationGraph(train_cost)
model_prior_mean = named_copy(
VariableFilter(applications=[noise_brick.apply],
name='model_prior_mean')(adapt_noise_cg)[0],
'model_prior_mean')
model_cost = named_copy(
VariableFilter(applications=[noise_brick.apply],
name='model_cost')(adapt_noise_cg)[0], 'model_cost')
model_prior_variance = named_copy(
VariableFilter(applications=[noise_brick.apply],
name='model_prior_variance')(adapt_noise_cg)[0],
'model_prior_variance')
regularized_cg = ComputationGraph(
[train_cost, model_cost] + regularized_cg.outputs +
[model_prior_mean, model_prior_variance])
primary_observables += [
regularized_cg.outputs[1], # model cost
regularized_cg.outputs[2], # task cost
regularized_cg.outputs[-2], # model prior mean
regularized_cg.outputs[-1]
] # model prior variance
# Model is weird class, we spend lots of time arguing with Bart
# what it should be. However it can already nice things, e.g.
# one extract all the parameters from the computation graphs
# and give them hierahical names. This help to notice when a
# because of some bug a parameter is not in the computation
# graph.
model = SpeechModel(train_cost)
if params:
logger.info("Load parameters from " + params)
# please note: we cannot use recognizer.load_params
# as it builds a new computation graph that dies not have
# shapred variables added by adaptive weight noise
param_values = load_parameter_values(params)
model.set_parameter_values(param_values)
parameters = model.get_parameter_dict()
logger.info("Parameters:\n" +
pprint.pformat([(key, parameters[key].get_value().shape)
for key in sorted(parameters.keys())],
width=120))
# Define the training algorithm.
clipping = StepClipping(train_conf['gradient_threshold'])
clipping.threshold.name = "gradient_norm_threshold"
rule_names = train_conf.get('rules', ['momentum'])
core_rules = []
if 'momentum' in rule_names:
logger.info("Using scaling and momentum for training")
core_rules.append(Momentum(train_conf['scale'],
train_conf['momentum']))
if 'adadelta' in rule_names:
logger.info("Using AdaDelta for training")
core_rules.append(
AdaDelta(train_conf['decay_rate'], train_conf['epsilon']))
max_norm_rules = []
if reg_config.get('max_norm', False) > 0:
logger.info("Apply MaxNorm")
maxnorm_subjects = VariableFilter(roles=[WEIGHT])(cg.parameters)
if reg_config.get('max_norm_exclude_lookup', False):
maxnorm_subjects = [
v for v in maxnorm_subjects
if not isinstance(get_brick(v), LookupTable)
]
logger.info("Parameters covered by MaxNorm:\n" + pprint.pformat(
[name for name, p in parameters.items() if p in maxnorm_subjects]))
logger.info("Parameters NOT covered by MaxNorm:\n" + pprint.pformat([
name for name, p in parameters.items() if not p in maxnorm_subjects
]))
max_norm_rules = [
Restrict(VariableClipping(reg_config['max_norm'], axis=0),
maxnorm_subjects)
]
burn_in = []
if train_conf.get('burn_in_steps', 0):
burn_in.append(BurnIn(num_steps=train_conf['burn_in_steps']))
algorithm = GradientDescent(
cost=train_cost,
parameters=parameters.values(),
gradients=gradients,
step_rule=CompositeRule(
[clipping] + core_rules + max_norm_rules +
# Parameters are not changed at all
# when nans are encountered.
[RemoveNotFinite(0.0)] + burn_in),
on_unused_sources='warn')
logger.debug("Scan Ops in the gradients")
gradient_cg = ComputationGraph(algorithm.gradients.values())
for op in ComputationGraph(gradient_cg).scans:
logger.debug(op)
# More variables for debugging: some of them can be added only
# after the `algorithm` object is created.
secondary_observables += list(regularized_cg.outputs)
if not 'train_cost' in [v.name for v in secondary_observables]:
secondary_observables += [train_cost]
secondary_observables += [
algorithm.total_step_norm, algorithm.total_gradient_norm,
clipping.threshold
]
for name, param in parameters.items():
num_elements = numpy.product(param.get_value().shape)
norm = param.norm(2) / num_elements**0.5
grad_norm = algorithm.gradients[param].norm(2) / num_elements**0.5
step_norm = algorithm.steps[param].norm(2) / num_elements**0.5
stats = tensor.stack(norm, grad_norm, step_norm, step_norm / grad_norm)
stats.name = name + '_stats'
secondary_observables.append(stats)
primary_observables += [
train_cost, algorithm.total_gradient_norm, algorithm.total_step_norm,
clipping.threshold, max_recording_length, max_attended_length,
max_attended_mask_length
]
validation_observables += [
rename(aggregation.mean(batch_cost, batch_size), cost.name),
rename(aggregation.sum_(batch_size), 'num_utterances'),
weights_entropy, weights_penalty
]
def attach_aggregation_schemes(variables):
# Aggregation specification has to be factored out as a separate
# function as it has to be applied at the very last stage
# separately to training and validation observables.
result = []
for var in variables:
if var.name == 'weights_penalty':
result.append(
rename(aggregation.mean(var, batch_size),
'weights_penalty_per_recording'))
elif var.name == 'weights_entropy':
result.append(
rename(aggregation.mean(var, labels_mask.sum()),
'weights_entropy_per_label'))
else:
result.append(var)
return result
mon_conf = config['monitoring']
# Build main loop.
logger.info("Initialize extensions")
extensions = []
if use_load_ext and params:
extensions.append(
Load(params, load_iteration_state=True, load_log=True))
if load_log and params:
extensions.append(LoadLog(params))
extensions += [
Timing(after_batch=True),
CGStatistics(),
#CodeVersion(['lvsr']),
]
extensions.append(
TrainingDataMonitoring(primary_observables, after_batch=True))
average_monitoring = TrainingDataMonitoring(
attach_aggregation_schemes(secondary_observables),
prefix="average",
every_n_batches=10)
extensions.append(average_monitoring)
validation = DataStreamMonitoring(
attach_aggregation_schemes(validation_observables),
data.get_stream("valid", shuffle=False),
prefix="valid").set_conditions(
before_first_epoch=not fast_start,
every_n_epochs=mon_conf['validate_every_epochs'],
every_n_batches=mon_conf['validate_every_batches'],
after_training=False)
extensions.append(validation)
per = PhonemeErrorRate(recognizer, data, **config['monitoring']['search'])
per_monitoring = DataStreamMonitoring(
[per],
data.get_stream("valid", batches=False, shuffle=False),
prefix="valid").set_conditions(
before_first_epoch=not fast_start,
every_n_epochs=mon_conf['search_every_epochs'],
every_n_batches=mon_conf['search_every_batches'],
after_training=False)
extensions.append(per_monitoring)
track_the_best_per = TrackTheBest(
per_monitoring.record_name(per)).set_conditions(
before_first_epoch=True, after_epoch=True)
track_the_best_cost = TrackTheBest(
validation.record_name(cost)).set_conditions(before_first_epoch=True,
after_epoch=True)
extensions += [track_the_best_cost, track_the_best_per]
extensions.append(
AdaptiveClipping(algorithm.total_gradient_norm.name,
clipping,
train_conf['gradient_threshold'],
decay_rate=0.998,
burnin_period=500))
extensions += [
SwitchOffLengthFilter(
data.length_filter,
after_n_batches=train_conf.get('stop_filtering')),
FinishAfter(after_n_batches=train_conf['num_batches'],
after_n_epochs=train_conf['num_epochs']).add_condition(
["after_batch"], _gradient_norm_is_none),
]
channels = [
# Plot 1: training and validation costs
[
average_monitoring.record_name(train_cost),
validation.record_name(cost)
],
# Plot 2: gradient norm,
[
average_monitoring.record_name(algorithm.total_gradient_norm),
average_monitoring.record_name(clipping.threshold)
],
# Plot 3: phoneme error rate
[per_monitoring.record_name(per)],
# Plot 4: training and validation mean weight entropy
[
average_monitoring._record_name('weights_entropy_per_label'),
validation._record_name('weights_entropy_per_label')
],
# Plot 5: training and validation monotonicity penalty
[
average_monitoring._record_name('weights_penalty_per_recording'),
validation._record_name('weights_penalty_per_recording')
]
]
if bokeh:
extensions += [
Plot(bokeh_name if bokeh_name else os.path.basename(save_path),
channels,
every_n_batches=10,
server_url=bokeh_server),
]
extensions += [
Checkpoint(save_path,
before_first_epoch=not fast_start,
after_epoch=True,
every_n_batches=train_conf.get('save_every_n_batches'),
save_separately=["model", "log"],
use_cpickle=True).add_condition(
['after_epoch'],
OnLogRecord(track_the_best_per.notification_name),
(root_path + "_best" + extension, )).add_condition(
['after_epoch'],
OnLogRecord(track_the_best_cost.notification_name),
(root_path + "_best_ll" + extension, )),
ProgressBar()
]
extensions.append(EmbedIPython(use_main_loop_run_caller_env=True))
if config['net']['criterion']['name'].startswith('mse'):
extensions.append(
LogInputsGains(labels, cg, recognizer.generator.readout.emitter,
data))
if train_conf.get('patience'):
patience_conf = train_conf['patience']
if not patience_conf.get('notification_names'):
# setdefault will not work for empty list
patience_conf['notification_names'] = [
track_the_best_per.notification_name,
track_the_best_cost.notification_name
]
extensions.append(Patience(**patience_conf))
extensions.append(
Printing(every_n_batches=1, attribute_filter=PrintingFilterList()))
return model, algorithm, data, extensions
def initialize_all(config, save_path, bokeh_name,
params, bokeh_server, bokeh, test_tag, use_load_ext,
load_log, fast_start):
root_path, extension = os.path.splitext(save_path)
data = Data(**config['data'])
train_conf = config['training']
recognizer = create_model(config, data, test_tag)
# Separate attention_params to be handled differently
# when regularization is applied
attention = recognizer.generator.transition.attention
attention_params = Selector(attention).get_parameters().values()
logger.info(
"Initialization schemes for all bricks.\n"
"Works well only in my branch with __repr__ added to all them,\n"
"there is an issue #463 in Blocks to do that properly.")
def show_init_scheme(cur):
result = dict()
for attr in dir(cur):
if attr.endswith('_init'):
result[attr] = getattr(cur, attr)
for child in cur.children:
result[child.name] = show_init_scheme(child)
return result
logger.info(pprint.pformat(show_init_scheme(recognizer)))
prediction, prediction_mask = add_exploration(recognizer, data, train_conf)
#
# Observables:
#
primary_observables = [] # monitored each batch
secondary_observables = [] # monitored every 10 batches
validation_observables = [] # monitored on the validation set
cg = recognizer.get_cost_graph(
batch=True, prediction=prediction, prediction_mask=prediction_mask)
labels, = VariableFilter(
applications=[recognizer.cost], name='labels')(cg)
labels_mask, = VariableFilter(
applications=[recognizer.cost], name='labels_mask')(cg)
gain_matrix = VariableFilter(
theano_name=RewardRegressionEmitter.GAIN_MATRIX)(cg)
if len(gain_matrix):
gain_matrix, = gain_matrix
primary_observables.append(
named_copy(gain_matrix.min(), 'min_gain'))
primary_observables.append(
named_copy(gain_matrix.max(), 'max_gain'))
batch_cost = cg.outputs[0].sum()
batch_size = named_copy(recognizer.recordings.shape[1], "batch_size")
# Assumes constant batch size. `aggregation.mean` is not used because
# of Blocks #514.
cost = batch_cost / batch_size
cost.name = "sequence_total_cost"
logger.info("Cost graph is built")
# Fetch variables useful for debugging.
# It is important not to use any aggregation schemes here,
# as it's currently impossible to spread the effect of
# regularization on their variables, see Blocks #514.
cost_cg = ComputationGraph(cost)
r = recognizer
energies, = VariableFilter(
applications=[r.generator.readout.readout], name="output_0")(
cost_cg)
bottom_output = VariableFilter(
applications=[r.bottom.apply], name="output")(
cost_cg)[-1]
attended, = VariableFilter(
applications=[r.generator.transition.apply], name="attended")(
cost_cg)
attended_mask, = VariableFilter(
applications=[r.generator.transition.apply], name="attended_mask")(
cost_cg)
weights, = VariableFilter(
applications=[r.generator.evaluate], name="weights")(
cost_cg)
max_recording_length = named_copy(r.recordings.shape[0],
"max_recording_length")
# To exclude subsampling related bugs
max_attended_mask_length = named_copy(attended_mask.shape[0],
"max_attended_mask_length")
max_attended_length = named_copy(attended.shape[0],
"max_attended_length")
max_num_phonemes = named_copy(labels.shape[0],
"max_num_phonemes")
min_energy = named_copy(energies.min(), "min_energy")
max_energy = named_copy(energies.max(), "max_energy")
mean_attended = named_copy(abs(attended).mean(),
"mean_attended")
mean_bottom_output = named_copy(abs(bottom_output).mean(),
"mean_bottom_output")
weights_penalty = named_copy(monotonicity_penalty(weights, labels_mask),
"weights_penalty")
weights_entropy = named_copy(entropy(weights, labels_mask),
"weights_entropy")
mask_density = named_copy(labels_mask.mean(),
"mask_density")
cg = ComputationGraph([
cost, weights_penalty, weights_entropy,
min_energy, max_energy,
mean_attended, mean_bottom_output,
batch_size, max_num_phonemes,
mask_density])
# Regularization. It is applied explicitly to all variables
# of interest, it could not be applied to the cost only as it
# would not have effect on auxiliary variables, see Blocks #514.
reg_config = config['regularization']
regularized_cg = cg
if reg_config.get('dropout'):
logger.info('apply dropout')
regularized_cg = apply_dropout(cg, [bottom_output], 0.5)
if reg_config.get('noise'):
logger.info('apply noise')
noise_subjects = [p for p in cg.parameters if p not in attention_params]
regularized_cg = apply_noise(cg, noise_subjects, reg_config['noise'])
train_cost = regularized_cg.outputs[0]
if reg_config.get("penalty_coof", .0) > 0:
# big warning!!!
# here we assume that:
# regularized_weights_penalty = regularized_cg.outputs[1]
train_cost = (train_cost +
reg_config.get("penalty_coof", .0) *
regularized_cg.outputs[1] / batch_size)
if reg_config.get("decay", .0) > 0:
train_cost = (train_cost + reg_config.get("decay", .0) *
l2_norm(VariableFilter(roles=[WEIGHT])(cg.parameters)) ** 2)
train_cost = named_copy(train_cost, 'train_cost')
gradients = None
if reg_config.get('adaptive_noise'):
logger.info('apply adaptive noise')
if ((reg_config.get("penalty_coof", .0) > 0) or
(reg_config.get("decay", .0) > 0)):
logger.error('using adaptive noise with alignment weight panalty '
'or weight decay is probably stupid')
train_cost, regularized_cg, gradients, noise_brick = apply_adaptive_noise(
cg, cg.outputs[0],
variables=cg.parameters,
num_examples=data.get_dataset('train').num_examples,
parameters=SpeechModel(regularized_cg.outputs[0]
).get_parameter_dict().values(),
**reg_config.get('adaptive_noise')
)
train_cost.name = 'train_cost'
adapt_noise_cg = ComputationGraph(train_cost)
model_prior_mean = named_copy(
VariableFilter(applications=[noise_brick.apply],
name='model_prior_mean')(adapt_noise_cg)[0],
'model_prior_mean')
model_cost = named_copy(
VariableFilter(applications=[noise_brick.apply],
name='model_cost')(adapt_noise_cg)[0],
'model_cost')
model_prior_variance = named_copy(
VariableFilter(applications=[noise_brick.apply],
name='model_prior_variance')(adapt_noise_cg)[0],
'model_prior_variance')
regularized_cg = ComputationGraph(
[train_cost, model_cost] +
regularized_cg.outputs +
[model_prior_mean, model_prior_variance])
primary_observables += [
regularized_cg.outputs[1], # model cost
regularized_cg.outputs[2], # task cost
regularized_cg.outputs[-2], # model prior mean
regularized_cg.outputs[-1]] # model prior variance
# Model is weird class, we spend lots of time arguing with Bart
# what it should be. However it can already nice things, e.g.
# one extract all the parameters from the computation graphs
# and give them hierahical names. This help to notice when a
# because of some bug a parameter is not in the computation
# graph.
model = SpeechModel(train_cost)
if params:
logger.info("Load parameters from " + params)
# please note: we cannot use recognizer.load_params
# as it builds a new computation graph that dies not have
# shapred variables added by adaptive weight noise
param_values = load_parameter_values(params)
model.set_parameter_values(param_values)
parameters = model.get_parameter_dict()
logger.info("Parameters:\n" +
pprint.pformat(
[(key, parameters[key].get_value().shape) for key
in sorted(parameters.keys())],
width=120))
# Define the training algorithm.
clipping = StepClipping(train_conf['gradient_threshold'])
clipping.threshold.name = "gradient_norm_threshold"
rule_names = train_conf.get('rules', ['momentum'])
core_rules = []
if 'momentum' in rule_names:
logger.info("Using scaling and momentum for training")
core_rules.append(Momentum(train_conf['scale'], train_conf['momentum']))
if 'adadelta' in rule_names:
logger.info("Using AdaDelta for training")
core_rules.append(AdaDelta(train_conf['decay_rate'], train_conf['epsilon']))
max_norm_rules = []
if reg_config.get('max_norm', False) > 0:
logger.info("Apply MaxNorm")
maxnorm_subjects = VariableFilter(roles=[WEIGHT])(cg.parameters)
if reg_config.get('max_norm_exclude_lookup', False):
maxnorm_subjects = [v for v in maxnorm_subjects
if not isinstance(get_brick(v), LookupTable)]
logger.info("Parameters covered by MaxNorm:\n"
+ pprint.pformat([name for name, p in parameters.items()
if p in maxnorm_subjects]))
logger.info("Parameters NOT covered by MaxNorm:\n"
+ pprint.pformat([name for name, p in parameters.items()
if not p in maxnorm_subjects]))
max_norm_rules = [
Restrict(VariableClipping(reg_config['max_norm'], axis=0),
maxnorm_subjects)]
burn_in = []
if train_conf.get('burn_in_steps', 0):
burn_in.append(
BurnIn(num_steps=train_conf['burn_in_steps']))
algorithm = GradientDescent(
cost=train_cost,
parameters=parameters.values(),
gradients=gradients,
step_rule=CompositeRule(
[clipping] + core_rules + max_norm_rules +
# Parameters are not changed at all
# when nans are encountered.
[RemoveNotFinite(0.0)] + burn_in),
on_unused_sources='warn')
logger.debug("Scan Ops in the gradients")
gradient_cg = ComputationGraph(algorithm.gradients.values())
for op in ComputationGraph(gradient_cg).scans:
logger.debug(op)
# More variables for debugging: some of them can be added only
# after the `algorithm` object is created.
secondary_observables += list(regularized_cg.outputs)
if not 'train_cost' in [v.name for v in secondary_observables]:
secondary_observables += [train_cost]
secondary_observables += [
algorithm.total_step_norm, algorithm.total_gradient_norm,
clipping.threshold]
for name, param in parameters.items():
num_elements = numpy.product(param.get_value().shape)
norm = param.norm(2) / num_elements ** 0.5
grad_norm = algorithm.gradients[param].norm(2) / num_elements ** 0.5
step_norm = algorithm.steps[param].norm(2) / num_elements ** 0.5
stats = tensor.stack(norm, grad_norm, step_norm, step_norm / grad_norm)
stats.name = name + '_stats'
secondary_observables.append(stats)
primary_observables += [
train_cost,
algorithm.total_gradient_norm,
algorithm.total_step_norm, clipping.threshold,
max_recording_length,
max_attended_length, max_attended_mask_length]
validation_observables += [
rename(aggregation.mean(batch_cost, batch_size), cost.name),
rename(aggregation.sum_(batch_size), 'num_utterances'),
weights_entropy, weights_penalty]
def attach_aggregation_schemes(variables):
# Aggregation specification has to be factored out as a separate
# function as it has to be applied at the very last stage
# separately to training and validation observables.
result = []
for var in variables:
if var.name == 'weights_penalty':
result.append(rename(aggregation.mean(var, batch_size),
'weights_penalty_per_recording'))
elif var.name == 'weights_entropy':
result.append(rename(aggregation.mean(var, labels_mask.sum()),
'weights_entropy_per_label'))
else:
result.append(var)
return result
mon_conf = config['monitoring']
# Build main loop.
logger.info("Initialize extensions")
extensions = []
if use_load_ext and params:
extensions.append(Load(params, load_iteration_state=True, load_log=True))
if load_log and params:
extensions.append(LoadLog(params))
extensions += [
Timing(after_batch=True),
CGStatistics(),
#CodeVersion(['lvsr']),
]
extensions.append(TrainingDataMonitoring(
primary_observables, after_batch=True))
average_monitoring = TrainingDataMonitoring(
attach_aggregation_schemes(secondary_observables),
prefix="average", every_n_batches=10)
extensions.append(average_monitoring)
validation = DataStreamMonitoring(
attach_aggregation_schemes(validation_observables),
data.get_stream("valid", shuffle=False), prefix="valid").set_conditions(
before_first_epoch=not fast_start,
every_n_epochs=mon_conf['validate_every_epochs'],
every_n_batches=mon_conf['validate_every_batches'],
after_training=False)
extensions.append(validation)
per = PhonemeErrorRate(recognizer, data, **config['monitoring']['search'])
per_monitoring = DataStreamMonitoring(
[per], data.get_stream("valid", batches=False, shuffle=False),
prefix="valid").set_conditions(
before_first_epoch=not fast_start,
every_n_epochs=mon_conf['search_every_epochs'],
every_n_batches=mon_conf['search_every_batches'],
after_training=False)
extensions.append(per_monitoring)
track_the_best_per = TrackTheBest(
per_monitoring.record_name(per)).set_conditions(
before_first_epoch=True, after_epoch=True)
track_the_best_cost = TrackTheBest(
validation.record_name(cost)).set_conditions(
before_first_epoch=True, after_epoch=True)
extensions += [track_the_best_cost, track_the_best_per]
extensions.append(AdaptiveClipping(
algorithm.total_gradient_norm.name,
clipping, train_conf['gradient_threshold'],
decay_rate=0.998, burnin_period=500))
extensions += [
SwitchOffLengthFilter(
data.length_filter,
after_n_batches=train_conf.get('stop_filtering')),
FinishAfter(after_n_batches=train_conf['num_batches'],
after_n_epochs=train_conf['num_epochs'])
.add_condition(["after_batch"], _gradient_norm_is_none),
]
channels = [
# Plot 1: training and validation costs
[average_monitoring.record_name(train_cost),
validation.record_name(cost)],
# Plot 2: gradient norm,
[average_monitoring.record_name(algorithm.total_gradient_norm),
average_monitoring.record_name(clipping.threshold)],
# Plot 3: phoneme error rate
[per_monitoring.record_name(per)],
# Plot 4: training and validation mean weight entropy
[average_monitoring._record_name('weights_entropy_per_label'),
validation._record_name('weights_entropy_per_label')],
# Plot 5: training and validation monotonicity penalty
[average_monitoring._record_name('weights_penalty_per_recording'),
validation._record_name('weights_penalty_per_recording')]]
if bokeh:
extensions += [
Plot(bokeh_name if bokeh_name
else os.path.basename(save_path),
channels,
every_n_batches=10,
server_url=bokeh_server),]
extensions += [
Checkpoint(save_path,
before_first_epoch=not fast_start, after_epoch=True,
every_n_batches=train_conf.get('save_every_n_batches'),
save_separately=["model", "log"],
use_cpickle=True)
.add_condition(
['after_epoch'],
OnLogRecord(track_the_best_per.notification_name),
(root_path + "_best" + extension,))
.add_condition(
['after_epoch'],
OnLogRecord(track_the_best_cost.notification_name),
(root_path + "_best_ll" + extension,)),
ProgressBar()]
extensions.append(EmbedIPython(use_main_loop_run_caller_env=True))
if config['net']['criterion']['name'].startswith('mse'):
extensions.append(
LogInputsGains(
labels, cg, recognizer.generator.readout.emitter, data))
if train_conf.get('patience'):
patience_conf = train_conf['patience']
if not patience_conf.get('notification_names'):
# setdefault will not work for empty list
patience_conf['notification_names'] = [
track_the_best_per.notification_name,
track_the_best_cost.notification_name]
extensions.append(Patience(**patience_conf))
extensions.append(Printing(every_n_batches=1,
attribute_filter=PrintingFilterList()))
return model, algorithm, data, extensions
def main(name, epochs, batch_size, learning_rate,
dim, mix_dim, old_model_name, max_length, bokeh, GRU, dropout,
depth, max_grad, step_method, epsilon, sample, skip, uniform, top):
#----------------------------------------------------------------------
datasource = name
def shnum(x):
""" Convert a positive float into a short tag-usable string
E.g.: 0 -> 0, 0.005 -> 53, 100 -> 1-2
"""
return '0' if x <= 0 else '%s%d' % (("%e"%x)[0], -np.floor(np.log10(x)))
jobname = "%s-%dX%dm%dd%dr%sb%de%s" % (datasource, depth, dim, mix_dim,
int(dropout*10),
shnum(learning_rate), batch_size,
shnum(epsilon))
if max_length != 600:
jobname += '-L%d'%max_length
if GRU:
jobname += 'g'
if max_grad != 5.:
jobname += 'G%g'%max_grad
if step_method != 'adam':
jobname += step_method
if skip:
jobname += 'D'
assert depth > 1
if top:
jobname += 'T'
assert depth > 1
if uniform > 0.:
jobname += 'u%d'%int(uniform*100)
if debug:
jobname += ".debug"
if sample:
print("Sampling")
else:
print("\nRunning experiment %s" % jobname)
if old_model_name:
print("starting from model %s"%old_model_name)
#----------------------------------------------------------------------
transitions = [GatedRecurrent(dim=dim) if GRU else LSTM(dim=dim)
for _ in range(depth)]
if depth > 1:
transition = RecurrentStack(transitions, name="transition",
fast=True, skip_connections=skip or top)
if skip:
source_names=['states'] + ['states_%d'%d for d in range(1,depth)]
else:
source_names=['states_%d'%(depth-1)]
else:
transition = transitions[0]
transition.name = "transition"
source_names=['states']
emitter = SketchEmitter(mix_dim=mix_dim,
epsilon=epsilon,
name="emitter")
readout = Readout(
readout_dim=emitter.get_dim('inputs'),
source_names=source_names,
emitter=emitter,
name="readout")
normal_inputs = [name for name in transition.apply.sequences
if 'mask' not in name]
fork = Fork(normal_inputs, prototype=Linear(use_bias=True))
generator = SequenceGenerator(readout=readout, transition=transition,
fork=fork)
# Initialization settings
if uniform > 0.:
generator.weights_init = Uniform(width=uniform*2.)
else:
generator.weights_init = OrthogonalGlorot()
generator.biases_init = Constant(0)
# Build the cost computation graph [steps, batch_size, 3]
x = T.tensor3('features', dtype=floatX)
if debug:
x.tag.test_value = np.ones((max_length,batch_size,3)).astype(floatX)
x = x[:max_length,:,:] # has to be after setting test_value
cost = generator.cost(x)
cost.name = "sequence_log_likelihood"
# Give an idea of what's going on
model = Model(cost)
params = model.get_params()
logger.info("Parameters:\n" +
pprint.pformat(
[(key, value.get_value().shape) for key, value
in params.items()],
width=120))
model_size = 0
for v in params.itervalues():
s = v.get_value().shape
model_size += s[0] * (s[1] if len(s) > 1 else 1)
logger.info("Total number of parameters %d"%model_size)
#------------------------------------------------------------
extensions = []
if old_model_name == 'continue':
extensions.append(LoadFromDump(jobname))
elif old_model_name:
# or you can just load the weights without state using:
old_params = LoadFromDump(old_model_name).manager.load_parameters()
model.set_param_values(old_params)
else:
# Initialize parameters
for brick in model.get_top_bricks():
brick.initialize()
if sample:
assert old_model_name and old_model_name != 'continue'
Sample(generator, steps=max_length, path=old_model_name).do(None)
exit(0)
#------------------------------------------------------------
# Define the training algorithm.
cg = ComputationGraph(cost)
if dropout > 0.:
from blocks.roles import INPUT, OUTPUT
dropout_target = VariableFilter(roles=[OUTPUT],
bricks=transitions,
name_regex='states')(cg.variables)
print('# dropout %d' % len(dropout_target))
cg = apply_dropout(cg, dropout_target, dropout)
opt_cost = cg.outputs[0]
else:
opt_cost = cost
if step_method == 'adam':
step_rule = Adam(learning_rate)
elif step_method == 'rmsprop':
step_rule = RMSProp(learning_rate, decay_rate=0.95)
elif step_method == 'adagrad':
step_rule = AdaGrad(learning_rate)
elif step_method == 'adadelta':
step_rule = AdaDelta()
elif step_method == 'scale':
step_rule = Scale(learning_rate)
else:
raise Exception('Unknown sttep method %s'%step_method)
step_rule = CompositeRule([StepClipping(max_grad), step_rule])
algorithm = GradientDescent(
cost=opt_cost, params=cg.parameters,
step_rule=step_rule)
#------------------------------------------------------------
observables = [cost]
# Fetch variables useful for debugging
(energies,) = VariableFilter(
applications=[generator.readout.readout],
name_regex="output")(cg.variables)
min_energy = named_copy(energies.min(), "min_energy")
max_energy = named_copy(energies.max(), "max_energy")
observables += [min_energy, max_energy]
# (activations,) = VariableFilter(
# applications=[generator.transition.apply],
# name=generator.transition.apply.states[0])(cg.variables)
# mean_activation = named_copy(abs(activations).mean(),
# "mean_activation")
# observables.append(mean_activation)
observables += [algorithm.total_step_norm, algorithm.total_gradient_norm]
for name, param in params.items():
observables.append(named_copy(
param.norm(2), name + "_norm"))
observables.append(named_copy(
algorithm.gradients[param].norm(2), name + "_grad_norm"))
#------------------------------------------------------------
datasource_fname = os.path.join(fuel.config.data_path, datasource,
datasource+'.hdf5')
train_ds = H5PYDataset(datasource_fname, #max_length=max_length,
which_set='train', sources=('features',),
load_in_memory=True)
train_stream = DataStream(train_ds,
iteration_scheme=ShuffledScheme(
train_ds.num_examples, batch_size))
test_ds = H5PYDataset(datasource_fname, #max_length=max_length,
which_set='test', sources=('features',),
load_in_memory=True)
test_stream = DataStream(test_ds,
iteration_scheme=SequentialScheme(
test_ds.num_examples, batch_size))
train_stream = Mapping(train_stream, _transpose)
test_stream = Mapping(test_stream, _transpose)
def stream_stats(ds, label):
itr = ds.get_epoch_iterator(as_dict=True)
batch_count = 0
examples_count = 0
for batch in itr:
batch_count += 1
examples_count += batch['features'].shape[1]
print('%s #batch %d #examples %d' %
(label, batch_count, examples_count))
stream_stats(train_stream, 'train')
stream_stats(test_stream, 'test')
extensions += [Timing(every_n_batches=10),
TrainingDataMonitoring(
observables, prefix="train",
every_n_batches=10),
DataStreamMonitoring(
[cost], # without dropout
test_stream,
prefix="test",
on_resumption=True,
after_epoch=False, # by default this is True
every_n_batches=100),
# all monitored data is ready so print it...
# (next steps may take more time and we want to see the
# results as soon as possible so print as soon as you can)
Printing(every_n_batches=10),
# perform multiple dumps at different intervals
# so if one of them breaks (has nan) we can hopefully
# find a model from few batches ago in the other
Dump(jobname, every_n_batches=11),
Dump(jobname+'.test', every_n_batches=100),
Sample(generator, steps=max_length,
path=jobname+'.test',
every_n_batches=100),
ProgressBar(),
FinishAfter(after_n_epochs=epochs)
# This shows a way to handle NaN emerging during
# training: simply finish it.
.add_condition("after_batch", _is_nan),
]
if bokeh:
from blocks.extensions.plot import Plot
extensions.append(Plot(
'sketch',
channels=[
['cost'],]))
# Construct the main loop and start training!
main_loop = MainLoop(
model=model,
data_stream=train_stream,
algorithm=algorithm,
extensions=extensions
)
main_loop.run()
def append(self, quantity, name=None):
if name is not None:
quantity = named_copy(quantity, name)
self.dikt.setdefault(quantity.name, []).append(quantity)
def main(save_to,
num_epochs,
feature_maps=None,
mlp_hiddens=None,
conv_sizes=None,
pool_sizes=None,
batch_size=500):
if feature_maps is None:
feature_maps = [20, 50]
if mlp_hiddens is None:
mlp_hiddens = [500]
if conv_sizes is None:
conv_sizes = [5, 5]
if pool_sizes is None:
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations,
1,
image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='full',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info(
"Input dim: {} {} {}".format(*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
logging.info("Layer {} dim: {} {} {}".format(i,
*layer.get_dim('output')))
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cost = named_copy(CategoricalCrossEntropy().apply(y.flatten(), probs),
'cost')
error_rate = named_copy(MisclassificationRate().apply(y.flatten(), probs),
'error_rate')
cg = ComputationGraph([cost, error_rate])
mnist_train = MNIST(("train", ))
mnist_train_stream = DataStream.default_stream(
mnist_train,
iteration_scheme=ShuffledScheme(mnist_train.num_examples, batch_size))
mnist_test = MNIST(("test", ))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=ShuffledScheme(mnist_test.num_examples, batch_size))
# Train with simple SGD
algorithm = GradientDescent(cost=cost,
parameters=cg.parameters,
step_rule=Scale(learning_rate=0.1))
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = [
Timing(),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring([cost, error_rate],
mnist_test_stream,
prefix="test"),
TrainingDataMonitoring([
cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)
],
prefix="train",
after_epoch=True),
Checkpoint(save_to),
ProgressBar(),
Printing()
]
model = Model(cost)
main_loop = MainLoop(algorithm,
mnist_train_stream,
model=model,
extensions=extensions)
main_loop.run()
def main(mode, save_path, num_batches, data_path=None):
reverser = WordReverser(100, len(char2code), name="reverser")
if mode == "train":
# Data processing pipeline
dataset_options = dict(dictionary=char2code, level="character",
preprocess=_lower)
if data_path:
dataset = TextFile(data_path, **dataset_options)
else:
dataset = OneBillionWord("training", [99], **dataset_options)
data_stream = dataset.get_example_stream()
data_stream = Filter(data_stream, _filter_long)
data_stream = Mapping(data_stream, reverse_words,
add_sources=("targets",))
data_stream = Batch(data_stream, iteration_scheme=ConstantScheme(10))
data_stream = Padding(data_stream)
data_stream = Mapping(data_stream, _transpose)
# Initialization settings
reverser.weights_init = IsotropicGaussian(0.1)
reverser.biases_init = Constant(0.0)
reverser.push_initialization_config()
reverser.encoder.weights_init = Orthogonal()
reverser.generator.transition.weights_init = Orthogonal()
# Build the cost computation graph
chars = tensor.lmatrix("features")
chars_mask = tensor.matrix("features_mask")
targets = tensor.lmatrix("targets")
targets_mask = tensor.matrix("targets_mask")
batch_cost = reverser.cost(
chars, chars_mask, targets, targets_mask).sum()
batch_size = named_copy(chars.shape[1], "batch_size")
cost = aggregation.mean(batch_cost, batch_size)
cost.name = "sequence_log_likelihood"
logger.info("Cost graph is built")
# Give an idea of what's going on
model = Model(cost)
params = model.get_params()
logger.info("Parameters:\n" +
pprint.pformat(
[(key, value.get_value().shape) for key, value
in params.items()],
width=120))
# Initialize parameters
for brick in model.get_top_bricks():
brick.initialize()
# Define the training algorithm.
cg = ComputationGraph(cost)
algorithm = GradientDescent(
cost=cost, params=cg.parameters,
step_rule=CompositeRule([StepClipping(10.0), Scale(0.01)]))
# Fetch variables useful for debugging
generator = reverser.generator
(energies,) = VariableFilter(
applications=[generator.readout.readout],
name_regex="output")(cg.variables)
(activations,) = VariableFilter(
applications=[generator.transition.apply],
name=generator.transition.apply.states[0])(cg.variables)
max_length = named_copy(chars.shape[0], "max_length")
cost_per_character = named_copy(
aggregation.mean(batch_cost, batch_size * max_length),
"character_log_likelihood")
min_energy = named_copy(energies.min(), "min_energy")
max_energy = named_copy(energies.max(), "max_energy")
mean_activation = named_copy(abs(activations).mean(),
"mean_activation")
observables = [
cost, min_energy, max_energy, mean_activation,
batch_size, max_length, cost_per_character,
algorithm.total_step_norm, algorithm.total_gradient_norm]
for name, param in params.items():
observables.append(named_copy(
param.norm(2), name + "_norm"))
observables.append(named_copy(
algorithm.gradients[param].norm(2), name + "_grad_norm"))
# Construct the main loop and start training!
average_monitoring = TrainingDataMonitoring(
observables, prefix="average", every_n_batches=10)
main_loop = MainLoop(
model=model,
data_stream=data_stream,
algorithm=algorithm,
extensions=[
Timing(),
TrainingDataMonitoring(observables, after_batch=True),
average_monitoring,
FinishAfter(after_n_batches=num_batches)
# This shows a way to handle NaN emerging during
# training: simply finish it.
.add_condition("after_batch", _is_nan),
# Saving the model and the log separately is convenient,
# because loading the whole pickle takes quite some time.
Checkpoint(save_path, every_n_batches=500,
save_separately=["model", "log"]),
Printing(every_n_batches=1)])
main_loop.run()
elif mode == "sample" or mode == "beam_search":
chars = tensor.lmatrix("input")
generated = reverser.generate(chars)
model = Model(generated)
logger.info("Loading the model..")
model.set_param_values(load_parameter_values(save_path))
def generate(input_):
"""Generate output sequences for an input sequence.
Incapsulates most of the difference between sampling and beam
search.
Returns
-------
outputs : list of lists
Trimmed output sequences.
costs : list
The negative log-likelihood of generating the respective
sequences.
"""
if mode == "beam_search":
samples, = VariableFilter(
bricks=[reverser.generator], name="outputs")(
ComputationGraph(generated[1]))
# NOTE: this will recompile beam search functions
# every time user presses Enter. Do not create
# a new `BeamSearch` object every time if
# speed is important for you.
beam_search = BeamSearch(input_.shape[1], samples)
outputs, costs = beam_search.search(
{chars: input_}, char2code['</S>'],
3 * input_.shape[0])
else:
_1, outputs, _2, _3, costs = (
model.get_theano_function()(input_))
outputs = list(outputs.T)
costs = list(costs.T)
for i in range(len(outputs)):
outputs[i] = list(outputs[i])
try:
true_length = outputs[i].index(char2code['</S>']) + 1
except ValueError:
true_length = len(outputs[i])
outputs[i] = outputs[i][:true_length]
costs[i] = costs[i][:true_length].sum()
return outputs, costs
while True:
line = input("Enter a sentence\n")
message = ("Enter the number of samples\n" if mode == "sample"
else "Enter the beam size\n")
batch_size = int(input(message))
encoded_input = [char2code.get(char, char2code["<UNK>"])
for char in line.lower().strip()]
encoded_input = ([char2code['<S>']] + encoded_input +
[char2code['</S>']])
print("Encoder input:", encoded_input)
target = reverse_words((encoded_input,))[0]
print("Target: ", target)
samples, costs = generate(
numpy.repeat(numpy.array(encoded_input)[:, None],
batch_size, axis=1))
messages = []
for sample, cost in equizip(samples, costs):
message = "({})".format(cost)
message += "".join(code2char[code] for code in sample)
if sample == target:
message += " CORRECT!"
messages.append((cost, message))
messages.sort(key=operator.itemgetter(0), reverse=True)
for _, message in messages:
print(message)
def main(save_to, num_epochs, feature_maps=None, mlp_hiddens=None,
conv_sizes=None, pool_sizes=None, batch_size=500):
if feature_maps is None:
feature_maps = [20, 50]
if mlp_hiddens is None:
mlp_hiddens = [500]
if conv_sizes is None:
conv_sizes = [5, 5]
if pool_sizes is None:
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations, 1, image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='full',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
logging.info("Layer {} dim: {} {} {}".format(
i, *layer.get_dim('output')))
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cost = named_copy(CategoricalCrossEntropy().apply(y.flatten(),
probs), 'cost')
error_rate = named_copy(MisclassificationRate().apply(y.flatten(), probs),
'error_rate')
cg = ComputationGraph([cost, error_rate])
mnist_train = MNIST(("train",))
mnist_train_stream = DataStream.default_stream(
mnist_train, iteration_scheme=ShuffledScheme(
mnist_train.num_examples, batch_size))
mnist_test = MNIST(("test",))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=ShuffledScheme(
mnist_test.num_examples, batch_size))
# Train with simple SGD
algorithm = GradientDescent(
cost=cost, params=cg.parameters,
step_rule=Scale(learning_rate=0.1))
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = [Timing(),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring(
[cost, error_rate],
mnist_test_stream,
prefix="test"),
TrainingDataMonitoring(
[cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)],
prefix="train",
after_epoch=True),
Checkpoint(save_to),
ProgressBar(),
Printing()]
model = Model(cost)
main_loop = MainLoop(
algorithm,
mnist_train_stream,
model=model,
extensions=extensions)
main_loop.run()
print(cg.inputs)
algorithm = GradientDescent(
cost=cost,
parameters=cg.parameters,
step_rule=CompositeRule([
StepClipping(10.0),
Scale(0.01),
]),
)
print("Defined Algorithm")
model = Model(cost)
print("Defined Model")
obs_max_length = named_copy(x.shape[0], "obs_max_length")
observables = [
cost,
obs_max_length,
#min_energy, max_energy,
#mean_activation,
]
main_loop = MainLoop(
model=model,
data_stream=data_stream,
algorithm=algorithm,
extensions=[
Timing(),
TrainingDataMonitoring(observables, after_batch=True),
#average_monitoring,
def main(mode, save_path, num_batches, from_dump):
if mode == "train":
# Experiment configuration
dimension = 100
readout_dimension = len(char2code)
# Data processing pipeline
data_stream = DataStreamMapping(
mapping=lambda data: tuple(array.T for array in data),
data_stream=PaddingDataStream(
BatchDataStream(
iteration_scheme=ConstantScheme(10),
data_stream=DataStreamMapping(
mapping=reverse_words,
add_sources=("targets", ),
data_stream=DataStreamFilter(
predicate=lambda data: len(data[0]) <= 100,
data_stream=OneBillionWord(
"training", [99],
char2code,
level="character",
preprocess=str.lower).get_default_stream())))))
# Build the model
chars = tensor.lmatrix("features")
chars_mask = tensor.matrix("features_mask")
targets = tensor.lmatrix("targets")
targets_mask = tensor.matrix("targets_mask")
encoder = Bidirectional(GatedRecurrent(dim=dimension,
activation=Tanh()),
weights_init=Orthogonal())
encoder.initialize()
fork = Fork([
name
for name in encoder.prototype.apply.sequences if name != 'mask'
],
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0))
fork.input_dim = dimension
fork.fork_dims = {name: dimension for name in fork.fork_names}
fork.initialize()
lookup = LookupTable(readout_dimension,
dimension,
weights_init=IsotropicGaussian(0.1))
lookup.initialize()
transition = Transition(activation=Tanh(),
dim=dimension,
attended_dim=2 * dimension,
name="transition")
attention = SequenceContentAttention(
state_names=transition.apply.states,
match_dim=dimension,
name="attention")
readout = LinearReadout(readout_dim=readout_dimension,
source_names=["states"],
emitter=SoftmaxEmitter(name="emitter"),
feedbacker=LookupFeedback(
readout_dimension, dimension),
name="readout")
generator = SequenceGenerator(readout=readout,
transition=transition,
attention=attention,
weights_init=IsotropicGaussian(0.1),
biases_init=Constant(0),
name="generator")
generator.push_initialization_config()
transition.weights_init = Orthogonal()
generator.initialize()
bricks = [encoder, fork, lookup, generator]
# Give an idea of what's going on
params = Selector(bricks).get_params()
logger.info("Parameters:\n" +
pprint.pformat([(key, value.get_value().shape)
for key, value in params.items()],
width=120))
# Build the cost computation graph
batch_cost = generator.cost(
targets,
targets_mask,
attended=encoder.apply(**dict_union(fork.apply(
lookup.lookup(chars), return_dict=True),
mask=chars_mask)),
attended_mask=chars_mask).sum()
batch_size = named_copy(chars.shape[1], "batch_size")
cost = aggregation.mean(batch_cost, batch_size)
cost.name = "sequence_log_likelihood"
logger.info("Cost graph is built")
# Fetch variables useful for debugging
max_length = named_copy(chars.shape[0], "max_length")
cost_per_character = named_copy(
aggregation.mean(batch_cost, batch_size * max_length),
"character_log_likelihood")
cg = ComputationGraph(cost)
energies = unpack(VariableFilter(application=readout.readout,
name="output")(cg.variables),
singleton=True)
min_energy = named_copy(energies.min(), "min_energy")
max_energy = named_copy(energies.max(), "max_energy")
(activations, ) = VariableFilter(
application=generator.transition.apply,
name="states")(cg.variables)
mean_activation = named_copy(activations.mean(), "mean_activation")
# Define the training algorithm.
algorithm = GradientDescent(cost=cost,
step_rule=CompositeRule([
GradientClipping(10.0),
SteepestDescent(0.01)
]))
observables = [
cost, min_energy, max_energy, mean_activation, batch_size,
max_length, cost_per_character, algorithm.total_step_norm,
algorithm.total_gradient_norm
]
for name, param in params.items():
observables.append(named_copy(param.norm(2), name + "_norm"))
observables.append(
named_copy(algorithm.gradients[param].norm(2),
name + "_grad_norm"))
main_loop = MainLoop(
model=bricks,
data_stream=data_stream,
algorithm=algorithm,
extensions=([LoadFromDump(from_dump)] if from_dump else []) + [
Timing(),
TrainingDataMonitoring(observables, after_every_batch=True),
TrainingDataMonitoring(
observables, prefix="average", every_n_batches=10),
FinishAfter(after_n_batches=num_batches).add_condition(
"after_batch", lambda log: math.isnan(
log.current_row.total_gradient_norm)),
Plot(os.path.basename(save_path),
[["average_" + cost.name],
["average_" + cost_per_character.name]],
every_n_batches=10),
SerializeMainLoop(save_path,
every_n_batches=500,
save_separately=["model", "log"]),
Printing(every_n_batches=1)
])
main_loop.run()
elif mode == "test":
with open(save_path, "rb") as source:
encoder, fork, lookup, generator = dill.load(source)
logger.info("Model is loaded")
chars = tensor.lmatrix("features")
generated = generator.generate(
n_steps=3 * chars.shape[0],
batch_size=chars.shape[1],
attended=encoder.apply(**dict_union(
fork.apply(lookup.lookup(chars), return_dict=True))),
attended_mask=tensor.ones(chars.shape))
sample_function = ComputationGraph(generated).get_theano_function()
logging.info("Sampling function is compiled")
while True:
# Python 2-3 compatibility
line = input("Enter a sentence\n")
batch_size = int(input("Enter a number of samples\n"))
encoded_input = [
char2code.get(char, char2code["<UNK>"])
for char in line.lower().strip()
]
encoded_input = ([char2code['<S>']] + encoded_input +
[char2code['</S>']])
print("Encoder input:", encoded_input)
target = reverse_words((encoded_input, ))[0]
print("Target: ", target)
states, samples, glimpses, weights, costs = sample_function(
numpy.repeat(numpy.array(encoded_input)[:, None],
batch_size,
axis=1))
messages = []
for i in range(samples.shape[1]):
sample = list(samples[:, i])
try:
true_length = sample.index(char2code['</S>']) + 1
except ValueError:
true_length = len(sample)
sample = sample[:true_length]
cost = costs[:true_length, i].sum()
message = "({})".format(cost)
message += "".join(code2char[code] for code in sample)
if sample == target:
message += " CORRECT!"
messages.append((cost, message))
messages.sort(key=lambda tuple_: -tuple_[0])
for _, message in messages:
print(message)
def train(config, save_path, bokeh_name,
params, bokeh_server, test_tag, use_load_ext,
load_log, fast_start, validation_epochs, validation_batches,
per_epochs, per_batches):
root_path, extension = os.path.splitext(save_path)
data = Data(**config['data'])
# Build the main brick and initialize all parameters.
recognizer = SpeechRecognizer(
data.recordings_source, data.labels_source,
data.eos_label,
data.num_features, data.num_labels,
name="recognizer",
data_prepend_eos=data.prepend_eos,
character_map=data.character_map,
**config["net"])
for brick_path, attribute_dict in sorted(
config['initialization'].items(),
key=lambda (k, v): -k.count('/')):
for attribute, value in attribute_dict.items():
brick, = Selector(recognizer).select(brick_path).bricks
setattr(brick, attribute, value)
brick.push_initialization_config()
recognizer.initialize()
# Separate attention_params to be handled differently
# when regularization is applied
attention = recognizer.generator.transition.attention
attention_params = Selector(attention).get_parameters().values()
logger.info(
"Initialization schemes for all bricks.\n"
"Works well only in my branch with __repr__ added to all them,\n"
"there is an issue #463 in Blocks to do that properly.")
def show_init_scheme(cur):
result = dict()
for attr in dir(cur):
if attr.endswith('_init'):
result[attr] = getattr(cur, attr)
for child in cur.children:
result[child.name] = show_init_scheme(child)
return result
logger.info(pprint.pformat(show_init_scheme(recognizer)))
if params:
logger.info("Load parameters from " + params)
recognizer.load_params(params)
if test_tag:
tensor.TensorVariable.__str__ = tensor.TensorVariable.__repr__
__stream = data.get_stream("train")
__data = next(__stream.get_epoch_iterator(as_dict=True))
recognizer.recordings.tag.test_value = __data[data.recordings_source]
recognizer.recordings_mask.tag.test_value = __data[data.recordings_source + '_mask']
recognizer.labels.tag.test_value = __data[data.labels_source]
recognizer.labels_mask.tag.test_value = __data[data.labels_source + '_mask']
theano.config.compute_test_value = 'warn'
batch_cost = recognizer.get_cost_graph().sum()
batch_size = named_copy(recognizer.recordings.shape[1], "batch_size")
# Assumes constant batch size. `aggregation.mean` is not used because
# of Blocks #514.
cost = batch_cost / batch_size
cost.name = "sequence_log_likelihood"
logger.info("Cost graph is built")
# Fetch variables useful for debugging.
# It is important not to use any aggregation schemes here,
# as it's currently impossible to spread the effect of
# regularization on their variables, see Blocks #514.
cost_cg = ComputationGraph(cost)
r = recognizer
energies, = VariableFilter(
applications=[r.generator.readout.readout], name="output_0")(
cost_cg)
bottom_output, = VariableFilter(
applications=[r.bottom.apply], name="output")(
cost_cg)
attended, = VariableFilter(
applications=[r.generator.transition.apply], name="attended")(
cost_cg)
attended_mask, = VariableFilter(
applications=[r.generator.transition.apply], name="attended_mask")(
cost_cg)
weights, = VariableFilter(
applications=[r.generator.evaluate], name="weights")(
cost_cg)
max_recording_length = named_copy(r.recordings.shape[0],
"max_recording_length")
# To exclude subsampling related bugs
max_attended_mask_length = named_copy(attended_mask.shape[0],
"max_attended_mask_length")
max_attended_length = named_copy(attended.shape[0],
"max_attended_length")
max_num_phonemes = named_copy(r.labels.shape[0],
"max_num_phonemes")
min_energy = named_copy(energies.min(), "min_energy")
max_energy = named_copy(energies.max(), "max_energy")
mean_attended = named_copy(abs(attended).mean(),
"mean_attended")
mean_bottom_output = named_copy(abs(bottom_output).mean(),
"mean_bottom_output")
weights_penalty = named_copy(monotonicity_penalty(weights, r.labels_mask),
"weights_penalty")
weights_entropy = named_copy(entropy(weights, r.labels_mask),
"weights_entropy")
mask_density = named_copy(r.labels_mask.mean(),
"mask_density")
cg = ComputationGraph([
cost, weights_penalty, weights_entropy,
min_energy, max_energy,
mean_attended, mean_bottom_output,
batch_size, max_num_phonemes,
mask_density])
# Regularization. It is applied explicitly to all variables
# of interest, it could not be applied to the cost only as it
# would not have effect on auxiliary variables, see Blocks #514.
reg_config = config['regularization']
regularized_cg = cg
if reg_config.get('dropout'):
logger.info('apply dropout')
regularized_cg = apply_dropout(cg, [bottom_output], 0.5)
if reg_config.get('noise'):
logger.info('apply noise')
noise_subjects = [p for p in cg.parameters if p not in attention_params]
regularized_cg = apply_noise(cg, noise_subjects, reg_config['noise'])
regularized_cost = regularized_cg.outputs[0]
regularized_weights_penalty = regularized_cg.outputs[1]
# Model is weird class, we spend lots of time arguing with Bart
# what it should be. However it can already nice things, e.g.
# one extract all the parameters from the computation graphs
# and give them hierahical names. This help to notice when a
# because of some bug a parameter is not in the computation
# graph.
model = SpeechModel(regularized_cost)
params = model.get_parameter_dict()
logger.info("Parameters:\n" +
pprint.pformat(
[(key, params[key].get_value().shape) for key
in sorted(params.keys())],
width=120))
# Define the training algorithm.
train_conf = config['training']
clipping = StepClipping(train_conf['gradient_threshold'])
clipping.threshold.name = "gradient_norm_threshold"
rule_names = train_conf.get('rules', ['momentum'])
core_rules = []
if 'momentum' in rule_names:
logger.info("Using scaling and momentum for training")
core_rules.append(Momentum(train_conf['scale'], train_conf['momentum']))
if 'adadelta' in rule_names:
logger.info("Using AdaDelta for training")
core_rules.append(AdaDelta(train_conf['decay_rate'], train_conf['epsilon']))
max_norm_rules = []
if reg_config.get('max_norm', False):
logger.info("Apply MaxNorm")
maxnorm_subjects = VariableFilter(roles=[WEIGHT])(cg.parameters)
if reg_config.get('max_norm_exclude_lookup', False):
maxnorm_subjects = [v for v in maxnorm_subjects
if not isinstance(get_brick(v), LookupTable)]
logger.info("Parameters covered by MaxNorm:\n"
+ pprint.pformat([name for name, p in params.items()
if p in maxnorm_subjects]))
logger.info("Parameters NOT covered by MaxNorm:\n"
+ pprint.pformat([name for name, p in params.items()
if not p in maxnorm_subjects]))
max_norm_rules = [
Restrict(VariableClipping(reg_config['max_norm'], axis=0),
maxnorm_subjects)]
algorithm = GradientDescent(
cost=regularized_cost +
reg_config.get("penalty_coof", .0) * regularized_weights_penalty / batch_size +
reg_config.get("decay", .0) *
l2_norm(VariableFilter(roles=[WEIGHT])(cg.parameters)) ** 2,
parameters=params.values(),
step_rule=CompositeRule(
[clipping] + core_rules + max_norm_rules +
# Parameters are not changed at all
# when nans are encountered.
[RemoveNotFinite(0.0)]))
# More variables for debugging: some of them can be added only
# after the `algorithm` object is created.
observables = regularized_cg.outputs
observables += [
algorithm.total_step_norm, algorithm.total_gradient_norm,
clipping.threshold]
for name, param in params.items():
num_elements = numpy.product(param.get_value().shape)
norm = param.norm(2) / num_elements ** 0.5
grad_norm = algorithm.gradients[param].norm(2) / num_elements ** 0.5
step_norm = algorithm.steps[param].norm(2) / num_elements ** 0.5
stats = tensor.stack(norm, grad_norm, step_norm, step_norm / grad_norm)
stats.name = name + '_stats'
observables.append(stats)
def attach_aggregation_schemes(variables):
# Aggregation specification has to be factored out as a separate
# function as it has to be applied at the very last stage
# separately to training and validation observables.
result = []
for var in variables:
if var.name == 'weights_penalty':
result.append(named_copy(aggregation.mean(var, batch_size),
'weights_penalty_per_recording'))
elif var.name == 'weights_entropy':
result.append(named_copy(aggregation.mean(
var, recognizer.labels_mask.sum()), 'weights_entropy_per_label'))
else:
result.append(var)
return result
# Build main loop.
logger.info("Initialize extensions")
extensions = []
if use_load_ext and params:
extensions.append(Load(params, load_iteration_state=True, load_log=True))
if load_log and params:
extensions.append(LoadLog(params))
extensions += [
Timing(after_batch=True),
CGStatistics(),
#CodeVersion(['lvsr']),
]
extensions.append(TrainingDataMonitoring(
[observables[0], algorithm.total_gradient_norm,
algorithm.total_step_norm, clipping.threshold,
max_recording_length,
max_attended_length, max_attended_mask_length], after_batch=True))
average_monitoring = TrainingDataMonitoring(
attach_aggregation_schemes(observables),
prefix="average", every_n_batches=10)
extensions.append(average_monitoring)
validation = DataStreamMonitoring(
attach_aggregation_schemes([cost, weights_entropy, weights_penalty]),
data.get_stream("valid"), prefix="valid").set_conditions(
before_first_epoch=not fast_start,
every_n_epochs=validation_epochs,
every_n_batches=validation_batches,
after_training=False)
extensions.append(validation)
recognizer.init_beam_search(10)
per = PhonemeErrorRate(recognizer, data.get_dataset("valid"))
per_monitoring = DataStreamMonitoring(
[per], data.get_stream("valid", batches=False, shuffle=False),
prefix="valid").set_conditions(
before_first_epoch=not fast_start,
every_n_epochs=per_epochs,
every_n_batches=per_batches,
after_training=False)
extensions.append(per_monitoring)
track_the_best_per = TrackTheBest(
per_monitoring.record_name(per)).set_conditions(
before_first_epoch=True, after_epoch=True)
track_the_best_likelihood = TrackTheBest(
validation.record_name(cost)).set_conditions(
before_first_epoch=True, after_epoch=True)
extensions += [track_the_best_likelihood, track_the_best_per]
extensions.append(AdaptiveClipping(
algorithm.total_gradient_norm.name,
clipping, train_conf['gradient_threshold'],
decay_rate=0.998, burnin_period=500))
extensions += [
SwitchOffLengthFilter(data.length_filter,
after_n_batches=train_conf.get('stop_filtering')),
FinishAfter(after_n_batches=train_conf['num_batches'],
after_n_epochs=train_conf['num_epochs'])
.add_condition(["after_batch"], _gradient_norm_is_none),
# Live plotting: requires launching `bokeh-server`
# and allows to see what happens online.
Plot(bokeh_name
if bokeh_name
else os.path.basename(save_path),
[# Plot 1: training and validation costs
[average_monitoring.record_name(regularized_cost),
validation.record_name(cost)],
# Plot 2: gradient norm,
[average_monitoring.record_name(algorithm.total_gradient_norm),
average_monitoring.record_name(clipping.threshold)],
# Plot 3: phoneme error rate
[per_monitoring.record_name(per)],
# Plot 4: training and validation mean weight entropy
[average_monitoring._record_name('weights_entropy_per_label'),
validation._record_name('weights_entropy_per_label')],
# Plot 5: training and validation monotonicity penalty
[average_monitoring._record_name('weights_penalty_per_recording'),
validation._record_name('weights_penalty_per_recording')]],
every_n_batches=10,
server_url=bokeh_server),
Checkpoint(save_path,
before_first_epoch=not fast_start, after_epoch=True,
every_n_batches=train_conf.get('save_every_n_batches'),
save_separately=["model", "log"],
use_cpickle=True)
.add_condition(
['after_epoch'],
OnLogRecord(track_the_best_per.notification_name),
(root_path + "_best" + extension,))
.add_condition(
['after_epoch'],
OnLogRecord(track_the_best_likelihood.notification_name),
(root_path + "_best_ll" + extension,)),
ProgressBar(),
Printing(every_n_batches=1,
attribute_filter=PrintingFilterList()
)]
# Save the config into the status
log = TrainingLog()
log.status['_config'] = repr(config)
main_loop = MainLoop(
model=model, log=log, algorithm=algorithm,
data_stream=data.get_stream("train"),
extensions=extensions)
main_loop.run()
def train(save_to, num_epochs, feature_maps=None, mlp_hiddens=None,
conv_sizes=None, pool_sizes=None, batch_size=500):
# Initialize the training set
train = CIFAR10(("train",))
train_stream = DataStream.default_stream(
train, iteration_scheme=ShuffledScheme(
train.num_examples, batch_size))
test = CIFAR10(("test",))
test_stream = DataStream.default_stream(
test,
iteration_scheme=ShuffledScheme(
test.num_examples, batch_size))
# ConvMLP Parameters
image_size = (32, 32)
num_channels = 3
num_conv = 3 # Number of Convolutional Layers
if feature_maps is None:
feature_maps = [20, 30, 30]
if not len(feature_maps) == num_conv:
raise ValueError('Must specify more feature maps')
if conv_sizes is None:
conv_sizes = [5] * num_conv
if pool_sizes is None:
pool_sizes = [2] * num_conv
if mlp_hiddens is None:
mlp_hiddens = [500]
output_size = 10
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = ConvMLP(conv_activations, num_channels, image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='full',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
for i in range(num_conv):
convnet.layers[i].weights_init = Uniform(width=.2)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
logging.info("Layer {} dim: {} {} {}".format(
i, *layer.get_dim('output')))
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cost = named_copy(CategoricalCrossEntropy().apply(y.flatten(),
probs), 'cost')
error_rate = named_copy(MisclassificationRate().apply(y.flatten(), probs),
'error_rate')
cg = ComputationGraph([cost, error_rate])
# Apply Dropout to outputs of rectifiers
from blocks.roles import OUTPUT
vs = VariableFilter(roles=[OUTPUT])(cg.variables)
vs1 = [v for v in vs if v.name.startswith('rectifier')]
vs1 = vs1[0: -2] # Only first two layers
cg = apply_dropout(cg, vs1, 0.5)
# Train with simple SGD
algorithm = GradientDescent(
cost=cost, parameters=cg.parameters,
step_rule=AdaDelta())
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = [Timing(),
FinishAfter(after_n_epochs=num_epochs),
DataStreamMonitoring(
[cost, error_rate],
test_stream,
prefix="test"),
TrainingDataMonitoring(
[cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)],
prefix="train",
after_epoch=True),
Checkpoint(save_to),
ProgressBar(),
Printing()]
model = Model(cost)
main_loop = MainLoop(
algorithm,
train_stream,
model=model,
extensions=extensions)
main_loop.run()
classifier_fn = 'convmlp_cifar10.zip'
with open(classifier_fn, 'w') as f:
dump(convnet, f)
def append(self, quantity, name=None):
if name is not None:
quantity = named_copy(quantity, name)
self.dikt.setdefault(quantity.name, []).append(quantity)
def main(mode, save_path, num_batches, data_path=None):
reverser = WordReverser(100, len(char2code), name="reverser")
if mode == "train":
# Data processing pipeline
dataset_options = dict(dictionary=char2code, level="character",
preprocess=_lower)
if data_path:
dataset = TextFile(data_path, **dataset_options)
else:
dataset = OneBillionWord("training", [99], **dataset_options)
data_stream = dataset.get_example_stream()
data_stream = Filter(data_stream, _filter_long)
data_stream = Mapping(data_stream, reverse_words,
add_sources=("targets",))
data_stream = Batch(data_stream, iteration_scheme=ConstantScheme(10))
data_stream = Padding(data_stream)
data_stream = Mapping(data_stream, _transpose)
# Initialization settings
reverser.weights_init = IsotropicGaussian(0.1)
reverser.biases_init = Constant(0.0)
reverser.push_initialization_config()
reverser.encoder.weghts_init = Orthogonal()
reverser.generator.transition.weights_init = Orthogonal()
# Build the cost computation graph
chars = tensor.lmatrix("features")
chars_mask = tensor.matrix("features_mask")
targets = tensor.lmatrix("targets")
targets_mask = tensor.matrix("targets_mask")
batch_cost = reverser.cost(
chars, chars_mask, targets, targets_mask).sum()
batch_size = named_copy(chars.shape[1], "batch_size")
cost = aggregation.mean(batch_cost, batch_size)
cost.name = "sequence_log_likelihood"
logger.info("Cost graph is built")
# Give an idea of what's going on
model = Model(cost)
params = model.get_params()
logger.info("Parameters:\n" +
pprint.pformat(
[(key, value.get_value().shape) for key, value
in params.items()],
width=120))
# Initialize parameters
for brick in model.get_top_bricks():
brick.initialize()
# Define the training algorithm.
cg = ComputationGraph(cost)
algorithm = GradientDescent(
cost=cost, params=cg.parameters,
step_rule=CompositeRule([StepClipping(10.0), Scale(0.01)]))
# Fetch variables useful for debugging
generator = reverser.generator
(energies,) = VariableFilter(
application=generator.readout.readout,
name="output")(cg.variables)
(activations,) = VariableFilter(
application=generator.transition.apply,
name=generator.transition.apply.states[0])(cg.variables)
max_length = named_copy(chars.shape[0], "max_length")
cost_per_character = named_copy(
aggregation.mean(batch_cost, batch_size * max_length),
"character_log_likelihood")
min_energy = named_copy(energies.min(), "min_energy")
max_energy = named_copy(energies.max(), "max_energy")
mean_activation = named_copy(abs(activations).mean(),
"mean_activation")
observables = [
cost, min_energy, max_energy, mean_activation,
batch_size, max_length, cost_per_character,
algorithm.total_step_norm, algorithm.total_gradient_norm]
for name, param in params.items():
observables.append(named_copy(
param.norm(2), name + "_norm"))
observables.append(named_copy(
algorithm.gradients[param].norm(2), name + "_grad_norm"))
# Construct the main loop and start training!
average_monitoring = TrainingDataMonitoring(
observables, prefix="average", every_n_batches=10)
main_loop = MainLoop(
model=model,
data_stream=data_stream,
algorithm=algorithm,
extensions=[
Timing(),
TrainingDataMonitoring(observables, after_batch=True),
average_monitoring,
FinishAfter(after_n_batches=num_batches)
# This shows a way to handle NaN emerging during
# training: simply finish it.
.add_condition("after_batch", _is_nan),
Plot(os.path.basename(save_path),
[[average_monitoring.record_name(cost)],
[average_monitoring.record_name(cost_per_character)]],
every_n_batches=10),
# Saving the model and the log separately is convenient,
# because loading the whole pickle takes quite some time.
Checkpoint(save_path, every_n_batches=500,
save_separately=["model", "log"]),
Printing(every_n_batches=1)])
main_loop.run()
elif mode == "sample" or mode == "beam_search":
chars = tensor.lmatrix("input")
generated = reverser.generate(chars)
model = Model(generated)
logger.info("Loading the model..")
model.set_param_values(load_parameter_values(save_path))
def generate(input_):
"""Generate output sequences for an input sequence.
Incapsulates most of the difference between sampling and beam
search.
Returns
-------
outputs : list of lists
Trimmed output sequences.
costs : list
The negative log-likelihood of generating the respective
sequences.
"""
if mode == "beam_search":
samples, = VariableFilter(
bricks=[reverser.generator], name="outputs")(
ComputationGraph(generated[1]))
# NOTE: this will recompile beam search functions
# every time user presses Enter. Do not create
# a new `BeamSearch` object every time if
# speed is important for you.
beam_search = BeamSearch(input_.shape[1], samples)
outputs, costs = beam_search.search(
{chars: input_}, char2code['</S>'],
3 * input_.shape[0])
else:
_1, outputs, _2, _3, costs = (
model.get_theano_function()(input_))
outputs = list(outputs.T)
costs = list(costs.T)
for i in range(len(outputs)):
outputs[i] = list(outputs[i])
try:
true_length = outputs[i].index(char2code['</S>']) + 1
except ValueError:
true_length = len(outputs[i])
outputs[i] = outputs[i][:true_length]
costs[i] = costs[i][:true_length].sum()
return outputs, costs
while True:
line = input("Enter a sentence\n")
message = ("Enter the number of samples\n" if mode == "sample"
else "Enter the beam size\n")
batch_size = int(input(message))
encoded_input = [char2code.get(char, char2code["<UNK>"])
for char in line.lower().strip()]
encoded_input = ([char2code['<S>']] + encoded_input +
[char2code['</S>']])
print("Encoder input:", encoded_input)
target = reverse_words((encoded_input,))[0]
print("Target: ", target)
samples, costs = generate(
numpy.repeat(numpy.array(encoded_input)[:, None],
batch_size, axis=1))
messages = []
for sample, cost in equizip(samples, costs):
message = "({})".format(cost)
message += "".join(code2char[code] for code in sample)
if sample == target:
message += " CORRECT!"
messages.append((cost, message))
messages.sort(key=operator.itemgetter(0), reverse=True)
for _, message in messages:
print(message)
h0_dim=32,
s0_dim=64,
h1_dim=32,
output_dim=2
)
main_brick.initialize()
prediction = main_brick.apply(input_)
prediction.name = 'output'
cost = tensor.sqr(target - prediction).sum(axis=2).mean(axis=1).mean(axis=0)
cost.name = 'training_cost'
# In[30]:
from blocks.utils import named_copy
validation_cost = named_copy(cost, 'validation_cost')
# ### Define training algorithm
# In[31]:
from blocks.graph import ComputationGraph
from blocks.algorithms import (
GradientDescent,
CompositeRule,
RMSProp,
StepClipping
)
print(str(p), p.dtype, p.shape.tag.test_value)
print("Created ComputationGraph, inputs:");
print(cg.inputs)
algorithm = GradientDescent(
cost=cost,
parameters=cg.parameters,
step_rule=CompositeRule( [StepClipping(10.0), Scale(0.01), ] ),
)
print("Defined Algorithm");
model = Model(cost)
print("Defined Model");
obs_max_length = named_copy(x.shape[0], "obs_max_length")
observables = [
cost,
obs_max_length,
#min_energy, max_energy,
#mean_activation,
]
main_loop = MainLoop(
model=model,
data_stream=data_stream,
algorithm=algorithm,
extensions=[
Timing(),
TrainingDataMonitoring(observables, after_batch=True),
#average_monitoring,