def test_serialization():
# Create a simple MLP to dump.
mlp = MLP(activations=[None, None],
dims=[10, 10, 10],
weights_init=Constant(1.),
use_bias=False)
mlp.initialize()
W = mlp.linear_transformations[1].W
W.set_value(W.get_value() * 2)
# Ensure warnings are raised when __main__ namespace objects are dumped.
foo.__module__ = '__main__'
import __main__
__main__.__dict__['foo'] = foo
mlp.foo = foo
with NamedTemporaryFile(delete=False) as f:
with warnings.catch_warnings(record=True) as w:
dump(mlp.foo, f)
assert len(w) == 1
assert '__main__' in str(w[-1].message)
# Check the parameters.
with NamedTemporaryFile(delete=False) as f:
dump(mlp, f, parameters=[mlp.children[0].W, mlp.children[1].W])
with open(f.name, 'rb') as ff:
numpy_data = load_parameters(ff)
assert set(numpy_data.keys()) == \
set(['/mlp/linear_0.W', '/mlp/linear_1.W'])
assert_allclose(numpy_data['/mlp/linear_0.W'], numpy.ones((10, 10)))
assert numpy_data['/mlp/linear_0.W'].dtype == theano.config.floatX
# Ensure that it can be unpickled.
with open(f.name, 'rb') as ff:
mlp = load(ff)
assert_allclose(mlp.linear_transformations[1].W.get_value(),
numpy.ones((10, 10)) * 2)
# Ensure that duplicate names are dealt with.
for child in mlp.children:
child.name = 'linear'
with NamedTemporaryFile(delete=False) as f:
dump(mlp, f, parameters=[mlp.children[0].W, mlp.children[1].W])
with open(f.name, 'rb') as ff:
numpy_data = load_parameters(ff)
assert set(numpy_data.keys()) == \
set(['/mlp/linear.W', '/mlp/linear.W_2'])
# Check when we don't dump the main object.
with NamedTemporaryFile(delete=False) as f:
dump(None, f, parameters=[mlp.children[0].W, mlp.children[1].W])
with tarfile.open(f.name, 'r') as tarball:
assert set(tarball.getnames()) == set(['_parameters'])
def test_serialization():
# Create a simple MLP to dump.
mlp = MLP(activations=[None, None], dims=[10, 10, 10],
weights_init=Constant(1.), use_bias=False)
mlp.initialize()
W = mlp.linear_transformations[1].W
W.set_value(W.get_value() * 2)
# Ensure warnings are raised when __main__ namespace objects are dumped.
foo.__module__ = '__main__'
import __main__
__main__.__dict__['foo'] = foo
mlp.foo = foo
with NamedTemporaryFile(delete=False) as f:
with warnings.catch_warnings(record=True) as w:
dump(mlp.foo, f)
assert len(w) == 1
assert '__main__' in str(w[-1].message)
# Check the parameters.
with NamedTemporaryFile(delete=False) as f:
dump(mlp, f, parameters=[mlp.children[0].W, mlp.children[1].W])
with open(f.name, 'rb') as ff:
numpy_data = load_parameters(ff)
assert set(numpy_data.keys()) == \
set(['/mlp/linear_0.W', '/mlp/linear_1.W'])
assert_allclose(numpy_data['/mlp/linear_0.W'], numpy.ones((10, 10)))
assert numpy_data['/mlp/linear_0.W'].dtype == theano.config.floatX
# Ensure that it can be unpickled.
with open(f.name, 'rb') as ff:
mlp = load(ff)
assert_allclose(mlp.linear_transformations[1].W.get_value(),
numpy.ones((10, 10)) * 2)
# Ensure that duplicate names are dealt with.
for child in mlp.children:
child.name = 'linear'
with NamedTemporaryFile(delete=False) as f:
dump(mlp, f, parameters=[mlp.children[0].W, mlp.children[1].W])
with open(f.name, 'rb') as ff:
numpy_data = load_parameters(ff)
assert set(numpy_data.keys()) == \
set(['/mlp/linear.W', '/mlp/linear.W_2'])
# Check when we don't dump the main object.
with NamedTemporaryFile(delete=False) as f:
dump(None, f, parameters=[mlp.children[0].W, mlp.children[1].W])
with tarfile.open(f.name, 'r') as tarball:
assert set(tarball.getnames()) == set(['_parameters'])
def load_to(self, main_loop):
with open(self.path, "rb") as source:
main_loop.model.set_parameter_values(load_parameters(source))
if self.load_iteration_state:
main_loop.iteration_state = load(source,
name='iteration_state')
if self.load_log:
main_loop.log = load(source, name='log')
def create_act_table(self, save_to, act_table):
batch_size = 500
image_size = (28, 28)
output_size = 10
convnet = create_lenet_5()
layers = convnet.layers
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outmap = OrderedDict((full_brick_name(get_brick(out)), out)
for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(
cg.variables))
# Generate pics for biases
biases = VariableFilter(roles=[BIAS])(cg.parameters)
# Generate parallel array, in the same order, for outputs
outs = [outmap[full_brick_name(get_brick(b))] for b in biases]
# Figure work count
error_rate = (MisclassificationRate().apply(y.flatten(), probs)
.copy(name='error_rate'))
max_activation_table = (MaxActivationTable().apply(
outs).copy(name='max_activation_table'))
max_activation_table.tag.aggregation_scheme = (
Concatenate(max_activation_table))
model = Model([
error_rate,
max_activation_table])
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
mnist_test_stream = DataStream.default_stream(
self.mnist_test,
iteration_scheme=SequentialScheme(
self.mnist_test.num_examples, batch_size))
evaluator = DatasetEvaluator([
error_rate,
max_activation_table
])
results = evaluator.evaluate(mnist_test_stream)
table = results['max_activation_table']
pickle.dump(table, open(act_table, 'wb'))
return table
def load_to(self, main_loop):
with open(self.path, "rb") as source:
main_loop.model.set_parameter_values(load_parameters(source))
if self.load_iteration_state or self.load_log:
loaded_main_loop = load(source)
if self.load_log:
main_loop.log = loaded_main_loop.log
if self.load_iteration_state:
main_loop.iteration_state = \
loaded_main_loop.iteration_state
def load_to(self, main_loop):
with open(self.path, "rb") as source:
main_loop.model.set_parameter_values(load_parameters(source))
if self.load_iteration_state or self.load_log:
loaded_main_loop = load(source)
if self.load_log:
main_loop.log = loaded_main_loop.log
if self.load_iteration_state:
main_loop.iteration_state = \
loaded_main_loop.iteration_state
def create_act_table(self, save_to, act_table):
batch_size = 500
image_size = (28, 28)
output_size = 10
convnet = create_lenet_5()
layers = convnet.layers
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outmap = OrderedDict(
(full_brick_name(get_brick(out)), out) for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(cg.variables))
# Generate pics for biases
biases = VariableFilter(roles=[BIAS])(cg.parameters)
# Generate parallel array, in the same order, for outputs
outs = [outmap[full_brick_name(get_brick(b))] for b in biases]
# Figure work count
error_rate = (MisclassificationRate().apply(
y.flatten(), probs).copy(name='error_rate'))
max_activation_table = (MaxActivationTable().apply(outs).copy(
name='max_activation_table'))
max_activation_table.tag.aggregation_scheme = (
Concatenate(max_activation_table))
model = Model([error_rate, max_activation_table])
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
mnist_test_stream = DataStream.default_stream(
self.mnist_test,
iteration_scheme=SequentialScheme(self.mnist_test.num_examples,
batch_size))
evaluator = DatasetEvaluator([error_rate, max_activation_table])
results = evaluator.evaluate(mnist_test_stream)
table = results['max_activation_table']
pickle.dump(table, open(act_table, 'wb'))
return table
def train(self):
error = tensor.neq(self.y.flatten(), self.y_hat.flatten() > 0.5).mean()
error.name = 'error'
self.error = error
experiment = Experiment(self.params['model_name'], self.train_stream)
experiment.cost = self.cost
experiment.set_adam(self.params['learning_rate'])
experiment.add_printing(after_epoch=True)
experiment.monitor_f_score(self.y,
self.y_hat,
average='macro',
threshold=self.params['threshold'])
experiment.monitor_auc_score(self.y, self.y_hat, average='macro')
experiment.add_timing()
experiment.extensions.append(
EarlyStopping('dev_f_score',
epochs=self.params['n_epochs'],
choose_best=max))
weights = VariableFilter(theano_name='W')(experiment.cg.variables)
experiment.regularize_max_norm(self.params['max_norms'], weights)
experiment.apply_dropout(self.params['dropout'])
experiment.track_best('dev_f_score',
save_path=self.params['model_name'] + '.tar',
choose_best=max)
experiment.track_best('dev_cost',
save_path=self.params['model_name'] +
'_cost.tar')
experiment.plot_channels(channels=[
['tra_f_score', 'dev_f_score'],
['tra_cost', 'dev_cost'],
],
url_bokeh='http://localhost:5006/',
before_first_epoch=True,
after_epoch=True)
experiment.add_monitored_vars([error])
experiment.add_norm_grads_vars()
experiment.monitor_stream(self.train_stream,
prefix='tra',
after_epoch=True)
experiment.monitor_stream(self.dev_stream, prefix='dev')
self.experiment = experiment
print('# of params for the model: {0}'.format(
experiment.get_num_params()))
main_loop = experiment.get_main_loop()
if not os.path.isfile(self.params['model_name'] + '.tar'):
main_loop.run()
with open(self.params['model_name'] + '.tar', "rb") as f:
print('loading saved model...')
main_loop.model.set_parameter_values(load_parameters(f))
def __init__(self, model_path='vgg.tar', synset_words='synset_words.txt'):
self.vgg_net = VGGNet()
x = theano.tensor.tensor4('x')
y_hat = self.vgg_net.apply(x)
cg = ComputationGraph(y_hat)
self.model = Model(y_hat)
with open(model_path, 'rb') as f:
self.model.set_parameter_values(load_parameters(f))
with open(synset_words) as f:
self.classes = numpy.array(f.read().splitlines())
self.predict = cg.get_theano_function()
fc15 = VariableFilter(
theano_name_regex='fc_15_apply_output')(cg.variables)[0]
self.fe_extractor = ComputationGraph(fc15).get_theano_function()
def do(self, callback_name, *args):
current_value = self.main_loop.log.current_row.get(self.track_var)
if current_value is None:
return
if current_value < self.best_value:
self.best_value = current_value
self.counter = 0
else:
self.counter += 1
# If nan, skip all the steps necessary to update the weights.
if numpy.isnan(current_value):
self.counter = self.patience + 1
if self.algorithm_buffers is None:
self.algorithm_buffers = [
x for x, y in self.main_loop.algorithm.step_rule_updates
]
self.algorithm_buffers = VariableFilter(roles=[ALGORITHM_BUFFER])(
self.algorithm_buffers)
if self.counter >= self.patience:
self.counter = 0
self.count_cuts += 1
with open(self.path, "rb") as source:
self.main_loop.model.set_parameter_values(
load_parameters(source))
self.main_loop.log.current_row[LOADED_FROM] = self.path
# Reset algorithm buffer
for var in self.algorithm_buffers:
var_val = var.get_value()
var.set_value(numpy.zeros(var_val.shape, dtype=var_val.dtype))
self.lr.set_value(float(self.cut_size * self.lr.get_value()))
if self.count_cuts >= self.num_cuts:
self.main_loop.log.current_row[
'training_finish_requested'] = True
def do(self, callback_name, *args):
current_value = self.main_loop.log.current_row.get(self.track_var)
if current_value is None:
return
if current_value < self.best_value:
self.best_value = current_value
self.counter = 0
else:
self.counter += 1
# If nan, skip all the steps necessary to update the weights.
if numpy.isnan(current_value):
self.counter = self.patience + 1
if self.algorithm_buffers is None:
self.algorithm_buffers = [
x for x, y in self.main_loop.algorithm.step_rule_updates]
self.algorithm_buffers = VariableFilter(
roles=[ALGORITHM_BUFFER])(self.algorithm_buffers)
if self.counter >= self.patience:
self.counter = 0
self.count_cuts += 1
with open(self.path, "rb") as source:
self.main_loop.model.set_parameter_values(
load_parameters(source))
self.main_loop.log.current_row[LOADED_FROM] = self.path
# Reset algorithm buffer
for var in self.algorithm_buffers:
var_val = var.get_value()
var.set_value(numpy.zeros(var_val.shape, dtype=var_val.dtype))
self.lr.set_value(float(self.cut_size * self.lr.get_value()))
if self.count_cuts >= self.num_cuts:
self.main_loop.log.current_row[
'training_finish_requested'] = True
def __init__(self, save_to):
batch_size = 500
image_size = (28, 28)
output_size = 10
convnet = create_lenet_5()
layers = convnet.layers
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(
i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
mnist_test = MNIST(("test",), sources=['features', 'targets'])
basis = create_fair_basis(mnist_test, 10, 10)
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outs = OrderedDict((full_brick_name(get_brick(out)), out)
for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(
cg.variables))
# Normalize input and apply the convnet
error_rate = (MisclassificationRate().apply(y.flatten(), probs)
.copy(name='error_rate'))
confusion = (ConfusionMatrix().apply(y.flatten(), probs)
.copy(name='confusion'))
confusion.tag.aggregation_scheme = Sum(confusion)
confusion_image = (ConfusionImage().apply(y.flatten(), probs, x)
.copy(name='confusion_image'))
confusion_image.tag.aggregation_scheme = Sum(confusion_image)
model = Model(
[error_rate, confusion, confusion_image] + list(outs.values()))
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
mnist_test = MNIST(("test",))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=SequentialScheme(
mnist_test.num_examples, batch_size))
self.model = model
self.mnist_test_stream = mnist_test_stream
self.evaluator = DatasetEvaluator(
[error_rate, confusion, confusion_image])
self.base_results = self.evaluator.evaluate(mnist_test_stream)
# TODO: allow target layer to be parameterized
self.target_layer = '/lenet/mlp/linear_0'
self.next_layer_param = '/lenet/mlp/linear_1.W'
self.base_sample = extract_sample(
outs[self.target_layer], mnist_test_stream)
self.base_param_value = (
model.get_parameter_dict()[
self.next_layer_param].get_value().copy())
def main(save_to):
batch_size = 365
feature_maps = [6, 16]
mlp_hiddens = [120, 84]
conv_sizes = [5, 5]
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations, 1, image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='valid',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
layers = convnet.layers
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(
i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
random_init = (numpy.random.rand(100, 1, 28, 28) * 128).astype('float32')
mnist_test = MNIST(("test",), sources=['features', 'targets'])
basis = create_fair_basis(mnist_test, 10, 10)
# state = mnist_test.open()
#
# basis = numpy.zeros((100, 1, 28, 28), dtype=theano.config.floatX)
# counters = [0] * 10
# index = 0
# while min(counters) < 10:
# feature, target = mnist_test.get_data(state=state, request=[index])
# target = target[0, 0]
# feature = feature / 256
# if counters[target] < 10:
# basis[target + counters[target] * 10, :, :, :] = feature[0, :, :, :]
# counters[target] += 1
# index += 1
# mnist_test.close(state=state)
# b = shared_floatx(basis)
# random_init = numpy.rand.random(100, 1000)
# r = shared_floatx(random_init)
# rn = r / r.norm(axis=1)
# x = tensor.dot(rn, tensor.shape_padright(b))
x = tensor.tensor4('features')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
outs = VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(cg.variables)
# Create an interior activation model
model = Model([probs] + outs)
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
fn = theano.function([x], outs)
results = fn(basis)
for snapshots, output in zip(results, outs):
layer = get_brick(output)
filmstrip = Filmstrip(
basis.shape[-2:], (snapshots.shape[1], snapshots.shape[0]),
background='purple')
if layer in layers:
fieldmap = layerarray_fieldmap(layers[0:layers.index(layer) + 1])
for unit in range(snapshots.shape[1]):
for index in range(snapshots.shape[0]):
mask = make_mask(basis.shape[-2:], fieldmap, numpy.clip(
snapshots[index, unit, :, :], 0, numpy.inf))
filmstrip.set_image((unit, index),
basis[index, :, :, :], mask)
filmstrip.save(layer.name + '_show.jpg')
ix_to_char, char_to_ix, vocab_size = get_metadata(hdf5_file)
if not args.primetext or len(args.primetext) == 0:
args.primetext = ix_to_char[numpy.random.randint(vocab_size)]
primetext = ''.join([ch for ch in args.primetext if ch in char_to_ix.keys()])
if len(primetext) == 0:
raise Exception('primetext characters are not in the vocabulary')
x_curr = numpy.expand_dims(
numpy.array([char_to_ix[ch] for ch in primetext], dtype='uint8'), axis=1)
print('Loading model from {0}...'.format(args.model))
x = tensor.matrix('features', dtype='uint8')
y = tensor.matrix('targets', dtype='uint8')
y_hat, cost, cells = nn_fprop(x, y, vocab_size, hidden_size, num_layers, model)
main_loop = MainLoop(algorithm=None, data_stream=None, model=Model(cost))
with open(args.model) as f:
main_loop.model.set_parameter_values(load_parameters(f))
bin_model = main_loop.model
activations = []
initial_states = []
for i in range(num_layers):
brick = [b for b in bin_model.get_top_bricks() if b.name==model+str(i)][0]
activations.extend(VariableFilter(theano_name=brick.name+'_apply_states')(bin_model.variables))
activations.extend(VariableFilter(theano_name=brick.name+'_apply_cells')(cells))
initial_states.extend(VariableFilter(roles=[roles.INITIAL_STATE])(brick.parameters))
#take activations of last element
activations = [act[-1].flatten() for act in activations]
states_as_params = [tensor.vector(dtype=initial.dtype) for initial in initial_states]
#Get prob. distribution of the last element in the last seq of the batch
fprop = theano.function([x] + states_as_params, activations + [y_hat[-1, -1, :]], givens=zip(initial_states, states_as_params))
def __init__(self, save_to):
batch_size = 500
image_size = (28, 28)
output_size = 10
convnet = create_lenet_5()
layers = convnet.layers
mnist_test = MNIST(("test", ), sources=['features', 'targets'])
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outmap = OrderedDict(
(full_brick_name(get_brick(out)), out) for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(cg.variables))
# Generate pics for biases
biases = VariableFilter(roles=[BIAS])(cg.parameters)
# Generate parallel array, in the same order, for outputs
outs = [outmap[full_brick_name(get_brick(b))] for b in biases]
# Figure work count
error_rate = (MisclassificationRate().apply(
y.flatten(), probs).copy(name='error_rate'))
sensitive_unit_count = (SensitiveUnitCount().apply(
y.flatten(), probs, biases).copy(name='sensitive_unit_count'))
sensitive_unit_count.tag.aggregation_scheme = (
Concatenate(sensitive_unit_count))
active_unit_count = (ActiveUnitCount().apply(outs).copy(
name='active_unit_count'))
active_unit_count.tag.aggregation_scheme = (
Concatenate(active_unit_count))
ignored_unit_count = (IgnoredUnitCount().apply(
y.flatten(), probs, biases, outs).copy(name='ignored_unit_count'))
ignored_unit_count.tag.aggregation_scheme = (
Concatenate(ignored_unit_count))
model = Model([
error_rate, sensitive_unit_count, active_unit_count,
ignored_unit_count
])
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
mnist_test = MNIST(("test", ))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=SequentialScheme(mnist_test.num_examples,
batch_size))
evaluator = DatasetEvaluator([
error_rate, sensitive_unit_count, active_unit_count,
ignored_unit_count
])
results = evaluator.evaluate(mnist_test_stream)
def save_ranked_image(scores, filename):
sorted_instances = scores.argsort()
filmstrip = Filmstrip(image_shape=(28, 28), grid_shape=(100, 100))
for i, index in enumerate(sorted_instances):
filmstrip.set_image((i // 100, i % 100),
mnist_test.get_data(request=index)[0])
filmstrip.save(filename)
save_ranked_image(results['sensitive_unit_count'], 'sensitive.jpg')
save_ranked_image(results['active_unit_count'], 'active.jpg')
save_ranked_image(results['ignored_unit_count'], 'ignored.jpg')
order = 34
alpha = 0.4
stage = 2
gamma = -1.0 / stage
save_dir = os.environ['RESULTS_DIR']
save_dir = os.path.join(save_dir,'blizzard/')
experiment_name = "baseline_sp_no_feedback"
num_sample = "06"
print "Sampling: " + experiment_name
with open(save_dir+"pkl/best_"+experiment_name+".tar", 'rb') as src:
parameters = load_parameters(src)
#####################################
# Model Construction
#####################################
from blocks.bricks import (Tanh, MLP,
Rectifier, Activation, Identity)
from blocks.bricks.sequence_generators import (
Readout, SequenceGenerator)
from blocks.bricks.recurrent import LSTM, RecurrentStack, GatedRecurrent
from play.bricks.custom import (DeepTransitionFeedback, GMMEmitter,
SPF0Emitter)
def evaluate_lm(config, tar_path, part, num_examples, dest_path, **kwargs):
c = config
if part not in ['valid', 'test_unseen', 'test']:
raise ValueError()
data, lm, _ = initialize_data_and_model(c)
words = T.ltensor3('words')
words_mask = T.matrix('words_mask')
costs = lm.apply(words, words_mask)
cg = Model(costs)
with open(tar_path) as src:
cg.set_parameter_values(load_parameters(src))
perplexities = VariableFilter(name_regex='perplexity.*')(cg)
mask_sums = [p.tag.aggregation_scheme.denominator for p in perplexities]
CEs = [p.tag.aggregation_scheme.numerator for p in perplexities]
proba_out, = VariableFilter(name='proba_out')(cg)
unk_ratios = VariableFilter(name_regex='unk_ratio.*')(cg)
#num_definitions, = VariableFilter(name='num_definitions')(cg)
print perplexities
print CEs
print mask_sums
name_to_aggregate = [p.name for p in perplexities]
for CE, mask_sum, name in zip(CEs, mask_sums, name_to_aggregate):
CE.name = name + "_num"
mask_sum.name = name + "_denom"
compute_l = CEs + mask_sums + unk_ratios
if part == 'test_unseen':
compute_l.append(proba_out)
compute = dict({p.name: p for p in compute_l})
print "to compute:", compute.keys()
predict_f = theano.function([words, words_mask], compute)
if part == 'test_unseen':
batch_size = 1
else:
batch_size = 128 # size of test_unseen
stream = data.get_stream(part, batch_size=batch_size, max_length=100)
raw_data = [] # list of dicts containing the inputs and computed outputs
i = 0
print "start computing"
for input_data in stream.get_epoch_iterator(as_dict=True):
if i and i % 100 == 0:
print "iteration:", i
words = input_data['words']
words_mask = input_data['words_mask']
to_save = predict_f(words, words_mask)
to_save.update(input_data)
raw_data.append(to_save)
i += 1
# aggregate in the log space
aggregated = Counter()
sum_mask_track = Counter()
for d in raw_data:
coef = d['words_mask'].sum() # over timesteps and batches
for name in name_to_aggregate:
aggregated[name] += d[name + "_num"]
sum_mask_track[name] += d[name + "_denom"]
for k, v in aggregated.iteritems():
print "k, v, m:", k, v, sum_mask_track[k]
aggregated[k] = np.exp(v / sum_mask_track[k])
n_params = sum([np.prod(p.shape.eval()) for p in cg.parameters])
aggregated['n_params'] = n_params
print "aggregated stats:", aggregated
print "# of parameters {}".format(n_params)
#TODO: check that different batch_size yields same validation error than
# end of training validation error.
# TODO: I think blocks aggreg is simply mean which should break
# when we use masks??? investigate
if not os.path.exists(dest_path):
os.makedirs(dest_path)
if part == 'test_unseen':
np.savez(
os.path.join(dest_path, "predictions"),
words=input_data['words'],
words_mask=input_data['words_mask'],
#unk_ratio = to_save['unk_ratio'],
#def_unk_ratio = to_save['def_unk_ratio'],
proba_out=to_save['languagemodel_apply_proba_out'],
vocab_in=lm._vocab.words[:c['num_input_words']],
vocab_out=lm._vocab.words[:c['num_output_words']])
json.dump(aggregated,
open(os.path.join(dest_path, "aggregates.json"), "w"),
sort_keys=True,
indent=2)
def load_params(self, path):
generated = self.get_generate_graph()
with open(path, 'r') as src:
param_values = load_parameters(src)
Model(generated['outputs']).set_parameter_values(param_values)
def main(save_to):
batch_size = 500
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
convnet = create_lenet_5()
mnist_test = MNIST(("test",), sources=['features', 'targets'])
basis_init = create_fair_basis(mnist_test, 10, 2)
# b = shared_floatx(basis)
# random_init = numpy.rand.random(100, 1000)
# r = shared_floatx(random_init)
# rn = r / r.norm(axis=1)
# x = tensor.dot(rn, tensor.shape_padright(b))
x = shared_floatx(basis_init)
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
outs = VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(cg.variables)
# Create an interior activation model
model = Model([probs] + outs)
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
learning_rate = shared_floatx(0.01, 'learning_rate')
unit = shared_floatx(0, 'unit', dtype='int64')
negate = False
suffix = '_negsynth.jpg' if negate else '_synth.jpg'
for output in outs:
layer = get_brick(output)
# For now, skip masks -for some reason they are always NaN
iterations = 10000
layername = layer.parents[0].name + '-' + layer.name
# if layername != 'noisylinear_2-linear':
# continue
dims = layer.get_dims(['output'])[0]
if negate:
measure = -output
else:
measure = output
measure = measure[(slice(0, basis_init.shape[0]), ) +
(slice(None),) * (measure.ndim - 1)]
if isinstance(dims, numbers.Integral):
dims = (dims, )
costvec = -tensor.log(tensor.nnet.softmax(
measure)[:,unit].flatten())
else:
flatout = measure.flatten(ndim=3)
maxout = flatout.max(axis=2)
costvec = -tensor.log(tensor.nnet.softmax(
maxout)[:,unit].flatten())
# Add a regularization to favor gray images.
# cost = costvec.sum() + (x - 0.5).norm(2) * (
# 10.0 / basis_init.shape[0])
cost = costvec.sum()
grad = gradient.grad(cost, x)
stepx = x - learning_rate * grad
normx = stepx / tensor.shape_padright(
stepx.flatten(ndim=2).max(axis=1), n_ones=3)
newx = tensor.clip(normx, 0, 1)
newx = newx[(slice(0, basis_init.shape[0]), ) +
(slice(None),) * (newx.ndim - 1)]
fn = theano.function([], [cost], updates=[(x, newx)])
filmstrip = Filmstrip(
basis_init.shape[-2:], (dims[0], basis_init.shape[0]),
background='red')
for u in range(dims[0]):
unit.set_value(u)
x.set_value(basis_init)
print('layer', layername, 'unit', u)
for index in range(iterations):
c = fn()[0]
if index % 1000 == 0:
print('cost', c)
result = x.get_value()
for i2 in range(basis_init.shape[0]):
filmstrip.set_image((u, i2), result[i2,:,:,:])
filmstrip.save(layername + suffix)
result = x.get_value()
for index in range(basis_init.shape[0]):
filmstrip.set_image((u, index), result[index,:,:,:])
filmstrip.save(layername + suffix)
def initialize_all(config, save_path, bokeh_name,
params, bokeh_server, bokeh, test_tag, use_load_ext,
load_log, fast_start):
root_path, extension = os.path.splitext(save_path)
data = Data(**config['data'])
train_conf = config['training']
recognizer = create_model(config, data, test_tag)
# Separate attention_params to be handled differently
# when regularization is applied
attention = recognizer.generator.transition.attention
attention_params = Selector(attention).get_parameters().values()
logger.info(
"Initialization schemes for all bricks.\n"
"Works well only in my branch with __repr__ added to all them,\n"
"there is an issue #463 in Blocks to do that properly.")
def show_init_scheme(cur):
result = dict()
for attr in dir(cur):
if attr.endswith('_init'):
result[attr] = getattr(cur, attr)
for child in cur.children:
result[child.name] = show_init_scheme(child)
return result
logger.info(pprint.pformat(show_init_scheme(recognizer)))
prediction, prediction_mask = add_exploration(recognizer, data, train_conf)
#
# Observables:
#
primary_observables = [] # monitored each batch
secondary_observables = [] # monitored every 10 batches
validation_observables = [] # monitored on the validation set
cg = recognizer.get_cost_graph(
batch=True, prediction=prediction, prediction_mask=prediction_mask)
labels, = VariableFilter(
applications=[recognizer.cost], name='labels')(cg)
labels_mask, = VariableFilter(
applications=[recognizer.cost], name='labels_mask')(cg)
gain_matrix = VariableFilter(
theano_name=RewardRegressionEmitter.GAIN_MATRIX)(cg)
if len(gain_matrix):
gain_matrix, = gain_matrix
primary_observables.append(
rename(gain_matrix.min(), 'min_gain'))
primary_observables.append(
rename(gain_matrix.max(), 'max_gain'))
batch_cost = cg.outputs[0].sum()
batch_size = rename(recognizer.labels.shape[1], "batch_size")
# Assumes constant batch size. `aggregation.mean` is not used because
# of Blocks #514.
cost = batch_cost / batch_size
cost.name = "sequence_total_cost"
logger.info("Cost graph is built")
# Fetch variables useful for debugging.
# It is important not to use any aggregation schemes here,
# as it's currently impossible to spread the effect of
# regularization on their variables, see Blocks #514.
cost_cg = ComputationGraph(cost)
r = recognizer
energies, = VariableFilter(
applications=[r.generator.readout.readout], name="output_0")(
cost_cg)
bottom_output = VariableFilter(
# We need name_regex instead of name because LookupTable calls itsoutput output_0
applications=[r.bottom.apply], name_regex="output")(
cost_cg)[-1]
attended, = VariableFilter(
applications=[r.generator.transition.apply], name="attended")(
cost_cg)
attended_mask, = VariableFilter(
applications=[r.generator.transition.apply], name="attended_mask")(
cost_cg)
weights, = VariableFilter(
applications=[r.generator.evaluate], name="weights")(
cost_cg)
max_recording_length = rename(bottom_output.shape[0],
"max_recording_length")
# To exclude subsampling related bugs
max_attended_mask_length = rename(attended_mask.shape[0],
"max_attended_mask_length")
max_attended_length = rename(attended.shape[0],
"max_attended_length")
max_num_phonemes = rename(labels.shape[0],
"max_num_phonemes")
min_energy = rename(energies.min(), "min_energy")
max_energy = rename(energies.max(), "max_energy")
mean_attended = rename(abs(attended).mean(),
"mean_attended")
mean_bottom_output = rename(abs(bottom_output).mean(),
"mean_bottom_output")
weights_penalty = rename(monotonicity_penalty(weights, labels_mask),
"weights_penalty")
weights_entropy = rename(entropy(weights, labels_mask),
"weights_entropy")
mask_density = rename(labels_mask.mean(),
"mask_density")
cg = ComputationGraph([
cost, weights_penalty, weights_entropy,
min_energy, max_energy,
mean_attended, mean_bottom_output,
batch_size, max_num_phonemes,
mask_density])
# Regularization. It is applied explicitly to all variables
# of interest, it could not be applied to the cost only as it
# would not have effect on auxiliary variables, see Blocks #514.
reg_config = config.get('regularization', dict())
regularized_cg = cg
if reg_config.get('dropout'):
logger.info('apply dropout')
regularized_cg = apply_dropout(cg, [bottom_output], 0.5)
if reg_config.get('noise'):
logger.info('apply noise')
noise_subjects = [p for p in cg.parameters if p not in attention_params]
regularized_cg = apply_noise(cg, noise_subjects, reg_config['noise'])
train_cost = regularized_cg.outputs[0]
if reg_config.get("penalty_coof", .0) > 0:
# big warning!!!
# here we assume that:
# regularized_weights_penalty = regularized_cg.outputs[1]
train_cost = (train_cost +
reg_config.get("penalty_coof", .0) *
regularized_cg.outputs[1] / batch_size)
if reg_config.get("decay", .0) > 0:
train_cost = (train_cost + reg_config.get("decay", .0) *
l2_norm(VariableFilter(roles=[WEIGHT])(cg.parameters)) ** 2)
train_cost = rename(train_cost, 'train_cost')
gradients = None
if reg_config.get('adaptive_noise'):
logger.info('apply adaptive noise')
if ((reg_config.get("penalty_coof", .0) > 0) or
(reg_config.get("decay", .0) > 0)):
logger.error('using adaptive noise with alignment weight panalty '
'or weight decay is probably stupid')
train_cost, regularized_cg, gradients, noise_brick = apply_adaptive_noise(
cg, cg.outputs[0],
variables=cg.parameters,
num_examples=data.get_dataset('train').num_examples,
parameters=Model(regularized_cg.outputs[0]).get_parameter_dict().values(),
**reg_config.get('adaptive_noise')
)
train_cost.name = 'train_cost'
adapt_noise_cg = ComputationGraph(train_cost)
model_prior_mean = rename(
VariableFilter(applications=[noise_brick.apply],
name='model_prior_mean')(adapt_noise_cg)[0],
'model_prior_mean')
model_cost = rename(
VariableFilter(applications=[noise_brick.apply],
name='model_cost')(adapt_noise_cg)[0],
'model_cost')
model_prior_variance = rename(
VariableFilter(applications=[noise_brick.apply],
name='model_prior_variance')(adapt_noise_cg)[0],
'model_prior_variance')
regularized_cg = ComputationGraph(
[train_cost, model_cost] +
regularized_cg.outputs +
[model_prior_mean, model_prior_variance])
primary_observables += [
regularized_cg.outputs[1], # model cost
regularized_cg.outputs[2], # task cost
regularized_cg.outputs[-2], # model prior mean
regularized_cg.outputs[-1]] # model prior variance
model = Model(train_cost)
if params:
logger.info("Load parameters from " + params)
# please note: we cannot use recognizer.load_params
# as it builds a new computation graph that dies not have
# shapred variables added by adaptive weight noise
with open(params, 'r') as src:
param_values = load_parameters(src)
model.set_parameter_values(param_values)
parameters = model.get_parameter_dict()
logger.info("Parameters:\n" +
pprint.pformat(
[(key, parameters[key].get_value().shape) for key
in sorted(parameters.keys())],
width=120))
# Define the training algorithm.
clipping = StepClipping(train_conf['gradient_threshold'])
clipping.threshold.name = "gradient_norm_threshold"
rule_names = train_conf.get('rules', ['momentum'])
core_rules = []
if 'momentum' in rule_names:
logger.info("Using scaling and momentum for training")
core_rules.append(Momentum(train_conf['scale'], train_conf['momentum']))
if 'adadelta' in rule_names:
logger.info("Using AdaDelta for training")
core_rules.append(AdaDelta(train_conf['decay_rate'], train_conf['epsilon']))
max_norm_rules = []
if reg_config.get('max_norm', False) > 0:
logger.info("Apply MaxNorm")
maxnorm_subjects = VariableFilter(roles=[WEIGHT])(cg.parameters)
if reg_config.get('max_norm_exclude_lookup', False):
maxnorm_subjects = [v for v in maxnorm_subjects
if not isinstance(get_brick(v), LookupTable)]
logger.info("Parameters covered by MaxNorm:\n"
+ pprint.pformat([name for name, p in parameters.items()
if p in maxnorm_subjects]))
logger.info("Parameters NOT covered by MaxNorm:\n"
+ pprint.pformat([name for name, p in parameters.items()
if not p in maxnorm_subjects]))
max_norm_rules = [
Restrict(VariableClipping(reg_config['max_norm'], axis=0),
maxnorm_subjects)]
burn_in = []
if train_conf.get('burn_in_steps', 0):
burn_in.append(
BurnIn(num_steps=train_conf['burn_in_steps']))
algorithm = GradientDescent(
cost=train_cost,
parameters=parameters.values(),
gradients=gradients,
step_rule=CompositeRule(
[clipping] + core_rules + max_norm_rules +
# Parameters are not changed at all
# when nans are encountered.
[RemoveNotFinite(0.0)] + burn_in),
on_unused_sources='warn')
logger.debug("Scan Ops in the gradients")
gradient_cg = ComputationGraph(algorithm.gradients.values())
for op in ComputationGraph(gradient_cg).scans:
logger.debug(op)
# More variables for debugging: some of them can be added only
# after the `algorithm` object is created.
secondary_observables += list(regularized_cg.outputs)
if not 'train_cost' in [v.name for v in secondary_observables]:
secondary_observables += [train_cost]
secondary_observables += [
algorithm.total_step_norm, algorithm.total_gradient_norm,
clipping.threshold]
for name, param in parameters.items():
num_elements = numpy.product(param.get_value().shape)
norm = param.norm(2) / num_elements ** 0.5
grad_norm = algorithm.gradients[param].norm(2) / num_elements ** 0.5
step_norm = algorithm.steps[param].norm(2) / num_elements ** 0.5
stats = tensor.stack(norm, grad_norm, step_norm, step_norm / grad_norm)
stats.name = name + '_stats'
secondary_observables.append(stats)
primary_observables += [
train_cost,
algorithm.total_gradient_norm,
algorithm.total_step_norm, clipping.threshold,
max_recording_length,
max_attended_length, max_attended_mask_length]
validation_observables += [
rename(aggregation.mean(batch_cost, batch_size), cost.name),
rename(aggregation.sum_(batch_size), 'num_utterances'),
weights_entropy, weights_penalty]
def attach_aggregation_schemes(variables):
# Aggregation specification has to be factored out as a separate
# function as it has to be applied at the very last stage
# separately to training and validation observables.
result = []
for var in variables:
if var.name == 'weights_penalty':
result.append(rename(aggregation.mean(var, batch_size),
'weights_penalty_per_recording'))
elif var.name == 'weights_entropy':
result.append(rename(aggregation.mean(var, labels_mask.sum()),
'weights_entropy_per_label'))
else:
result.append(var)
return result
mon_conf = config['monitoring']
# Build main loop.
logger.info("Initialize extensions")
extensions = []
if use_load_ext and params:
extensions.append(Load(params, load_iteration_state=True, load_log=True))
if load_log and params:
extensions.append(LoadLog(params))
extensions += [
Timing(after_batch=True),
CGStatistics(),
#CodeVersion(['lvsr']),
]
extensions.append(TrainingDataMonitoring(
primary_observables, after_batch=True))
average_monitoring = TrainingDataMonitoring(
attach_aggregation_schemes(secondary_observables),
prefix="average", every_n_batches=10)
extensions.append(average_monitoring)
validation = DataStreamMonitoring(
attach_aggregation_schemes(validation_observables),
data.get_stream("valid", shuffle=False), prefix="valid").set_conditions(
before_first_epoch=not fast_start,
every_n_epochs=mon_conf['validate_every_epochs'],
every_n_batches=mon_conf['validate_every_batches'],
after_training=False)
extensions.append(validation)
per = PhonemeErrorRate(recognizer, data,
**config['monitoring']['search'])
per_monitoring = DataStreamMonitoring(
[per], data.get_stream("valid", batches=False, shuffle=False),
prefix="valid").set_conditions(
before_first_epoch=not fast_start,
every_n_epochs=mon_conf['search_every_epochs'],
every_n_batches=mon_conf['search_every_batches'],
after_training=False)
extensions.append(per_monitoring)
track_the_best_per = TrackTheBest(
per_monitoring.record_name(per)).set_conditions(
before_first_epoch=True, after_epoch=True)
track_the_best_cost = TrackTheBest(
validation.record_name(cost)).set_conditions(
before_first_epoch=True, after_epoch=True)
extensions += [track_the_best_cost, track_the_best_per]
extensions.append(AdaptiveClipping(
algorithm.total_gradient_norm.name,
clipping, train_conf['gradient_threshold'],
decay_rate=0.998, burnin_period=500))
extensions += [
SwitchOffLengthFilter(
data.length_filter,
after_n_batches=train_conf.get('stop_filtering')),
FinishAfter(after_n_batches=train_conf.get('num_batches'),
after_n_epochs=train_conf.get('num_epochs'))
.add_condition(["after_batch"], _gradient_norm_is_none),
]
channels = [
# Plot 1: training and validation costs
[average_monitoring.record_name(train_cost),
validation.record_name(cost)],
# Plot 2: gradient norm,
[average_monitoring.record_name(algorithm.total_gradient_norm),
average_monitoring.record_name(clipping.threshold)],
# Plot 3: phoneme error rate
[per_monitoring.record_name(per)],
# Plot 4: training and validation mean weight entropy
[average_monitoring._record_name('weights_entropy_per_label'),
validation._record_name('weights_entropy_per_label')],
# Plot 5: training and validation monotonicity penalty
[average_monitoring._record_name('weights_penalty_per_recording'),
validation._record_name('weights_penalty_per_recording')]]
if bokeh:
extensions += [
Plot(bokeh_name if bokeh_name
else os.path.basename(save_path),
channels,
every_n_batches=10,
server_url=bokeh_server),]
extensions += [
Checkpoint(save_path,
before_first_epoch=not fast_start, after_epoch=True,
every_n_batches=train_conf.get('save_every_n_batches'),
save_separately=["model", "log"],
use_cpickle=True)
.add_condition(
['after_epoch'],
OnLogRecord(track_the_best_per.notification_name),
(root_path + "_best" + extension,))
.add_condition(
['after_epoch'],
OnLogRecord(track_the_best_cost.notification_name),
(root_path + "_best_ll" + extension,)),
ProgressBar()]
extensions.append(EmbedIPython(use_main_loop_run_caller_env=True))
if config['net']['criterion']['name'].startswith('mse'):
extensions.append(
LogInputsGains(
labels, cg, recognizer.generator.readout.emitter, data))
if train_conf.get('patience'):
patience_conf = train_conf['patience']
if not patience_conf.get('notification_names'):
# setdefault will not work for empty list
patience_conf['notification_names'] = [
track_the_best_per.notification_name,
track_the_best_cost.notification_name]
extensions.append(Patience(**patience_conf))
extensions.append(Printing(every_n_batches=1,
attribute_filter=PrintingFilterList()))
return model, algorithm, data, extensions
def main(save_to, hist_file):
batch_size = 365
feature_maps = [6, 16]
mlp_hiddens = [120, 84]
conv_sizes = [5, 5]
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations,
1,
image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='valid',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info(
"Input dim: {} {} {}".format(*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
mnist_test = MNIST(("test", ), sources=['features', 'targets'])
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
error_rate = (MisclassificationRate().apply(y.flatten(),
probs).copy(name='error_rate'))
confusion = (ConfusionMatrix().apply(y.flatten(),
probs).copy(name='confusion'))
confusion.tag.aggregation_scheme = Sum(confusion)
model = Model([error_rate, confusion])
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outs = OrderedDict(
(full_brick_name(get_brick(out)), out) for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(model.variables))
# Load histogram information
with open(hist_file, 'rb') as handle:
histograms = pickle.load(handle)
# Corpora
mnist_train = MNIST(("train", ))
mnist_train_stream = DataStream.default_stream(
mnist_train,
iteration_scheme=ShuffledScheme(mnist_train.num_examples, batch_size))
mnist_test = MNIST(("test", ))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=ShuffledScheme(mnist_test.num_examples, batch_size))
# Probe the given layer
target_layer = '/lenet/mlp/linear_0'
next_layer_param = '/lenet/mlp/linear_1.W'
sample = extract_sample(outs[target_layer], mnist_test_stream)
print('sample shape', sample.shape)
# Figure neurons to ablate
hist = histograms[('linear_1', 'b')]
targets = [i for i in range(hist.shape[1]) if hist[2, i] * hist[7, i] < 0]
print('ablating', len(targets), ':', targets)
# Now adjust the next layer weights based on the probe
param = model.get_parameter_dict()[next_layer_param]
print('param shape', param.get_value().shape)
new_weights = ablate_inputs(targets,
sample,
param.get_value(),
compensate=False)
param.set_value(new_weights)
# Evaluation pass
evaluator = DatasetEvaluator([error_rate, confusion])
print(evaluator.evaluate(mnist_test_stream))
def main(save_to):
batch_size = 365
feature_maps = [6, 16]
mlp_hiddens = [120, 84]
conv_sizes = [5, 5]
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations, 1, image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='valid',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(
i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
random_init = (numpy.random.rand(100, 1, 28, 28) * 128).astype('float32')
layers = [l for l in convnet.layers if isinstance(l, Convolutional)]
mnist_test = MNIST(("test",), sources=['features', 'targets'])
basis_init = create_fair_basis(mnist_test, 10, 50)
basis_set = make_shifted_basis(basis_init, convnet, layers)
for layer, basis in zip(layers, basis_set):
# basis is 5d:
# (probed_units, base_cases, 1-c, 28-y, 28-x)
b = shared_floatx(basis)
# coefficients is 2d:
# (probed_units, base_cases)
coefficients = shared_floatx(
numpy.ones(basis.shape[0:2],
dtype=theano.config.floatX))
# prod is 5d: (probed_units, base_cases, 1-c, 28-y, 28-x)
prod = tensor.shape_padright(coefficients, 3) * b
# x is 4d: (probed_units, 1-c, 28-y, 28-x)
ux = prod.sum(axis=1)
x = tensor.clip(ux /
tensor.shape_padright(ux.flatten(ndim=2).max(axis=1), 3),
0, 1)
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
outs = VariableFilter(
roles=[OUTPUT], bricks=[layer])(cg.variables)
# Create an interior activation model
model = Model([probs] + outs)
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
learning_rate = shared_floatx(0.03, 'learning_rate')
# We will try to do all units at once.
# unit = shared_floatx(0, 'unit', dtype='int64')
# But we are only doing one layer at once.
output = outs[0]
dims = layer.get_dims(['output'])[0]
if isinstance(dims, numbers.Integral):
# FC case: output is 2d: (probed_units, units)
dims = (dims, )
unitrange = tensor.arange(dims[0])
costvec = -tensor.log(
tensor.nnet.softmax(output)[unitrange, unitrage].
flatten())
else:
# Conv case: output is 4d: (probed_units, units, y, x)
unitrange = tensor.arange(dims[0])
print('dims is', dims)
costvec = -tensor.log(tensor.nnet.softmax(output[
unitrange, unitrange, dims[1] // 2, dims[2] // 2]).
flatten())
cost = costvec.sum()
# grad is dims (probed_units, basis_size)
grad = gradient.grad(cost, coefficients)
stepc = coefficients # - learning_rate * grad
newc = stepc / tensor.shape_padright(stepc.mean(axis=1))
fn = theano.function([], [cost, x], updates=[(coefficients, newc)])
filmstrip = Filmstrip(
random_init.shape[-2:], (dims[0], 1),
background='red')
layer = get_brick(output)
learning_rate.set_value(0.1)
for index in range(20000):
c, result = fn()
if index % 1000 == 0:
learning_rate.set_value(numpy.cast[theano.config.floatX](
learning_rate.get_value() * 0.8))
print('cost', c)
for u in range(dims[0]):
filmstrip.set_image((u, 0), result[u,:,:,:])
filmstrip.save(layer.name + '_stroke.jpg')
for u in range(dims[0]):
filmstrip.set_image((u, 0), result[u,:,:,:])
filmstrip.save(layer.name + '_stroke.jpg')
def main(save_to):
batch_size = 365
feature_maps = [6, 16]
mlp_hiddens = [120, 84]
conv_sizes = [5, 5]
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations, 1, image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='valid',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(
i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
x = tensor.tensor4('features')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
outs = VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(cg.variables)
# Create an interior activation model
model = Model([probs] + outs)
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
algorithm = MaximumActivationSearch(outputs=outs)
# Use the mnist test set, unshuffled
mnist_test = MNIST(("test",), sources=['features'])
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=SequentialScheme(
mnist_test.num_examples, batch_size))
extensions = [Timing(),
FinishAfter(after_n_epochs=1),
DataStreamMonitoring(
[],
mnist_test_stream,
prefix="test"),
Checkpoint("maxact.tar"),
ProgressBar(),
Printing()]
main_loop = MainLoop(
algorithm,
mnist_test_stream,
model=model,
extensions=extensions)
main_loop.run()
examples = mnist_test.get_example_stream()
example = examples.get_data(0)[0]
layers = convnet.layers
for output, record in algorithm.maximum_activations.items():
layer = get_brick(output)
activations, indices, snapshots = (
r.get_value() if r else None for r in record[1:])
filmstrip = Filmstrip(
example.shape[-2:], (indices.shape[1], indices.shape[0]),
background='blue')
if layer in layers:
fieldmap = layerarray_fieldmap(layers[0:layers.index(layer) + 1])
for unit in range(indices.shape[1]):
for index in range(100):
mask = make_mask(example.shape[-2:], fieldmap, numpy.clip(
snapshots[index, unit, :, :], 0, numpy.inf))
imagenum = indices[index, unit, 0]
filmstrip.set_image((unit, index),
examples.get_data(imagenum)[0], mask)
else:
for unit in range(indices.shape[1]):
for index in range(100):
imagenum = indices[index, unit]
filmstrip.set_image((unit, index),
examples.get_data(imagenum)[0])
filmstrip.save(layer.name + '_maxact.jpg')
def load_params(self, path):
generated = self.get_generate_graph()
with open(path, 'r') as src:
param_values = load_parameters(src)
Model(generated['outputs']).set_parameter_values(param_values)
def load_params(self, path):
cg = self.get_cost_graph()
with open(path, 'r') as src:
param_values = load_parameters(src)
Model(cg.outputs).set_parameter_values(param_values)
def main(save_to):
batch_size = 365
feature_maps = [6, 16]
mlp_hiddens = [120, 84]
conv_sizes = [5, 5]
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations,
1,
image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='valid',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
layers = convnet.layers
logging.info(
"Input dim: {} {} {}".format(*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
random_init = (numpy.random.rand(100, 1, 28, 28) * 128).astype('float32')
mnist_test = MNIST(("test", ), sources=['features', 'targets'])
basis = create_fair_basis(mnist_test, 10, 10)
# state = mnist_test.open()
#
# basis = numpy.zeros((100, 1, 28, 28), dtype=theano.config.floatX)
# counters = [0] * 10
# index = 0
# while min(counters) < 10:
# feature, target = mnist_test.get_data(state=state, request=[index])
# target = target[0, 0]
# feature = feature / 256
# if counters[target] < 10:
# basis[target + counters[target] * 10, :, :, :] = feature[0, :, :, :]
# counters[target] += 1
# index += 1
# mnist_test.close(state=state)
# b = shared_floatx(basis)
# random_init = numpy.rand.random(100, 1000)
# r = shared_floatx(random_init)
# rn = r / r.norm(axis=1)
# x = tensor.dot(rn, tensor.shape_padright(b))
x = tensor.tensor4('features')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
outs = VariableFilter(roles=[OUTPUT], bricks=[Convolutional,
Linear])(cg.variables)
# Create an interior activation model
model = Model([probs] + outs)
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
fn = theano.function([x], outs)
results = fn(basis)
for snapshots, output in zip(results, outs):
layer = get_brick(output)
filmstrip = Filmstrip(basis.shape[-2:],
(snapshots.shape[1], snapshots.shape[0]),
background='purple')
if layer in layers:
fieldmap = layerarray_fieldmap(layers[0:layers.index(layer) + 1])
for unit in range(snapshots.shape[1]):
for index in range(snapshots.shape[0]):
mask = make_mask(
basis.shape[-2:], fieldmap,
numpy.clip(snapshots[index, unit, :, :], 0, numpy.inf))
filmstrip.set_image((unit, index), basis[index, :, :, :],
mask)
filmstrip.save(layer.name + '_show.jpg')
def train_extractive_qa(new_training_job, config, save_path, params,
fast_start, fuel_server, seed):
if seed:
fuel.config.default_seed = seed
blocks.config.config.default_seed = seed
root_path = os.path.join(save_path, 'training_state')
extension = '.tar'
tar_path = root_path + extension
best_tar_path = root_path + '_best' + extension
c = config
data, qam = initialize_data_and_model(c)
if theano.config.compute_test_value != 'off':
test_value_data = next(
data.get_stream('train', shuffle=True, batch_size=4,
max_length=5).get_epoch_iterator(as_dict=True))
for var in qam.input_vars.values():
var.tag.test_value = test_value_data[var.name]
costs = qam.apply_with_default_vars()
cost = rename(costs.mean(), 'mean_cost')
cg = Model(cost)
if params:
logger.debug("Load parameters from {}".format(params))
with open(params) as src:
cg.set_parameter_values(load_parameters(src))
length = rename(qam.contexts.shape[1], 'length')
batch_size = rename(qam.contexts.shape[0], 'batch_size')
predicted_begins, = VariableFilter(name='predicted_begins')(cg)
predicted_ends, = VariableFilter(name='predicted_ends')(cg)
exact_match, = VariableFilter(name='exact_match')(cg)
exact_match_ratio = rename(exact_match.mean(), 'exact_match_ratio')
context_unk_ratio, = VariableFilter(name='context_unk_ratio')(cg)
monitored_vars = [
length, batch_size, cost, exact_match_ratio, context_unk_ratio
]
if c['dict_path']:
def_unk_ratio, = VariableFilter(name='def_unk_ratio')(cg)
num_definitions = rename(qam.input_vars['defs'].shape[0],
'num_definitions')
max_definition_length = rename(qam.input_vars['defs'].shape[1],
'max_definition_length')
monitored_vars.extend(
[def_unk_ratio, num_definitions, max_definition_length])
if c['def_word_gating'] == 'self_attention':
def_gates = VariableFilter(name='def_gates')(cg)
def_gates_min = tensor.minimum(*[x.min() for x in def_gates])
def_gates_max = tensor.maximum(*[x.max() for x in def_gates])
monitored_vars.extend([
rename(def_gates_min, 'def_gates_min'),
rename(def_gates_max, 'def_gates_max')
])
text_match_ratio = TextMatchRatio(data_path=os.path.join(
fuel.config.data_path[0], 'squad/dev-v1.1.json'),
requires=[
predicted_begins, predicted_ends,
tensor.ltensor3('contexts_text'),
tensor.lmatrix('q_ids')
],
name='text_match_ratio')
parameters = cg.get_parameter_dict()
trained_parameters = parameters.values()
if c['embedding_path']:
logger.debug("Exclude word embeddings from the trained parameters")
trained_parameters = [
p for p in trained_parameters if not p == qam.embeddings_var()
]
if c['train_only_def_part']:
def_reading_parameters = qam.def_reading_parameters()
trained_parameters = [
p for p in trained_parameters if p in def_reading_parameters
]
logger.info("Cost parameters" + "\n" + pprint.pformat([
" ".join(
(key, str(parameters[key].get_value().shape),
'trained' if parameters[key] in trained_parameters else 'frozen'))
for key in sorted(parameters.keys())
],
width=120))
# apply dropout to the training cost and to all the variables
# that we monitor during training
train_cost = cost
train_monitored_vars = list(monitored_vars)
if c['dropout']:
regularized_cg = ComputationGraph([cost] + train_monitored_vars)
# Dima: the dropout that I implemented first
bidir_outputs, = VariableFilter(bricks=[Bidirectional],
roles=[OUTPUT])(cg)
readout_layers = VariableFilter(bricks=[Rectifier], roles=[OUTPUT])(cg)
dropout_vars = [bidir_outputs] + readout_layers
logger.debug("applying dropout to {}".format(", ".join(
[v.name for v in dropout_vars])))
regularized_cg = apply_dropout(regularized_cg, dropout_vars,
c['dropout'])
# a new dropout with exactly same mask at different steps
emb_vars = VariableFilter(roles=[EMBEDDINGS])(regularized_cg)
emb_dropout_mask = get_dropout_mask(emb_vars[0], c['emb_dropout'])
if c['emb_dropout_type'] == 'same_mask':
regularized_cg = apply_dropout2(regularized_cg,
emb_vars,
c['emb_dropout'],
dropout_mask=emb_dropout_mask)
elif c['emb_dropout_type'] == 'regular':
regularized_cg = apply_dropout(regularized_cg, emb_vars,
c['emb_dropout'])
else:
raise ValueError("unknown dropout type {}".format(
c['emb_dropout_type']))
train_cost = regularized_cg.outputs[0]
train_monitored_vars = regularized_cg.outputs[1:]
rules = []
if c['grad_clip_threshold']:
rules.append(StepClipping(c['grad_clip_threshold']))
rules.append(Adam(learning_rate=c['learning_rate'], beta1=c['momentum']))
algorithm = GradientDescent(cost=train_cost,
parameters=trained_parameters,
step_rule=CompositeRule(rules))
if c['grad_clip_threshold']:
train_monitored_vars.append(algorithm.total_gradient_norm)
if c['monitor_parameters']:
train_monitored_vars.extend(parameter_stats(parameters, algorithm))
training_stream = data.get_stream('train',
batch_size=c['batch_size'],
shuffle=True,
max_length=c['max_length'])
original_training_stream = training_stream
if fuel_server:
# the port will be configured by the StartFuelServer extension
training_stream = ServerDataStream(
sources=training_stream.sources,
produces_examples=training_stream.produces_examples)
extensions = [
LoadNoUnpickling(tar_path, load_iteration_state=True,
load_log=True).set_conditions(
before_training=not new_training_job),
StartFuelServer(original_training_stream,
os.path.join(save_path, 'stream.pkl'),
before_training=fuel_server),
Timing(every_n_batches=c['mon_freq_train']),
TrainingDataMonitoring(train_monitored_vars,
prefix="train",
every_n_batches=c['mon_freq_train']),
]
validation = DataStreamMonitoring(
[text_match_ratio] + monitored_vars,
data.get_stream('dev',
batch_size=c['batch_size_valid'],
raw_text=True,
q_ids=True),
prefix="dev").set_conditions(before_training=not fast_start,
after_epoch=True)
dump_predictions = DumpPredictions(save_path,
text_match_ratio,
before_training=not fast_start,
after_epoch=True)
track_the_best_exact = TrackTheBest(
validation.record_name(exact_match_ratio),
choose_best=max).set_conditions(before_training=True, after_epoch=True)
track_the_best_text = TrackTheBest(
validation.record_name(text_match_ratio),
choose_best=max).set_conditions(before_training=True, after_epoch=True)
extensions.extend([
validation, dump_predictions, track_the_best_exact, track_the_best_text
])
# We often use pretrained word embeddings and we don't want
# to load and save them every time. To avoid that, we use
# save_main_loop=False, we only save the trained parameters,
# and we save the log and the iterations state separately
# in the tar file.
extensions.extend([
Checkpoint(tar_path,
parameters=trained_parameters,
save_main_loop=False,
save_separately=['log', 'iteration_state'],
before_training=not fast_start,
every_n_epochs=c['save_freq_epochs'],
every_n_batches=c['save_freq_batches'],
after_training=not fast_start).add_condition(
['after_batch', 'after_epoch'],
OnLogRecord(track_the_best_text.notification_name),
(best_tar_path, )),
DumpTensorflowSummaries(save_path,
after_epoch=True,
every_n_batches=c['mon_freq_train'],
after_training=True),
RetrievalPrintStats(retrieval=data._retrieval,
every_n_batches=c['mon_freq_train'],
before_training=not fast_start),
Printing(after_epoch=True, every_n_batches=c['mon_freq_train']),
FinishAfter(after_n_batches=c['n_batches'],
after_n_epochs=c['n_epochs']),
Annealing(c['annealing_learning_rate'],
after_n_epochs=c['annealing_start_epoch']),
LoadNoUnpickling(best_tar_path,
after_n_epochs=c['annealing_start_epoch'])
])
main_loop = MainLoop(algorithm,
training_stream,
model=Model(cost),
extensions=extensions)
main_loop.run()
def main(save_to, hist_file):
batch_size = 365
feature_maps = [6, 16]
mlp_hiddens = [120, 84]
conv_sizes = [5, 5]
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations, 1, image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='valid',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(
i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
mnist_test = MNIST(("test",), sources=['features', 'targets'])
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
error_rate = (MisclassificationRate().apply(y.flatten(), probs)
.copy(name='error_rate'))
confusion = (ConfusionMatrix().apply(y.flatten(), probs)
.copy(name='confusion'))
confusion.tag.aggregation_scheme = Sum(confusion)
model = Model([error_rate, confusion])
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outs = OrderedDict((full_brick_name(get_brick(out)), out)
for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(
model.variables))
# Load histogram information
with open(hist_file, 'rb') as handle:
histograms = pickle.load(handle)
# Corpora
mnist_train = MNIST(("train",))
mnist_train_stream = DataStream.default_stream(
mnist_train, iteration_scheme=ShuffledScheme(
mnist_train.num_examples, batch_size))
mnist_test = MNIST(("test",))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=ShuffledScheme(
mnist_test.num_examples, batch_size))
# Probe the given layer
target_layer = '/lenet/mlp/linear_0'
next_layer_param = '/lenet/mlp/linear_1.W'
sample = extract_sample(outs[target_layer], mnist_test_stream)
print('sample shape', sample.shape)
# Figure neurons to ablate
hist = histograms[('linear_1', 'b')]
targets = [i for i in range(hist.shape[1])
if hist[2, i] * hist[7, i] < 0]
print('ablating', len(targets), ':', targets)
# Now adjust the next layer weights based on the probe
param = model.get_parameter_dict()[next_layer_param]
print('param shape', param.get_value().shape)
new_weights = ablate_inputs(
targets,
sample,
param.get_value(),
compensate=False)
param.set_value(new_weights)
# Evaluation pass
evaluator = DatasetEvaluator([error_rate, confusion])
print(evaluator.evaluate(mnist_test_stream))
def evaluate_extractive_qa(config,
tar_path,
part,
num_examples,
dest_path,
qids=None,
dataset=None):
if not dest_path:
dest_path = os.path.join(os.path.dirname(tar_path), 'predictions.json')
log_path = os.path.splitext(dest_path)[0] + '_log.json'
if qids:
qids = qids.split(',')
if dataset:
dataset = SQuADDataset(dataset, ('all', ))
c = config
data, qam = initialize_data_and_model(c)
costs = qam.apply_with_default_vars()
cg = Model(costs)
with open(tar_path) as src:
cg.set_parameter_values(load_parameters(src))
predicted_begins, = VariableFilter(name='predicted_begins')(cg)
predicted_ends, = VariableFilter(name='predicted_ends')(cg)
compute = {'begins': predicted_begins, 'ends': predicted_ends}
if c['coattention']:
d2q_att_weights, = VariableFilter(name='d2q_att_weights')(cg)
q2d_att_weights, = VariableFilter(name='q2d_att_weights')(cg)
compute.update({'d2q': d2q_att_weights, 'q2d': q2d_att_weights})
compute['costs'] = costs
predict_func = theano.function(qam.input_vars.values(), compute)
logger.debug("Ready to evaluate")
done_examples = 0
num_correct = 0
def print_stats():
print('EXACT MATCH RATIO: {}'.format(num_correct /
float(done_examples)))
predictions = {}
log = {}
stream = data.get_stream(part,
batch_size=1,
shuffle=part == 'train',
raw_text=True,
q_ids=True,
dataset=dataset)
for example in stream.get_epoch_iterator(as_dict=True):
if done_examples == num_examples:
break
q_id = vec2str(example['q_ids'][0])
if qids and not q_id in qids:
continue
example['contexts_text'] = [map(vec2str, example['contexts_text'][0])]
example['questions_text'] = [
map(vec2str, example['questions_text'][0])
]
feed = dict(example)
del feed['q_ids']
del feed['contexts_text']
del feed['questions_text']
del feed['contexts_text_mask']
result = predict_func(**feed)
correct_answer_span = slice(example['answer_begins'][0],
example['answer_ends'][0])
predicted_answer_span = slice(result['begins'][0], result['ends'][0])
correct_answer = example['contexts_text'][0][correct_answer_span]
answer = example['contexts_text'][0][predicted_answer_span]
is_correct = correct_answer_span == predicted_answer_span
context = example['contexts_text'][0]
question = example['questions_text'][0]
context_def_map = example['contexts_def_map']
# pretty print
outcome = 'correct' if is_correct else 'wrong'
print('#{}'.format(done_examples))
print(u"CONTEXT:", detokenize(context))
print(u"QUESTION:", detokenize(question))
print(u"RIGHT ANSWER: {}".format(detokenize(correct_answer)))
print(
u"ANSWER (span=[{}, {}], {}):".format(predicted_answer_span.start,
predicted_answer_span.stop,
outcome), detokenize(answer))
print(u"COST: {}".format(float(result['costs'][0])))
print(u"DEFINITIONS AVAILABLE FOR:")
for pos in set(context_def_map[:, 1]):
print(context[pos])
print()
# update statistics
done_examples += 1
num_correct += is_correct
# save the results
predictions[q_id] = detokenize(answer)
log_entry = {
'context': context,
'question': question,
'answer': answer,
'correct_answer': correct_answer,
'cost': float(result['costs'][0])
}
if c['coattention']:
log_entry['d2q'] = cPickle.dumps(result['d2q'][0])
log_entry['q2d'] = cPickle.dumps(result['q2d'][0])
log[q_id] = log_entry
if done_examples % 100 == 0:
print_stats()
print_stats()
with open(log_path, 'w') as dst:
json.dump(log, dst, indent=2, sort_keys=True)
with open(dest_path, 'w') as dst:
json.dump(predictions, dst, indent=2, sort_keys=True)
def main(save_to):
batch_size = 365
feature_maps = [6, 16]
mlp_hiddens = [120, 84]
conv_sizes = [5, 5]
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations,
1,
image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='valid',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info(
"Input dim: {} {} {}".format(*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
random_init = (numpy.random.rand(100, 1, 28, 28) * 128).astype('float32')
layers = [l for l in convnet.layers if isinstance(l, Convolutional)]
mnist_test = MNIST(("test", ), sources=['features', 'targets'])
basis_init = create_fair_basis(mnist_test, 10, 50)
basis_set = make_shifted_basis(basis_init, convnet, layers)
for layer, basis in zip(layers, basis_set):
# basis is 5d:
# (probed_units, base_cases, 1-c, 28-y, 28-x)
b = shared_floatx(basis)
# coefficients is 2d:
# (probed_units, base_cases)
coefficients = shared_floatx(
numpy.ones(basis.shape[0:2], dtype=theano.config.floatX))
# prod is 5d: (probed_units, base_cases, 1-c, 28-y, 28-x)
prod = tensor.shape_padright(coefficients, 3) * b
# x is 4d: (probed_units, 1-c, 28-y, 28-x)
ux = prod.sum(axis=1)
x = tensor.clip(
ux / tensor.shape_padright(ux.flatten(ndim=2).max(axis=1), 3), 0,
1)
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
outs = VariableFilter(roles=[OUTPUT], bricks=[layer])(cg.variables)
# Create an interior activation model
model = Model([probs] + outs)
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
learning_rate = shared_floatx(0.03, 'learning_rate')
# We will try to do all units at once.
# unit = shared_floatx(0, 'unit', dtype='int64')
# But we are only doing one layer at once.
output = outs[0]
dims = layer.get_dims(['output'])[0]
if isinstance(dims, numbers.Integral):
# FC case: output is 2d: (probed_units, units)
dims = (dims, )
unitrange = tensor.arange(dims[0])
costvec = -tensor.log(
tensor.nnet.softmax(output)[unitrange, unitrage].flatten())
else:
# Conv case: output is 4d: (probed_units, units, y, x)
unitrange = tensor.arange(dims[0])
print('dims is', dims)
costvec = -tensor.log(
tensor.nnet.softmax(output[unitrange, unitrange, dims[1] // 2,
dims[2] // 2]).flatten())
cost = costvec.sum()
# grad is dims (probed_units, basis_size)
grad = gradient.grad(cost, coefficients)
stepc = coefficients # - learning_rate * grad
newc = stepc / tensor.shape_padright(stepc.mean(axis=1))
fn = theano.function([], [cost, x], updates=[(coefficients, newc)])
filmstrip = Filmstrip(random_init.shape[-2:], (dims[0], 1),
background='red')
layer = get_brick(output)
learning_rate.set_value(0.1)
for index in range(20000):
c, result = fn()
if index % 1000 == 0:
learning_rate.set_value(numpy.cast[theano.config.floatX](
learning_rate.get_value() * 0.8))
print('cost', c)
for u in range(dims[0]):
filmstrip.set_image((u, 0), result[u, :, :, :])
filmstrip.save(layer.name + '_stroke.jpg')
for u in range(dims[0]):
filmstrip.set_image((u, 0), result[u, :, :, :])
filmstrip.save(layer.name + '_stroke.jpg')
def train_model(new_training_job, config, save_path, params, fast_start,
fuel_server, seed):
c = config
if seed:
fuel.config.default_seed = seed
blocks.config.config.default_seed = seed
data, model = initialize_data_and_model(config, train_phase=True)
# full main loop can be saved...
main_loop_path = os.path.join(save_path, 'main_loop.tar')
# or only state (log + params) which can be useful not to pickle embeddings
state_path = os.path.join(save_path, 'training_state.tar')
stream_path = os.path.join(save_path, 'stream.pkl')
best_tar_path = os.path.join(save_path, "best_model.tar")
keys = tensor.lmatrix('keys')
n_identical_keys = tensor.lvector('n_identical_keys')
words = tensor.ltensor3('words')
words_mask = tensor.matrix('words_mask')
if theano.config.compute_test_value != 'off':
#TODO
test_value_data = next(
data.get_stream('train', batch_size=4,
max_length=5).get_epoch_iterator())
words.tag.test_value = test_value_data[0]
words_mask.tag.test_value = test_value_data[1]
if use_keys(c) and use_n_identical_keys(c):
costs = model.apply(words,
words_mask,
keys,
n_identical_keys,
train_phase=True)
elif use_keys(c):
costs = model.apply(words, words_mask, keys, train_phase=True)
else:
costs = model.apply(words, words_mask, train_phase=True)
cost = rename(costs.mean(), 'mean_cost')
cg = Model(cost)
if params:
logger.debug("Load parameters from {}".format(params))
with open(params) as src:
cg.set_parameter_values(load_parameters(src))
length = rename(words.shape[1], 'length')
perplexity, = VariableFilter(name='perplexity')(cg)
monitored_vars = [length, cost, perplexity]
if c['proximity_coef']:
proximity_term, = VariableFilter(name='proximity_term')(cg)
monitored_vars.append(proximity_term)
print "inputs of the model:", cg.inputs
parameters = cg.get_parameter_dict()
trained_parameters = parameters.values()
saved_parameters = parameters.values()
if c['embedding_path']:
if c['freeze_pretrained']:
logger.debug(
"Exclude pretrained encoder embeddings from the trained parameters"
)
to_freeze = 'main'
elif c['provide_targets']:
logger.debug(
"Exclude pretrained targets from the trained parameters")
to_freeze = 'target'
trained_parameters = [
p for p in trained_parameters
if not p == model.get_def_embeddings_params(to_freeze)
]
saved_parameters = [
p for p in saved_parameters
if not p == model.get_def_embeddings_params(to_freeze)
]
logger.info("Cost parameters" + "\n" + pprint.pformat([
" ".join(
(key, str(parameters[key].get_value().shape),
'trained' if parameters[key] in trained_parameters else 'frozen'))
for key in sorted(parameters.keys())
],
width=120))
rules = []
if c['grad_clip_threshold']:
rules.append(StepClipping(c['grad_clip_threshold']))
rules.append(Adam(learning_rate=c['learning_rate'], beta1=c['momentum']))
algorithm = GradientDescent(cost=cost,
parameters=trained_parameters,
step_rule=CompositeRule(rules))
train_monitored_vars = list(monitored_vars)
if c['grad_clip_threshold']:
train_monitored_vars.append(algorithm.total_gradient_norm)
if c['monitor_parameters']:
train_monitored_vars.extend(parameter_stats(parameters, algorithm))
# We use a completely random seed on purpose. With Fuel server
# it's currently not possible to restore the state of the training
# stream. That's why it's probably better to just have it stateless.
stream_seed = numpy.random.randint(0, 10000000) if fuel_server else None
training_stream = data.get_stream(
'train',
batch_size=c['batch_size'],
max_length=c['max_length'],
seed=stream_seed,
remove_keys=not use_keys(c),
remove_n_identical_keys=not use_n_identical_keys(c))
print "trainin_stream will contains sources:", training_stream.sources
original_training_stream = training_stream
if fuel_server:
# the port will be configured by the StartFuelServer extension
training_stream = ServerDataStream(
sources=training_stream.sources,
produces_examples=training_stream.produces_examples)
validate = c['mon_freq_valid'] > 0
if validate:
valid_stream = data.get_stream(
'valid',
batch_size=c['batch_size_valid'],
max_length=c['max_length'],
seed=stream_seed,
remove_keys=not use_keys(c),
remove_n_identical_keys=not use_n_identical_keys(c))
validation = DataStreamMonitoring(
monitored_vars, valid_stream,
prefix="valid").set_conditions(before_first_epoch=not fast_start,
on_resumption=True,
every_n_batches=c['mon_freq_valid'])
track_the_best = TrackTheBest(validation.record_name(cost),
choose_best=min).set_conditions(
on_resumption=True,
after_epoch=True,
every_n_batches=c['mon_freq_valid'])
# don't save them the entire main loop to avoid pickling everything
if c['fast_checkpoint']:
cp_path = state_path
load = (LoadNoUnpickling(cp_path,
load_iteration_state=True,
load_log=True).set_conditions(
before_training=not new_training_job))
cp_args = {
'save_main_loop': False,
'save_separately': ['log', 'iteration_state'],
'parameters': saved_parameters
}
else:
cp_path = main_loop_path
load = (Load(cp_path, load_iteration_state=True,
load_log=True).set_conditions(
before_training=not new_training_job))
cp_args = {
'save_separately': ['iteration_state'],
'parameters': saved_parameters
}
checkpoint = Checkpoint(cp_path,
before_training=not fast_start,
every_n_batches=c['save_freq_batches'],
after_training=not fast_start,
**cp_args)
if c['checkpoint_every_n_batches'] > 0 or c[
'checkpoint_every_n_epochs'] > 0:
intermediate_cp = IntermediateCheckpoint(
cp_path,
every_n_epochs=c['checkpoint_every_n_epochs'],
every_n_batches=c['checkpoint_every_n_batches'],
after_training=False,
**cp_args)
if validate:
checkpoint = checkpoint.add_condition(
['after_batch', 'after_epoch'],
OnLogRecord(track_the_best.notification_name), (best_tar_path, ))
extensions = [
load,
StartFuelServer(original_training_stream,
stream_path,
before_training=fuel_server),
Timing(every_n_batches=c['mon_freq_train'])
]
extensions.extend([
TrainingDataMonitoring(train_monitored_vars,
prefix="train",
every_n_batches=c['mon_freq_train']),
])
if validate:
extensions.extend([validation, track_the_best])
extensions.append(checkpoint)
if c['checkpoint_every_n_batches'] > 0 or c[
'checkpoint_every_n_epochs'] > 0:
extensions.append(intermediate_cp)
extensions.extend(
[Printing(on_resumption=True, every_n_batches=c['mon_freq_train'])])
if validate and c['n_valid_early'] > 0:
extensions.append(
FinishIfNoImprovementAfter(track_the_best.notification_name,
iterations=c['n_valid_early'] *
c['mon_freq_valid'],
every_n_batches=c['mon_freq_valid']))
extensions.append(FinishAfter(after_n_epochs=c['n_epochs']))
logger.info("monitored variables during training:" + "\n" +
pprint.pformat(train_monitored_vars, width=120))
logger.info("monitored variables during valid:" + "\n" +
pprint.pformat(monitored_vars, width=120))
main_loop = MainLoop(algorithm,
training_stream,
model=Model(cost),
extensions=extensions)
main_loop.run()
def main(mode, save_to, num_epochs, load_params=None,
feature_maps=None, mlp_hiddens=None,
conv_sizes=None, pool_sizes=None, stride=None, repeat_times=None,
batch_size=None, num_batches=None, algo=None,
test_set=None, valid_examples=None,
dropout=None, max_norm=None, weight_decay=None,
batch_norm=None):
if feature_maps is None:
feature_maps = [20, 50, 50]
if mlp_hiddens is None:
mlp_hiddens = [500]
if conv_sizes is None:
conv_sizes = [5, 5, 5]
if pool_sizes is None:
pool_sizes = [2, 2, 2]
if repeat_times is None:
repeat_times = [1, 1, 1]
if batch_size is None:
batch_size = 500
if valid_examples is None:
valid_examples = 2500
if stride is None:
stride = 1
if test_set is None:
test_set = 'test'
if algo is None:
algo = 'rmsprop'
if batch_norm is None:
batch_norm = False
image_size = (128, 128)
output_size = 2
if (len(feature_maps) != len(conv_sizes) or
len(feature_maps) != len(pool_sizes) or
len(feature_maps) != len(repeat_times)):
raise ValueError("OMG, inconsistent arguments")
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations, 3, image_size,
stride=stride,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
repeat_times=repeat_times,
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='full',
batch_norm=batch_norm,
weights_init=Glorot(),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.initialize()
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(
i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
single_x = tensor.tensor3('image_features')
x = tensor.tensor4('image_features')
single_y = tensor.lvector('targets')
y = tensor.lmatrix('targets')
# Training
with batch_normalization(convnet):
probs = convnet.apply(x)
cost = (CategoricalCrossEntropy().apply(y.flatten(), probs)
.copy(name='cost'))
error_rate = (MisclassificationRate().apply(y.flatten(), probs)
.copy(name='error_rate'))
cg = ComputationGraph([cost, error_rate])
extra_updates = []
if batch_norm: # batch norm:
logger.debug("Apply batch norm")
pop_updates = get_batch_normalization_updates(cg)
# p stands for population mean
# m stands for minibatch
alpha = 0.005
extra_updates = [(p, m * alpha + p * (1 - alpha))
for p, m in pop_updates]
population_statistics = [p for p, m in extra_updates]
if dropout:
relu_outputs = VariableFilter(bricks=[Rectifier], roles=[OUTPUT])(cg)
cg = apply_dropout(cg, relu_outputs, dropout)
cost, error_rate = cg.outputs
if weight_decay:
logger.debug("Apply weight decay {}".format(weight_decay))
cost += weight_decay * l2_norm(cg.parameters)
cost.name = 'cost'
# Validation
valid_probs = convnet.apply_5windows(single_x)
valid_cost = (CategoricalCrossEntropy().apply(single_y, valid_probs)
.copy(name='cost'))
valid_error_rate = (MisclassificationRate().apply(
single_y, valid_probs).copy(name='error_rate'))
model = Model([cost, error_rate])
if load_params:
logger.info("Loaded params from {}".format(load_params))
with open(load_params, 'r') as src:
model.set_parameter_values(load_parameters(src))
# Training stream with random cropping
train = DogsVsCats(("train",), subset=slice(None, 25000 - valid_examples, None))
train_str = DataStream(
train, iteration_scheme=ShuffledScheme(train.num_examples, batch_size))
train_str = add_transformers(train_str, random_crop=True)
# Validation stream without cropping
valid = DogsVsCats(("train",), subset=slice(25000 - valid_examples, None, None))
valid_str = DataStream(
valid, iteration_scheme=SequentialExampleScheme(valid.num_examples))
valid_str = add_transformers(valid_str)
if mode == 'train':
directory, _ = os.path.split(sys.argv[0])
env = dict(os.environ)
env['THEANO_FLAGS'] = 'floatX=float32'
port = numpy.random.randint(1025, 10000)
server = subprocess.Popen(
[directory + '/server.py',
str(25000 - valid_examples), str(batch_size), str(port)],
env=env, stderr=subprocess.STDOUT)
train_str = ServerDataStream(
('image_features', 'targets'), produces_examples=False,
port=port)
save_to_base, save_to_extension = os.path.splitext(save_to)
# Train with simple SGD
if algo == 'rmsprop':
step_rule = RMSProp(decay_rate=0.999, learning_rate=0.0003)
elif algo == 'adam':
step_rule = Adam()
else:
assert False
if max_norm:
conv_params = VariableFilter(bricks=[Convolutional], roles=[WEIGHT])(cg)
linear_params = VariableFilter(bricks=[Linear], roles=[WEIGHT])(cg)
step_rule = CompositeRule(
[step_rule,
Restrict(VariableClipping(max_norm, axis=0), linear_params),
Restrict(VariableClipping(max_norm, axis=(1, 2, 3)), conv_params)])
algorithm = GradientDescent(
cost=cost, parameters=model.parameters,
step_rule=step_rule)
algorithm.add_updates(extra_updates)
# `Timing` extension reports time for reading data, aggregating a batch
# and monitoring;
# `ProgressBar` displays a nice progress bar during training.
extensions = [Timing(every_n_batches=100),
FinishAfter(after_n_epochs=num_epochs,
after_n_batches=num_batches),
DataStreamMonitoring(
[valid_cost, valid_error_rate],
valid_str,
prefix="valid"),
TrainingDataMonitoring(
[cost, error_rate,
aggregation.mean(algorithm.total_gradient_norm)],
prefix="train",
after_epoch=True),
TrackTheBest("valid_error_rate"),
Checkpoint(save_to, save_separately=['log'],
parameters=cg.parameters +
(population_statistics if batch_norm else []),
before_training=True, after_epoch=True)
.add_condition(
['after_epoch'],
OnLogRecord("valid_error_rate_best_so_far"),
(save_to_base + '_best' + save_to_extension,)),
Printing(every_n_batches=100)]
model = Model(cost)
main_loop = MainLoop(
algorithm,
train_str,
model=model,
extensions=extensions)
try:
main_loop.run()
finally:
server.terminate()
elif mode == 'test':
classify = theano.function([single_x], valid_probs.argmax())
test = DogsVsCats((test_set,))
test_str = DataStream(
test, iteration_scheme=SequentialExampleScheme(test.num_examples))
test_str = add_transformers(test_str)
correct = 0
with open(save_to, 'w') as dst:
print("id", "label", sep=',', file=dst)
for index, example in enumerate(test_str.get_epoch_iterator()):
image = example[0]
prediction = classify(image)
print(index + 1, classify(image), sep=',', file=dst)
if len(example) > 1 and prediction == example[1]:
correct += 1
print(correct / float(test.num_examples))
else:
assert False
def main(save_to):
batch_size = 500
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
convnet = create_lenet_5()
mnist_test = MNIST(("test", ), sources=['features', 'targets'])
basis_init = create_fair_basis(mnist_test, 10, 2)
# b = shared_floatx(basis)
# random_init = numpy.rand.random(100, 1000)
# r = shared_floatx(random_init)
# rn = r / r.norm(axis=1)
# x = tensor.dot(rn, tensor.shape_padright(b))
x = shared_floatx(basis_init)
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
outs = VariableFilter(roles=[OUTPUT], bricks=[Convolutional,
Linear])(cg.variables)
# Create an interior activation model
model = Model([probs] + outs)
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
learning_rate = shared_floatx(0.01, 'learning_rate')
unit = shared_floatx(0, 'unit', dtype='int64')
negate = False
suffix = '_negsynth.jpg' if negate else '_synth.jpg'
for output in outs:
layer = get_brick(output)
# For now, skip masks -for some reason they are always NaN
iterations = 10000
layername = layer.parents[0].name + '-' + layer.name
# if layername != 'noisylinear_2-linear':
# continue
dims = layer.get_dims(['output'])[0]
if negate:
measure = -output
else:
measure = output
measure = measure[(slice(0, basis_init.shape[0]), ) + (slice(None), ) *
(measure.ndim - 1)]
if isinstance(dims, numbers.Integral):
dims = (dims, )
costvec = -tensor.log(
tensor.nnet.softmax(measure)[:, unit].flatten())
else:
flatout = measure.flatten(ndim=3)
maxout = flatout.max(axis=2)
costvec = -tensor.log(
tensor.nnet.softmax(maxout)[:, unit].flatten())
# Add a regularization to favor gray images.
# cost = costvec.sum() + (x - 0.5).norm(2) * (
# 10.0 / basis_init.shape[0])
cost = costvec.sum()
grad = gradient.grad(cost, x)
stepx = x - learning_rate * grad
normx = stepx / tensor.shape_padright(
stepx.flatten(ndim=2).max(axis=1), n_ones=3)
newx = tensor.clip(normx, 0, 1)
newx = newx[(slice(0, basis_init.shape[0]), ) + (slice(None), ) *
(newx.ndim - 1)]
fn = theano.function([], [cost], updates=[(x, newx)])
filmstrip = Filmstrip(basis_init.shape[-2:],
(dims[0], basis_init.shape[0]),
background='red')
for u in range(dims[0]):
unit.set_value(u)
x.set_value(basis_init)
print('layer', layername, 'unit', u)
for index in range(iterations):
c = fn()[0]
if index % 1000 == 0:
print('cost', c)
result = x.get_value()
for i2 in range(basis_init.shape[0]):
filmstrip.set_image((u, i2), result[i2, :, :, :])
filmstrip.save(layername + suffix)
result = x.get_value()
for index in range(basis_init.shape[0]):
filmstrip.set_image((u, index), result[index, :, :, :])
filmstrip.save(layername + suffix)
def initialize_all(config, save_path, bokeh_name, params, bokeh_server, bokeh,
test_tag, use_load_ext, load_log, fast_start):
root_path, extension = os.path.splitext(save_path)
data = Data(**config['data'])
train_conf = config['training']
recognizer = create_model(config, data, test_tag)
# Separate attention_params to be handled differently
# when regularization is applied
attention = recognizer.generator.transition.attention
attention_params = Selector(attention).get_parameters().values()
logger.info(
"Initialization schemes for all bricks.\n"
"Works well only in my branch with __repr__ added to all them,\n"
"there is an issue #463 in Blocks to do that properly.")
def show_init_scheme(cur):
result = dict()
for attr in dir(cur):
if attr.endswith('_init'):
result[attr] = getattr(cur, attr)
for child in cur.children:
result[child.name] = show_init_scheme(child)
return result
logger.info(pprint.pformat(show_init_scheme(recognizer)))
prediction, prediction_mask = add_exploration(recognizer, data, train_conf)
#
# Observables:
#
primary_observables = [] # monitored each batch
secondary_observables = [] # monitored every 10 batches
validation_observables = [] # monitored on the validation set
cg = recognizer.get_cost_graph(batch=True,
prediction=prediction,
prediction_mask=prediction_mask)
labels, = VariableFilter(applications=[recognizer.cost], name='labels')(cg)
labels_mask, = VariableFilter(applications=[recognizer.cost],
name='labels_mask')(cg)
gain_matrix = VariableFilter(
theano_name=RewardRegressionEmitter.GAIN_MATRIX)(cg)
if len(gain_matrix):
gain_matrix, = gain_matrix
primary_observables.append(rename(gain_matrix.min(), 'min_gain'))
primary_observables.append(rename(gain_matrix.max(), 'max_gain'))
batch_cost = cg.outputs[0].sum()
batch_size = rename(recognizer.labels.shape[1], "batch_size")
# Assumes constant batch size. `aggregation.mean` is not used because
# of Blocks #514.
cost = batch_cost / batch_size
cost.name = "sequence_total_cost"
logger.info("Cost graph is built")
# Fetch variables useful for debugging.
# It is important not to use any aggregation schemes here,
# as it's currently impossible to spread the effect of
# regularization on their variables, see Blocks #514.
cost_cg = ComputationGraph(cost)
r = recognizer
energies, = VariableFilter(applications=[r.generator.readout.readout],
name="output_0")(cost_cg)
bottom_output = VariableFilter(
# We need name_regex instead of name because LookupTable calls itsoutput output_0
applications=[r.bottom.apply],
name_regex="output")(cost_cg)[-1]
attended, = VariableFilter(applications=[r.generator.transition.apply],
name="attended")(cost_cg)
attended_mask, = VariableFilter(applications=[
r.generator.transition.apply
],
name="attended_mask")(cost_cg)
weights, = VariableFilter(applications=[r.generator.evaluate],
name="weights")(cost_cg)
from blocks.roles import AUXILIARY
l2_cost, = VariableFilter(roles=[AUXILIARY],
theano_name='l2_cost_aux')(cost_cg)
cost_forward, = VariableFilter(roles=[AUXILIARY],
theano_name='costs_forward_aux')(cost_cg)
max_recording_length = rename(bottom_output.shape[0],
"max_recording_length")
# To exclude subsampling related bugs
max_attended_mask_length = rename(attended_mask.shape[0],
"max_attended_mask_length")
max_attended_length = rename(attended.shape[0], "max_attended_length")
max_num_phonemes = rename(labels.shape[0], "max_num_phonemes")
min_energy = rename(energies.min(), "min_energy")
max_energy = rename(energies.max(), "max_energy")
mean_attended = rename(abs(attended).mean(), "mean_attended")
mean_bottom_output = rename(
abs(bottom_output).mean(), "mean_bottom_output")
weights_penalty = rename(monotonicity_penalty(weights, labels_mask),
"weights_penalty")
weights_entropy = rename(entropy(weights, labels_mask), "weights_entropy")
mask_density = rename(labels_mask.mean(), "mask_density")
cg = ComputationGraph([
cost, weights_penalty, weights_entropy, min_energy, max_energy,
mean_attended, mean_bottom_output, batch_size, max_num_phonemes,
mask_density
])
# Regularization. It is applied explicitly to all variables
# of interest, it could not be applied to the cost only as it
# would not have effect on auxiliary variables, see Blocks #514.
reg_config = config.get('regularization', dict())
regularized_cg = cg
if reg_config.get('dropout'):
logger.info('apply dropout')
regularized_cg = apply_dropout(cg, [bottom_output], 0.5)
if reg_config.get('noise'):
logger.info('apply noise')
noise_subjects = [
p for p in cg.parameters if p not in attention_params
]
regularized_cg = apply_noise(cg, noise_subjects, reg_config['noise'])
train_cost = regularized_cg.outputs[0]
if reg_config.get("penalty_coof", .0) > 0:
# big warning!!!
# here we assume that:
# regularized_weights_penalty = regularized_cg.outputs[1]
train_cost = (train_cost + reg_config.get("penalty_coof", .0) *
regularized_cg.outputs[1] / batch_size)
if reg_config.get("decay", .0) > 0:
train_cost = (
train_cost + reg_config.get("decay", .0) *
l2_norm(VariableFilter(roles=[WEIGHT])(cg.parameters))**2)
train_cost = rename(train_cost, 'train_cost')
gradients = None
if reg_config.get('adaptive_noise'):
logger.info('apply adaptive noise')
if ((reg_config.get("penalty_coof", .0) > 0)
or (reg_config.get("decay", .0) > 0)):
logger.error('using adaptive noise with alignment weight panalty '
'or weight decay is probably stupid')
train_cost, regularized_cg, gradients, noise_brick = apply_adaptive_noise(
cg,
cg.outputs[0],
variables=cg.parameters,
num_examples=data.get_dataset('train').num_examples,
parameters=Model(
regularized_cg.outputs[0]).get_parameter_dict().values(),
**reg_config.get('adaptive_noise'))
train_cost.name = 'train_cost'
adapt_noise_cg = ComputationGraph(train_cost)
model_prior_mean = rename(
VariableFilter(applications=[noise_brick.apply],
name='model_prior_mean')(adapt_noise_cg)[0],
'model_prior_mean')
model_cost = rename(
VariableFilter(applications=[noise_brick.apply],
name='model_cost')(adapt_noise_cg)[0], 'model_cost')
model_prior_variance = rename(
VariableFilter(applications=[noise_brick.apply],
name='model_prior_variance')(adapt_noise_cg)[0],
'model_prior_variance')
regularized_cg = ComputationGraph(
[train_cost, model_cost] + regularized_cg.outputs +
[model_prior_mean, model_prior_variance])
primary_observables += [
regularized_cg.outputs[1], # model cost
regularized_cg.outputs[2], # task cost
regularized_cg.outputs[-2], # model prior mean
regularized_cg.outputs[-1]
] # model prior variance
model = Model(train_cost)
if params:
logger.info("Load parameters from " + params)
# please note: we cannot use recognizer.load_params
# as it builds a new computation graph that dies not have
# shapred variables added by adaptive weight noise
with open(params, 'r') as src:
param_values = load_parameters(src)
model.set_parameter_values(param_values)
parameters = model.get_parameter_dict()
logger.info("Parameters:\n" +
pprint.pformat([(key, parameters[key].get_value().shape)
for key in sorted(parameters.keys())],
width=120))
# Define the training algorithm.
clipping = StepClipping(train_conf['gradient_threshold'])
clipping.threshold.name = "gradient_norm_threshold"
rule_names = train_conf.get('rules', ['momentum'])
core_rules = []
if 'momentum' in rule_names:
logger.info("Using scaling and momentum for training")
core_rules.append(Momentum(train_conf['scale'],
train_conf['momentum']))
if 'adadelta' in rule_names:
logger.info("Using AdaDelta for training")
core_rules.append(
AdaDelta(train_conf['decay_rate'], train_conf['epsilon']))
max_norm_rules = []
if reg_config.get('max_norm', False) > 0:
logger.info("Apply MaxNorm")
maxnorm_subjects = VariableFilter(roles=[WEIGHT])(cg.parameters)
if reg_config.get('max_norm_exclude_lookup', False):
maxnorm_subjects = [
v for v in maxnorm_subjects
if not isinstance(get_brick(v), LookupTable)
]
logger.info("Parameters covered by MaxNorm:\n" + pprint.pformat(
[name for name, p in parameters.items() if p in maxnorm_subjects]))
logger.info("Parameters NOT covered by MaxNorm:\n" + pprint.pformat([
name for name, p in parameters.items() if not p in maxnorm_subjects
]))
max_norm_rules = [
Restrict(VariableClipping(reg_config['max_norm'], axis=0),
maxnorm_subjects)
]
burn_in = []
if train_conf.get('burn_in_steps', 0):
burn_in.append(BurnIn(num_steps=train_conf['burn_in_steps']))
algorithm = GradientDescent(
cost=train_cost,
parameters=parameters.values(),
gradients=gradients,
step_rule=CompositeRule(
[clipping] + core_rules + max_norm_rules +
# Parameters are not changed at all
# when nans are encountered.
[RemoveNotFinite(0.0)] + burn_in),
on_unused_sources='warn')
logger.debug("Scan Ops in the gradients")
gradient_cg = ComputationGraph(algorithm.gradients.values())
for op in ComputationGraph(gradient_cg).scans:
logger.debug(op)
# More variables for debugging: some of them can be added only
# after the `algorithm` object is created.
secondary_observables += list(regularized_cg.outputs)
if not 'train_cost' in [v.name for v in secondary_observables]:
secondary_observables += [train_cost]
secondary_observables += [
algorithm.total_step_norm, algorithm.total_gradient_norm,
clipping.threshold
]
for name, param in parameters.items():
num_elements = numpy.product(param.get_value().shape)
norm = param.norm(2) / num_elements**0.5
grad_norm = algorithm.gradients[param].norm(2) / num_elements**0.5
step_norm = algorithm.steps[param].norm(2) / num_elements**0.5
stats = tensor.stack(norm, grad_norm, step_norm, step_norm / grad_norm)
stats.name = name + '_stats'
secondary_observables.append(stats)
primary_observables += [
train_cost, algorithm.total_gradient_norm, algorithm.total_step_norm,
clipping.threshold, max_recording_length, max_attended_length,
max_attended_mask_length
]
validation_observables += [
rename(aggregation.mean(batch_cost, batch_size), cost.name),
rename(aggregation.sum_(batch_size), 'num_utterances'),
weights_entropy, weights_penalty
]
def attach_aggregation_schemes(variables):
# Aggregation specification has to be factored out as a separate
# function as it has to be applied at the very last stage
# separately to training and validation observables.
result = []
for var in variables:
if var.name == 'weights_penalty':
result.append(
rename(aggregation.mean(var, batch_size),
'weights_penalty_per_recording'))
elif var.name == 'weights_entropy':
result.append(
rename(aggregation.mean(var, labels_mask.sum()),
'weights_entropy_per_label'))
else:
result.append(var)
return result
mon_conf = config['monitoring']
# Build main loop.
logger.info("Initialize extensions")
extensions = []
if use_load_ext and params:
extensions.append(
Load(params, load_iteration_state=True, load_log=True))
if load_log and params:
extensions.append(LoadLog(params))
extensions += [
Timing(after_batch=True),
CGStatistics(),
#CodeVersion(['lvsr']),
]
extensions.append(
TrainingDataMonitoring(primary_observables + [l2_cost, cost_forward],
after_batch=True))
average_monitoring = TrainingDataMonitoring(
attach_aggregation_schemes(secondary_observables),
prefix="average",
every_n_batches=10)
extensions.append(average_monitoring)
validation = DataStreamMonitoring(
attach_aggregation_schemes(validation_observables +
[l2_cost, cost_forward]),
data.get_stream("valid", shuffle=False),
prefix="valid").set_conditions(
before_first_epoch=not fast_start,
every_n_epochs=mon_conf['validate_every_epochs'],
every_n_batches=mon_conf['validate_every_batches'],
after_training=False)
extensions.append(validation)
per = PhonemeErrorRate(recognizer, data, **config['monitoring']['search'])
per_monitoring = DataStreamMonitoring(
[per],
data.get_stream("valid", batches=False, shuffle=False),
prefix="valid").set_conditions(
before_first_epoch=not fast_start,
every_n_epochs=mon_conf['search_every_epochs'],
every_n_batches=mon_conf['search_every_batches'],
after_training=False)
extensions.append(per_monitoring)
track_the_best_per = TrackTheBest(
per_monitoring.record_name(per)).set_conditions(
before_first_epoch=True, after_epoch=True)
track_the_best_cost = TrackTheBest(
validation.record_name(cost)).set_conditions(before_first_epoch=True,
after_epoch=True)
extensions += [track_the_best_cost, track_the_best_per]
extensions.append(
AdaptiveClipping(algorithm.total_gradient_norm.name,
clipping,
train_conf['gradient_threshold'],
decay_rate=0.998,
burnin_period=500))
extensions += [
SwitchOffLengthFilter(
data.length_filter,
after_n_batches=train_conf.get('stop_filtering')),
FinishAfter(after_n_batches=train_conf.get('num_batches'),
after_n_epochs=train_conf.get('num_epochs')).add_condition(
["after_batch"], _gradient_norm_is_none),
]
channels = [
# Plot 1: training and validation costs
[
average_monitoring.record_name(train_cost),
validation.record_name(cost)
],
# Plot 2: gradient norm,
[
average_monitoring.record_name(algorithm.total_gradient_norm),
average_monitoring.record_name(clipping.threshold)
],
# Plot 3: phoneme error rate
[per_monitoring.record_name(per)],
# Plot 4: training and validation mean weight entropy
[
average_monitoring._record_name('weights_entropy_per_label'),
validation._record_name('weights_entropy_per_label')
],
# Plot 5: training and validation monotonicity penalty
[
average_monitoring._record_name('weights_penalty_per_recording'),
validation._record_name('weights_penalty_per_recording')
]
]
if bokeh:
extensions += [
Plot(bokeh_name if bokeh_name else os.path.basename(save_path),
channels,
every_n_batches=10,
server_url=bokeh_server),
]
extensions += [
Checkpoint(save_path,
before_first_epoch=not fast_start,
after_epoch=True,
every_n_batches=train_conf.get('save_every_n_batches'),
save_separately=["model", "log"],
use_cpickle=True).add_condition(
['after_epoch'],
OnLogRecord(track_the_best_per.notification_name),
(root_path + "_best" + extension, )).add_condition(
['after_epoch'],
OnLogRecord(track_the_best_cost.notification_name),
(root_path + "_best_ll" + extension, )),
ProgressBar()
]
extensions.append(EmbedIPython(use_main_loop_run_caller_env=True))
if config['net']['criterion']['name'].startswith('mse'):
extensions.append(
LogInputsGains(labels, cg, recognizer.generator.readout.emitter,
data))
if train_conf.get('patience'):
patience_conf = train_conf['patience']
if not patience_conf.get('notification_names'):
# setdefault will not work for empty list
patience_conf['notification_names'] = [
track_the_best_per.notification_name,
track_the_best_cost.notification_name
]
extensions.append(Patience(**patience_conf))
extensions.append(
Printing(every_n_batches=1, attribute_filter=PrintingFilterList()))
return model, algorithm, data, extensions
def main(mode, save_path, num_batches, data_path=None):
reverser = WordReverser(100, len(char2code), name="reverser")
if mode == "train":
# Data processing pipeline
dataset_options = dict(dictionary=char2code, level="character",
preprocess=_lower)
if data_path:
dataset = TextFile(data_path, **dataset_options)
else:
dataset = OneBillionWord("training", [99], **dataset_options)
data_stream = dataset.get_example_stream()
data_stream = Filter(data_stream, _filter_long)
data_stream = Mapping(data_stream, reverse_words,
add_sources=("targets",))
data_stream = Batch(data_stream, iteration_scheme=ConstantScheme(10))
data_stream = Padding(data_stream)
data_stream = Mapping(data_stream, _transpose)
# Initialization settings
reverser.weights_init = IsotropicGaussian(0.1)
reverser.biases_init = Constant(0.0)
reverser.push_initialization_config()
reverser.encoder.weights_init = Orthogonal()
reverser.generator.transition.weights_init = Orthogonal()
# Build the cost computation graph
chars = tensor.lmatrix("features")
chars_mask = tensor.matrix("features_mask")
targets = tensor.lmatrix("targets")
targets_mask = tensor.matrix("targets_mask")
batch_cost = reverser.cost(
chars, chars_mask, targets, targets_mask).sum()
batch_size = chars.shape[1].copy(name="batch_size")
cost = aggregation.mean(batch_cost, batch_size)
cost.name = "sequence_log_likelihood"
logger.info("Cost graph is built")
# Give an idea of what's going on
model = Model(cost)
parameters = model.get_parameter_dict()
logger.info("Parameters:\n" +
pprint.pformat(
[(key, value.get_value().shape) for key, value
in parameters.items()],
width=120))
# Initialize parameters
for brick in model.get_top_bricks():
brick.initialize()
# Define the training algorithm.
cg = ComputationGraph(cost)
algorithm = GradientDescent(
cost=cost, parameters=cg.parameters,
step_rule=CompositeRule([StepClipping(10.0), Scale(0.01)]))
# Fetch variables useful for debugging
generator = reverser.generator
(energies,) = VariableFilter(
applications=[generator.readout.readout],
name_regex="output")(cg.variables)
(activations,) = VariableFilter(
applications=[generator.transition.apply],
name=generator.transition.apply.states[0])(cg.variables)
max_length = chars.shape[0].copy(name="max_length")
cost_per_character = aggregation.mean(
batch_cost, batch_size * max_length).copy(
name="character_log_likelihood")
min_energy = energies.min().copy(name="min_energy")
max_energy = energies.max().copy(name="max_energy")
mean_activation = abs(activations).mean().copy(
name="mean_activation")
observables = [
cost, min_energy, max_energy, mean_activation,
batch_size, max_length, cost_per_character,
algorithm.total_step_norm, algorithm.total_gradient_norm]
for name, parameter in parameters.items():
observables.append(parameter.norm(2).copy(name + "_norm"))
observables.append(algorithm.gradients[parameter].norm(2).copy(
name + "_grad_norm"))
# Construct the main loop and start training!
average_monitoring = TrainingDataMonitoring(
observables, prefix="average", every_n_batches=10)
main_loop = MainLoop(
model=model,
data_stream=data_stream,
algorithm=algorithm,
extensions=[
Timing(),
TrainingDataMonitoring(observables, after_batch=True),
average_monitoring,
FinishAfter(after_n_batches=num_batches)
# This shows a way to handle NaN emerging during
# training: simply finish it.
.add_condition(["after_batch"], _is_nan),
# Saving the model and the log separately is convenient,
# because loading the whole pickle takes quite some time.
Checkpoint(save_path, every_n_batches=500,
save_separately=["model", "log"]),
Printing(every_n_batches=1)])
main_loop.run()
elif mode == "sample" or mode == "beam_search":
chars = tensor.lmatrix("input")
generated = reverser.generate(chars)
model = Model(generated)
logger.info("Loading the model..")
with open(save_path, 'rb') as f:
model.set_parameter_values(load_parameters(f))
def generate(input_):
"""Generate output sequences for an input sequence.
Incapsulates most of the difference between sampling and beam
search.
Returns
-------
outputs : list of lists
Trimmed output sequences.
costs : list
The negative log-likelihood of generating the respective
sequences.
"""
if mode == "beam_search":
samples, = VariableFilter(
applications=[reverser.generator.generate], name="outputs")(
ComputationGraph(generated[1]))
# NOTE: this will recompile beam search functions
# every time user presses Enter. Do not create
# a new `BeamSearch` object every time if
# speed is important for you.
beam_search = BeamSearch(samples)
outputs, costs = beam_search.search(
{chars: input_}, char2code['</S>'],
3 * input_.shape[0])
else:
_1, outputs, _2, _3, costs = (
model.get_theano_function()(input_))
outputs = list(outputs.T)
costs = list(costs.T)
for i in range(len(outputs)):
outputs[i] = list(outputs[i])
try:
true_length = outputs[i].index(char2code['</S>']) + 1
except ValueError:
true_length = len(outputs[i])
outputs[i] = outputs[i][:true_length]
costs[i] = costs[i][:true_length].sum()
return outputs, costs
while True:
try:
line = input("Enter a sentence\n")
message = ("Enter the number of samples\n" if mode == "sample"
else "Enter the beam size\n")
batch_size = int(input(message))
except EOFError:
break
except Exception:
traceback.print_exc()
continue
encoded_input = [char2code.get(char, char2code["<UNK>"])
for char in line.lower().strip()]
encoded_input = ([char2code['<S>']] + encoded_input +
[char2code['</S>']])
print("Encoder input:", encoded_input)
target = reverse_words((encoded_input,))[0]
print("Target: ", target)
samples, costs = generate(
numpy.repeat(numpy.array(encoded_input)[:, None],
batch_size, axis=1))
messages = []
for sample, cost in equizip(samples, costs):
message = "({})".format(cost)
message += "".join(code2char[code] for code in sample)
if sample == target:
message += " CORRECT!"
messages.append((cost, message))
messages.sort(key=operator.itemgetter(0), reverse=True)
for _, message in messages:
print(message)
def train_language_model(new_training_job, config, save_path, params,
fast_start, fuel_server, seed):
c = config
if seed:
fuel.config.default_seed = seed
blocks.config.config.default_seed = seed
data, lm, retrieval = initialize_data_and_model(config)
# full main loop can be saved...
main_loop_path = os.path.join(save_path, 'main_loop.tar')
# or only state (log + params) which can be useful not to pickle embeddings
state_path = os.path.join(save_path, 'training_state.tar')
stream_path = os.path.join(save_path, 'stream.pkl')
best_tar_path = os.path.join(save_path, "best_model.tar")
words = tensor.ltensor3('words')
words_mask = tensor.matrix('words_mask')
if theano.config.compute_test_value != 'off':
test_value_data = next(
data.get_stream('train', batch_size=4,
max_length=5).get_epoch_iterator())
words.tag.test_value = test_value_data[0]
words_mask.tag.test_value = test_value_data[1]
costs, updates = lm.apply(words, words_mask)
cost = rename(costs.mean(), 'mean_cost')
cg = Model(cost)
if params:
logger.debug("Load parameters from {}".format(params))
with open(params) as src:
cg.set_parameter_values(load_parameters(src))
length = rename(words.shape[1], 'length')
perplexity, = VariableFilter(name='perplexity')(cg)
perplexities = VariableFilter(name_regex='perplexity.*')(cg)
monitored_vars = [length, cost] + perplexities
if c['dict_path']:
num_definitions, = VariableFilter(name='num_definitions')(cg)
monitored_vars.extend([num_definitions])
parameters = cg.get_parameter_dict()
trained_parameters = parameters.values()
saved_parameters = parameters.values()
if c['embedding_path']:
logger.debug("Exclude word embeddings from the trained parameters")
trained_parameters = [
p for p in trained_parameters
if not p == lm.get_def_embeddings_params()
]
saved_parameters = [
p for p in saved_parameters
if not p == lm.get_def_embeddings_params()
]
if c['cache_size'] != 0:
logger.debug("Enable fake recursivity for looking up embeddings")
trained_parameters = [
p for p in trained_parameters if not p == lm.get_cache_params()
]
logger.info("Cost parameters" + "\n" + pprint.pformat([
" ".join(
(key, str(parameters[key].get_value().shape),
'trained' if parameters[key] in trained_parameters else 'frozen'))
for key in sorted(parameters.keys())
],
width=120))
rules = []
if c['grad_clip_threshold']:
rules.append(StepClipping(c['grad_clip_threshold']))
rules.append(Adam(learning_rate=c['learning_rate'], beta1=c['momentum']))
algorithm = GradientDescent(cost=cost,
parameters=trained_parameters,
step_rule=CompositeRule(rules))
if c['cache_size'] != 0:
algorithm.add_updates(updates)
train_monitored_vars = list(monitored_vars)
if c['grad_clip_threshold']:
train_monitored_vars.append(algorithm.total_gradient_norm)
word_emb_RMS, = VariableFilter(name='word_emb_RMS')(cg)
main_rnn_in_RMS, = VariableFilter(name='main_rnn_in_RMS')(cg)
train_monitored_vars.extend([word_emb_RMS, main_rnn_in_RMS])
if c['monitor_parameters']:
train_monitored_vars.extend(parameter_stats(parameters, algorithm))
# We use a completely random seed on purpose. With Fuel server
# it's currently not possible to restore the state of the training
# stream. That's why it's probably better to just have it stateless.
stream_seed = numpy.random.randint(0, 10000000) if fuel_server else None
training_stream = data.get_stream('train',
batch_size=c['batch_size'],
max_length=c['max_length'],
seed=stream_seed)
valid_stream = data.get_stream('valid',
batch_size=c['batch_size_valid'],
max_length=c['max_length'],
seed=stream_seed)
original_training_stream = training_stream
if fuel_server:
# the port will be configured by the StartFuelServer extension
training_stream = ServerDataStream(
sources=training_stream.sources,
produces_examples=training_stream.produces_examples)
validation = DataStreamMonitoring(monitored_vars,
valid_stream,
prefix="valid").set_conditions(
before_first_epoch=not fast_start,
on_resumption=True,
every_n_batches=c['mon_freq_valid'])
track_the_best = TrackTheBest(validation.record_name(perplexity),
choose_best=min).set_conditions(
on_resumption=True,
after_epoch=True,
every_n_batches=c['mon_freq_valid'])
# don't save them the entire main loop to avoid pickling everything
if c['fast_checkpoint']:
load = (LoadNoUnpickling(state_path,
load_iteration_state=True,
load_log=True).set_conditions(
before_training=not new_training_job))
cp_args = {
'save_main_loop': False,
'save_separately': ['log', 'iteration_state'],
'parameters': saved_parameters
}
checkpoint = Checkpoint(state_path,
before_training=not fast_start,
every_n_batches=c['save_freq_batches'],
after_training=not fast_start,
**cp_args)
if c['checkpoint_every_n_batches']:
intermediate_cp = IntermediateCheckpoint(
state_path,
every_n_batches=c['checkpoint_every_n_batches'],
after_training=False,
**cp_args)
else:
load = (Load(main_loop_path, load_iteration_state=True,
load_log=True).set_conditions(
before_training=not new_training_job))
cp_args = {
'save_separately': ['iteration_state'],
'parameters': saved_parameters
}
checkpoint = Checkpoint(main_loop_path,
before_training=not fast_start,
every_n_batches=c['save_freq_batches'],
after_training=not fast_start,
**cp_args)
if c['checkpoint_every_n_batches']:
intermediate_cp = IntermediateCheckpoint(
main_loop_path,
every_n_batches=c['checkpoint_every_n_batches'],
after_training=False,
**cp_args)
checkpoint = checkpoint.add_condition(
['after_batch', 'after_epoch'],
OnLogRecord(track_the_best.notification_name), (best_tar_path, ))
extensions = [
load,
StartFuelServer(original_training_stream,
stream_path,
before_training=fuel_server),
Timing(every_n_batches=c['mon_freq_train'])
]
if retrieval:
extensions.append(
RetrievalPrintStats(retrieval=retrieval,
every_n_batches=c['mon_freq_train'],
before_training=not fast_start))
extensions.extend([
TrainingDataMonitoring(train_monitored_vars,
prefix="train",
every_n_batches=c['mon_freq_train']),
validation, track_the_best, checkpoint
])
if c['checkpoint_every_n_batches']:
extensions.append(intermediate_cp)
extensions.extend([
DumpTensorflowSummaries(save_path,
every_n_batches=c['mon_freq_train'],
after_training=True),
Printing(on_resumption=True, every_n_batches=c['mon_freq_train']),
FinishIfNoImprovementAfter(track_the_best.notification_name,
iterations=50 * c['mon_freq_valid'],
every_n_batches=c['mon_freq_valid']),
FinishAfter(after_n_batches=c['n_batches'])
])
logger.info("monitored variables during training:" + "\n" +
pprint.pformat(train_monitored_vars, width=120))
logger.info("monitored variables during valid:" + "\n" +
pprint.pformat(monitored_vars, width=120))
main_loop = MainLoop(algorithm,
training_stream,
model=Model(cost),
extensions=extensions)
main_loop.run()
def generate_embeddings(config,
tar_path,
part,
dest_path,
format_,
average=False,
encoder_embeddings=None,
**kwargs):
"""
generate embeddings for all the defintions, average them and serialize OR
if encoder_embeddings, serialize the models' encoder embeddings
config: name of the config of the model
tar_path: tar path of the model parameters
part: part of the dataset (should be either 'train', 'valid', 'test' or 'all')
dest_path: directory where the serialized embeddings will be written
format: either 'dict' or 'glove'
encoder_embeddings: None, 'only', 'mixed', 'if_missing'
- None: don't include encoder embeddings
- 'only': don't read any data, just serialize the encoder embeddings
- 'mixed': add the encoder embeddings to the list of definition embeddings
- 'if_missing': add the encoder embeddings when there is no corresponding def
average: if true, multi-prototype embeddings will be averaged
"""
if not os.path.exists(dest_path):
os.makedirs(dest_path)
c = config
data, model = initialize_data_and_model(c, train_phase=False)
words = T.ltensor3('words')
words_mask = T.matrix('words_mask')
keys = T.lmatrix('keys')
n_identical_keys = T.lvector('n_identical_keys')
sym_args = [words, words_mask]
if format_ not in ['dict', 'glove']:
raise ValueError("format should be either: dict, glove")
if not c['encoder'] and encoder_embeddings != 'only':
raise ValueError('Error: this model does not have an encoder.')
if use_keys(c):
sym_args.append(keys)
if use_n_identical_keys(c):
sym_args.append(n_identical_keys)
costs = model.apply(*sym_args, train_phase=False)
cg = Model(costs)
with open(tar_path) as src:
cg.set_parameter_values(load_parameters(src))
if encoder_embeddings:
if encoder_embeddings == 'only' and not c['encoder']:
embeddings_array = model.get_def_embeddings_params('key').eval()
else:
embeddings_array = model.get_def_embeddings_params('main').eval()
entries = model.get_embeddings_entries()
enc_embeddings = {
e: np.asarray(a)
for e, a in zip(entries, embeddings_array)
}
if encoder_embeddings == 'only':
serialize_embeddings(enc_embeddings, format_, dest_path,
"encoder_embeddings")
return 0
embeddings_var, = VariableFilter(name='embeddings')(cg)
compute = dict({"embeddings": embeddings_var})
if c['proximity_coef'] != 0:
prox_var, = VariableFilter(name='proximity_term')(cg)
compute["proximity_term"] = prox_var
print "sym args", sym_args
predict_f = theano.function(sym_args, compute)
batch_size = 256 # size of test_unseen
stream = data.get_stream(part,
batch_size=batch_size,
max_length=c['max_length'],
remove_keys=False,
remove_n_identical_keys=False)
raw_data = [] # list of dicts containing the inputs and computed outputs
i = 0
vocab = model._vocab
print "start computing"
embeddings = defaultdict(list)
for input_data in stream.get_epoch_iterator(as_dict=True):
if i % 10 == 0:
print "iteration:", i
words = input_data['words']
words_mask = input_data['words_mask']
keys = input_data['keys']
n_identical_keys = input_data['n_identical_keys']
args = [words, words_mask]
if use_keys(c):
args.append(keys)
if use_n_identical_keys(c):
args.append(n_identical_keys)
to_save = predict_f(*args)
for k, h in zip(keys, to_save['embeddings']):
key = vec2str(k)
if encoder_embeddings == 'if_missing':
try:
del enc_embeddings[key]
except KeyError:
pass
embeddings[key].append(h)
i += 1
if encoder_embeddings in ['mixed', 'if_missing']:
for k, e in enc_embeddings.iteritems():
embeddings[k].append(e)
if encoder_embeddings == 'mixed':
prefix_fname = 'mix_e_'
elif encoder_embeddings == 'if_missing':
prefix_fname = 'if_mis_e_'
else:
prefix_fname = ''
# combine:
if average:
mean_embeddings = {}
for k in embeddings.keys():
mean_embeddings[k] = np.mean(np.asarray(embeddings[k]), axis=0)
serialize_embeddings(mean_embeddings, format_, dest_path,
prefix_fname + "mean_embeddings")
else:
serialize_embeddings(embeddings, format_, dest_path,
prefix_fname + "embeddings")
saved_args = cPickle.load(f)
assert saved_args.dataset == args.dataset
if args.use_last:
params_mode = 'last_'
else:
params_mode = 'best_'
args.samples_name = params_mode + args.samples_name
with open(
os.path.join(args.save_dir, "pkl",
params_mode + args.experiment_name + ".tar"),
'rb') as src:
parameters = load_parameters(src)
test_stream = parrot_stream(args.dataset,
saved_args.use_speaker, ('test', ),
args.num_samples,
args.num_steps,
sorting_mult=1,
labels_type=saved_args.labels_type,
raw_data=args.plot_raw)
data_tr = next(test_stream.get_epoch_iterator())
data_tr = {source: data for source, data in zip(test_stream.sources, data_tr)}
print "Loaded sources from test_stream: ", data_tr.keys()
features_tr = data_tr.get('features', None)
features_mask_tr = data_tr.get('features_mask', None)
def train_snli_model(new_training_job,
config,
save_path,
params,
fast_start,
fuel_server,
seed,
model='simple'):
if config['exclude_top_k'] > config['num_input_words'] and config[
'num_input_words'] > 0:
raise Exception("Some words have neither word nor def embedding")
c = config
logger = configure_logger(name="snli_baseline_training",
log_file=os.path.join(save_path, "log.txt"))
if not os.path.exists(save_path):
logger.info("Start a new job")
os.mkdir(save_path)
else:
logger.info("Continue an existing job")
with open(os.path.join(save_path, "cmd.txt"), "w") as f:
f.write(" ".join(sys.argv))
# Make data paths nice
for path in [
'dict_path', 'embedding_def_path', 'embedding_path', 'vocab',
'vocab_def', 'vocab_text'
]:
if c.get(path, ''):
if not os.path.isabs(c[path]):
c[path] = os.path.join(fuel.config.data_path[0], c[path])
main_loop_path = os.path.join(save_path, 'main_loop.tar')
main_loop_best_val_path = os.path.join(save_path, 'main_loop_best_val.tar')
stream_path = os.path.join(save_path, 'stream.pkl')
# Save config to save_path
json.dump(config, open(os.path.join(save_path, "config.json"), "w"))
if model == 'simple':
nli_model, data, used_dict, used_retrieval, _ = _initialize_simple_model_and_data(
c)
elif model == 'esim':
nli_model, data, used_dict, used_retrieval, _ = _initialize_esim_model_and_data(
c)
else:
raise NotImplementedError()
# Compute cost
s1, s2 = T.lmatrix('sentence1'), T.lmatrix('sentence2')
if c['dict_path']:
assert os.path.exists(c['dict_path'])
s1_def_map, s2_def_map = T.lmatrix('sentence1_def_map'), T.lmatrix(
'sentence2_def_map')
def_mask = T.fmatrix("def_mask")
defs = T.lmatrix("defs")
else:
s1_def_map, s2_def_map = None, None
def_mask = None
defs = None
s1_mask, s2_mask = T.fmatrix('sentence1_mask'), T.fmatrix('sentence2_mask')
y = T.ivector('label')
cg = {}
for train_phase in [True, False]:
# NOTE: Please don't change outputs of cg
if train_phase:
with batch_normalization(nli_model):
pred = nli_model.apply(s1,
s1_mask,
s2,
s2_mask,
def_mask=def_mask,
defs=defs,
s1_def_map=s1_def_map,
s2_def_map=s2_def_map,
train_phase=train_phase)
else:
pred = nli_model.apply(s1,
s1_mask,
s2,
s2_mask,
def_mask=def_mask,
defs=defs,
s1_def_map=s1_def_map,
s2_def_map=s2_def_map,
train_phase=train_phase)
cost = CategoricalCrossEntropy().apply(y.flatten(), pred)
error_rate = MisclassificationRate().apply(y.flatten(), pred)
cg[train_phase] = ComputationGraph([cost, error_rate])
# Weight decay (TODO: Make it less bug prone)
if model == 'simple':
weights_to_decay = VariableFilter(
bricks=[dense for dense, relu, bn in nli_model._mlp],
roles=[WEIGHT])(cg[True].variables)
weight_decay = np.float32(c['l2']) * sum(
(w**2).sum() for w in weights_to_decay)
elif model == 'esim':
weight_decay = 0.0
else:
raise NotImplementedError()
final_cost = cg[True].outputs[0] + weight_decay
final_cost.name = 'final_cost'
# Add updates for population parameters
if c.get("bn", True):
pop_updates = get_batch_normalization_updates(cg[True])
extra_updates = [(p, m * 0.1 + p * (1 - 0.1)) for p, m in pop_updates]
else:
pop_updates = []
extra_updates = []
if params:
logger.debug("Load parameters from {}".format(params))
with open(params) as src:
loaded_params = load_parameters(src)
cg[True].set_parameter_values(loaded_params)
for param, m in pop_updates:
param.set_value(loaded_params[get_brick(
param).get_hierarchical_name(param)])
if os.path.exists(os.path.join(save_path, "main_loop.tar")):
logger.warning("Manually loading BN stats :(")
with open(os.path.join(save_path, "main_loop.tar")) as src:
loaded_params = load_parameters(src)
for param, m in pop_updates:
param.set_value(
loaded_params[get_brick(param).get_hierarchical_name(param)])
if theano.config.compute_test_value != 'off':
test_value_data = next(
data.get_stream('train', batch_size=4).get_epoch_iterator())
s1.tag.test_value = test_value_data[0]
s1_mask.tag.test_value = test_value_data[1]
s2.tag.test_value = test_value_data[2]
s2_mask.tag.test_value = test_value_data[3]
y.tag.test_value = test_value_data[4]
# Freeze embeddings
if not c['train_emb']:
frozen_params = [
p for E in nli_model.get_embeddings_lookups() for p in E.parameters
]
train_params = [p for p in cg[True].parameters]
assert len(set(frozen_params) & set(train_params)) > 0
else:
frozen_params = []
if not c.get('train_def_emb', 1):
frozen_params_def = [
p for E in nli_model.get_def_embeddings_lookups()
for p in E.parameters
]
train_params = [p for p in cg[True].parameters]
assert len(set(frozen_params_def) & set(train_params)) > 0
frozen_params += frozen_params_def
train_params = [p for p in cg[True].parameters if p not in frozen_params]
train_params_keys = [
get_brick(p).get_hierarchical_name(p) for p in train_params
]
# Optimizer
algorithm = GradientDescent(cost=final_cost,
on_unused_sources='ignore',
parameters=train_params,
step_rule=Adam(learning_rate=c['lr']))
algorithm.add_updates(extra_updates)
m = Model(final_cost)
parameters = m.get_parameter_dict() # Blocks version mismatch
logger.info("Trainable parameters" + "\n" +
pprint.pformat([(key, parameters[key].get_value().shape)
for key in sorted(train_params_keys)],
width=120))
logger.info("# of parameters {}".format(
sum([
np.prod(parameters[key].get_value().shape)
for key in sorted(train_params_keys)
])))
### Monitored args ###
train_monitored_vars = [final_cost] + cg[True].outputs
monitored_vars = cg[False].outputs
val_acc = monitored_vars[1]
to_monitor_names = [
'def_unk_ratio', 's1_merged_input_rootmean2', 's1_def_mean_rootmean2',
's1_gate_rootmean2', 's1_compose_gate_rootmean2'
]
for k in to_monitor_names:
train_v, valid_v = VariableFilter(name=k)(
cg[True]), VariableFilter(name=k)(cg[False])
if len(train_v):
logger.info("Adding {} tracking".format(k))
train_monitored_vars.append(train_v[0])
monitored_vars.append(valid_v[0])
else:
logger.warning("Didnt find {} in cg".format(k))
if c['monitor_parameters']:
for name in train_params_keys:
param = parameters[name]
num_elements = numpy.product(param.get_value().shape)
norm = param.norm(2) / num_elements
grad_norm = algorithm.gradients[param].norm(2) / num_elements
step_norm = algorithm.steps[param].norm(2) / num_elements
stats = tensor.stack(norm, grad_norm, step_norm,
step_norm / grad_norm)
stats.name = name + '_stats'
train_monitored_vars.append(stats)
regular_training_stream = data.get_stream('train',
batch_size=c['batch_size'],
seed=seed)
if fuel_server:
# the port will be configured by the StartFuelServer extension
training_stream = ServerDataStream(
sources=regular_training_stream.sources,
hwm=100,
produces_examples=regular_training_stream.produces_examples)
else:
training_stream = regular_training_stream
### Build extensions ###
extensions = [
# Load(main_loop_path, load_iteration_state=True, load_log=True)
# .set_conditions(before_training=not new_training_job),
StartFuelServer(regular_training_stream,
stream_path,
hwm=100,
script_path=os.path.join(
os.path.dirname(__file__),
"../bin/start_fuel_server.py"),
before_training=fuel_server),
Timing(every_n_batches=c['mon_freq']),
ProgressBar(),
RetrievalPrintStats(retrieval=used_retrieval,
every_n_batches=c['mon_freq_valid'],
before_training=not fast_start),
Timestamp(),
TrainingDataMonitoring(train_monitored_vars,
prefix="train",
every_n_batches=c['mon_freq']),
]
if c['layout'] == 'snli':
validation = DataStreamMonitoring(monitored_vars,
data.get_stream('valid',
batch_size=14,
seed=seed),
before_training=not fast_start,
on_resumption=True,
after_training=True,
every_n_batches=c['mon_freq_valid'],
prefix='valid')
extensions.append(validation)
elif c['layout'] == 'mnli':
validation = DataStreamMonitoring(monitored_vars,
data.get_stream('valid_matched',
batch_size=14,
seed=seed),
every_n_batches=c['mon_freq_valid'],
on_resumption=True,
after_training=True,
prefix='valid_matched')
validation_mismatched = DataStreamMonitoring(
monitored_vars,
data.get_stream('valid_mismatched', batch_size=14, seed=seed),
every_n_batches=c['mon_freq_valid'],
before_training=not fast_start,
on_resumption=True,
after_training=True,
prefix='valid_mismatched')
extensions.extend([validation, validation_mismatched])
else:
raise NotImplementedError()
# Similarity trackers for embeddings
if len(c.get('vocab_def', '')):
retrieval_vocab = Vocabulary(c['vocab_def'])
else:
retrieval_vocab = data.vocab
retrieval_all = Retrieval(vocab_text=retrieval_vocab,
dictionary=used_dict,
max_def_length=c['max_def_length'],
exclude_top_k=0,
max_def_per_word=c['max_def_per_word'])
for name in [
's1_word_embeddings', 's1_dict_word_embeddings',
's1_translated_word_embeddings'
]:
variables = VariableFilter(name=name)(cg[False])
if len(variables):
s1_emb = variables[0]
logger.info("Adding similarity tracking for " + name)
# A bit sloppy about downcast
if "dict" in name:
embedder = construct_dict_embedder(theano.function(
[s1, defs, def_mask, s1_def_map],
s1_emb,
allow_input_downcast=True),
vocab=data.vocab,
retrieval=retrieval_all)
extensions.append(
SimilarityWordEmbeddingEval(
embedder=embedder,
prefix=name,
every_n_batches=c['mon_freq_valid'],
before_training=not fast_start))
else:
embedder = construct_embedder(theano.function(
[s1], s1_emb, allow_input_downcast=True),
vocab=data.vocab)
extensions.append(
SimilarityWordEmbeddingEval(
embedder=embedder,
prefix=name,
every_n_batches=c['mon_freq_valid'],
before_training=not fast_start))
track_the_best = TrackTheBest(validation.record_name(val_acc),
before_training=not fast_start,
every_n_epochs=c['save_freq_epochs'],
after_training=not fast_start,
every_n_batches=c['mon_freq_valid'],
choose_best=min)
extensions.append(track_the_best)
# Special care for serializing embeddings
if len(c.get('embedding_path', '')) or len(c.get('embedding_def_path',
'')):
extensions.insert(
0,
LoadNoUnpickling(main_loop_path,
load_iteration_state=True,
load_log=True).set_conditions(
before_training=not new_training_job))
extensions.append(
Checkpoint(main_loop_path,
parameters=train_params + [p for p, m in pop_updates],
save_main_loop=False,
save_separately=['log', 'iteration_state'],
before_training=not fast_start,
every_n_epochs=c['save_freq_epochs'],
after_training=not fast_start).add_condition(
['after_batch', 'after_epoch'],
OnLogRecord(track_the_best.notification_name),
(main_loop_best_val_path, )))
else:
extensions.insert(
0,
Load(main_loop_path, load_iteration_state=True,
load_log=True).set_conditions(
before_training=not new_training_job))
extensions.append(
Checkpoint(main_loop_path,
parameters=cg[True].parameters +
[p for p, m in pop_updates],
before_training=not fast_start,
every_n_epochs=c['save_freq_epochs'],
after_training=not fast_start).add_condition(
['after_batch', 'after_epoch'],
OnLogRecord(track_the_best.notification_name),
(main_loop_best_val_path, )))
extensions.extend([
DumpCSVSummaries(save_path,
every_n_batches=c['mon_freq_valid'],
after_training=True),
DumpTensorflowSummaries(save_path,
after_epoch=True,
every_n_batches=c['mon_freq_valid'],
after_training=True),
Printing(every_n_batches=c['mon_freq_valid']),
PrintMessage(msg="save_path={}".format(save_path),
every_n_batches=c['mon_freq']),
FinishAfter(after_n_batches=c['n_batches']).add_condition(
['after_batch'],
OnLogStatusExceed('iterations_done', c['n_batches']))
])
logger.info(extensions)
### Run training ###
if "VISDOM_SERVER" in os.environ:
print("Running visdom server")
ret = subprocess.Popen([
os.path.join(os.path.dirname(__file__), "../visdom_plotter.py"),
"--visdom-server={}".format(os.environ['VISDOM_SERVER']),
"--folder={}".format(save_path)
])
time.sleep(0.1)
if ret.returncode is not None:
raise Exception()
atexit.register(lambda: os.kill(ret.pid, signal.SIGINT))
model = Model(cost)
for p, m in pop_updates:
model._parameter_dict[get_brick(p).get_hierarchical_name(p)] = p
main_loop = MainLoop(algorithm,
training_stream,
model=model,
extensions=extensions)
assert os.path.exists(save_path)
main_loop.run()
Globals.read_alphabet(f_vocab);
print("Vocab:",Globals.char2code, file=sys.stderr);
Globals.read_lookup(f_test);
print("Lookup size:", len(Globals.lookup), file=sys.stderr);
#print("Test:",f_test,file=sys.stderr);
m = MorphGen(100, len(Globals.char2code));
chars = tensor.lmatrix("input")
generated = m.generate(chars)
model = Model(generated)
# Load model
with open(f_model, 'rb') as f: #{
model.set_parameter_values(load_parameters(f))
#}
f_in = open(f_test);
total = 0.0;
correct = 0.0;
samples, = VariableFilter(applications=[m.generator.generate], name="outputs")(ComputationGraph(generated[1]))
# NOTE: this will recompile beam search functions every time user presses Enter. Do not create
# a new `BeamSearch` object every time if speed is important for you.
beam_search = BeamSearch(samples);
for line in f_in.readlines(): #{
inp = _tokenise(line);
form = '|'.join(line.strip().split('|||')[1:]);
encoded_input = _encode(inp);
# print(inp,'→',encoded_input, sys.stderr);
def evaluate(c, tar_path, *args, **kwargs):
"""
Performs rudimentary evaluation of SNLI/MNLI run
* Runs on valid and test given network
* Saves all predictions
* Saves embedding matrix
* Saves results.json and predictions.csv
"""
# Load and configure
model = kwargs['model']
assert c.endswith("json")
c = json.load(open(c))
# Very ugly absolute path fix
ABS_PATHS = [
"data/", "/mnt/users/jastrzebski/local/dict_based_learning/data/",
"/data/cf9ffb48-61bd-40dc-a011-b2e7e5acfd72/"
]
from six import string_types
for abs_path in ABS_PATHS:
for k in c:
if isinstance(c[k], string_types):
if c[k].startswith(abs_path):
c[k] = c[k][len(abs_path):]
# Make data paths nice
for path in [
'dict_path', 'embedding_def_path', 'embedding_path', 'vocab',
'vocab_def', 'vocab_text'
]:
if c.get(path, ''):
if not os.path.isabs(c[path]):
c[path] = os.path.join(fuel.config.data_path[0], c[path])
logging.info("Updating config with " + str(kwargs))
c.update(**kwargs)
# NOTE: This assures we don't miss crucial definition for some def heavy words
# usually it is a good idea
c['max_def_per_word'] = c['max_def_per_word'] * 2
assert tar_path.endswith("tar")
dest_path = os.path.dirname(tar_path)
prefix = os.path.splitext(os.path.basename(tar_path))[0]
s1_decoded, s2_decoded = T.lmatrix('sentence1'), T.lmatrix('sentence2')
if c['dict_path']:
s1_def_map, s2_def_map = T.lmatrix('sentence1_def_map'), T.lmatrix(
'sentence2_def_map')
def_mask = T.fmatrix("def_mask")
defs = T.lmatrix("defs")
else:
s1_def_map, s2_def_map = None, None
def_mask = None
defs = None
s1_mask, s2_mask = T.fmatrix('sentence1_mask'), T.fmatrix('sentence2_mask')
if model == 'simple':
model, data, used_dict, used_retrieval, used_vocab = _initialize_simple_model_and_data(
c)
elif model == 'esim':
model, data, used_dict, used_retrieval, used_vocab = _initialize_esim_model_and_data(
c)
else:
raise NotImplementedError()
pred = model.apply(s1_decoded,
s1_mask,
s2_decoded,
s2_mask,
def_mask=def_mask,
defs=defs,
s1_def_map=s1_def_map,
s2_def_map=s2_def_map,
train_phase=False)
cg = ComputationGraph([pred])
if c.get("bn", True):
bn_params = [
p for p in VariableFilter(bricks=[BatchNormalization])(cg)
if hasattr(p, "set_value")
]
else:
bn_params = []
# Load model
model = Model(cg.outputs)
parameters = model.get_parameter_dict() # Blocks version mismatch
logging.info(
"Trainable parameters" + "\n" +
pprint.pformat([(key, parameters[key].get_value().shape)
for key in sorted([
get_brick(param).get_hierarchical_name(param)
for param in cg.parameters
])],
width=120))
logging.info("# of parameters {}".format(
sum([
np.prod(parameters[key].get_value().shape) for key in sorted([
get_brick(param).get_hierarchical_name(param)
for param in cg.parameters
])
])))
with open(tar_path) as src:
params = load_parameters(src)
loaded_params_set = set(params.keys())
model_params_set = set([
get_brick(param).get_hierarchical_name(param)
for param in cg.parameters
])
logging.info("Loaded extra parameters")
logging.info(loaded_params_set - model_params_set)
logging.info("Missing parameters")
logging.info(model_params_set - loaded_params_set)
model.set_parameter_values(params)
if c.get("bn", True):
logging.info("Loading " + str([
get_brick(param).get_hierarchical_name(param)
for param in bn_params
]))
for param in bn_params:
param.set_value(
params[get_brick(param).get_hierarchical_name(param)])
for p in bn_params:
model._parameter_dict[get_brick(p).get_hierarchical_name(p)] = p
# Read logs
logs = pd.read_csv(os.path.join(dest_path, "logs.csv"))
best_val_acc = logs['valid_misclassificationrate_apply_error_rate'].min()
logging.info("Best measured valid acc: " + str(best_val_acc))
# NOTE(kudkudak): We need this to have comparable mean rank and embedding scores
reference_vocab = Vocabulary(
os.path.join(fuel.config.data_path[0], c['data_path'], 'vocab.txt'))
vocab_all = Vocabulary(
os.path.join(
fuel.config.data_path[0], c['data_path'],
'vocab_all.txt')) # Can include OOV words, which is interesting
retrieval_all = Retrieval(vocab_text=used_vocab,
dictionary=used_dict,
max_def_length=c['max_def_length'],
exclude_top_k=0,
max_def_per_word=c['max_def_per_word'])
# logging.info("Calculating dict and word embeddings for vocab.txt and vocab_all.txt")
# for name in ['s1_word_embeddings', 's1_dict_word_embeddings']:
# variables = VariableFilter(name=name)(cg)
# if len(variables):
# s1_emb = variables[0]
# # A bit sloppy about downcast
#
# if "dict" in name:
# embedder = construct_dict_embedder(
# theano.function([s1_decoded, defs, def_mask, s1_def_map], s1_emb, allow_input_downcast=True),
# vocab=data.vocab, retrieval=retrieval_all)
# else:
# embedder = construct_embedder(theano.function([s1_decoded], s1_emb, allow_input_downcast=True),
# vocab=data.vocab)
#
# for v_name, v in [("vocab_all", vocab_all), ("vocab", reference_vocab)]:
# logging.info("Calculating {} embeddings for {}".format(name, v_name))
# Predict
predict_fnc = theano.function(cg.inputs, pred)
results = {}
batch_size = 14
for subset in ['valid', 'test']:
logging.info("Predicting on " + subset)
stream = data.get_stream(subset, batch_size=batch_size, seed=778)
it = stream.get_epoch_iterator()
rows = []
for ex in tqdm.tqdm(it, total=10000 / batch_size):
ex = dict(zip(stream.sources, ex))
inp = [ex[v.name] for v in cg.inputs]
prob = predict_fnc(*inp)
label_pred = np.argmax(prob, axis=1)
for id in range(len(prob)):
s1_decoded = used_vocab.decode(ex['sentence1'][id]).split()
s2_decoded = used_vocab.decode(ex['sentence2'][id]).split()
assert used_vocab == data.vocab
s1_decoded = [
'*' + w + '*'
if used_vocab.word_to_id(w) > c['num_input_words'] else w
for w in s1_decoded
]
s2_decoded = [
'*' + w + '*'
if used_vocab.word_to_id(w) > c['num_input_words'] else w
for w in s2_decoded
]
# Different difficulty metrics
# text_unk_percentage
s1_no_pad = [w for w in ex['sentence1'][id] if w != 0]
s2_no_pad = [w for w in ex['sentence2'][id] if w != 0]
s1_unk_percentage = sum([
1. for w in s1_no_pad if w == used_vocab.unk
]) / len(s1_no_pad)
s2_unk_percentage = sum([
1. for w in s1_no_pad if w == used_vocab.unk
]) / len(s2_no_pad)
# mean freq word
s1_mean_freq = np.mean([
0 if w == data.vocab.unk else used_vocab._id_to_freq[w]
for w in s1_no_pad
])
s2_mean_freq = np.mean([
0 if w == data.vocab.unk else used_vocab._id_to_freq[w]
for w in s2_no_pad
])
# mean rank word (UNK is max rank)
# NOTE(kudkudak): Will break if we reindex unk between vocabs :P
s1_mean_rank = np.mean([
reference_vocab.size() if reference_vocab.word_to_id(
used_vocab.id_to_word(w)) == reference_vocab.unk else
reference_vocab.word_to_id(used_vocab.id_to_word(w))
for w in s1_no_pad
])
s2_mean_rank = np.mean([
reference_vocab.size() if reference_vocab.word_to_id(
used_vocab.id_to_word(w)) == reference_vocab.unk else
reference_vocab.word_to_id(used_vocab.id_to_word(w))
for w in s2_no_pad
])
rows.append({
"pred": label_pred[id],
"true_label": ex['label'][id],
"s1": ' '.join(s1_decoded),
"s2": ' '.join(s2_decoded),
"s1_unk_percentage": s1_unk_percentage,
"s2_unk_percentage": s2_unk_percentage,
"s1_mean_freq": s1_mean_freq,
"s2_mean_freq": s2_mean_freq,
"s1_mean_rank": s1_mean_rank,
"s2_mean_rank": s2_mean_rank,
"p_0": prob[id, 0],
"p_1": prob[id, 1],
"p_2": prob[id, 2]
})
preds = pd.DataFrame(rows, columns=rows[0].keys())
preds.to_csv(
os.path.join(dest_path,
prefix + '_predictions_{}.csv'.format(subset)))
results[subset] = {}
results[subset]['misclassification'] = 1 - np.mean(
preds.pred == preds.true_label)
if subset == "valid" and np.abs(
(1 - np.mean(preds.pred == preds.true_label)) -
best_val_acc) > 0.001:
logging.error("!!!")
logging.error(
"Found different best_val_acc. Probably due to changed specification of the model class."
)
logging.error("Discrepancy {}".format(
(1 - np.mean(preds.pred == preds.true_label)) - best_val_acc))
logging.error("!!!")
logging.info(results)
json.dump(results,
open(os.path.join(dest_path, prefix + '_results.json'), "w"))