def test_dataset_evaluators():
X = theano.tensor.matrix('X')
brick = TestBrick(name='test_brick')
Y = brick.apply(X)
graph = ComputationGraph([Y])
monitor_variables = [v for v in graph.auxiliary_variables]
validator = DatasetEvaluator(monitor_variables)
data = [
numpy.arange(1, 5, dtype=floatX).reshape(2, 2),
numpy.arange(10, 16, dtype=floatX).reshape(3, 2)
]
data_stream = IterableDataset(dict(X=data)).get_example_stream()
values = validator.evaluate(data_stream)
assert values['test_brick_apply_V_squared'] == 4
numpy.testing.assert_allclose(values['test_brick_apply_mean_row_mean'],
numpy.vstack(data).mean())
per_batch_mean = numpy.mean([batch.mean() for batch in data])
numpy.testing.assert_allclose(
values['test_brick_apply_mean_batch_element'], per_batch_mean)
with assert_raises(Exception) as ar:
data_stream = IterableDataset(dict(X2=data)).get_example_stream()
validator.evaluate(data_stream)
assert "Not all data sources" in ar.exception.args[0]
def __init__(self, variables, data_stream, prefix=None, **kwargs):
kwargs.setdefault("after_every_epoch", True)
kwargs.setdefault("before_first_epoch", True)
super(DataStreamMonitoring, self).__init__(**kwargs)
self._evaluator = DatasetEvaluator(variables)
self.data_stream = data_stream
self.prefix = prefix
def _test_mean_like_aggregator(scheme, func):
"""Common test function for both Mean and Perplexity."""
features = numpy.array([[0, 3], [2, 9], [2, 4], [5, 1], [6, 7]],
dtype=theano.config.floatX)
num_examples = features.shape[0]
batch_size = 2
dataset = IndexableDataset(OrderedDict([('features', features)]))
data_stream = DataStream(dataset,
iteration_scheme=SequentialScheme(
num_examples, batch_size))
x = tensor.matrix('features')
y = (x**0.5).sum(axis=0)
y.name = 'y'
z = y.sum()
z.name = 'z'
y.tag.aggregation_scheme = scheme(y, x.shape[0])
z.tag.aggregation_scheme = scheme(z, x.shape[0])
y_desired = func((features**0.5).mean(axis=0))
z_desired = func((features**0.5).sum(axis=1).mean(axis=0))
assert_allclose(
DatasetEvaluator([y]).evaluate(data_stream)['y'],
numpy.array(y_desired, dtype=theano.config.floatX))
assert_allclose(
DatasetEvaluator([z]).evaluate(data_stream)['z'],
numpy.array(z_desired, dtype=theano.config.floatX))
def test_mean_aggregator():
num_examples = 4
batch_size = 2
features = numpy.array([[0, 3], [2, 9], [2, 4], [5, 1]],
dtype=theano.config.floatX)
dataset = IndexableDataset(OrderedDict([('features', features)]))
data_stream = DataStream(dataset,
iteration_scheme=SequentialScheme(
num_examples, batch_size))
x = tensor.matrix('features')
y = (x**2).mean(axis=0)
y.name = 'y'
z = y.sum()
z.name = 'z'
y.tag.aggregation_scheme = Mean(y, 1.)
z.tag.aggregation_scheme = Mean(z, 1.)
assert_allclose(
DatasetEvaluator([y]).evaluate(data_stream)['y'],
numpy.array([8.25, 26.75], dtype=theano.config.floatX))
assert_allclose(
DatasetEvaluator([z]).evaluate(data_stream)['z'],
numpy.array([35], dtype=theano.config.floatX))
def __init__(self, output_vars, train_data_stream, test_data_streams,
**kwargs):
super(FinalTestMonitoring, self).__init__(**kwargs)
if not isinstance(test_data_streams, dict):
self.tst_streams = {self.prefix, test_data_streams}
else:
self.tst_streams = test_data_streams
self._tst_evaluator = DatasetEvaluator(output_vars)
def create_act_table(self, save_to, act_table):
batch_size = 500
image_size = (28, 28)
output_size = 10
convnet = create_lenet_5()
layers = convnet.layers
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outmap = OrderedDict((full_brick_name(get_brick(out)), out)
for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(
cg.variables))
# Generate pics for biases
biases = VariableFilter(roles=[BIAS])(cg.parameters)
# Generate parallel array, in the same order, for outputs
outs = [outmap[full_brick_name(get_brick(b))] for b in biases]
# Figure work count
error_rate = (MisclassificationRate().apply(y.flatten(), probs)
.copy(name='error_rate'))
max_activation_table = (MaxActivationTable().apply(
outs).copy(name='max_activation_table'))
max_activation_table.tag.aggregation_scheme = (
Concatenate(max_activation_table))
model = Model([
error_rate,
max_activation_table])
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
mnist_test_stream = DataStream.default_stream(
self.mnist_test,
iteration_scheme=SequentialScheme(
self.mnist_test.num_examples, batch_size))
evaluator = DatasetEvaluator([
error_rate,
max_activation_table
])
results = evaluator.evaluate(mnist_test_stream)
table = results['max_activation_table']
pickle.dump(table, open(act_table, 'wb'))
return table
def evaluate(self):
evaluator = DatasetEvaluator([self.cost, self.error] +
self.experiment.get_quantitites_vars())
print('subset\tcost\terror\tf_score\tauc')
for split in ['train', 'dev', 'test']:
stream = getattr(self, split + '_stream')
print('{0}\t{cost}\t{error}\t{f_score}\t{auc}'.format(
split, **evaluator.evaluate(stream)))
y_prob, y_test = self.get_targets(self.test_stream)
report_performance(y_test, y_prob, self.params['threshold'])
def __init__(self,
variables,
data_stream,
updates=None,
noise_parameters=None,
**kwargs):
kwargs.setdefault("after_epoch", True)
kwargs.setdefault("before_first_epoch", True)
super(DataStreamMonitoring, self).__init__(**kwargs)
self._evaluator = DatasetEvaluator(variables, updates)
self.data_stream = data_stream
self.noise_parameters = noise_parameters
def create_act_table(self, save_to, act_table):
batch_size = 500
image_size = (28, 28)
output_size = 10
convnet = create_lenet_5()
layers = convnet.layers
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outmap = OrderedDict(
(full_brick_name(get_brick(out)), out) for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(cg.variables))
# Generate pics for biases
biases = VariableFilter(roles=[BIAS])(cg.parameters)
# Generate parallel array, in the same order, for outputs
outs = [outmap[full_brick_name(get_brick(b))] for b in biases]
# Figure work count
error_rate = (MisclassificationRate().apply(
y.flatten(), probs).copy(name='error_rate'))
max_activation_table = (MaxActivationTable().apply(outs).copy(
name='max_activation_table'))
max_activation_table.tag.aggregation_scheme = (
Concatenate(max_activation_table))
model = Model([error_rate, max_activation_table])
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
mnist_test_stream = DataStream.default_stream(
self.mnist_test,
iteration_scheme=SequentialScheme(self.mnist_test.num_examples,
batch_size))
evaluator = DatasetEvaluator([error_rate, max_activation_table])
results = evaluator.evaluate(mnist_test_stream)
table = results['max_activation_table']
pickle.dump(table, open(act_table, 'wb'))
return table
def __init__(self,
variables,
data_stream,
sharedDataTrain,
sharedDataActualTest,
updates=None,
saveEveryXIteration=10,
**kwargs):
super(DataStreamMonitoringPlot, self).__init__(**kwargs)
self._evaluator = DatasetEvaluator(variables, updates)
self.data_stream = data_stream
self.dataTrain = sharedDataTrain
self.dataTest = sharedDataActualTest
self.saveEveryXIteration = saveEveryXIteration
self.curTime = time.time()
class NoisyDataStreamMonitoring(DataStreamMonitoring):
def __init__(self,
variables,
data_stream,
updates=None,
noise_parameters=None,
**kwargs):
kwargs.setdefault("after_epoch", True)
kwargs.setdefault("before_first_epoch", True)
super(DataStreamMonitoring, self).__init__(**kwargs)
self._evaluator = DatasetEvaluator(variables, updates)
self.data_stream = data_stream
self.noise_parameters = noise_parameters
def do(self, callback_name, *args):
"""Write the values of monitored variables to the log."""
logger.info("Monitoring on auxiliary data started")
saved = [(p, p.get_value()) for p in self.noise_parameters]
for (p, v) in saved:
p.set_value(np.zeros(v.shape, dtype=v.dtype))
value_dict = self._evaluator.evaluate(self.data_stream)
self.add_records(self.main_loop.log, value_dict.items())
for (p, v) in saved:
p.set_value(v)
logger.info("Monitoring on auxiliary data finished")
def build_evaluator(model_name):
config = importlib.import_module('.%s' % model_name, 'config')
config.batch_size = 1
config.shuffle_questions = False
# Build datastream
valid_path = os.path.join(os.getcwd(),
"squad_rare/dev-v1.0_tokenized.json")
vocab_path = os.path.join(os.getcwd(), "squad_rare/vocab.txt")
import data
ds, valid_stream = data.setup_squad_datastream(valid_path, vocab_path,
config)
dump_path = os.path.join("model_params", model_name + ".pkl")
# Build model
m = config.Model(config, ds.vocab_size)
model = Model(m.sgd_cost)
if os.path.isfile(dump_path):
with open(dump_path, 'r') as f:
print "Analysing %s from best dump" % (model_name)
model.set_parameter_values(cPickle.load(f))
else:
print "Analysing %s with random parameters" % (model_name)
evaluator = DatasetEvaluator(m.analyse_vars)
return evaluator, valid_stream, ds
class DataStreamMonitoring(SimpleExtension):
"""Monitors values of Theano variables on a data stream.
By default monitoring is done before the first and after every epoch.
Parameters
----------
variables : list of :class:`~tensor.TensorVariable`
The variables to monitor. The variable names are used as record
names in the logs.
data_stream : instance of :class:`.DataStream`
The data stream to monitor on. A data epoch is requested
each time monitoring is done.
prefix : str, optional
A prefix to add to the names when adding records to the log. An
underscore will be used to separate the prefix.
"""
PREFIX_SEPARATOR = '_'
def __init__(self, variables, data_stream, prefix=None, **kwargs):
kwargs.setdefault("after_every_epoch", True)
kwargs.setdefault("before_first_epoch", True)
super(DataStreamMonitoring, self).__init__(**kwargs)
self._evaluator = DatasetEvaluator(variables)
self.data_stream = data_stream
self.prefix = prefix
def do(self, callback_name, *args):
"""Write the values of monitored variables to the log."""
logger.info("Monitoring on auxiliary data started")
value_dict = self._evaluator.evaluate(self.data_stream)
_add_records(self.main_loop.log, self.prefix, value_dict.items())
logger.info("Monitoring on auxiliary data finished")
class DataStreamMonitoringPlot(SimpleExtension, MonitoringExtension):
def __init__(self,
variables,
data_stream,
sharedDataTrain,
sharedDataActualTest,
updates=None,
saveEveryXIteration=10,
**kwargs):
super(DataStreamMonitoringPlot, self).__init__(**kwargs)
self._evaluator = DatasetEvaluator(variables, updates)
self.data_stream = data_stream
self.dataTrain = sharedDataTrain
self.dataTest = sharedDataActualTest
self.saveEveryXIteration = saveEveryXIteration
self.curTime = time.time()
def do(self, callback_name, *args):
log = self.main_loop.log
iteration = log.status['iterations_done']
if iteration % self.saveEveryXIteration == 0:
print("time took is : " + str(time.time() - self.curTime))
self.switchData(self.dataTrain, self.dataTest)
value_dict = self._evaluator.evaluate(self.data_stream)
self.switchData(self.dataTrain, self.dataTest)
self.add_records(self.main_loop.log, value_dict.items())
#switch pointers between x and y
def switchData(self, x, y):
for key in x:
temp = x[key].get_value(borrow=True)
x[key].set_value(y[key].get_value(borrow=True), borrow=True)
y[key].set_value(temp, borrow=True)
def __init__(self, output_vars, train_data_stream, test_data_streams, **kwargs):
super(FinalTestMonitoring, self).__init__(**kwargs)
if not isinstance(test_data_streams, dict):
self.tst_streams = {self.prefix, test_data_streams}
else:
self.tst_streams = test_data_streams
self._tst_evaluator = DatasetEvaluator(output_vars)
def __init__(self, variables, data_stream,
updates=None, noise_parameters=None, **kwargs):
kwargs.setdefault("after_epoch", True)
kwargs.setdefault("before_first_epoch", True)
super(DataStreamMonitoring, self).__init__(**kwargs)
self._evaluator = DatasetEvaluator(variables, updates)
self.data_stream = data_stream
self.noise_parameters = noise_parameters
def __init__(self, output_vars, train_data_stream, test_data_stream,
**kwargs):
output_vars = self.replicate_vars(output_vars)
super(FinalTestMonitoring, self).__init__(**kwargs)
self.trn_stream = train_data_stream
self.tst_stream = test_data_stream
bn_ps, bn_share, output_vars_replaced = self._get_bn_params(
output_vars)
if self._bn:
updates = self._get_updates(bn_ps, bn_share)
trn_evaluator = DatasetEvaluator(bn_ps, updates=updates)
else:
trn_evaluator = None
self._trn_evaluator = trn_evaluator
self._tst_evaluator = DatasetEvaluator(output_vars_replaced)
def test_min_max_aggregators():
num_examples = 4
batch_size = 2
features = numpy.array([[2, 3], [2, 9], [2, 4], [5, 1]],
dtype=theano.config.floatX)
dataset = IndexableDataset(OrderedDict([('features', features)]))
data_stream = DataStream(dataset,
iteration_scheme=SequentialScheme(
num_examples, batch_size))
x = tensor.matrix('features')
y = (x**2).sum(axis=0)
y.name = 'y'
z = y.min()
z.name = 'z'
y.tag.aggregation_scheme = Maximum(y)
z.tag.aggregation_scheme = Minimum(z)
assert_allclose(
DatasetEvaluator([y]).evaluate(data_stream)['y'],
numpy.array([29, 90], dtype=theano.config.floatX))
assert_allclose(
DatasetEvaluator([z]).evaluate(data_stream)['z'],
numpy.array([8], dtype=theano.config.floatX))
# Make sure accumulators are reset.
features = numpy.array([[2, 1], [1, 3], [1, -1], [2.5, 1]],
dtype=theano.config.floatX)
dataset = IndexableDataset(OrderedDict([('features', features)]))
data_stream = DataStream(dataset,
iteration_scheme=SequentialScheme(
num_examples, batch_size))
assert_allclose(
DatasetEvaluator([y]).evaluate(data_stream)['y'],
numpy.array([7.25, 10], dtype=theano.config.floatX))
assert_allclose(
DatasetEvaluator([z]).evaluate(data_stream)['z'],
numpy.array([2], dtype=theano.config.floatX))
class FinalTestMonitoring(SimpleExtension, MonitoringExtension,
TestMonitoring):
"""Monitors validation and test set data with batch norm
Calculates the training set statistics for batch normalization and adds
them to the model before calculating the validation and test set values.
This is done in two steps: First the training set is iterated and the
statistics are saved in shared variables, then the model iterates through
the test/validation set using the saved shared variables.
When the training set is iterated, it is done for the full set, layer by
layer so that the statistics are correct. This is expensive for very deep
models, in which case some approximation could be in order
"""
def __init__(self, output_vars, train_data_stream, test_data_stream,
**kwargs):
output_vars = self.replicate_vars(output_vars)
super(FinalTestMonitoring, self).__init__(**kwargs)
self.trn_stream = train_data_stream
self.tst_stream = test_data_stream
bn_ps, bn_share, output_vars_replaced = self._get_bn_params(
output_vars)
if self._bn:
updates = self._get_updates(bn_ps, bn_share)
trn_evaluator = DatasetEvaluator(bn_ps, updates=updates)
else:
trn_evaluator = None
self._trn_evaluator = trn_evaluator
self._tst_evaluator = DatasetEvaluator(output_vars_replaced)
def _get_updates(self, bn_ps, bn_share):
cg = ComputationGraph(bn_ps)
# Only store updates that relate to params or the counter
updates = OrderedDict([(up, cg.updates[up]) for up in cg.updates
if up.name == 'counter' or up in bn_share])
assert self._counter == self._param_from_updates(cg.updates, 'counter')
assert self._counter_max == self._param_from_updates(
cg.updates, 'counter_max')
assert len(updates) == len(bn_ps) + 1, \
'Counter or var missing from update'
return updates
def do(self, which_callback, *args):
"""Write the values of monitored variables to the log."""
assert not which_callback == "after_batch", "Do not monitor each mb"
# Run on train data and get the statistics
if self._bn:
self._counter_max.set_value(np.float32(np.inf))
self.reset_counter()
self._trn_evaluator.evaluate(self.trn_stream)
self.reset_counter()
value_dict = self._tst_evaluator.evaluate(self.tst_stream)
self.add_records(self.main_loop.log, list(value_dict.items()))
class FinalTestMonitoring(SimpleExtension, MonitoringExtension, TestMonitoring):
"""Monitors validation and test set data with batch norm
Calculates the training set statistics for batch normalization and adds
them to the model before calculating the validation and test set values.
This is done in two steps: First the training set is iterated and the
statistics are saved in shared variables, then the model iterates through
the test/validation set using the saved shared variables.
When the training set is iterated, it is done for the full set, layer by
layer so that the statistics are correct. This is expensive for very deep
models, in which case some approximation could be in order
"""
def __init__(self, output_vars, train_data_stream, test_data_stream,
**kwargs):
output_vars = self.replicate_vars(output_vars)
super(FinalTestMonitoring, self).__init__(**kwargs)
self.trn_stream = train_data_stream
self.tst_stream = test_data_stream
bn_ps, bn_share, output_vars_replaced = self._get_bn_params(output_vars)
if self._bn:
updates = self._get_updates(bn_ps, bn_share)
trn_evaluator = DatasetEvaluator(bn_ps, updates=updates)
else:
trn_evaluator = None
self._trn_evaluator = trn_evaluator
self._tst_evaluator = DatasetEvaluator(output_vars_replaced)
def _get_updates(self, bn_ps, bn_share):
cg = ComputationGraph(bn_ps)
# Only store updates that relate to params or the counter
updates = OrderedDict([(up, cg.updates[up]) for up in
cg.updates if up.name == 'counter' or
up in bn_share])
assert self._counter == self._param_from_updates(cg.updates, 'counter')
assert self._counter_max == self._param_from_updates(cg.updates,
'counter_max')
assert len(updates) == len(bn_ps) + 1, \
'Counter or var missing from update'
return updates
def do(self, which_callback, *args):
"""Write the values of monitored variables to the log."""
assert not which_callback == "after_batch", "Do not monitor each mb"
# Run on train data and get the statistics
if self._bn:
self._counter_max.set_value(np.float32(np.inf))
self.reset_counter()
self._trn_evaluator.evaluate(self.trn_stream)
self.reset_counter()
value_dict = self._tst_evaluator.evaluate(self.tst_stream)
self.add_records(self.main_loop.log, value_dict.items())
def test_dataset_evaluators():
X = theano.tensor.vector('X')
Y = theano.tensor.vector('Y')
data = [numpy.arange(1, 7, dtype=theano.config.floatX).reshape(3, 2),
numpy.arange(11, 17, dtype=theano.config.floatX).reshape(3, 2)]
data_stream = IterableDataset(dict(X=data[0],
Y=data[1])).get_example_stream()
validator = DatasetEvaluator([
CrossEntropy(requires=[X, Y],
name="monitored_cross_entropy0"),
# to test two same quantities and make sure that state will be reset
CrossEntropy(requires=[X, Y],
name="monitored_cross_entropy1"),
CategoricalCrossEntropy().apply(X, Y), ])
values = validator.evaluate(data_stream)
numpy.testing.assert_allclose(
values['monitored_cross_entropy1'],
values['categoricalcrossentropy_apply_cost'])
def test_dataset_evaluators():
X = theano.tensor.vector('X')
Y = theano.tensor.vector('Y')
data = [
numpy.arange(1, 7, dtype=theano.config.floatX).reshape(3, 2),
numpy.arange(11, 17, dtype=theano.config.floatX).reshape(3, 2)
]
data_stream = IterableDataset(dict(X=data[0],
Y=data[1])).get_example_stream()
validator = DatasetEvaluator([
CrossEntropy(requires=[X, Y], name="monitored_cross_entropy0"),
# to test two same quantities and make sure that state will be reset
CrossEntropy(requires=[X, Y], name="monitored_cross_entropy1"),
CategoricalCrossEntropy().apply(X, Y),
])
values = validator.evaluate(data_stream)
numpy.testing.assert_allclose(values['monitored_cross_entropy1'],
values['categoricalcrossentropy_apply_cost'])
class ActpicExtension(SimpleExtension):
def __init__(self, actpic_variables=None, pics=None, case_labels=None,
label_count=None, data_stream=None, rectify=False, **kwargs):
center_val = 0.5
self.input_pics = pics
# self.batch_size = batch_size
# self.label_count = label_count
self.actpic_variables = actpic_variables
# attributes pics: (cases, picy, picx) to (cases, labels, picy, picx)
# attributed_pics = tensor.batched_tensordot(
# tensor.extra_ops.to_one_hot(case_labels.flatten(), label_count),
# pics[:, 0, :, :], axes=0)
zeroed_pics = pics - 0.5
attributed_pics = tensor.batched_tensordot(
tensor.extra_ops.to_one_hot(
case_labels.flatten(), label_count),
zeroed_pics[:, 0, :, :],
axes=0)
self.actpics = [self._create_actpic_image_for(
name + '_actpic', var, attributed_pics, rectify)
for name, var in self.actpic_variables.items()]
self.evaluator = DatasetEvaluator(self.actpics)
self.data_stream = data_stream
self.results = None
super(ActpicExtension, self).__init__(**kwargs)
def do(self, callback_name, *args):
self.parse_args(callback_name, args)
self.results = self.evaluator.evaluate(self.data_stream)
def _create_actpic_image_for(self, name, var, pics, rectify):
# var is (cases, unit1, unit2, ...)
# pics is (cases, labels, picy, picx)
# output is (unit1, unit2... labels, picy, picx)
# TODO: for convolutions, use gaussian to take contribution
# just around the receptive field of the neuron. For now we
# just take the whole image as contribution as follows.
while var.ndim > 2:
var = var.sum(axis=var.ndim - 1)
if rectify == -1:
var = -tensor.nnet.relu(-var)
elif rectify:
var = tensor.nnet.relu(var)
return tensor.tensordot(var, pics, axes=[[0],[0]]).copy(name=name)
def get_picdata(self):
result = OrderedDict()
for name, actpic in self.results.items():
layername = name[:-7] # trim '_actpic' from the end
result[layername] = actpic
return result
def __init__(self, actpic_variables=None, pics=None, case_labels=None,
label_count=None, data_stream=None, rectify=False, **kwargs):
center_val = 0.5
self.input_pics = pics
# self.batch_size = batch_size
# self.label_count = label_count
self.actpic_variables = actpic_variables
# attributes pics: (cases, picy, picx) to (cases, labels, picy, picx)
# attributed_pics = tensor.batched_tensordot(
# tensor.extra_ops.to_one_hot(case_labels.flatten(), label_count),
# pics[:, 0, :, :], axes=0)
zeroed_pics = pics - 0.5
attributed_pics = tensor.batched_tensordot(
tensor.extra_ops.to_one_hot(
case_labels.flatten(), label_count),
zeroed_pics[:, 0, :, :],
axes=0)
self.actpics = [self._create_actpic_image_for(
name + '_actpic', var, attributed_pics, rectify)
for name, var in self.actpic_variables.items()]
self.evaluator = DatasetEvaluator(self.actpics)
self.data_stream = data_stream
self.results = None
super(ActpicExtension, self).__init__(**kwargs)
def test_concatenate_aggregator():
num_examples = 4
batch_size = 2
features = numpy.array([[2, 3], [2, 9], [2, 4], [5, 1]],
dtype=theano.config.floatX)
dataset = IndexableDataset(OrderedDict([('features', features)]))
data_stream = DataStream(dataset,
iteration_scheme=SequentialScheme(
num_examples, batch_size))
x = tensor.matrix('features')
y = x.sum(axis=0).copy('y')
z = y.sum(axis=0).copy('z')
y.tag.aggregation_scheme = Concatenate(y)
z.tag.aggregation_scheme = Concatenate(z)
assert_allclose(
DatasetEvaluator([y]).evaluate(data_stream)['y'],
numpy.array([[4, 12], [7, 5]], dtype=theano.config.floatX))
assert_allclose(
DatasetEvaluator([z]).evaluate(data_stream)['z'],
numpy.array([16, 12], dtype=theano.config.floatX))
def test_dataset_evaluators():
X = theano.tensor.matrix('X')
brick = TestBrick(name='test_brick')
Y = brick.apply(X)
graph = ComputationGraph([Y])
monitor_variables = [v for v in graph.auxiliary_variables]
validator = DatasetEvaluator(monitor_variables)
data = [numpy.arange(1, 5, dtype=theano.config.floatX).reshape(2, 2),
numpy.arange(10, 16, dtype=theano.config.floatX).reshape(3, 2)]
data_stream = IterableDataset(dict(X=data)).get_example_stream()
values = validator.evaluate(data_stream)
assert values['test_brick_apply_V_squared'] == 4
numpy.testing.assert_allclose(
values['test_brick_apply_mean_row_mean'], numpy.vstack(data).mean())
per_batch_mean = numpy.mean([batch.mean() for batch in data])
numpy.testing.assert_allclose(
values['test_brick_apply_mean_batch_element'], per_batch_mean)
with assert_raises(Exception) as ar:
data_stream = IterableDataset(dict(X2=data)).get_example_stream()
validator.evaluate(data_stream)
assert "Not all data sources" in ar.exception.args[0]
def __init__(self, output_vars, train_data_stream, test_data_stream,
**kwargs):
output_vars = self.replicate_vars(output_vars)
super(FinalTestMonitoring, self).__init__(**kwargs)
self.trn_stream = train_data_stream
self.tst_stream = test_data_stream
bn_ps, bn_share, output_vars_replaced = self._get_bn_params(output_vars)
if self._bn:
updates = self._get_updates(bn_ps, bn_share)
trn_evaluator = DatasetEvaluator(bn_ps, updates=updates)
else:
trn_evaluator = None
self._trn_evaluator = trn_evaluator
self._tst_evaluator = DatasetEvaluator(output_vars_replaced)
class FinalTestMonitoring(SimpleExtension, MonitoringExtension):
"""Monitors validation and test set data with batch norm
Calculates the training set statistics for batch normalization and adds
them to the model before calculating the validation and test set values.
This is done in two steps: First the training set is iterated and the
statistics are saved in shared variables, then the model iterates through
the test/validation set using the saved shared variables.
When the training set is iterated, it is done for the full set, layer by
layer so that the statistics are correct. This is expensive for very deep
models, in which case some approximation could be in order
"""
def __init__(self, output_vars, train_data_stream, test_data_streams,
**kwargs):
super(FinalTestMonitoring, self).__init__(**kwargs)
if not isinstance(test_data_streams, dict):
self.tst_streams = {self.prefix, test_data_streams}
else:
self.tst_streams = test_data_streams
self._tst_evaluator = DatasetEvaluator(output_vars)
def do(self, which_callback, *args):
"""Write the values of monitored variables to the log."""
# Run on train data and get the statistics
logger.info("Evaluating final test/validation monitor...")
for prefix, tst_stream in self.tst_streams.iteritems():
value_dict = self._tst_evaluator.evaluate(tst_stream)
self.add_records(self.main_loop.log, value_dict.items(), prefix)
def _record_name(self, name, prefix=None):
"""The record name for a variable name."""
PREFIX_SEPARATOR = '_'
prefix = prefix or self.prefix
return prefix + PREFIX_SEPARATOR + name if prefix else name
def add_records(self, log, record_tuples, prefix=None):
"""Helper function to add monitoring records to the log."""
for name, value in record_tuples:
if not name:
raise ValueError("monitor variable without name")
log.current_row[self._record_name(name, prefix)] = value
class FinalTestMonitoring(SimpleExtension, MonitoringExtension):
"""Monitors validation and test set data with batch norm
Calculates the training set statistics for batch normalization and adds
them to the model before calculating the validation and test set values.
This is done in two steps: First the training set is iterated and the
statistics are saved in shared variables, then the model iterates through
the test/validation set using the saved shared variables.
When the training set is iterated, it is done for the full set, layer by
layer so that the statistics are correct. This is expensive for very deep
models, in which case some approximation could be in order
"""
def __init__(self, output_vars, train_data_stream, test_data_streams, **kwargs):
super(FinalTestMonitoring, self).__init__(**kwargs)
if not isinstance(test_data_streams, dict):
self.tst_streams = {self.prefix, test_data_streams}
else:
self.tst_streams = test_data_streams
self._tst_evaluator = DatasetEvaluator(output_vars)
def do(self, which_callback, *args):
"""Write the values of monitored variables to the log."""
# Run on train data and get the statistics
logger.info("Evaluating final test/validation monitor...")
for prefix, tst_stream in self.tst_streams.iteritems():
value_dict = self._tst_evaluator.evaluate(tst_stream)
self.add_records(self.main_loop.log, value_dict.items(), prefix)
def _record_name(self, name, prefix=None):
"""The record name for a variable name."""
PREFIX_SEPARATOR = '_'
prefix = prefix or self.prefix
return prefix + PREFIX_SEPARATOR + name if prefix else name
def add_records(self, log, record_tuples, prefix=None):
"""Helper function to add monitoring records to the log."""
for name, value in record_tuples:
if not name:
raise ValueError("monitor variable without name")
log.current_row[self._record_name(name, prefix)] = value
class NoisyDataStreamMonitoring(DataStreamMonitoring):
def __init__(self, variables, data_stream,
updates=None, noise_parameters=None, **kwargs):
kwargs.setdefault("after_epoch", True)
kwargs.setdefault("before_first_epoch", True)
super(DataStreamMonitoring, self).__init__(**kwargs)
self._evaluator = DatasetEvaluator(variables, updates)
self.data_stream = data_stream
self.noise_parameters = noise_parameters
def do(self, callback_name, *args):
"""Write the values of monitored variables to the log."""
logger.info("Monitoring on auxiliary data started")
saved = [(p, p.get_value()) for p in self.noise_parameters]
for (p, v) in saved:
p.set_value(np.zeros(v.shape, dtype=v.dtype))
value_dict = self._evaluator.evaluate(self.data_stream)
self.add_records(self.main_loop.log, value_dict.items())
for (p, v) in saved:
p.set_value(v)
logger.info("Monitoring on auxiliary data finished")
class DataStreamMonitoring(SimpleExtension, MonitoringExtension):
"""Monitors Theano variables and monitored-quantities on a data stream.
By default monitoring is done before the first and after every epoch.
Parameters
----------
variables : list of :class:`~tensor.TensorVariable` and
:class:`MonitoredQuantity`
The variables to monitor. The variable names are used as record
names in the logs.
updates : list of tuples or :class:`~collections.OrderedDict` or None
:class:`~tensor.TensorSharedVariable` updates to be performed
during evaluation. This parameter is only for Theano variables.
Be careful not to update any model parameters as this is not
intended to alter your model in any meaningful way. A typical
use case of this option arises when the theano function used
for evaluation contains a call to :func:`~theano.scan` which
might have returned shared variable updates.
data_stream : instance of :class:`.DataStream`
The data stream to monitor on. A data epoch is requested
each time monitoring is done.
"""
PREFIX_SEPARATOR = '_'
def __init__(self, variables, data_stream, updates=None, **kwargs):
kwargs.setdefault("after_epoch", True)
kwargs.setdefault("before_first_epoch", True)
super(DataStreamMonitoring, self).__init__(**kwargs)
self._evaluator = DatasetEvaluator(variables, updates)
self.data_stream = data_stream
def do(self, callback_name, *args):
"""Write the values of monitored variables to the log."""
logger.info("Monitoring on auxiliary data started")
value_dict = self._evaluator.evaluate(self.data_stream)
self.add_records(self.main_loop.log, value_dict.items())
logger.info("Monitoring on auxiliary data finished")
def __init__(self, variables, data_stream, path, tolerance, updates=None, parameters=None, save_separately=None,
save_main_loop=True, use_cpickle=False, strict_mode=True, **kwargs):
if len(variables) != 1:
raise ValueError("Please specify exactly one variable (packed in a list)!")
# DatastreamMonitoring part
kwargs.setdefault("after_epoch", True)
kwargs.setdefault("before_first_epoch", True)
super(EarlyStopping, self).__init__(**kwargs)
self._evaluator = DatasetEvaluator(variables, updates)
self.data_stream = data_stream
# Checkpoint part
super(EarlyStopping, self).__init__(**kwargs)
self.path = path
self.parameters = parameters
self.save_separately = save_separately
self.save_main_loop = save_main_loop
self.use_cpickle = use_cpickle
# additional quantities for early stopping
self.tolerance = tolerance
self.n_since_improvement = 0
self.last_value = float("inf")
self.best_value = float("inf")
self.strict_mode = strict_mode
bstats_mean[k] = v / float(nbatches)
#for popstat, value in updates:
#popstat.set_value(bstats_mean[popstat.name])
new_popstats = dict((popstat, popstat.get_value()) for popstat, _ in updates)
from blocks.monitoring.evaluators import DatasetEvaluator
results = dict()
for situation in "training inference".split():
results[situation] = dict()
outputs, = [
extension._evaluator.theano_variables
for extension in main_loop.extensions
if getattr(extension, "prefix", None) == "valid_%s" % situation
]
evaluator = DatasetEvaluator(outputs)
for which_set in "train valid test".split():
if which_set == "test":
results[situation][which_set] = evaluator.evaluate(
get_stream(which_set=which_set, batch_size=5000))
else:
results[situation][which_set] = evaluator.evaluate(
get_stream(which_set=which_set, batch_size=5000))
results["proper_test"] = evaluator.evaluate(
get_stream(which_set="test", batch_size=1000))
print 'Results: ', results["proper_test"]
import cPickle
cPickle.dump(
dict(results=results, old_popstats=old_popstats,
new_popstats=new_popstats),
def evaluate(args, main_loop):
# load parameters of trained model
trained_main_loop = load(args.evaluate)
transfer_parameters(trained_main_loop, main_loop)
del trained_main_loop
# extract population statistic updates
updates = [
update for update in main_loop.algorithm.updates
# FRAGILE
if re.search("_(mean|var)$", update[0].name)
]
print updates
old_popstats = dict(
(popstat, popstat.get_value()) for popstat, _ in updates)
# baseline doesn't need all this
if updates:
train_stream = get_stream(which_set="train",
batch_size=1000,
length=args.length)
nbatches = len(list(train_stream.get_epoch_iterator()))
# destructure moving average expression to construct a new expression
new_updates = []
for popstat, value in updates:
# FRAGILE
assert value.owner.op.scalar_op == theano.scalar.add
terms = value.owner.inputs
# right multiplicand of second term is popstat
assert popstat in theano.gof.graph.ancestors(
[terms[1].owner.inputs[1]])
# right multiplicand of first term is batchstat
batchstat = terms[0].owner.inputs[1]
old_popstats[popstat] = popstat.get_value()
# FRAGILE: assume population statistics not used in computation of batch statistics
# otherwise popstat should always have a reasonable value
popstat.set_value(0 * popstat.get_value(borrow=True))
new_updates.append(
(popstat, popstat + batchstat / float(nbatches)))
# FRAGILE: assume all the other algorithm updates are unneeded for computation of batch statistics
estimate_fn = theano.function(main_loop.algorithm.inputs, [],
updates=new_updates,
on_unused_input="warn")
print("averaging batch statistics over", nbatches, "batches")
for batch in train_stream.get_epoch_iterator(as_dict=True):
estimate_fn(**batch)
sys.stdout.write(".")
sys.stdout.flush()
print
new_popstats = dict(
(popstat, popstat.get_value()) for popstat, _ in updates)
from blocks.monitoring.evaluators import DatasetEvaluator
results = dict()
for situation in "training inference".split():
results[situation] = dict()
outputs, = [
extension._evaluator.theano_variables
for extension in main_loop.extensions
if getattr(extension, "prefix", None) == "valid_%s" % situation
]
evaluator = DatasetEvaluator(outputs)
for which_set in "valid test".split():
print(situation, which_set)
results[situation][which_set] = OrderedDict(
(length,
evaluator.evaluate(
get_stream(
which_set=which_set, batch_size=100, length=length)))
for length in [1000])
try:
results["proper_test"] = evaluator.evaluate(
get_stream(which_set="test", batch_size=1, length=5 * 10**6))
except:
# that will probably run out of memory
pass
import cPickle
cPickle.dump(
dict(results=results,
old_popstats=old_popstats,
new_popstats=new_popstats),
open(sys.argv[1] + "_popstat_results.pkl", "w"))
def __init__(self, save_to):
batch_size = 500
image_size = (28, 28)
output_size = 10
convnet = create_lenet_5()
layers = convnet.layers
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(
i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
mnist_test = MNIST(("test",), sources=['features', 'targets'])
basis = create_fair_basis(mnist_test, 10, 10)
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outs = OrderedDict((full_brick_name(get_brick(out)), out)
for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(
cg.variables))
# Normalize input and apply the convnet
error_rate = (MisclassificationRate().apply(y.flatten(), probs)
.copy(name='error_rate'))
confusion = (ConfusionMatrix().apply(y.flatten(), probs)
.copy(name='confusion'))
confusion.tag.aggregation_scheme = Sum(confusion)
confusion_image = (ConfusionImage().apply(y.flatten(), probs, x)
.copy(name='confusion_image'))
confusion_image.tag.aggregation_scheme = Sum(confusion_image)
model = Model(
[error_rate, confusion, confusion_image] + list(outs.values()))
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
mnist_test = MNIST(("test",))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=SequentialScheme(
mnist_test.num_examples, batch_size))
self.model = model
self.mnist_test_stream = mnist_test_stream
self.evaluator = DatasetEvaluator(
[error_rate, confusion, confusion_image])
self.base_results = self.evaluator.evaluate(mnist_test_stream)
# TODO: allow target layer to be parameterized
self.target_layer = '/lenet/mlp/linear_0'
self.next_layer_param = '/lenet/mlp/linear_1.W'
self.base_sample = extract_sample(
outs[self.target_layer], mnist_test_stream)
self.base_param_value = (
model.get_parameter_dict()[
self.next_layer_param].get_value().copy())
o4 = Logistic().apply(n4)
a5 = l5.apply(o4)
n5, M5, S5 = normalize(a5, output_dim = 10)
probs = Softmax().apply(n5)
statistics_list=[(M1,S1,a1), (M2,S2,a2), (M3,S3,a3), (M4,S4,a4), (M5,S5,a5)]
# initialize_variables
# for variable (M,S) in variables:
# compute M and S in the whole data.
if normalization == 'bn2':
for m,s,var in statistics_list:
var.tag.aggregation_scheme = MeanAndVariance(var, var.shape[0], axis = 0)
init_mn, init_var = DatasetEvaluator([var]).evaluate(stream_train)[var.name]
m.set_value(init_mn.astype(floatX))
s.set_value(sqrt(init_var).astype(floatX))
cost = CategoricalCrossEntropy().apply(y.flatten(), probs)
cost.name = 'cost'
error_rate = MisclassificationRate().apply(y.flatten(), probs)
error_rate.name = 'error_rate'
cg = ComputationGraph([cost])
parameters = cg.parameters
# add gradient descent to M,S
if normalization == 'bn2':
for m,s,var in statistics_list:
parameters.extend([m,s])
#for popstat, value in updates:
#popstat.set_value(bstats_mean[popstat.name])
new_popstats = dict((popstat, popstat.get_value()) for popstat, _ in updates)
from blocks.monitoring.evaluators import DatasetEvaluator
results = dict()
for situation in "training inference".split():
results[situation] = dict()
outputs, = [
extension._evaluator.theano_variables
for extension in main_loop.extensions
if getattr(extension, "prefix", None) == "valid_%s" % situation]
evaluator = DatasetEvaluator(outputs)
for which_set in "train valid test".split():
if which_set == "test":
results[situation][which_set] = evaluator.evaluate(get_stream(which_set=which_set,
batch_size=5000))
else:
results[situation][which_set] = evaluator.evaluate(get_stream(which_set=which_set,
batch_size=5000))
results["proper_test"] = evaluator.evaluate(
get_stream(
which_set="test",
batch_size=1000))
print 'Results: ', results["proper_test"]
import cPickle
cPickle.dump(dict(results=results,
class EarlyStopping(SimpleExtension, MonitoringExtension):
"""
Extension that tracks some quantity (PLEASE use cost on the validation set here!) and creates a checkpoint whenever
a new best-value-so-far has been achieved. In addition, training is stopped if performance hasn't improved since n
epochs. Note: "Improvement" is not needed over the current best value, but only over the last time we measured.
You can call this as often as you like. Default is after every epoch. Note that if you want to call it more often
(e.g. using every_n_batches), you might wanna increase n (i.e. the tolerance for no improvement) accordingly.
ALSO NOTE that this will combine both a DatastreamMonitoring on the passed quantity as well as a kinda FinishAfter
extension, so you don't need to pass those to the MainLoop. Of course you still could -- another monitoring would
just do the work again, which isn't that dramatic (albeit useless), while an "explicit" FinishAfter could be used to
set a maximum on the number of epochs we want to train for, in addition to the early stopping.
"""
def __init__(self, variables, data_stream, path, tolerance, updates=None, parameters=None, save_separately=None,
save_main_loop=True, use_cpickle=False, strict_mode=True, **kwargs):
if len(variables) != 1:
raise ValueError("Please specify exactly one variable (packed in a list)!")
# DatastreamMonitoring part
kwargs.setdefault("after_epoch", True)
kwargs.setdefault("before_first_epoch", True)
super(EarlyStopping, self).__init__(**kwargs)
self._evaluator = DatasetEvaluator(variables, updates)
self.data_stream = data_stream
# Checkpoint part
super(EarlyStopping, self).__init__(**kwargs)
self.path = path
self.parameters = parameters
self.save_separately = save_separately
self.save_main_loop = save_main_loop
self.use_cpickle = use_cpickle
# additional quantities for early stopping
self.tolerance = tolerance
self.n_since_improvement = 0
self.last_value = float("inf")
self.best_value = float("inf")
self.strict_mode = strict_mode
def do(self, callback_name, *args):
value_dict = self.do_monitoring()
# see if we have an improvement from the best (or last) measure
# check out this awesome code duplication
current_value = value_dict.values()[0]
if self.strict_mode:
if current_value < self.best_value:
logger.info("Got an improvement over best measure; tolerance reset.")
self.n_since_improvement = 0
else:
self.n_since_improvement += 1
if self.tolerance - self.n_since_improvement >= 0:
logger.info("No improvement over best measure! Tolerating " +
str(self.tolerance - self.n_since_improvement) + " more...")
else:
if current_value < self.last_value:
# if so, we reset our counter of how-long-since-improvement
logger.info("Got an improvement over last measure; tolerance reset.")
self.n_since_improvement = 0
else:
self.n_since_improvement += 1
if self.tolerance - self.n_since_improvement >= 0:
logger.info("No improvement since the last measure! Tolerating " +
str(self.tolerance - self.n_since_improvement) + " more...")
self.last_value = current_value
# if we have an improvement over the best yet, we store that and make a checkpoint
if current_value < self.best_value:
logger.info("Got a new best value! Saving model...")
self.do_checkpoint(callback_name, *args)
self.best_value = current_value
# went too long without improvement? Giving up is the obvious solution.
if self.n_since_improvement > self.tolerance:
logger.info("Thou hast exceeded that tolerance of mine for thy miserable performance!")
# Note that the way this comparison is set up enforces the following interpretation on the tolerance
# parameter: The number of times in a row we will accept no improvement. E.g. if it set to 0, we will stop
# if we *ever* get no improvement on the tracked value.
self.main_loop.log.current_row['training_finish_requested'] = True
def do_monitoring(self):
logger.info("Monitoring on auxiliary data started")
value_dict = self._evaluator.evaluate(self.data_stream)
self.add_records(self.main_loop.log, value_dict.items())
logger.info("Monitoring on auxiliary data finished")
return value_dict
def do_checkpoint(self, callback_name, *args):
logger.info("Checkpointing has started")
_, from_user = self.parse_args(callback_name, args)
try:
path = self.path
if from_user:
path, = from_user
to_add = None
if self.save_separately:
to_add = {attr: getattr(self.main_loop, attr) for attr in
self.save_separately}
if self.parameters is None:
if hasattr(self.main_loop, 'model'):
self.parameters = self.main_loop.model.parameters
object_ = None
if self.save_main_loop:
object_ = self.main_loop
secure_dump(object_, path,
dump_function=dump_and_add_to_dump,
parameters=self.parameters,
to_add=to_add,
use_cpickle=self.use_cpickle)
except Exception:
path = None
raise
finally:
already_saved_to = self.main_loop.log.current_row.get(SAVED_TO, ())
self.main_loop.log.current_row[SAVED_TO] = (already_saved_to +
(path,))
logger.info("Checkpointing has finished")
def evaluate(args, main_loop):
# load parameters of trained model
trained_main_loop = load(args.evaluate)
transfer_parameters(trained_main_loop, main_loop)
del trained_main_loop
# extract population statistic updates
updates = [
update
for update in main_loop.algorithm.updates
# FRAGILE
if re.search("_(mean|var)$", update[0].name)
]
print updates
old_popstats = dict((popstat, popstat.get_value()) for popstat, _ in updates)
# baseline doesn't need all this
if updates:
train_stream = get_stream(which_set="train", batch_size=1000, length=args.length)
nbatches = len(list(train_stream.get_epoch_iterator()))
# destructure moving average expression to construct a new expression
new_updates = []
for popstat, value in updates:
# FRAGILE
assert value.owner.op.scalar_op == theano.scalar.add
terms = value.owner.inputs
# right multiplicand of second term is popstat
assert popstat in theano.gof.graph.ancestors([terms[1].owner.inputs[1]])
# right multiplicand of first term is batchstat
batchstat = terms[0].owner.inputs[1]
old_popstats[popstat] = popstat.get_value()
# FRAGILE: assume population statistics not used in computation of batch statistics
# otherwise popstat should always have a reasonable value
popstat.set_value(0 * popstat.get_value(borrow=True))
new_updates.append((popstat, popstat + batchstat / float(nbatches)))
# FRAGILE: assume all the other algorithm updates are unneeded for computation of batch statistics
estimate_fn = theano.function(main_loop.algorithm.inputs, [], updates=new_updates, on_unused_input="warn")
print ("averaging batch statistics over", nbatches, "batches")
for batch in train_stream.get_epoch_iterator(as_dict=True):
estimate_fn(**batch)
sys.stdout.write(".")
sys.stdout.flush()
print
new_popstats = dict((popstat, popstat.get_value()) for popstat, _ in updates)
from blocks.monitoring.evaluators import DatasetEvaluator
results = dict()
for situation in "training inference".split():
results[situation] = dict()
outputs, = [
extension._evaluator.theano_variables
for extension in main_loop.extensions
if getattr(extension, "prefix", None) == "valid_%s" % situation
]
evaluator = DatasetEvaluator(outputs)
for which_set in "valid test".split():
print (situation, which_set)
results[situation][which_set] = OrderedDict(
(length, evaluator.evaluate(get_stream(which_set=which_set, batch_size=100, length=length)))
for length in [1000]
)
try:
results["proper_test"] = evaluator.evaluate(get_stream(which_set="test", batch_size=1, length=5 * 10 ** 6))
except:
# that will probably run out of memory
pass
import cPickle
cPickle.dump(
dict(results=results, old_popstats=old_popstats, new_popstats=new_popstats),
open(sys.argv[1] + "_popstat_results.pkl", "w"),
)
def train_lstm(train, test, input_dim,
hidden_dimension, columns, epochs,
save_file, execution_name, batch_size, plot):
stream_train = build_stream(train, batch_size, columns)
stream_test = build_stream(test, batch_size, columns)
# The train stream will return (TimeSequence, BatchSize, Dimensions) for
# and the train test will return (TimeSequence, BatchSize, 1)
x = T.tensor3('x')
y = T.tensor3('y')
y = y.reshape((y.shape[1], y.shape[0], y.shape[2]))
# input_dim = 6
# output_dim = 1
linear_lstm = LinearLSTM(input_dim, 1, hidden_dimension,
# print_intermediate=True,
print_attrs=['__str__', 'shape'])
y_hat = linear_lstm.apply(x)
linear_lstm.initialize()
c_test = AbsolutePercentageError().apply(y, y_hat)
c_test.name = 'mape'
c = SquaredError().apply(y, y_hat)
c.name = 'cost'
cg = ComputationGraph(c_test)
def one_perc_min(current_value, best_value):
if (1 - best_value / current_value) > 0.01:
return best_value
else:
return current_value
extensions = []
extensions.append(DataStreamMonitoring(variables=[c, c_test],
data_stream=stream_test,
prefix='test',
after_epoch=False,
every_n_epochs=100))
extensions.append(TrainingDataMonitoring(variables=[c_test],
prefix='train',
after_epoch=True))
extensions.append(FinishAfter(after_n_epochs=epochs))
# extensions.append(Printing())
# extensions.append(ProgressBar())
extensions.append(TrackTheBest('test_mape', choose_best=one_perc_min))
extensions.append(TrackTheBest('test_cost', choose_best=one_perc_min))
extensions.append(FinishIfNoImprovementAfter('test_cost_best_so_far', epochs=500))
# Save only parameters, not the whole main loop and only when best_test_cost is updated
checkpoint = Checkpoint(save_file, save_main_loop=False, after_training=False)
checkpoint.add_condition(['after_epoch'], predicate=OnLogRecord('test_cost_best_so_far'))
extensions.append(checkpoint)
if BOKEH_AVAILABLE and plot:
extensions.append(Plot(execution_name, channels=[[ # 'train_cost',
'test_cost']]))
step_rule = Adam()
algorithm = GradientDescent(cost=c_test, parameters=cg.parameters, step_rule=step_rule)
main_loop = MainLoop(algorithm, stream_train, model=Model(c_test), extensions=extensions)
main_loop.run()
test_mape = 0
if main_loop.log.status.get('best_test_mape', None) is None:
with open(save_file, 'rb') as f:
parameters = load_parameters(f)
model = main_loop.model
model.set_parameter_values(parameters)
ev = DatasetEvaluator([c_test])
test_mape = ev.evaluate(stream_test)['mape']
else:
test_mape = main_loop.log.status['best_test_mape']
return test_mape, main_loop.log.status['epochs_done']
def main(save_to, hist_file):
batch_size = 365
feature_maps = [6, 16]
mlp_hiddens = [120, 84]
conv_sizes = [5, 5]
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations,
1,
image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='valid',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info(
"Input dim: {} {} {}".format(*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
mnist_test = MNIST(("test", ), sources=['features', 'targets'])
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
error_rate = (MisclassificationRate().apply(y.flatten(),
probs).copy(name='error_rate'))
confusion = (ConfusionMatrix().apply(y.flatten(),
probs).copy(name='confusion'))
confusion.tag.aggregation_scheme = Sum(confusion)
model = Model([error_rate, confusion])
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outs = OrderedDict(
(full_brick_name(get_brick(out)), out) for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(model.variables))
# Load histogram information
with open(hist_file, 'rb') as handle:
histograms = pickle.load(handle)
# Corpora
mnist_train = MNIST(("train", ))
mnist_train_stream = DataStream.default_stream(
mnist_train,
iteration_scheme=ShuffledScheme(mnist_train.num_examples, batch_size))
mnist_test = MNIST(("test", ))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=ShuffledScheme(mnist_test.num_examples, batch_size))
# Probe the given layer
target_layer = '/lenet/mlp/linear_0'
next_layer_param = '/lenet/mlp/linear_1.W'
sample = extract_sample(outs[target_layer], mnist_test_stream)
print('sample shape', sample.shape)
# Figure neurons to ablate
hist = histograms[('linear_1', 'b')]
targets = [i for i in range(hist.shape[1]) if hist[2, i] * hist[7, i] < 0]
print('ablating', len(targets), ':', targets)
# Now adjust the next layer weights based on the probe
param = model.get_parameter_dict()[next_layer_param]
print('param shape', param.get_value().shape)
new_weights = ablate_inputs(targets,
sample,
param.get_value(),
compensate=False)
param.set_value(new_weights)
# Evaluation pass
evaluator = DatasetEvaluator([error_rate, confusion])
print(evaluator.evaluate(mnist_test_stream))
def main(save_to, hist_file):
batch_size = 365
feature_maps = [6, 16]
mlp_hiddens = [120, 84]
conv_sizes = [5, 5]
pool_sizes = [2, 2]
image_size = (28, 28)
output_size = 10
# The above are from LeCun's paper. The blocks example had:
# feature_maps = [20, 50]
# mlp_hiddens = [500]
# Use ReLUs everywhere and softmax for the final prediction
conv_activations = [Rectifier() for _ in feature_maps]
mlp_activations = [Rectifier() for _ in mlp_hiddens] + [Softmax()]
convnet = LeNet(conv_activations, 1, image_size,
filter_sizes=zip(conv_sizes, conv_sizes),
feature_maps=feature_maps,
pooling_sizes=zip(pool_sizes, pool_sizes),
top_mlp_activations=mlp_activations,
top_mlp_dims=mlp_hiddens + [output_size],
border_mode='valid',
weights_init=Uniform(width=.2),
biases_init=Constant(0))
# We push initialization config to set different initialization schemes
# for convolutional layers.
convnet.push_initialization_config()
convnet.layers[0].weights_init = Uniform(width=.2)
convnet.layers[1].weights_init = Uniform(width=.09)
convnet.top_mlp.linear_transformations[0].weights_init = Uniform(width=.08)
convnet.top_mlp.linear_transformations[1].weights_init = Uniform(width=.11)
convnet.initialize()
logging.info("Input dim: {} {} {}".format(
*convnet.children[0].get_dim('input_')))
for i, layer in enumerate(convnet.layers):
if isinstance(layer, Activation):
logging.info("Layer {} ({})".format(
i, layer.__class__.__name__))
else:
logging.info("Layer {} ({}) dim: {} {} {}".format(
i, layer.__class__.__name__, *layer.get_dim('output')))
mnist_test = MNIST(("test",), sources=['features', 'targets'])
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
error_rate = (MisclassificationRate().apply(y.flatten(), probs)
.copy(name='error_rate'))
confusion = (ConfusionMatrix().apply(y.flatten(), probs)
.copy(name='confusion'))
confusion.tag.aggregation_scheme = Sum(confusion)
model = Model([error_rate, confusion])
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outs = OrderedDict((full_brick_name(get_brick(out)), out)
for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(
model.variables))
# Load histogram information
with open(hist_file, 'rb') as handle:
histograms = pickle.load(handle)
# Corpora
mnist_train = MNIST(("train",))
mnist_train_stream = DataStream.default_stream(
mnist_train, iteration_scheme=ShuffledScheme(
mnist_train.num_examples, batch_size))
mnist_test = MNIST(("test",))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=ShuffledScheme(
mnist_test.num_examples, batch_size))
# Probe the given layer
target_layer = '/lenet/mlp/linear_0'
next_layer_param = '/lenet/mlp/linear_1.W'
sample = extract_sample(outs[target_layer], mnist_test_stream)
print('sample shape', sample.shape)
# Figure neurons to ablate
hist = histograms[('linear_1', 'b')]
targets = [i for i in range(hist.shape[1])
if hist[2, i] * hist[7, i] < 0]
print('ablating', len(targets), ':', targets)
# Now adjust the next layer weights based on the probe
param = model.get_parameter_dict()[next_layer_param]
print('param shape', param.get_value().shape)
new_weights = ablate_inputs(
targets,
sample,
param.get_value(),
compensate=False)
param.set_value(new_weights)
# Evaluation pass
evaluator = DatasetEvaluator([error_rate, confusion])
print(evaluator.evaluate(mnist_test_stream))
def __init__(self, save_to):
batch_size = 500
image_size = (28, 28)
output_size = 10
convnet = create_lenet_5()
layers = convnet.layers
mnist_test = MNIST(("test", ), sources=['features', 'targets'])
x = tensor.tensor4('features')
y = tensor.lmatrix('targets')
# Normalize input and apply the convnet
probs = convnet.apply(x)
cg = ComputationGraph([probs])
def full_brick_name(brick):
return '/'.join([''] + [b.name for b in brick.get_unique_path()])
# Find layer outputs to probe
outmap = OrderedDict(
(full_brick_name(get_brick(out)), out) for out in VariableFilter(
roles=[OUTPUT], bricks=[Convolutional, Linear])(cg.variables))
# Generate pics for biases
biases = VariableFilter(roles=[BIAS])(cg.parameters)
# Generate parallel array, in the same order, for outputs
outs = [outmap[full_brick_name(get_brick(b))] for b in biases]
# Figure work count
error_rate = (MisclassificationRate().apply(
y.flatten(), probs).copy(name='error_rate'))
sensitive_unit_count = (SensitiveUnitCount().apply(
y.flatten(), probs, biases).copy(name='sensitive_unit_count'))
sensitive_unit_count.tag.aggregation_scheme = (
Concatenate(sensitive_unit_count))
active_unit_count = (ActiveUnitCount().apply(outs).copy(
name='active_unit_count'))
active_unit_count.tag.aggregation_scheme = (
Concatenate(active_unit_count))
ignored_unit_count = (IgnoredUnitCount().apply(
y.flatten(), probs, biases, outs).copy(name='ignored_unit_count'))
ignored_unit_count.tag.aggregation_scheme = (
Concatenate(ignored_unit_count))
model = Model([
error_rate, sensitive_unit_count, active_unit_count,
ignored_unit_count
])
# Load it with trained parameters
params = load_parameters(open(save_to, 'rb'))
model.set_parameter_values(params)
mnist_test = MNIST(("test", ))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=SequentialScheme(mnist_test.num_examples,
batch_size))
evaluator = DatasetEvaluator([
error_rate, sensitive_unit_count, active_unit_count,
ignored_unit_count
])
results = evaluator.evaluate(mnist_test_stream)
def save_ranked_image(scores, filename):
sorted_instances = scores.argsort()
filmstrip = Filmstrip(image_shape=(28, 28), grid_shape=(100, 100))
for i, index in enumerate(sorted_instances):
filmstrip.set_image((i // 100, i % 100),
mnist_test.get_data(request=index)[0])
filmstrip.save(filename)
save_ranked_image(results['sensitive_unit_count'], 'sensitive.jpg')
save_ranked_image(results['active_unit_count'], 'active.jpg')
save_ranked_image(results['ignored_unit_count'], 'ignored.jpg')
on_unused_input="warn")
for batch in train_stream.get_epoch_iterator(as_dict=True):
estimate_fn(**batch)
new_popstats = dict((popstat, popstat.get_value()) for popstat, _ in updates)
from blocks.monitoring.evaluators import DatasetEvaluator
results = dict()
for situation in "training inference".split():
results[situation] = dict()
outputs, = [
extension._evaluator.theano_variables
for extension in main_loop.extensions
if getattr(extension, "prefix", None) == "valid_%s" % situation
]
evaluator = DatasetEvaluator(outputs)
for which_set in "train valid test".split():
results[situation][which_set] = OrderedDict(
(length,
evaluator.evaluate(
get_stream(which_set=which_set,
batch_size=100,
augment=False,
length=length))) for length in
[50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000])
results["proper_test"] = evaluator.evaluate(
get_stream(which_set="test", batch_size=1, length=446184))
import cPickle
cPickle.dump(
estimate_fn = theano.function(main_loop.algorithm.inputs, [],
updates=new_updates, on_unused_input="warn")
for batch in train_stream.get_epoch_iterator(as_dict=True):
estimate_fn(**batch)
new_popstats = dict((popstat, popstat.get_value()) for popstat, _ in updates)
from blocks.monitoring.evaluators import DatasetEvaluator
results = dict()
for situation in "training inference".split():
results[situation] = dict()
outputs, = [
extension._evaluator.theano_variables
for extension in main_loop.extensions
if getattr(extension, "prefix", None) == "valid_%s" % situation]
evaluator = DatasetEvaluator(outputs)
for which_set in "train valid test".split():
results[situation][which_set] = OrderedDict(
(length, evaluator.evaluate(get_stream(
which_set=which_set,
batch_size=100,
augment=False,
length=length)))
for length in [50, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000])
results["proper_test"] = evaluator.evaluate(
get_stream(
which_set="test",
batch_size=1,
length=446184))
