class TrainingDataMonitoring(SimpleExtension, MonitoringExtension):
"""Monitors values of Theano variables on training batches.
Use this extension to monitor a quantity on every training batch
cheaply. It integrates with the training algorithm in order to avoid
recomputing same things several times. For instance, if you are
training a network and you want to log the norm of the gradient on
every batch, the backpropagation will only be done once. By
controlling the frequency with which the :meth:`do` method is called,
you can aggregate the monitored variables, e.g. only log the gradient
norm average over an epoch.
Parameters
----------
variables : list of :class:`~tensor.TensorVariable`
The variables to monitor. The variable names are used as record
names in the logs.
Notes
-----
All the monitored variables are evaluated _before_ the parameter
update.
Requires the training algorithm to be an instance of
:class:`.DifferentiableCostMinimizer`.
"""
def __init__(self, variables, **kwargs):
kwargs.setdefault("before_training", True)
super(TrainingDataMonitoring, self).__init__(**kwargs)
self._buffer = AggregationBuffer(variables, use_take_last=True)
self._last_time_called = -1
def do(self, callback_name, *args):
"""Initializes the buffer or commits the values to the log.
What this method does depends on from what callback it is called.
When called within `before_training`, it initializes the
aggregation buffer and instructs the training algorithm what
additional computations should be carried at each step by adding
corresponding updates to it. In all other cases it writes
aggregated values of the monitored variables to the log.
"""
if callback_name == 'before_training':
if not isinstance(self.main_loop.algorithm,
DifferentiableCostMinimizer):
raise ValueError
self.main_loop.algorithm.add_updates(
self._buffer.accumulation_updates)
self._buffer.initialize_aggregators()
else:
if (self.main_loop.status['iterations_done'] ==
self._last_time_called):
raise Exception("TrainingDataMonitoring.do should be invoked"
" no more than once per iteration")
self._last_time_called = self.main_loop.status['iterations_done']
self.add_records(self.main_loop.log,
self._buffer.get_aggregated_values().items())
self._buffer.initialize_aggregators()
def __init__(self,
variables,
mini_batch_size,
state_updates,
dataset,
updates=None):
theano_variables = []
monitored_quantities = []
for variable in variables:
if isinstance(variable, MonitoredQuantity):
monitored_quantities.append(variable)
else:
theano_variables.append(variable)
self.theano_variables = theano_variables
self.monitored_quantities = monitored_quantities
variable_names = [v.name for v in variables]
if len(set(variable_names)) < len(variables):
raise ValueError("variables should have different names")
self.theano_buffer = AggregationBuffer(theano_variables)
self.monitored_quantities_buffer = MonitoredQuantityBuffer(
monitored_quantities)
self.dataset = dataset
self.updates = updates
self.mini_batch_size = mini_batch_size
self._compile(state_updates)
def __init__(self, variables, **kwargs):
kwargs.setdefault("before_training", True)
super(TrainingDataMonitoring, self).__init__(**kwargs)
self.add_condition(['after_batch'], arguments=('just_aggregate',))
self._non_variables = []
self._variables = []
for variable_or_not in variables:
if isinstance(variable_or_not, theano.Variable):
self._variables.append(variable_or_not)
elif isinstance(variable_or_not, MonitoredQuantity):
self._non_variables.append(variable_or_not)
else:
raise ValueError("can not monitor {}".format(variable_or_not))
self._non_variables = MonitoredQuantityBuffer(self._non_variables)
self._required_for_non_variables = AggregationBuffer(
[take_last(v) for v in self._non_variables.requires])
self._variables = AggregationBuffer(
self._variables, use_take_last=True)
self._last_time_called = -1
def __init__(self, variables, **kwargs):
kwargs.setdefault("before_training", True)
super(TrainingDataMonitoring, self).__init__(**kwargs)
self.add_condition(['after_batch'], arguments=('just_aggregate', ))
self._non_variables = []
self._variables = []
for variable_or_not in variables:
if isinstance(variable_or_not, theano.Variable):
self._variables.append(variable_or_not)
elif isinstance(variable_or_not, MonitoredQuantity):
self._non_variables.append(variable_or_not)
else:
raise ValueError("can not monitor {}".format(variable_or_not))
self._non_variables = MonitoredQuantityBuffer(self._non_variables)
self._required_for_non_variables = AggregationBuffer(
[take_last(v) for v in self._non_variables.requires])
self._variables = AggregationBuffer(self._variables,
use_take_last=True)
self._last_time_called = -1
def __init__(self,
picsources=None,
pattern=None,
title=None,
data=None,
graph=None,
graph_len=None,
unit_order=None,
**kwargs):
kwargs.setdefault("before_training", True)
self.picsources = picsources
self.count = 0
if pattern is None:
pattern = 'pics/syn/%s_%04d.jpg'
self.pattern = pattern
self.unit_order = unit_order
self.title = title
self.data = data
self.graph = graph
self.graph_len = graph_len
self.graph_data = None
# Now create an AggregationBuffer for theano variables to monitor
self.variables = AggregationBuffer(data, use_take_last=True)
super(SaveImages, self).__init__(**kwargs)
def __init__(self, picsources=None, pattern=None,
title=None, data=None, graph=None, graph_len=None,
unit_order=None, **kwargs):
kwargs.setdefault("before_training", True)
self.picsources = picsources
self.count = 0
if pattern is None:
pattern = 'pics/syn/%s_%04d.jpg'
self.pattern = pattern
self.unit_order = unit_order
self.title = title
self.data = data
self.graph = graph
self.graph_len = graph_len
self.graph_data = None
# Now create an AggregationBuffer for theano variables to monitor
self.variables = AggregationBuffer(data, use_take_last=True)
super(SaveImages, self).__init__(**kwargs)
def __init__(self, variables, updates=None):
theano_variables = []
monitored_quantities = []
for variable in variables:
if isinstance(variable, MonitoredQuantity):
monitored_quantities.append(variable)
else:
theano_variables.append(variable)
self.theano_variables = theano_variables
self.monitored_quantities = monitored_quantities
variable_names = [v.name for v in variables]
if len(set(variable_names)) < len(variables):
raise ValueError("variables should have different names")
self.theano_buffer = AggregationBuffer(theano_variables)
self.monitored_quantities_buffer = MonitoredQuantityBuffer(
monitored_quantities)
self.updates = updates
self._compile()
state_to_compare_1 = list(filter(lambda x: x.name == 'sequencegenerator_cost_matrix_states#1', sc.state_variables))[0]
state_to_compare_2 = list(filter(lambda x: x.name == 'sequencegenerator_cost_matrix_states#2', sc.state_variables))[0] # notice: python2 filter seems to return a list, but anyway
# The thing that I feed into the parameters argument I copied from some blocks-examples thing. the good old computation
# graph and then cg.parameters should work as well.
# On the step rule: This means gradient clipping (threshold passed in as a command line argument) followed by Adam
# performed on the clipped gradient.
# If you don't know what gradient clipping is: Define threshold T; if the length of the gradient is > T, scale it such
# that its length is equal to T. This serves to make exploding gradients less severe.
algorithm = GradientDescent(cost=cross_ent, parameters=cost_model.parameters,
step_rule=CompositeRule(components=[StepClipping(threshold=args.clipping), Adam()]),
on_unused_sources="ignore")
aggr_0 = AggregationBuffer(variables=[state_to_compare_0], use_take_last=True)
aggr_0.initialize_aggregators()
check_value_0 = shared(zeros((network.transitions[-1].dim), dtype='float32')) # TODO: is it necessary or recommendable to add these to the computation graph is constants or sth like that?
def modifier_function_0(iterations_done, old_value):
"""
Note about this method: The accumulator seems to get the state vector in random order
:param iterations_done:
:param old_value:
:return:
"""
values = aggr_0.get_aggregated_values()
new_value = values[state_to_compare_0.name]
aggr_0.initialize_aggregators() # TODO what's the purpose of that? I observed them do it in the monitoring extensions after every request
#print('0:OLD ..:', old_value)
#print('0:NEW in:', new_value[-1][0], new_value[0][0], sep='\n')
class TrainingDataMonitoring(SimpleExtension, MonitoringExtension):
"""Monitors values of Theano variables on training batches.
Use this extension to monitor a quantity on every training batch
cheaply. It integrates with the training algorithm in order to avoid
recomputing same things several times. For instance, if you are
training a network and you want to log the norm of the gradient on
every batch, the backpropagation will only be done once. By
controlling the frequency with which the :meth:`do` method is called,
you can aggregate the monitored variables, e.g. only log the gradient
norm average over an epoch.
Parameters
----------
variables : list of :class:`~tensor.TensorVariable` or
:class:`~blocks.monitoring.aggregation.MonitoredQuantity`
The variables or non-Theano quantities to monitor.
The variable names are used as record names in the logs.
Notes
-----
All the monitored variables are evaluated _before_ the parameter
update.
Requires the training algorithm to be an instance of
:class:`.DifferentiableCostMinimizer`.
"""
def __init__(self, variables, **kwargs):
kwargs.setdefault("before_training", True)
super(TrainingDataMonitoring, self).__init__(**kwargs)
self.add_condition(['after_batch'], arguments=('just_aggregate',))
self._non_variables = []
self._variables = []
for variable_or_not in variables:
if isinstance(variable_or_not, theano.Variable):
self._variables.append(variable_or_not)
elif isinstance(variable_or_not, MonitoredQuantity):
self._non_variables.append(variable_or_not)
else:
raise ValueError("can not monitor {}".format(variable_or_not))
self._non_variables = MonitoredQuantityBuffer(self._non_variables)
self._required_for_non_variables = AggregationBuffer(
[take_last(v) for v in self._non_variables.requires])
self._variables = AggregationBuffer(
self._variables, use_take_last=True)
self._last_time_called = -1
def do(self, callback_name, *args):
"""Initializes the buffer or commits the values to the log.
What this method does depends on from what callback it is called
and with which arguments. When called within `before_training`, it
initializes the aggregation buffer and instructs the training
algorithm what additional computations should be carried at each
step by adding corresponding updates to it. In most_other cases it
writes aggregated values of the monitored variables to the log. An
exception is when an argument `just_aggregate` is given: in this
cases it updates the values of monitored non-Theano quantities, but
does not write anything to the log.
"""
data, args = self.parse_args(callback_name, args)
if callback_name == 'before_training':
if not isinstance(self.main_loop.algorithm,
DifferentiableCostMinimizer):
raise ValueError
self.main_loop.algorithm.add_updates(
self._variables.accumulation_updates)
self.main_loop.algorithm.add_updates(
self._required_for_non_variables.accumulation_updates)
self._variables.initialize_aggregators()
self._required_for_non_variables.initialize_aggregators()
self._non_variables.initialize_quantities()
else:
# When called first time at any iterations, update
# monitored non-Theano quantities
if (self.main_loop.status['iterations_done'] >
self._last_time_called):
self._non_variables.aggregate_quantities(
list(self._required_for_non_variables
.get_aggregated_values().values()))
self._required_for_non_variables.initialize_aggregators()
self._last_time_called = (
self.main_loop.status['iterations_done'])
# If only called to update non-Theano quantities,
# do just that
if args == ('just_aggregate',):
return
# Otherwise, also output current values of from the accumulators
# to the log.
self.add_records(
self.main_loop.log,
self._variables.get_aggregated_values().items())
self._variables.initialize_aggregators()
self.add_records(
self.main_loop.log,
self._non_variables.get_aggregated_values().items())
self._non_variables.initialize_quantities()
class TrainingDataMonitoring(SimpleExtension, MonitoringExtension):
"""Monitors values of Theano variables on training batches.
Use this extension to monitor a quantity on every training batch
cheaply. It integrates with the training algorithm in order to avoid
recomputing same things several times. For instance, if you are
training a network and you want to log the norm of the gradient on
every batch, the backpropagation will only be done once. By
controlling the frequency with which the :meth:`do` method is called,
you can aggregate the monitored variables, e.g. only log the gradient
norm average over an epoch.
Parameters
----------
variables : list of :class:`~tensor.TensorVariable` or
:class:`~blocks.monitoring.aggregation.MonitoredQuantity`
The variables or non-Theano quantities to monitor.
The variable names are used as record names in the logs.
Notes
-----
All the monitored variables are evaluated _before_ the parameter
update.
Requires the training algorithm to be an instance of
:class:`.DifferentiableCostMinimizer`.
"""
def __init__(self, variables, **kwargs):
kwargs.setdefault("before_training", True)
super(TrainingDataMonitoring, self).__init__(**kwargs)
self.add_condition(['after_batch'], arguments=('just_aggregate', ))
self._non_variables = []
self._variables = []
for variable_or_not in variables:
if isinstance(variable_or_not, theano.Variable):
self._variables.append(variable_or_not)
elif isinstance(variable_or_not, MonitoredQuantity):
self._non_variables.append(variable_or_not)
else:
raise ValueError("can not monitor {}".format(variable_or_not))
self._non_variables = MonitoredQuantityBuffer(self._non_variables)
self._required_for_non_variables = AggregationBuffer(
[take_last(v) for v in self._non_variables.requires])
self._variables = AggregationBuffer(self._variables,
use_take_last=True)
self._last_time_called = -1
def do(self, callback_name, *args):
"""Initializes the buffer or commits the values to the log.
What this method does depends on from what callback it is called
and with which arguments. When called within `before_training`, it
initializes the aggregation buffer and instructs the training
algorithm what additional computations should be carried at each
step by adding corresponding updates to it. In most_other cases it
writes aggregated values of the monitored variables to the log. An
exception is when an argument `just_aggregate` is given: in this
cases it updates the values of monitored non-Theano quantities, but
does not write anything to the log.
"""
data, args = self.parse_args(callback_name, args)
if callback_name == 'before_training':
if not isinstance(self.main_loop.algorithm,
DifferentiableCostMinimizer):
raise ValueError
self.main_loop.algorithm.add_updates(
self._variables.accumulation_updates)
self.main_loop.algorithm.add_updates(
self._required_for_non_variables.accumulation_updates)
self._variables.initialize_aggregators()
self._required_for_non_variables.initialize_aggregators()
self._non_variables.initialize_quantities()
else:
# When called first time at any iterations, update
# monitored non-Theano quantities
if (self.main_loop.status['iterations_done'] >
self._last_time_called):
self._non_variables.aggregate_quantities(
list(self._required_for_non_variables.
get_aggregated_values().values()))
self._required_for_non_variables.initialize_aggregators()
self._last_time_called = (
self.main_loop.status['iterations_done'])
# If only called to update non-Theano quantities,
# do just that
if args == ('just_aggregate', ):
return
# Otherwise, also output current values of from the accumulators
# to the log.
self.add_records(self.main_loop.log,
self._variables.get_aggregated_values().items())
self._variables.initialize_aggregators()
self.add_records(
self.main_loop.log,
self._non_variables.get_aggregated_values().items())
self._non_variables.initialize_quantities()
class SaveImages(SimpleExtension):
def __init__(self,
picsources=None,
pattern=None,
title=None,
data=None,
graph=None,
graph_len=None,
unit_order=None,
**kwargs):
kwargs.setdefault("before_training", True)
self.picsources = picsources
self.count = 0
if pattern is None:
pattern = 'pics/syn/%s_%04d.jpg'
self.pattern = pattern
self.unit_order = unit_order
self.title = title
self.data = data
self.graph = graph
self.graph_len = graph_len
self.graph_data = None
# Now create an AggregationBuffer for theano variables to monitor
self.variables = AggregationBuffer(data, use_take_last=True)
super(SaveImages, self).__init__(**kwargs)
def do(self, callback_name, *args):
self.parse_args(callback_name, args)
if (callback_name == 'before_training'):
self.main_loop.algorithm.add_updates(
self.variables.accumulation_updates)
self.variables.initialize_aggregators()
else:
title = self.title
if self.data:
values = dict(
(k, float(v))
for k, v in self.variables.get_aggregated_values().items())
values['i'] = self.count
title = title.format(**values)
graph = None
if self.graph:
if self.graph_data is None:
self.graph_data = numpy.array(())
self.graph_data = numpy.append(self.graph_data,
values[self.graph])
graph = self.graph_data / self.graph_data.max()
filename = self.pattern % ('composite', self.count)
self.count += 1
picdata = [ps.get_picdata() for ps in self.picsources]
self.save_composite_image(title=title,
graph=graph,
graph_len=self.graph_len,
picdata=picdata,
unit_order=self.unit_order,
filename=filename,
aspect_ratio=16.0 / 9.0)
self.variables.initialize_aggregators()
def save_composite_image(self,
title=None,
graph=None,
graph_len=None,
picdata=None,
filename=None,
aspect_ratio=None,
unit_order=None):
if filename is None:
pattern = 'synpic.jpg'
unit_count = 0
layer_count = 0
if graph is not None:
unit_count += 4 # TODO: make configurable
if title is not None:
unit_count += 1
merged = OrderedDict([(k, [d[k] for d in picdata])
for k in picdata[0].keys()])
unit_width = 0
for name, d in merged.items():
for dat in d:
if len(dat.shape) != 4:
raise NotImplementedError('%s has %s dimensions' %
(name, dat.shape))
unit_count += dat.shape[0]
unit_width = max(unit_width, dat.shape[1])
layer_count += 1
unit_width += 1
column_height, column_count = plan_grid(unit_count + layer_count,
aspect_ratio, dat.shape[-2:],
(1, unit_width))
filmstrip = Filmstrip(image_shape=dat.shape[-2:],
grid_shape=(column_height,
column_count * unit_width))
pos = 0
if graph is not None:
col, row = divmod(pos, column_height)
filmstrip.plot_graph((row, col * unit_width + 1),
(4, unit_width - 1), graph, graph_len)
pos += 4
if title is not None:
col, row = divmod(pos, column_height)
filmstrip.set_text((row, col * unit_width + unit_width // 2),
title)
pos += 1
for layername, d in merged.items():
units = d[0].shape[0]
col, row = divmod(pos, column_height)
filmstrip.set_text((row, col * unit_width + unit_width // 2),
layername)
pos += 1
if unit_order:
ordering = unit_order[layername]
else:
ordering = range(units)
scales = [dat.std() * 5 for dat in d]
for unit in ordering:
for dat, scale in zip(d, scales):
col, row = divmod(pos, column_height)
filmstrip.set_text((row, col * unit_width), "%d:" % unit)
im = dat[unit, :, :, :]
# imin = im.min()
# imax = im.max()
# scale = (imax - imin) * 0.7
im = im / (scale + 1e-9) + 0.5
for label in range(im.shape[0]):
filmstrip.set_image(
(row, label + 1 + col * unit_width),
im[label, :, :])
pos += 1
filmstrip.save(filename)
class myTrainingDataMonitoring(SimpleExtension, MonitoringExtension):
def __init__(self, variables, saveEveryXIteration, **kwargs):
self.saveEveryXIteration = saveEveryXIteration
kwargs.setdefault("before_training", True)
super(myTrainingDataMonitoring, self).__init__(**kwargs)
self.add_condition(['after_batch'], arguments=('just_aggregate', ))
self._non_variables = []
self._variables = []
for variable_or_not in variables:
if isinstance(variable_or_not, theano.Variable):
self._variables.append(variable_or_not)
elif isinstance(variable_or_not, MonitoredQuantity):
self._non_variables.append(variable_or_not)
else:
raise ValueError("can not monitor {}".format(variable_or_not))
self._non_variables = MonitoredQuantityBuffer(self._non_variables)
self._required_for_non_variables = AggregationBuffer(
[take_last(v) for v in self._non_variables.requires])
self._variables = AggregationBuffer(self._variables,
use_take_last=True)
self._last_time_called = -1
def do(self, callback_name, *args):
data, args = self.parse_args(callback_name, args)
if callback_name == 'before_training':
if not isinstance(self.main_loop.algorithm,
DifferentiableCostMinimizer):
raise ValueError
self.main_loop.algorithm.add_updates(
self._variables.accumulation_updates)
self.main_loop.algorithm.add_updates(
self._required_for_non_variables.accumulation_updates)
self._variables.initialize_aggregators()
self._required_for_non_variables.initialize_aggregators()
self._non_variables.initialize_quantities()
else:
log = self.main_loop.log
iteration = log.status['iterations_done']
if iteration % self.saveEveryXIteration == 0:
# When called first time at any iterations, update
# monitored non-Theano quantities
if (self.main_loop.status['iterations_done'] >
self._last_time_called):
self._non_variables.aggregate_quantities(
list(self._required_for_non_variables.
get_aggregated_values().values()))
self._required_for_non_variables.initialize_aggregators()
self._last_time_called = (
self.main_loop.status['iterations_done'])
# If only called to update non-Theano quantities,
# do just that
if args == ('just_aggregate', ):
return
# Otherwise, also output current values of from the accumulators
# to the log.
self.add_records(
self.main_loop.log,
self._variables.get_aggregated_values().items())
self._variables.initialize_aggregators()
self.add_records(
self.main_loop.log,
self._non_variables.get_aggregated_values().items())
self._non_variables.initialize_quantities()
def __init__(self, variables, **kwargs):
kwargs.setdefault("before_training", True)
super(TrainingDataMonitoring, self).__init__(**kwargs)
self._buffer = AggregationBuffer(variables, use_take_last=True)
self._last_time_called = -1
class DatasetEvaluator(object):
"""A DatasetEvaluator evaluates many Theano variables or other quantities.
The DatasetEvaluator provides a do-it-all method, :meth:`evaluate`,
which computes values of ``variables`` on a dataset.
Alternatively, methods :meth:`initialize_aggregators`,
:meth:`process_batch`, :meth:`get_aggregated_values` can be used with a
custom loop over data.
The values computed on subsets of the given dataset are aggregated
using the :class:`AggregationScheme`s provided in the
`aggregation_scheme` tags. If no tag is given, the value is **averaged
over minibatches**. However, care is taken to ensure that variables
which do not depend on data are not unnecessarily recomputed.
Parameters
----------
variables : list of :class:`~tensor.TensorVariable` and
:class:`MonitoredQuantity`
The variable names are used as record names in the logs. Hence, all
the names must be different.
Each variable can be tagged with an :class:`AggregationScheme` that
specifies how the value can be computed for a data set by
aggregating minibatches.
updates : list of tuples or :class:`~collections.OrderedDict` or None
:class:`~tensor.TensorSharedVariable` updates to be performed
during evaluation. This parameter is only for Theano variables.
Be careful not to update any model parameters as this is not
intended to alter your model in any meaningfullway. A typical
use case of this option arises when the theano function used
for evaluation contains a call to:function:`~theano.scan` which
might have returned shared variable updates.
"""
def __init__(self, variables, mini_batch_size, state_updates,
dataset, updates=None):
theano_variables = []
monitored_quantities = []
for variable in variables:
if isinstance(variable, MonitoredQuantity):
monitored_quantities.append(variable)
else:
theano_variables.append(variable)
self.theano_variables = theano_variables
self.monitored_quantities = monitored_quantities
variable_names = [v.name for v in variables]
if len(set(variable_names)) < len(variables):
raise ValueError("variables should have different names")
self.theano_buffer = AggregationBuffer(theano_variables)
self.monitored_quantities_buffer = MonitoredQuantityBuffer(
monitored_quantities)
self.dataset = dataset
self.updates = updates
self.mini_batch_size = mini_batch_size
self._compile(state_updates)
def _compile(self, state_updates):
"""Compiles Theano functions.
.. todo::
The current compilation method does not account for updates
attached to `ComputationGraph` elements. Compiling should
be out-sourced to `ComputationGraph` to deal with it.
"""
inputs = []
outputs = []
updates = None
givens, f_updates = carry_hidden_state(state_updates,
self.mini_batch_size,
reset=not(has_indices(self.dataset)))
if self.theano_buffer.accumulation_updates:
updates = OrderedDict()
updates.update(self.theano_buffer.accumulation_updates)
if self.updates:
updates.update(self.updates)
inputs += self.theano_buffer.inputs
inputs += self.monitored_quantities_buffer.inputs
outputs = self.monitored_quantities_buffer.requires
if inputs != []:
self.unique_inputs = list(set(inputs))
updates.update(f_updates)
self._accumulate_fun = theano.function(self.unique_inputs,
outputs,
givens=givens,
updates=updates)
else:
self._accumulate_fun = None
def initialize_aggregators(self):
self.theano_buffer.initialize_aggregators()
self.monitored_quantities_buffer.initialize()
def process_batch(self, batch):
try:
input_names = [v.name for v in self.unique_inputs]
batch = dict_subset(batch, input_names)
except KeyError:
reraise_as(
"Not all data sources required for monitoring were"
" provided. The list of required data sources:"
" {}.".format(input_names))
if self._accumulate_fun is not None:
numerical_values = self._accumulate_fun(**batch)
self.monitored_quantities_buffer.accumulate_quantities(
numerical_values)
def get_aggregated_values(self):
values = self.theano_buffer.get_aggregated_values()
values.update(
self.monitored_quantities_buffer.get_aggregated_values())
return values
def evaluate(self, data_stream):
"""Compute the variables over a data stream.
Parameters
----------
data_stream : instance of :class:`.DataStream`
The data stream. Only the first epoch of data is used.
Returns
-------
A mapping from record names to the values computed on the provided
dataset.
"""
self.initialize_aggregators()
if self._accumulate_fun is not None:
for batch in data_stream.get_epoch_iterator(as_dict=True):
self.process_batch(batch)
else:
logger.debug(
'Only data independent variables were given,'
'will not iterate the over data!')
return self.get_aggregated_values()
class SaveImages(SimpleExtension):
def __init__(self, picsources=None, pattern=None,
title=None, data=None, graph=None, graph_len=None,
unit_order=None, **kwargs):
kwargs.setdefault("before_training", True)
self.picsources = picsources
self.count = 0
if pattern is None:
pattern = 'pics/syn/%s_%04d.jpg'
self.pattern = pattern
self.unit_order = unit_order
self.title = title
self.data = data
self.graph = graph
self.graph_len = graph_len
self.graph_data = None
# Now create an AggregationBuffer for theano variables to monitor
self.variables = AggregationBuffer(data, use_take_last=True)
super(SaveImages, self).__init__(**kwargs)
def do(self, callback_name, *args):
self.parse_args(callback_name, args)
if (callback_name == 'before_training'):
self.main_loop.algorithm.add_updates(
self.variables.accumulation_updates)
self.variables.initialize_aggregators()
else:
title = self.title
if self.data:
values = dict((k, float(v))
for k, v in self.variables.get_aggregated_values().items())
values['i'] = self.count
title = title.format(**values)
graph = None
if self.graph:
if self.graph_data is None:
self.graph_data = numpy.array(())
self.graph_data = numpy.append(
self.graph_data, values[self.graph])
graph = self.graph_data / self.graph_data.max()
filename = self.pattern % ('composite', self.count)
self.count += 1
picdata = [ps.get_picdata() for ps in self.picsources]
self.save_composite_image(
title=title, graph=graph, graph_len=self.graph_len,
picdata=picdata, unit_order=self.unit_order,
filename=filename, aspect_ratio=16.0/9.0)
self.variables.initialize_aggregators()
def save_composite_image(self,
title=None, graph=None, graph_len=None, picdata=None,
filename=None, aspect_ratio=None, unit_order=None):
if filename is None:
pattern = 'synpic.jpg'
unit_count = 0
layer_count = 0
if graph is not None:
unit_count += 4 # TODO: make configurable
if title is not None:
unit_count += 1
merged = OrderedDict([
(k, [d[k] for d in picdata]) for k in picdata[0].keys()])
unit_width = 0
for name, d in merged.items():
for dat in d:
if len(dat.shape) != 4:
raise NotImplementedError('%s has %s dimensions' % (
name, dat.shape))
unit_count += dat.shape[0]
unit_width = max(unit_width, dat.shape[1])
layer_count += 1
unit_width += 1
column_height, column_count = plan_grid(unit_count + layer_count,
aspect_ratio, dat.shape[-2:], (1, unit_width))
filmstrip = Filmstrip(image_shape=dat.shape[-2:],
grid_shape=(column_height, column_count * unit_width))
pos = 0
if graph is not None:
col, row = divmod(pos, column_height)
filmstrip.plot_graph((row, col * unit_width + 1),
(4, unit_width - 1),
graph, graph_len)
pos += 4
if title is not None:
col, row = divmod(pos, column_height)
filmstrip.set_text((row, col * unit_width + unit_width // 2),
title)
pos += 1
for layername, d in merged.items():
units = d[0].shape[0]
col, row = divmod(pos, column_height)
filmstrip.set_text((row, col * unit_width + unit_width // 2),
layername)
pos += 1
if unit_order:
ordering = unit_order[layername]
else:
ordering = range(units)
scales = [dat.std() * 5 for dat in d]
for unit in ordering:
for dat, scale in zip(d, scales):
col, row = divmod(pos, column_height)
filmstrip.set_text((row, col * unit_width), "%d:" % unit)
im = dat[unit, :, :, :]
# imin = im.min()
# imax = im.max()
# scale = (imax - imin) * 0.7
im = im / (scale + 1e-9) + 0.5
for label in range(im.shape[0]):
filmstrip.set_image((row, label + 1 +
col * unit_width), im[label, :, :])
pos += 1
filmstrip.save(filename)
class DatasetEvaluator(object):
"""A DatasetEvaluator evaluates many Theano variables or other quantities.
The DatasetEvaluator provides a do-it-all method, :meth:`evaluate`,
which computes values of ``variables`` on a dataset.
Alternatively, methods :meth:`initialize_aggregators`,
:meth:`process_batch`, :meth:`get_aggregated_values` can be used with a
custom loop over data.
The values computed on subsets of the given dataset are aggregated
using the :class:`AggregationScheme`s provided in the
`aggregation_scheme` tags. If no tag is given, the value is **averaged
over minibatches**. However, care is taken to ensure that variables
which do not depend on data are not unnecessarily recomputed.
Parameters
----------
variables : list of :class:`~tensor.TensorVariable` and
:class:`MonitoredQuantity`
The variable names are used as record names in the logs. Hence, all
the names must be different.
Each variable can be tagged with an :class:`AggregationScheme` that
specifies how the value can be computed for a data set by
aggregating minibatches.
updates : list of tuples or :class:`~collections.OrderedDict` or None
:class:`~tensor.TensorSharedVariable` updates to be performed
during evaluation. This parameter is only for Theano variables.
Be careful not to update any model parameters as this is not
intended to alter your model in any meaningfullway. A typical
use case of this option arises when the theano function used
for evaluation contains a call to:function:`~theano.scan` which
might have returned shared variable updates.
"""
def __init__(self,
variables,
mini_batch_size,
state_updates,
updates=None):
theano_variables = []
monitored_quantities = []
for variable in variables:
if isinstance(variable, MonitoredQuantity):
monitored_quantities.append(variable)
else:
theano_variables.append(variable)
self.theano_variables = theano_variables
self.monitored_quantities = monitored_quantities
variable_names = [v.name for v in variables]
if len(set(variable_names)) < len(variables):
raise ValueError("variables should have different names")
self.theano_buffer = AggregationBuffer(theano_variables)
self.monitored_quantities_buffer = MonitoredQuantityBuffer(
monitored_quantities)
self.updates = updates
self.mini_batch_size = mini_batch_size
self._compile(state_updates)
def _compile(self, state_updates):
"""Compiles Theano functions.
.. todo::
The current compilation method does not account for updates
attached to `ComputationGraph` elements. Compiling should
be out-sourced to `ComputationGraph` to deal with it.
"""
inputs = []
outputs = []
updates = None
state_vars = [
theano.shared(
numpy.zeros((self.mini_batch_size, v.shape[1].eval()),
dtype=numpy.float32), v.name + '-gen')
for v, _ in state_updates
]
givens = [(v, x) for (v, _), x in zip(state_updates, state_vars)]
f_updates = [(x, upd)
for x, (_, upd) in zip(state_vars, state_updates)]
if self.theano_buffer.accumulation_updates:
updates = OrderedDict()
updates.update(self.theano_buffer.accumulation_updates)
if self.updates:
updates.update(self.updates)
inputs += self.theano_buffer.inputs
inputs += self.monitored_quantities_buffer.inputs
outputs = self.monitored_quantities_buffer.requires
if inputs != []:
self.unique_inputs = list(set(inputs))
updates.update(f_updates)
self._accumulate_fun = theano.function(self.unique_inputs,
outputs,
givens=givens,
updates=updates)
else:
self._accumulate_fun = None
def initialize_aggregators(self):
self.theano_buffer.initialize_aggregators()
self.monitored_quantities_buffer.initialize()
def process_batch(self, batch):
try:
input_names = [v.name for v in self.unique_inputs]
batch = dict_subset(batch, input_names)
except KeyError:
reraise_as("Not all data sources required for monitoring were"
" provided. The list of required data sources:"
" {}.".format(input_names))
if self._accumulate_fun is not None:
numerical_values = self._accumulate_fun(**batch)
self.monitored_quantities_buffer.accumulate_quantities(
numerical_values)
def get_aggregated_values(self):
values = self.theano_buffer.get_aggregated_values()
values.update(self.monitored_quantities_buffer.get_aggregated_values())
return values
def evaluate(self, data_stream):
"""Compute the variables over a data stream.
Parameters
----------
data_stream : instance of :class:`.DataStream`
The data stream. Only the first epoch of data is used.
Returns
-------
A mapping from record names to the values computed on the provided
dataset.
"""
self.initialize_aggregators()
if self._accumulate_fun is not None:
for batch in data_stream.get_epoch_iterator(as_dict=True):
self.process_batch(batch)
else:
logger.debug('Only data independent variables were given,'
'will not iterate the over data!')
return self.get_aggregated_values()