def __init__(self,
variables,
mini_batch_size,
state_updates,
dataset,
updates=None):
theano_variables = []
monitored_quantities = []
for variable in variables:
if isinstance(variable, MonitoredQuantity):
monitored_quantities.append(variable)
else:
theano_variables.append(variable)
self.theano_variables = theano_variables
self.monitored_quantities = monitored_quantities
variable_names = [v.name for v in variables]
if len(set(variable_names)) < len(variables):
raise ValueError("variables should have different names")
self.theano_buffer = AggregationBuffer(theano_variables)
self.monitored_quantities_buffer = MonitoredQuantityBuffer(
monitored_quantities)
self.dataset = dataset
self.updates = updates
self.mini_batch_size = mini_batch_size
self._compile(state_updates)
def __init__(self, variables, **kwargs):
kwargs.setdefault("before_training", True)
super(TrainingDataMonitoring, self).__init__(**kwargs)
self.add_condition(['after_batch'], arguments=('just_aggregate', ))
self._non_variables = []
self._variables = []
for variable_or_not in variables:
if isinstance(variable_or_not, theano.Variable):
self._variables.append(variable_or_not)
elif isinstance(variable_or_not, MonitoredQuantity):
self._non_variables.append(variable_or_not)
else:
raise ValueError("can not monitor {}".format(variable_or_not))
self._non_variables = MonitoredQuantityBuffer(self._non_variables)
self._required_for_non_variables = AggregationBuffer(
[take_last(v) for v in self._non_variables.requires])
self._variables = AggregationBuffer(self._variables,
use_take_last=True)
self._last_time_called = -1
def __init__(self, variables, updates=None):
theano_variables = []
monitored_quantities = []
for variable in variables:
if isinstance(variable, MonitoredQuantity):
monitored_quantities.append(variable)
else:
theano_variables.append(variable)
self.theano_variables = theano_variables
self.monitored_quantities = monitored_quantities
variable_names = [v.name for v in variables]
if len(set(variable_names)) < len(variables):
raise ValueError("variables should have different names")
self.theano_buffer = AggregationBuffer(theano_variables)
self.monitored_quantities_buffer = MonitoredQuantityBuffer(
monitored_quantities)
self.updates = updates
self._compile()
def __init__(self, variables, **kwargs):
kwargs.setdefault("before_training", True)
super(TrainingDataMonitoring, self).__init__(**kwargs)
self.add_condition(['after_batch'], arguments=('just_aggregate',))
self._non_variables = []
self._variables = []
for variable_or_not in variables:
if isinstance(variable_or_not, theano.Variable):
self._variables.append(variable_or_not)
elif isinstance(variable_or_not, MonitoredQuantity):
self._non_variables.append(variable_or_not)
else:
raise ValueError("can not monitor {}".format(variable_or_not))
self._non_variables = MonitoredQuantityBuffer(self._non_variables)
self._required_for_non_variables = AggregationBuffer(
[take_last(v) for v in self._non_variables.requires])
self._variables = AggregationBuffer(
self._variables, use_take_last=True)
self._last_time_called = -1
class TrainingDataMonitoring(SimpleExtension, MonitoringExtension):
"""Monitors values of Theano variables on training batches.
Use this extension to monitor a quantity on every training batch
cheaply. It integrates with the training algorithm in order to avoid
recomputing same things several times. For instance, if you are
training a network and you want to log the norm of the gradient on
every batch, the backpropagation will only be done once. By
controlling the frequency with which the :meth:`do` method is called,
you can aggregate the monitored variables, e.g. only log the gradient
norm average over an epoch.
Parameters
----------
variables : list of :class:`~tensor.TensorVariable` or
:class:`~blocks.monitoring.aggregation.MonitoredQuantity`
The variables or non-Theano quantities to monitor.
The variable names are used as record names in the logs.
Notes
-----
All the monitored variables are evaluated _before_ the parameter
update.
Requires the training algorithm to be an instance of
:class:`.DifferentiableCostMinimizer`.
"""
def __init__(self, variables, **kwargs):
kwargs.setdefault("before_training", True)
super(TrainingDataMonitoring, self).__init__(**kwargs)
self.add_condition(['after_batch'], arguments=('just_aggregate',))
self._non_variables = []
self._variables = []
for variable_or_not in variables:
if isinstance(variable_or_not, theano.Variable):
self._variables.append(variable_or_not)
elif isinstance(variable_or_not, MonitoredQuantity):
self._non_variables.append(variable_or_not)
else:
raise ValueError("can not monitor {}".format(variable_or_not))
self._non_variables = MonitoredQuantityBuffer(self._non_variables)
self._required_for_non_variables = AggregationBuffer(
[take_last(v) for v in self._non_variables.requires])
self._variables = AggregationBuffer(
self._variables, use_take_last=True)
self._last_time_called = -1
def do(self, callback_name, *args):
"""Initializes the buffer or commits the values to the log.
What this method does depends on from what callback it is called
and with which arguments. When called within `before_training`, it
initializes the aggregation buffer and instructs the training
algorithm what additional computations should be carried at each
step by adding corresponding updates to it. In most_other cases it
writes aggregated values of the monitored variables to the log. An
exception is when an argument `just_aggregate` is given: in this
cases it updates the values of monitored non-Theano quantities, but
does not write anything to the log.
"""
data, args = self.parse_args(callback_name, args)
if callback_name == 'before_training':
if not isinstance(self.main_loop.algorithm,
DifferentiableCostMinimizer):
raise ValueError
self.main_loop.algorithm.add_updates(
self._variables.accumulation_updates)
self.main_loop.algorithm.add_updates(
self._required_for_non_variables.accumulation_updates)
self._variables.initialize_aggregators()
self._required_for_non_variables.initialize_aggregators()
self._non_variables.initialize_quantities()
else:
# When called first time at any iterations, update
# monitored non-Theano quantities
if (self.main_loop.status['iterations_done'] >
self._last_time_called):
self._non_variables.aggregate_quantities(
list(self._required_for_non_variables
.get_aggregated_values().values()))
self._required_for_non_variables.initialize_aggregators()
self._last_time_called = (
self.main_loop.status['iterations_done'])
# If only called to update non-Theano quantities,
# do just that
if args == ('just_aggregate',):
return
# Otherwise, also output current values of from the accumulators
# to the log.
self.add_records(
self.main_loop.log,
self._variables.get_aggregated_values().items())
self._variables.initialize_aggregators()
self.add_records(
self.main_loop.log,
self._non_variables.get_aggregated_values().items())
self._non_variables.initialize_quantities()
class TrainingDataMonitoring(SimpleExtension, MonitoringExtension):
"""Monitors values of Theano variables on training batches.
Use this extension to monitor a quantity on every training batch
cheaply. It integrates with the training algorithm in order to avoid
recomputing same things several times. For instance, if you are
training a network and you want to log the norm of the gradient on
every batch, the backpropagation will only be done once. By
controlling the frequency with which the :meth:`do` method is called,
you can aggregate the monitored variables, e.g. only log the gradient
norm average over an epoch.
Parameters
----------
variables : list of :class:`~tensor.TensorVariable` or
:class:`~blocks.monitoring.aggregation.MonitoredQuantity`
The variables or non-Theano quantities to monitor.
The variable names are used as record names in the logs.
Notes
-----
All the monitored variables are evaluated _before_ the parameter
update.
Requires the training algorithm to be an instance of
:class:`.DifferentiableCostMinimizer`.
"""
def __init__(self, variables, **kwargs):
kwargs.setdefault("before_training", True)
super(TrainingDataMonitoring, self).__init__(**kwargs)
self.add_condition(['after_batch'], arguments=('just_aggregate', ))
self._non_variables = []
self._variables = []
for variable_or_not in variables:
if isinstance(variable_or_not, theano.Variable):
self._variables.append(variable_or_not)
elif isinstance(variable_or_not, MonitoredQuantity):
self._non_variables.append(variable_or_not)
else:
raise ValueError("can not monitor {}".format(variable_or_not))
self._non_variables = MonitoredQuantityBuffer(self._non_variables)
self._required_for_non_variables = AggregationBuffer(
[take_last(v) for v in self._non_variables.requires])
self._variables = AggregationBuffer(self._variables,
use_take_last=True)
self._last_time_called = -1
def do(self, callback_name, *args):
"""Initializes the buffer or commits the values to the log.
What this method does depends on from what callback it is called
and with which arguments. When called within `before_training`, it
initializes the aggregation buffer and instructs the training
algorithm what additional computations should be carried at each
step by adding corresponding updates to it. In most_other cases it
writes aggregated values of the monitored variables to the log. An
exception is when an argument `just_aggregate` is given: in this
cases it updates the values of monitored non-Theano quantities, but
does not write anything to the log.
"""
data, args = self.parse_args(callback_name, args)
if callback_name == 'before_training':
if not isinstance(self.main_loop.algorithm,
DifferentiableCostMinimizer):
raise ValueError
self.main_loop.algorithm.add_updates(
self._variables.accumulation_updates)
self.main_loop.algorithm.add_updates(
self._required_for_non_variables.accumulation_updates)
self._variables.initialize_aggregators()
self._required_for_non_variables.initialize_aggregators()
self._non_variables.initialize_quantities()
else:
# When called first time at any iterations, update
# monitored non-Theano quantities
if (self.main_loop.status['iterations_done'] >
self._last_time_called):
self._non_variables.aggregate_quantities(
list(self._required_for_non_variables.
get_aggregated_values().values()))
self._required_for_non_variables.initialize_aggregators()
self._last_time_called = (
self.main_loop.status['iterations_done'])
# If only called to update non-Theano quantities,
# do just that
if args == ('just_aggregate', ):
return
# Otherwise, also output current values of from the accumulators
# to the log.
self.add_records(self.main_loop.log,
self._variables.get_aggregated_values().items())
self._variables.initialize_aggregators()
self.add_records(
self.main_loop.log,
self._non_variables.get_aggregated_values().items())
self._non_variables.initialize_quantities()
class myTrainingDataMonitoring(SimpleExtension, MonitoringExtension):
def __init__(self, variables, saveEveryXIteration, **kwargs):
self.saveEveryXIteration = saveEveryXIteration
kwargs.setdefault("before_training", True)
super(myTrainingDataMonitoring, self).__init__(**kwargs)
self.add_condition(['after_batch'], arguments=('just_aggregate', ))
self._non_variables = []
self._variables = []
for variable_or_not in variables:
if isinstance(variable_or_not, theano.Variable):
self._variables.append(variable_or_not)
elif isinstance(variable_or_not, MonitoredQuantity):
self._non_variables.append(variable_or_not)
else:
raise ValueError("can not monitor {}".format(variable_or_not))
self._non_variables = MonitoredQuantityBuffer(self._non_variables)
self._required_for_non_variables = AggregationBuffer(
[take_last(v) for v in self._non_variables.requires])
self._variables = AggregationBuffer(self._variables,
use_take_last=True)
self._last_time_called = -1
def do(self, callback_name, *args):
data, args = self.parse_args(callback_name, args)
if callback_name == 'before_training':
if not isinstance(self.main_loop.algorithm,
DifferentiableCostMinimizer):
raise ValueError
self.main_loop.algorithm.add_updates(
self._variables.accumulation_updates)
self.main_loop.algorithm.add_updates(
self._required_for_non_variables.accumulation_updates)
self._variables.initialize_aggregators()
self._required_for_non_variables.initialize_aggregators()
self._non_variables.initialize_quantities()
else:
log = self.main_loop.log
iteration = log.status['iterations_done']
if iteration % self.saveEveryXIteration == 0:
# When called first time at any iterations, update
# monitored non-Theano quantities
if (self.main_loop.status['iterations_done'] >
self._last_time_called):
self._non_variables.aggregate_quantities(
list(self._required_for_non_variables.
get_aggregated_values().values()))
self._required_for_non_variables.initialize_aggregators()
self._last_time_called = (
self.main_loop.status['iterations_done'])
# If only called to update non-Theano quantities,
# do just that
if args == ('just_aggregate', ):
return
# Otherwise, also output current values of from the accumulators
# to the log.
self.add_records(
self.main_loop.log,
self._variables.get_aggregated_values().items())
self._variables.initialize_aggregators()
self.add_records(
self.main_loop.log,
self._non_variables.get_aggregated_values().items())
self._non_variables.initialize_quantities()
class DatasetEvaluator(object):
"""A DatasetEvaluator evaluates many Theano variables or other quantities.
The DatasetEvaluator provides a do-it-all method, :meth:`evaluate`,
which computes values of ``variables`` on a dataset.
Alternatively, methods :meth:`initialize_aggregators`,
:meth:`process_batch`, :meth:`get_aggregated_values` can be used with a
custom loop over data.
The values computed on subsets of the given dataset are aggregated
using the :class:`AggregationScheme`s provided in the
`aggregation_scheme` tags. If no tag is given, the value is **averaged
over minibatches**. However, care is taken to ensure that variables
which do not depend on data are not unnecessarily recomputed.
Parameters
----------
variables : list of :class:`~tensor.TensorVariable` and
:class:`MonitoredQuantity`
The variable names are used as record names in the logs. Hence, all
the names must be different.
Each variable can be tagged with an :class:`AggregationScheme` that
specifies how the value can be computed for a data set by
aggregating minibatches.
updates : list of tuples or :class:`~collections.OrderedDict` or None
:class:`~tensor.TensorSharedVariable` updates to be performed
during evaluation. This parameter is only for Theano variables.
Be careful not to update any model parameters as this is not
intended to alter your model in any meaningfullway. A typical
use case of this option arises when the theano function used
for evaluation contains a call to:function:`~theano.scan` which
might have returned shared variable updates.
"""
def __init__(self, variables, mini_batch_size, state_updates,
dataset, updates=None):
theano_variables = []
monitored_quantities = []
for variable in variables:
if isinstance(variable, MonitoredQuantity):
monitored_quantities.append(variable)
else:
theano_variables.append(variable)
self.theano_variables = theano_variables
self.monitored_quantities = monitored_quantities
variable_names = [v.name for v in variables]
if len(set(variable_names)) < len(variables):
raise ValueError("variables should have different names")
self.theano_buffer = AggregationBuffer(theano_variables)
self.monitored_quantities_buffer = MonitoredQuantityBuffer(
monitored_quantities)
self.dataset = dataset
self.updates = updates
self.mini_batch_size = mini_batch_size
self._compile(state_updates)
def _compile(self, state_updates):
"""Compiles Theano functions.
.. todo::
The current compilation method does not account for updates
attached to `ComputationGraph` elements. Compiling should
be out-sourced to `ComputationGraph` to deal with it.
"""
inputs = []
outputs = []
updates = None
givens, f_updates = carry_hidden_state(state_updates,
self.mini_batch_size,
reset=not(has_indices(self.dataset)))
if self.theano_buffer.accumulation_updates:
updates = OrderedDict()
updates.update(self.theano_buffer.accumulation_updates)
if self.updates:
updates.update(self.updates)
inputs += self.theano_buffer.inputs
inputs += self.monitored_quantities_buffer.inputs
outputs = self.monitored_quantities_buffer.requires
if inputs != []:
self.unique_inputs = list(set(inputs))
updates.update(f_updates)
self._accumulate_fun = theano.function(self.unique_inputs,
outputs,
givens=givens,
updates=updates)
else:
self._accumulate_fun = None
def initialize_aggregators(self):
self.theano_buffer.initialize_aggregators()
self.monitored_quantities_buffer.initialize()
def process_batch(self, batch):
try:
input_names = [v.name for v in self.unique_inputs]
batch = dict_subset(batch, input_names)
except KeyError:
reraise_as(
"Not all data sources required for monitoring were"
" provided. The list of required data sources:"
" {}.".format(input_names))
if self._accumulate_fun is not None:
numerical_values = self._accumulate_fun(**batch)
self.monitored_quantities_buffer.accumulate_quantities(
numerical_values)
def get_aggregated_values(self):
values = self.theano_buffer.get_aggregated_values()
values.update(
self.monitored_quantities_buffer.get_aggregated_values())
return values
def evaluate(self, data_stream):
"""Compute the variables over a data stream.
Parameters
----------
data_stream : instance of :class:`.DataStream`
The data stream. Only the first epoch of data is used.
Returns
-------
A mapping from record names to the values computed on the provided
dataset.
"""
self.initialize_aggregators()
if self._accumulate_fun is not None:
for batch in data_stream.get_epoch_iterator(as_dict=True):
self.process_batch(batch)
else:
logger.debug(
'Only data independent variables were given,'
'will not iterate the over data!')
return self.get_aggregated_values()
class DatasetEvaluator(object):
"""A DatasetEvaluator evaluates many Theano variables or other quantities.
The DatasetEvaluator provides a do-it-all method, :meth:`evaluate`,
which computes values of ``variables`` on a dataset.
Alternatively, methods :meth:`initialize_aggregators`,
:meth:`process_batch`, :meth:`get_aggregated_values` can be used with a
custom loop over data.
The values computed on subsets of the given dataset are aggregated
using the :class:`AggregationScheme`s provided in the
`aggregation_scheme` tags. If no tag is given, the value is **averaged
over minibatches**. However, care is taken to ensure that variables
which do not depend on data are not unnecessarily recomputed.
Parameters
----------
variables : list of :class:`~tensor.TensorVariable` and
:class:`MonitoredQuantity`
The variable names are used as record names in the logs. Hence, all
the names must be different.
Each variable can be tagged with an :class:`AggregationScheme` that
specifies how the value can be computed for a data set by
aggregating minibatches.
updates : list of tuples or :class:`~collections.OrderedDict` or None
:class:`~tensor.TensorSharedVariable` updates to be performed
during evaluation. This parameter is only for Theano variables.
Be careful not to update any model parameters as this is not
intended to alter your model in any meaningfullway. A typical
use case of this option arises when the theano function used
for evaluation contains a call to:function:`~theano.scan` which
might have returned shared variable updates.
"""
def __init__(self,
variables,
mini_batch_size,
state_updates,
updates=None):
theano_variables = []
monitored_quantities = []
for variable in variables:
if isinstance(variable, MonitoredQuantity):
monitored_quantities.append(variable)
else:
theano_variables.append(variable)
self.theano_variables = theano_variables
self.monitored_quantities = monitored_quantities
variable_names = [v.name for v in variables]
if len(set(variable_names)) < len(variables):
raise ValueError("variables should have different names")
self.theano_buffer = AggregationBuffer(theano_variables)
self.monitored_quantities_buffer = MonitoredQuantityBuffer(
monitored_quantities)
self.updates = updates
self.mini_batch_size = mini_batch_size
self._compile(state_updates)
def _compile(self, state_updates):
"""Compiles Theano functions.
.. todo::
The current compilation method does not account for updates
attached to `ComputationGraph` elements. Compiling should
be out-sourced to `ComputationGraph` to deal with it.
"""
inputs = []
outputs = []
updates = None
state_vars = [
theano.shared(
numpy.zeros((self.mini_batch_size, v.shape[1].eval()),
dtype=numpy.float32), v.name + '-gen')
for v, _ in state_updates
]
givens = [(v, x) for (v, _), x in zip(state_updates, state_vars)]
f_updates = [(x, upd)
for x, (_, upd) in zip(state_vars, state_updates)]
if self.theano_buffer.accumulation_updates:
updates = OrderedDict()
updates.update(self.theano_buffer.accumulation_updates)
if self.updates:
updates.update(self.updates)
inputs += self.theano_buffer.inputs
inputs += self.monitored_quantities_buffer.inputs
outputs = self.monitored_quantities_buffer.requires
if inputs != []:
self.unique_inputs = list(set(inputs))
updates.update(f_updates)
self._accumulate_fun = theano.function(self.unique_inputs,
outputs,
givens=givens,
updates=updates)
else:
self._accumulate_fun = None
def initialize_aggregators(self):
self.theano_buffer.initialize_aggregators()
self.monitored_quantities_buffer.initialize()
def process_batch(self, batch):
try:
input_names = [v.name for v in self.unique_inputs]
batch = dict_subset(batch, input_names)
except KeyError:
reraise_as("Not all data sources required for monitoring were"
" provided. The list of required data sources:"
" {}.".format(input_names))
if self._accumulate_fun is not None:
numerical_values = self._accumulate_fun(**batch)
self.monitored_quantities_buffer.accumulate_quantities(
numerical_values)
def get_aggregated_values(self):
values = self.theano_buffer.get_aggregated_values()
values.update(self.monitored_quantities_buffer.get_aggregated_values())
return values
def evaluate(self, data_stream):
"""Compute the variables over a data stream.
Parameters
----------
data_stream : instance of :class:`.DataStream`
The data stream. Only the first epoch of data is used.
Returns
-------
A mapping from record names to the values computed on the provided
dataset.
"""
self.initialize_aggregators()
if self._accumulate_fun is not None:
for batch in data_stream.get_epoch_iterator(as_dict=True):
self.process_batch(batch)
else:
logger.debug('Only data independent variables were given,'
'will not iterate the over data!')
return self.get_aggregated_values()
