def evaluate(network, iter, scorers=(), out_name='', targets_name='targets',
mask_name=None):
"""Evaluate one or more scores for a network.
This tool can be used to evaluate scores of a trained network on test
data.
Args:
network (brainstorm.structure.Network): Network to be evaluated.
iter (brainstorm.DataIterator): A data iterator which produces the
data on which the scores are computed.
scorers (tuple[brainstorm.scorers.Scorer]): A list or tuple of Scorers.
out_name (Optional[str]): Name of the network output which is scored
against the targets.
targets_name (Optional[str]): Name of the targets data provided by the
data iterator (``iter``).
mask_name (Optional[str]): Name of the mask data provided by the
data iterator (``iter``).
"""
iterator = iter(handler=network.handler)
scores = {scorer.__name__: [] for scorer in scorers}
for n in network.get_loss_values():
scores[n] = []
for _ in run_network(network, iterator):
network.forward_pass()
gather_losses_and_scores(
network, scorers, scores, out_name=out_name,
targets_name=targets_name, mask_name=mask_name)
return aggregate_losses_and_scores(scores, network, scorers)
def extract_and_save(network, iter, buffer_names, file_name):
"""Save the desired buffer values of a network to an HDF5 file.
In particular, this tool can be used to save the predictions of a
network on a dataset.
In general, any number of internal, input or output buffers of the network
can be extracted.
Examples:
>>> getter = Minibatches(100, default=x_test)
>>> extract_and_save(network,
... getter,
... ['Output.outputs.probabilities',
... 'Hid1.internals.H'],
... 'network_features.hdf5')
Args:
network (brainstorm.structure.Network): Network using which the features
should be generated.
iter (brainstorm.DataIterator): A data iterator which produces the
data on which the features are
computed.
buffer_names (list[unicode]): Name of the buffer views to be saved (in
dotted notation). See example.
file_name (unicode): Name of the hdf5 file (including extension) in
which the features should be saved.
"""
iterator = iter(handler=network.handler)
if isinstance(buffer_names, six.string_types):
buffer_names = [buffer_names]
nr_items = 0
ds = []
with h5py.File(file_name, "w") as f:
f.attrs.create("info", get_brainstorm_info())
f.attrs.create("format", b"Buffers file v1.0")
for _ in run_network(network, iterator, all_inputs=False):
network.forward_pass()
first_pass = False if len(ds) > 0 else True
for num, buffer_name in enumerate(buffer_names):
data = network.get(buffer_name)
if num == 0:
nr_items += data.shape[1]
if first_pass:
ds.append(
f.create_dataset(
buffer_name,
data.shape,
data.dtype,
chunks=data.shape,
maxshape=(data.shape[0], None) + data.shape[2:],
)
)
ds[-1][:] = data
else:
ds[num].resize(size=nr_items, axis=1)
ds[num][:, nr_items - data.shape[1] : nr_items, ...] = data
def extract_and_save(network, iter, buffer_names, file_name):
"""Save the desired buffer values of a network to an HDF5 file.
In particular, this tool can be used to save the predictions of a
network on a dataset.
In general, any number of internal, input or output buffers of the network
can be extracted.
Examples:
>>> getter = Minibatches(100, default=x_test)
>>> extract_and_save(network,
... getter,
... ['Output.outputs.predictions',
... 'Hid1.internals.H'],
... 'network_features.hdf5')
Args:
network (brainstorm.structure.Network):
Network using which the features should be generated.
iter (brainstorm.DataIterator):
A data iterator which produces the data on which the features are
computed.
buffer_names (list[unicode]):
Name of the buffer views to be saved (in dotted notation).
See example.
file_name (unicode):
Name of the hdf5 file (including extension) in which the features
should be saved.
"""
iterator = iter(handler=network.handler)
if isinstance(buffer_names, six.string_types):
buffer_names = [buffer_names]
nr_items = 0
ds = []
with h5py.File(file_name, 'w') as f:
f.attrs.create('info', get_brainstorm_info())
f.attrs.create('format', b'Buffers file v1.0')
for _ in run_network(network, iterator, all_inputs=False):
network.forward_pass()
first_pass = False if len(ds) > 0 else True
for num, buffer_name in enumerate(buffer_names):
data = network.get(buffer_name)
if num == 0:
nr_items += data.shape[1]
if first_pass:
ds.append(
f.create_dataset(buffer_name,
data.shape,
data.dtype,
chunks=data.shape,
maxshape=(data.shape[0], None) +
data.shape[2:]))
ds[-1][:] = data
else:
ds[num].resize(size=nr_items, axis=1)
ds[num][:, nr_items - data.shape[1]:nr_items, ...] = data
def extract(network, iter, buffer_names):
"""Apply the network to some data and return the requested buffers.
Batches are returned as a list and as they are provided by the data
iterator, i.e. if you want results per-batch (per sequence), use an
online iterator / minibatch iterator with a minibatch size of 1."""
iterator = iter(handler=network.handler)
if isinstance(buffer_names, six.string_types):
buffer_names = [buffer_names]
num_items = 0
ds = []
returnData = []
for _ in run_network(network, iterator, all_inputs=False):
network.forward_pass()
returnBuffers = []
for buffer_name in buffer_names:
data = network.get(buffer_name)
returnBuffers.append(data)
returnData.append(returnBuffers)
return returnData
def extract(network, iter, buffer_names):
"""Apply the network to some data and return the requested buffers.
Batches are returned as a dictionary, with one entry for each
requested buffer, with the data in (T, B, ...) order.
Args:
network (brainstorm.structure.Network):
Network using which the features should be generated.
iter (brainstorm.DataIterator):
A data iterator which produces the data on which the features are
computed.
buffer_names (list[unicode]):
Name of the buffer views to be saved (in dotted notation).
Returns:
dict[unicode, np.ndarray]
"""
iterator = iter(handler=network.handler)
if isinstance(buffer_names, six.string_types):
buffer_names = [buffer_names]
nr_examples = iter.data_shapes.values()[0][1]
return_data = {}
nr_items = 0
for _ in run_network(network, iterator, all_inputs=False):
network.forward_pass()
first_pass = False if len(return_data) == len(buffer_names) else True
for num, buffer_name in enumerate(buffer_names):
data = network.get(buffer_name)
if num == 0:
nr_items += data.shape[1]
if first_pass:
data_shape = (data.shape[0], nr_examples) + data.shape[2:]
return_data[buffer_name] = np.zeros(data_shape, data.dtype)
return_data[buffer_name][:, nr_items - data.shape[1]:nr_items] = \
data
return return_data
