def extract_and_save(network, iter, buffer_names, file_name):
"""Save the desired buffer values of a network to an HDF5 file.
In particular, this tool can be used to save the predictions of a
network on a dataset.
In general, any number of internal, input or output buffers of the network
can be extracted.
Examples:
>>> getter = Minibatches(100, default=x_test)
>>> extract_and_save(network,
... getter,
... ['Output.outputs.predictions',
... 'Hid1.internals.H'],
... 'network_features.hdf5')
Args:
network (brainstorm.structure.Network):
Network using which the features should be generated.
iter (brainstorm.DataIterator):
A data iterator which produces the data on which the features are
computed.
buffer_names (list[unicode]):
Name of the buffer views to be saved (in dotted notation).
See example.
file_name (unicode):
Name of the hdf5 file (including extension) in which the features
should be saved.
"""
iterator = iter(handler=network.handler)
if isinstance(buffer_names, six.string_types):
buffer_names = [buffer_names]
nr_items = 0
ds = []
with h5py.File(file_name, 'w') as f:
f.attrs.create('info', get_brainstorm_info())
f.attrs.create('format', b'Buffers file v1.0')
for _ in run_network(network, iterator, all_inputs=False):
network.forward_pass()
first_pass = False if len(ds) > 0 else True
for num, buffer_name in enumerate(buffer_names):
data = network.get(buffer_name)
if num == 0:
nr_items += data.shape[1]
if first_pass:
ds.append(
f.create_dataset(buffer_name,
data.shape,
data.dtype,
chunks=data.shape,
maxshape=(data.shape[0], None) +
data.shape[2:]))
ds[-1][:] = data
else:
ds[num].resize(size=nr_items, axis=1)
ds[num][:, nr_items - data.shape[1]:nr_items, ...] = data
def extract_and_save(network, iter, buffer_names, file_name):
"""Save the desired buffer values of a network to an HDF5 file.
In particular, this tool can be used to save the predictions of a
network on a dataset.
In general, any number of internal, input or output buffers of the network
can be extracted.
Examples:
>>> getter = Minibatches(100, default=x_test)
>>> extract_and_save(network,
... getter,
... ['Output.outputs.probabilities',
... 'Hid1.internals.H'],
... 'network_features.hdf5')
Args:
network (brainstorm.structure.Network): Network using which the features
should be generated.
iter (brainstorm.DataIterator): A data iterator which produces the
data on which the features are
computed.
buffer_names (list[unicode]): Name of the buffer views to be saved (in
dotted notation). See example.
file_name (unicode): Name of the hdf5 file (including extension) in
which the features should be saved.
"""
iterator = iter(handler=network.handler)
if isinstance(buffer_names, six.string_types):
buffer_names = [buffer_names]
nr_items = 0
ds = []
with h5py.File(file_name, "w") as f:
f.attrs.create("info", get_brainstorm_info())
f.attrs.create("format", b"Buffers file v1.0")
for _ in run_network(network, iterator, all_inputs=False):
network.forward_pass()
first_pass = False if len(ds) > 0 else True
for num, buffer_name in enumerate(buffer_names):
data = network.get(buffer_name)
if num == 0:
nr_items += data.shape[1]
if first_pass:
ds.append(
f.create_dataset(
buffer_name,
data.shape,
data.dtype,
chunks=data.shape,
maxshape=(data.shape[0], None) + data.shape[2:],
)
)
ds[-1][:] = data
else:
ds[num].resize(size=nr_items, axis=1)
ds[num][:, nr_items - data.shape[1] : nr_items, ...] = data
def save_as_hdf5(self, filename, comment=''):
"""
Save this network as an HDF5 file.
The file will contain a description of this network and the parameters.
Args:
filename (str):
Name of the file this network should be saved to.
All directories have to exist already.
comment (Optional[str]):
An optional comment that will be saved inside the file.
"""
with h5py.File(filename, 'w') as f:
f.attrs.create('info', get_brainstorm_info())
f.attrs.create('format', b'Network file v1.0')
if comment:
f.attrs.create('comment', comment.encode())
description = get_description(self)
f['description'] = json.dumps(description).encode()
f.create_dataset('parameters',
compression='gzip',
data=self.get('parameters'))
def save_as_hdf5(self, filename, comment=''):
"""
Save this network as an HDF5 file.
The file will contain a description of this network and the parameters.
Args:
filename (str):
Name of the file this network should be saved to.
All directories have to exist already.
comment (Optional[str]):
An optional comment that will be saved inside the file.
"""
with h5py.File(filename, 'w') as f:
f.attrs.create('info', get_brainstorm_info())
f.attrs.create('format', b'Network file v1.0')
if comment:
f.attrs.create('comment', comment.encode())
description = get_description(self)
f['description'] = json.dumps(description).encode()
f.create_dataset(
'parameters', compression='gzip',
data=self.get('parameters'))
def __call__(self, epoch_nr, update_nr, net, stepper, logs):
with h5py.File(self.filename, 'w') as f:
f.attrs.create('info', get_brainstorm_info())
f.attrs.create('format', b'Logs file v1.0')
SaveLogs._save_recursively(f, logs)
def __call__(self, epoch_nr, update_nr, net, stepper, logs):
with h5py.File(self.filename, 'w') as f:
f.attrs.create('info', get_brainstorm_info())
f.attrs.create('format', b'Logs file v1.0')
SaveLogs._save_recursively(f, logs)
