def extract_and_save(network, iter, buffer_names, file_name): """Save the desired buffer values of a network to an HDF5 file. In particular, this tool can be used to save the predictions of a network on a dataset. In general, any number of internal, input or output buffers of the network can be extracted. Examples: >>> getter = Minibatches(100, default=x_test) >>> extract_and_save(network, ... getter, ... ['Output.outputs.predictions', ... 'Hid1.internals.H'], ... 'network_features.hdf5') Args: network (brainstorm.structure.Network): Network using which the features should be generated. iter (brainstorm.DataIterator): A data iterator which produces the data on which the features are computed. buffer_names (list[unicode]): Name of the buffer views to be saved (in dotted notation). See example. file_name (unicode): Name of the hdf5 file (including extension) in which the features should be saved. """ iterator = iter(handler=network.handler) if isinstance(buffer_names, six.string_types): buffer_names = [buffer_names] nr_items = 0 ds = [] with h5py.File(file_name, 'w') as f: f.attrs.create('info', get_brainstorm_info()) f.attrs.create('format', b'Buffers file v1.0') for _ in run_network(network, iterator, all_inputs=False): network.forward_pass() first_pass = False if len(ds) > 0 else True for num, buffer_name in enumerate(buffer_names): data = network.get(buffer_name) if num == 0: nr_items += data.shape[1] if first_pass: ds.append( f.create_dataset(buffer_name, data.shape, data.dtype, chunks=data.shape, maxshape=(data.shape[0], None) + data.shape[2:])) ds[-1][:] = data else: ds[num].resize(size=nr_items, axis=1) ds[num][:, nr_items - data.shape[1]:nr_items, ...] = data
def extract_and_save(network, iter, buffer_names, file_name): """Save the desired buffer values of a network to an HDF5 file. In particular, this tool can be used to save the predictions of a network on a dataset. In general, any number of internal, input or output buffers of the network can be extracted. Examples: >>> getter = Minibatches(100, default=x_test) >>> extract_and_save(network, ... getter, ... ['Output.outputs.probabilities', ... 'Hid1.internals.H'], ... 'network_features.hdf5') Args: network (brainstorm.structure.Network): Network using which the features should be generated. iter (brainstorm.DataIterator): A data iterator which produces the data on which the features are computed. buffer_names (list[unicode]): Name of the buffer views to be saved (in dotted notation). See example. file_name (unicode): Name of the hdf5 file (including extension) in which the features should be saved. """ iterator = iter(handler=network.handler) if isinstance(buffer_names, six.string_types): buffer_names = [buffer_names] nr_items = 0 ds = [] with h5py.File(file_name, "w") as f: f.attrs.create("info", get_brainstorm_info()) f.attrs.create("format", b"Buffers file v1.0") for _ in run_network(network, iterator, all_inputs=False): network.forward_pass() first_pass = False if len(ds) > 0 else True for num, buffer_name in enumerate(buffer_names): data = network.get(buffer_name) if num == 0: nr_items += data.shape[1] if first_pass: ds.append( f.create_dataset( buffer_name, data.shape, data.dtype, chunks=data.shape, maxshape=(data.shape[0], None) + data.shape[2:], ) ) ds[-1][:] = data else: ds[num].resize(size=nr_items, axis=1) ds[num][:, nr_items - data.shape[1] : nr_items, ...] = data
def save_as_hdf5(self, filename, comment=''): """ Save this network as an HDF5 file. The file will contain a description of this network and the parameters. Args: filename (str): Name of the file this network should be saved to. All directories have to exist already. comment (Optional[str]): An optional comment that will be saved inside the file. """ with h5py.File(filename, 'w') as f: f.attrs.create('info', get_brainstorm_info()) f.attrs.create('format', b'Network file v1.0') if comment: f.attrs.create('comment', comment.encode()) description = get_description(self) f['description'] = json.dumps(description).encode() f.create_dataset('parameters', compression='gzip', data=self.get('parameters'))
def save_as_hdf5(self, filename, comment=''): """ Save this network as an HDF5 file. The file will contain a description of this network and the parameters. Args: filename (str): Name of the file this network should be saved to. All directories have to exist already. comment (Optional[str]): An optional comment that will be saved inside the file. """ with h5py.File(filename, 'w') as f: f.attrs.create('info', get_brainstorm_info()) f.attrs.create('format', b'Network file v1.0') if comment: f.attrs.create('comment', comment.encode()) description = get_description(self) f['description'] = json.dumps(description).encode() f.create_dataset( 'parameters', compression='gzip', data=self.get('parameters'))
def __call__(self, epoch_nr, update_nr, net, stepper, logs): with h5py.File(self.filename, 'w') as f: f.attrs.create('info', get_brainstorm_info()) f.attrs.create('format', b'Logs file v1.0') SaveLogs._save_recursively(f, logs)