def evaluate(network, iter, scorers=(), out_name='', targets_name='targets', mask_name=None): """Evaluate one or more scores for a network. This tool can be used to evaluate scores of a trained network on test data. Args: network (brainstorm.structure.Network): Network to be evaluated. iter (brainstorm.DataIterator): A data iterator which produces the data on which the scores are computed. scorers (tuple[brainstorm.scorers.Scorer]): A list or tuple of Scorers. out_name (Optional[str]): Name of the network output which is scored against the targets. targets_name (Optional[str]): Name of the targets data provided by the data iterator (``iter``). mask_name (Optional[str]): Name of the mask data provided by the data iterator (``iter``). """ iterator = iter(handler=network.handler) scores = {scorer.__name__: [] for scorer in scorers} for n in network.get_loss_values(): scores[n] = [] for _ in run_network(network, iterator): network.forward_pass() gather_losses_and_scores( network, scorers, scores, out_name=out_name, targets_name=targets_name, mask_name=mask_name) return aggregate_losses_and_scores(scores, network, scorers)
def extract_and_save(network, iter, buffer_names, file_name): """Save the desired buffer values of a network to an HDF5 file. In particular, this tool can be used to save the predictions of a network on a dataset. In general, any number of internal, input or output buffers of the network can be extracted. Examples: >>> getter = Minibatches(100, default=x_test) >>> extract_and_save(network, ... getter, ... ['Output.outputs.probabilities', ... 'Hid1.internals.H'], ... 'network_features.hdf5') Args: network (brainstorm.structure.Network): Network using which the features should be generated. iter (brainstorm.DataIterator): A data iterator which produces the data on which the features are computed. buffer_names (list[unicode]): Name of the buffer views to be saved (in dotted notation). See example. file_name (unicode): Name of the hdf5 file (including extension) in which the features should be saved. """ iterator = iter(handler=network.handler) if isinstance(buffer_names, six.string_types): buffer_names = [buffer_names] nr_items = 0 ds = [] with h5py.File(file_name, "w") as f: f.attrs.create("info", get_brainstorm_info()) f.attrs.create("format", b"Buffers file v1.0") for _ in run_network(network, iterator, all_inputs=False): network.forward_pass() first_pass = False if len(ds) > 0 else True for num, buffer_name in enumerate(buffer_names): data = network.get(buffer_name) if num == 0: nr_items += data.shape[1] if first_pass: ds.append( f.create_dataset( buffer_name, data.shape, data.dtype, chunks=data.shape, maxshape=(data.shape[0], None) + data.shape[2:], ) ) ds[-1][:] = data else: ds[num].resize(size=nr_items, axis=1) ds[num][:, nr_items - data.shape[1] : nr_items, ...] = data
def extract_and_save(network, iter, buffer_names, file_name): """Save the desired buffer values of a network to an HDF5 file. In particular, this tool can be used to save the predictions of a network on a dataset. In general, any number of internal, input or output buffers of the network can be extracted. Examples: >>> getter = Minibatches(100, default=x_test) >>> extract_and_save(network, ... getter, ... ['Output.outputs.predictions', ... 'Hid1.internals.H'], ... 'network_features.hdf5') Args: network (brainstorm.structure.Network): Network using which the features should be generated. iter (brainstorm.DataIterator): A data iterator which produces the data on which the features are computed. buffer_names (list[unicode]): Name of the buffer views to be saved (in dotted notation). See example. file_name (unicode): Name of the hdf5 file (including extension) in which the features should be saved. """ iterator = iter(handler=network.handler) if isinstance(buffer_names, six.string_types): buffer_names = [buffer_names] nr_items = 0 ds = [] with h5py.File(file_name, 'w') as f: f.attrs.create('info', get_brainstorm_info()) f.attrs.create('format', b'Buffers file v1.0') for _ in run_network(network, iterator, all_inputs=False): network.forward_pass() first_pass = False if len(ds) > 0 else True for num, buffer_name in enumerate(buffer_names): data = network.get(buffer_name) if num == 0: nr_items += data.shape[1] if first_pass: ds.append( f.create_dataset(buffer_name, data.shape, data.dtype, chunks=data.shape, maxshape=(data.shape[0], None) + data.shape[2:])) ds[-1][:] = data else: ds[num].resize(size=nr_items, axis=1) ds[num][:, nr_items - data.shape[1]:nr_items, ...] = data
def extract(network, iter, buffer_names): """Apply the network to some data and return the requested buffers. Batches are returned as a list and as they are provided by the data iterator, i.e. if you want results per-batch (per sequence), use an online iterator / minibatch iterator with a minibatch size of 1.""" iterator = iter(handler=network.handler) if isinstance(buffer_names, six.string_types): buffer_names = [buffer_names] num_items = 0 ds = [] returnData = [] for _ in run_network(network, iterator, all_inputs=False): network.forward_pass() returnBuffers = [] for buffer_name in buffer_names: data = network.get(buffer_name) returnBuffers.append(data) returnData.append(returnBuffers) return returnData
def extract(network, iter, buffer_names): """Apply the network to some data and return the requested buffers. Batches are returned as a dictionary, with one entry for each requested buffer, with the data in (T, B, ...) order. Args: network (brainstorm.structure.Network): Network using which the features should be generated. iter (brainstorm.DataIterator): A data iterator which produces the data on which the features are computed. buffer_names (list[unicode]): Name of the buffer views to be saved (in dotted notation). Returns: dict[unicode, np.ndarray] """ iterator = iter(handler=network.handler) if isinstance(buffer_names, six.string_types): buffer_names = [buffer_names] nr_examples = iter.data_shapes.values()[0][1] return_data = {} nr_items = 0 for _ in run_network(network, iterator, all_inputs=False): network.forward_pass() first_pass = False if len(return_data) == len(buffer_names) else True for num, buffer_name in enumerate(buffer_names): data = network.get(buffer_name) if num == 0: nr_items += data.shape[1] if first_pass: data_shape = (data.shape[0], nr_examples) + data.shape[2:] return_data[buffer_name] = np.zeros(data_shape, data.dtype) return_data[buffer_name][:, nr_items - data.shape[1]:nr_items] = \ data return return_data