def test_connection_extraction(self): connection = layers.Input(2) > layers.Sigmoid(3) self.assertIs(extract_connection(connection), connection) network = algorithms.GradientDescent(connection) self.assertIs(extract_connection(network), connection) list_of_layers = [layers.Input(2), layers.Sigmoid(3)] actual_connection = extract_connection(list_of_layers) self.assertEqual(len(actual_connection), 2) self.assertEqual(actual_connection.input_shape, (2, )) self.assertEqual(actual_connection.output_shape, (3, )) with self.assertRaisesRegexp(TypeError, "Invalid input type"): extract_connection(object)
def save_json(connection, filepath, indent=None): """ Save network parameters in JSON format. Parameters ---------- {save_dict.connection} filepath : str Path to the JSON file that stores network parameters. indent : int or None Indentation that would be specified for the output JSON. Intentation equal to `2` or `4` makes it easy to read raw text files. The `None` value disables indentation which means that everything will be stored compactly. Defaults to `None`. Examples -------- >>> from neupy import layers, storage >>> >>> connection = layers.Input(10) > layers.Softmax(3) >>> storage.save_json(connection, '/path/to/parameters.json') """ connection = extract_connection(connection) data = save_dict(connection) with open(filepath, 'w') as f: # Without extra data processor we won't be able to dump # numpy array into json without raising an error. # `json` will have issues with numpy array encoding convert_numpy_array_to_list_recursively(data) return json.dump(data, f, indent=indent, default=repr)
def save_json(connection, filepath, indent=None): """ Save network parameters in JSON format. Parameters ---------- {save_dict.connection} filepath : str Path to the JSON file that stores network parameters. indent : int or None Indentation that would be specified for the output JSON. Intentation equal to `2` or `4` makes it easy to read raw text files. The `None` value disables indentation which means that everything will be stored compactly. Defaults to `None`. Notes ----- Install `ujson` library in order to speed up saving and loading procedure. Examples -------- >>> from neupy import layers, storage >>> >>> connection = layers.Input(10) > layers.Softmax(3) >>> storage.save_json(connection, '/path/to/parameters.json') """ connection = extract_connection(connection) data = save_dict(connection) with open(filepath, 'w') as f: dump_with_fastest_json_module(data, f)
def save_pickle(connection, filepath, python_compatible=True): """ Save layer parameters in pickle file. Parameters ---------- {save_dict.connection} filepath : str Path to the pickle file that stores network parameters. python_compatible : bool If `True` pickled object would be compatible with Python 2 and 3 (pickle protocol equalt to `2`). If `False` then value would be pickled as highest protocol (`pickle.HIGHEST_PROTOCOL`). Defaults to `True`. Examples -------- >>> from neupy import layers, storage >>> >>> connection = layers.Input(10) > layers.Softmax(3) >>> storage.save_pickle(connection, '/path/to/parameters.pickle') """ connection = extract_connection(connection) data = save_dict(connection) with open(filepath, 'wb+') as f: # Protocol 2 is compatible for both python versions protocol = pickle.HIGHEST_PROTOCOL if python_compatible else 2 pickle.dump(data, f, protocol)
def load_hdf5(connection, filepath, ignore_missed=False, load_by='names_or_order'): """ Load network parameters from HDF5 file. Parameters ---------- {load_dict.connection} filepath : str Path to HDF5 file that will store network parameters. {load_dict.ignore_missed} {load_dict.load_by} Raises ------ {load_dict.Raises} Examples -------- >>> from neupy import layers, storage >>> >>> connection = layers.Input(10) > layers.Softmax(3) >>> storage.load_hdf5(connection, '/path/to/parameters.hdf5') """ hdf5 = load_hdf5_module() connection = extract_connection(connection) data = {} with hdf5.File(filepath, mode='r') as f: data['metadata'] = json.loads(f.attrs['metadata']) data['graph'] = json.loads(f.attrs['graph']) data['layers'] = [] layer_names = json.loads(f.attrs['layer_names']) for layer_name in layer_names: layer_group = f[layer_name] layer = {'name': layer_name} for attrname, attrvalue in layer_group.attrs.items(): try: layer[attrname] = json.loads(attrvalue) except ValueError: layer[attrname] = attrvalue layer['parameters'] = {} for param_name, parameter in layer_group.items(): layer['parameters'][param_name] = { 'value': parameter.value, 'trainable': parameter.attrs['trainable'], } data['layers'].append(layer) load_dict(connection, data, ignore_missed, load_by)
def save_hdf5(connection, filepath): """ Save network parameters in HDF5 format. Parameters ---------- {save_dict.connection} filepath : str Path to the HDF5 file that stores network parameters. Examples -------- >>> from neupy import layers, storage >>> >>> connection = layers.Input(10) > layers.Softmax(3) >>> storage.save_hdf5(connection, '/path/to/parameters.hdf5') """ hdf5 = load_hdf5_module() connection = extract_connection(connection) data = save_dict(connection) with hdf5.File(filepath, mode='w') as f: layer_names = [] for layer in data['layers']: layer_name = layer['name'] layer_group = f.create_group(layer_name) for attrname, attrvalue in layer.items(): if attrname != 'parameters': layer_group.attrs[attrname] = json.dumps( attrvalue, default=repr) for param_name, param in layer['parameters'].items(): dataset = layer_group.create_dataset( param_name, data=param['value']) dataset.attrs['trainable'] = param['trainable'] layer_names.append(layer_name) f.attrs['metadata'] = json.dumps(data['metadata']) f.attrs['graph'] = json.dumps(data['graph']) f.attrs['layer_names'] = json.dumps(layer_names)
def load_pickle(connection, filepath, ignore_missing=False, load_by='names_or_order', skip_validation=True): """ Load and set parameters for layers from the specified filepath. Parameters ---------- {load_dict.connection} filepath : str Path to pickle file that will store network parameters. {load_dict.ignore_missing} {load_dict.load_by} {load_dict.skip_validation} Raises ------ {load_dict.Raises} Examples -------- >>> from neupy import layers, storage >>> >>> connection = layers.Input(10) > layers.Softmax(3) >>> storage.load_pickle(connection, '/path/to/parameters.pickle') """ connection = extract_connection(connection) with open(filepath, 'rb') as f: # Specify encoding for python 3 in order to be able to # read files that has been created in python 2 options = {'encoding': 'latin1'} if six.PY3 else {} data = pickle.load(f, **options) load_dict(connection, data, ignore_missing, load_by)
def load_json(connection, filepath, ignore_missed=False, load_by='names_or_order'): """ Load network parameters from JSON file. Parameters ---------- {load_dict.connection} filepath : str Path to JSON file that will store network parameters. {load_dict.ignore_missed} {load_dict.load_by} Raises ------ {load_dict.Raises} Notes ----- Install `ujson` library in order to speed up saving and loading procedure. Examples -------- >>> from neupy import layers, storage >>> >>> connection = layers.Input(10) > layers.Softmax(3) >>> storage.load_json(connection, '/path/to/parameters.json') """ connection = extract_connection(connection) data = save_dict(connection) with open(filepath, 'r') as f: data = load_with_fastest_json_module(f) load_dict(connection, data, ignore_missed, load_by)
def load_pickle(connection, filepath, ignore_missed=False, load_by='names_or_order'): """ Load and set parameters for layers from the specified filepath. Parameters ---------- {load_dict.connection} filepath : str Path to pickle file that will store network parameters. {load_dict.ignore_missed} {load_dict.load_by} Raises ------ {load_dict.Raises} Examples -------- >>> from neupy import layers, storage >>> >>> connection = layers.Input(10) > layers.Softmax(3) >>> storage.load_pickle(connection, '/path/to/parameters.pickle') """ connection = extract_connection(connection) with open(filepath, 'rb') as f: if six.PY3: # Specify encoding for python 3 in order to be able to # read files that has been created in python 2 data = pickle.load(f, encoding='latin1') # skip coverage else: data = pickle.load(f) # skip coverage load_dict(connection, data, ignore_missed, load_by)
def load_json(connection, filepath, ignore_missing=False, load_by='names_or_order', skip_validation=True): """ Load network parameters from JSON file. Parameters ---------- {load_dict.connection} filepath : str Path to JSON file that will store network parameters. {load_dict.ignore_missing} {load_dict.load_by} {load_dict.skip_validation} Raises ------ {load_dict.Raises} Examples -------- >>> from neupy import layers, storage >>> >>> connection = layers.Input(10) > layers.Softmax(3) >>> storage.load_json(connection, '/path/to/parameters.json') """ connection = extract_connection(connection) data = save_dict(connection) with open(filepath, 'r') as f: data = json.load(f) load_dict(connection, data, ignore_missing, load_by)
def mixture_of_experts(networks, gating_layer=None): """ Generates mixture of experts architecture from the set of networks that has the same input and output shapes. Mixture of experts learns to how to mix results from different networks in order to get better performances. It adds gating layer that using input data tries to figure out which of the networks will make better contribution to the final result. The final result mixes from all networks using different weights. The higher the weight the larger contribution from the individual layer. Parameters ---------- networks : list of connections or networks These networks will be combine into mixture of experts. Every network should have single 1D input layer and single output layer. Another restriction is that all networks should expect the same input and output layers. gating_layer : None or layer In case if value equal to `None` that the following layer will be created. .. code-block:: python gating_layer = layers.Softmax(len(networks)) Output from the gating layer should be 1D and equal to the number of networks. Raises ------ ValueError In case if there is some problem with input networks or custom gating layer. Returns ------- connection Mixture of experts network that combine all networks into single one and adds gating layer to it. Examples -------- >>> from neupy import layers, algorithms, architectures >>> >>> network = architectures.mixture_of_experts([ ... layers.join( ... layers.Input(10), ... layers.Relu(5), ... ), ... layers.join( ... layers.Input(10), ... layers.Relu(33), ... layers.Relu(5), ... ), ... layers.join( ... layers.Input(10), ... layers.Relu(12), ... layers.Relu(25), ... layers.Relu(5), ... ), ... ]) >>> network 10 -> [... 12 layers ...] -> 5 >>> >>> gdnet = algorithms.Momentum(network, step=0.1) """ if not isinstance(networks, (list, tuple)): raise ValueError("Networks should be specified as a list") connections = [] for index, network in enumerate(networks): connection = extract_connection(network) check_if_connection_is_valid(connection, index) connections.append(connection) check_if_connections_compatible(connections) first_connection = connections[0] n_features = first_connection.input_shape[0] n_layers_to_combine = len(connections) if gating_layer is None: gating_layer = layers.Softmax(n_layers_to_combine) check_if_gating_layer_valid(gating_layer, n_layers_to_combine) return layers.join( layers.Input(n_features), # Note: Gating network should be specified # as a first parameter. [gating_layer] + connections, layers.GatedAverage(), )
def layer_structure(connection, ignore_layers=None, filepath=None, show=True): """ Draw graphical representation of the layer connection structure in form of directional graph. Parameters ---------- connection : BaseLayer instance, BaseNetwork instance ignore_layers : list or None List of layer types that needs to be excluded from the plot. Defaults to ``None``. filepath : str or None Path to the file that stores graph. ``None`` means that file will be saved in temporary file. Defaults to ``None``. show : bool ``True`` opens PDF file. Defaults to ``True``. Examples -------- >>> from neupy import layers, plots >>> >>> connection = layers.Input(10) > layers.Sigmoid(1) >>> plots.layer_structure(connection) """ connection = extract_connection(connection) if ignore_layers is None: ignore_layers = [] if filepath is None: filepath = tempfile.mktemp() ignore_layers = [ResidualConnection] + ignore_layers forward_graph = connection.graph.forward_graph forward_graph = exclude_layer_from_graph(forward_graph, ignore_layers) digraph = graphviz.Digraph() for layer in forward_graph.keys(): digraph.node(layer_uid(layer), str(layer)) output_id = 1 for from_layer, to_layers in forward_graph.items(): for to_layer in to_layers: digraph.edge(layer_uid(from_layer), layer_uid(to_layer), label=format_label(from_layer.output_shape)) if not to_layers: output = 'output-{}'.format(output_id) digraph.node(output, 'Output #{}'.format(output_id)) digraph.edge(layer_uid(from_layer), output, label=" {}".format(from_layer.output_shape)) output_id += 1 digraph.render(filepath, view=show)
def save_dict(connection): """ Save network into the dictionary. Parameters ---------- connection : network, list of layer or connection Returns ------- dict Saved parameters and information about network in dictionary using specific format. Learn more about the NeuPy's storage format in the official documentation. Examples -------- >>> from neupy import layers, storage >>> >>> connection = layers.Input(10) > layers.Softmax(3) >>> layers_data = storage.save_dict(connection) >>> >>> layers_data.keys() ['layers', 'graph', 'metadata'] """ connection = extract_connection(connection) data = { 'metadata': { 'language': 'python', 'library': 'neupy', 'version': neupy.__version__, 'created': strftime("%a, %d %b %Y %H:%M:%S %Z", gmtime()), 'theano_float': theano.config.floatX, }, # Make it as a list in order to save the right order # of paramters, otherwise it can be convert to the dictionary. 'graph': connection.graph.layer_names_only(), 'layers': [], } for layer in connection: parameters = {} configs = {} for attrname, parameter in layer.parameters.items(): parameters[attrname] = { 'value': asfloat(parameter.get_value()), 'trainable': parameter.trainable, } for option_name in layer.options: if option_name not in parameters: configs[option_name] = getattr(layer, option_name) data['layers'].append({ 'class_name': layer.__class__.__name__, 'input_shape': layer.input_shape, 'output_shape': layer.output_shape, 'name': layer.name, 'parameters': parameters, 'configs': configs, }) return data
def load_dict(connection, data, ignore_missed=False, load_by='names_or_order'): """ Load network connections from dictionary. Parameters ---------- connection : network, list of layer or connection data : dict Dictionary that stores network parameters. ignore_missed : bool ``False`` means that error will be triggered in case if some of the layers doesn't have storage parameters in the specified source. Defaults to ``False``. load_by : {{``names``, ``order``, ``names_or_order``}} Defines strategy that will be used during parameter loading - ``names`` - Matches layers in the network with stored layer using their names. - ``order`` - Matches layers in the network with stored layer using exect order of layers. - ``names_or_order`` - Matches layers in the network with stored layer trying to do it first using the same names and then matching them sequentialy. Defaults to ``names_or_order``. Raises ------ ValueError Happens in case if `ignore_missed=False` and there is no parameters for some of the layers. """ if load_by not in ('names', 'order', 'names_or_order'): raise ValueError( "Invalid value for the `load_by` argument: {}. Should be " "one of the following values: names, order, names_or_order." "".format(load_by)) validate_data_structure(data) connection = extract_connection(connection) # We are only interested in layers that has parameters layers = data['layers'] layers_data = [l for l in layers if l['parameters']] layers_conn = [l for l in connection if l.parameters] if not ignore_missed and len(layers_data) != len(layers_conn): raise ParameterLoaderError( "Couldn't load parameters from the dictionary. Connection " "has {} layers with parameters whether stored data has {}" "".format(len(layers_data), len(layers_conn))) if load_by == 'names': load_dict_by_names(layers_conn, layers_data, ignore_missed) elif load_by == 'order': load_dict_sequentially(layers_conn, layers_data) else: try: # First we try to load parameters using there names as # identifiers. Names are more reliable identifiers than # order of layers in the network load_dict_by_names(layers_conn, layers_data, ignore_missed) except ParameterLoaderError: # If we couldn't load data using layer names we will try to # compare layers in sequence one by one. Even if names are # different networks can be the same and order of parameters # should also be the same load_dict_sequentially(layers_conn, layers_data) # We need to initalize connection, to make sure # that each layer will generate shared variables # and validate connections connection.initialize()