def test_connection_extraction(self):
connection = layers.Input(2) > layers.Sigmoid(3)
self.assertIs(extract_connection(connection), connection)
network = algorithms.GradientDescent(connection)
self.assertIs(extract_connection(network), connection)
list_of_layers = [layers.Input(2), layers.Sigmoid(3)]
actual_connection = extract_connection(list_of_layers)
self.assertEqual(len(actual_connection), 2)
self.assertEqual(actual_connection.input_shape, (2, ))
self.assertEqual(actual_connection.output_shape, (3, ))
with self.assertRaisesRegexp(TypeError, "Invalid input type"):
extract_connection(object)
def save_json(connection, filepath, indent=None):
"""
Save network parameters in JSON format.
Parameters
----------
{save_dict.connection}
filepath : str
Path to the JSON file that stores network parameters.
indent : int or None
Indentation that would be specified for the output JSON.
Intentation equal to `2` or `4` makes it easy to read raw
text files. The `None` value disables indentation which means
that everything will be stored compactly. Defaults to `None`.
Examples
--------
>>> from neupy import layers, storage
>>>
>>> connection = layers.Input(10) > layers.Softmax(3)
>>> storage.save_json(connection, '/path/to/parameters.json')
"""
connection = extract_connection(connection)
data = save_dict(connection)
with open(filepath, 'w') as f:
# Without extra data processor we won't be able to dump
# numpy array into json without raising an error.
# `json` will have issues with numpy array encoding
convert_numpy_array_to_list_recursively(data)
return json.dump(data, f, indent=indent, default=repr)
def save_json(connection, filepath, indent=None):
"""
Save network parameters in JSON format.
Parameters
----------
{save_dict.connection}
filepath : str
Path to the JSON file that stores network parameters.
indent : int or None
Indentation that would be specified for the output JSON.
Intentation equal to `2` or `4` makes it easy to read raw
text files. The `None` value disables indentation which means
that everything will be stored compactly. Defaults to `None`.
Notes
-----
Install `ujson` library in order to speed up saving and
loading procedure.
Examples
--------
>>> from neupy import layers, storage
>>>
>>> connection = layers.Input(10) > layers.Softmax(3)
>>> storage.save_json(connection, '/path/to/parameters.json')
"""
connection = extract_connection(connection)
data = save_dict(connection)
with open(filepath, 'w') as f:
dump_with_fastest_json_module(data, f)
def save_pickle(connection, filepath, python_compatible=True):
"""
Save layer parameters in pickle file.
Parameters
----------
{save_dict.connection}
filepath : str
Path to the pickle file that stores network parameters.
python_compatible : bool
If `True` pickled object would be compatible with
Python 2 and 3 (pickle protocol equalt to `2`).
If `False` then value would be pickled as highest
protocol (`pickle.HIGHEST_PROTOCOL`).
Defaults to `True`.
Examples
--------
>>> from neupy import layers, storage
>>>
>>> connection = layers.Input(10) > layers.Softmax(3)
>>> storage.save_pickle(connection, '/path/to/parameters.pickle')
"""
connection = extract_connection(connection)
data = save_dict(connection)
with open(filepath, 'wb+') as f:
# Protocol 2 is compatible for both python versions
protocol = pickle.HIGHEST_PROTOCOL if python_compatible else 2
pickle.dump(data, f, protocol)
def load_hdf5(connection,
filepath,
ignore_missed=False,
load_by='names_or_order'):
"""
Load network parameters from HDF5 file.
Parameters
----------
{load_dict.connection}
filepath : str
Path to HDF5 file that will store network parameters.
{load_dict.ignore_missed}
{load_dict.load_by}
Raises
------
{load_dict.Raises}
Examples
--------
>>> from neupy import layers, storage
>>>
>>> connection = layers.Input(10) > layers.Softmax(3)
>>> storage.load_hdf5(connection, '/path/to/parameters.hdf5')
"""
hdf5 = load_hdf5_module()
connection = extract_connection(connection)
data = {}
with hdf5.File(filepath, mode='r') as f:
data['metadata'] = json.loads(f.attrs['metadata'])
data['graph'] = json.loads(f.attrs['graph'])
data['layers'] = []
layer_names = json.loads(f.attrs['layer_names'])
for layer_name in layer_names:
layer_group = f[layer_name]
layer = {'name': layer_name}
for attrname, attrvalue in layer_group.attrs.items():
try:
layer[attrname] = json.loads(attrvalue)
except ValueError:
layer[attrname] = attrvalue
layer['parameters'] = {}
for param_name, parameter in layer_group.items():
layer['parameters'][param_name] = {
'value': parameter.value,
'trainable': parameter.attrs['trainable'],
}
data['layers'].append(layer)
load_dict(connection, data, ignore_missed, load_by)
def save_hdf5(connection, filepath):
"""
Save network parameters in HDF5 format.
Parameters
----------
{save_dict.connection}
filepath : str
Path to the HDF5 file that stores network parameters.
Examples
--------
>>> from neupy import layers, storage
>>>
>>> connection = layers.Input(10) > layers.Softmax(3)
>>> storage.save_hdf5(connection, '/path/to/parameters.hdf5')
"""
hdf5 = load_hdf5_module()
connection = extract_connection(connection)
data = save_dict(connection)
with hdf5.File(filepath, mode='w') as f:
layer_names = []
for layer in data['layers']:
layer_name = layer['name']
layer_group = f.create_group(layer_name)
for attrname, attrvalue in layer.items():
if attrname != 'parameters':
layer_group.attrs[attrname] = json.dumps(
attrvalue, default=repr)
for param_name, param in layer['parameters'].items():
dataset = layer_group.create_dataset(
param_name, data=param['value'])
dataset.attrs['trainable'] = param['trainable']
layer_names.append(layer_name)
f.attrs['metadata'] = json.dumps(data['metadata'])
f.attrs['graph'] = json.dumps(data['graph'])
f.attrs['layer_names'] = json.dumps(layer_names)
def load_pickle(connection,
filepath,
ignore_missing=False,
load_by='names_or_order',
skip_validation=True):
"""
Load and set parameters for layers from the
specified filepath.
Parameters
----------
{load_dict.connection}
filepath : str
Path to pickle file that will store network parameters.
{load_dict.ignore_missing}
{load_dict.load_by}
{load_dict.skip_validation}
Raises
------
{load_dict.Raises}
Examples
--------
>>> from neupy import layers, storage
>>>
>>> connection = layers.Input(10) > layers.Softmax(3)
>>> storage.load_pickle(connection, '/path/to/parameters.pickle')
"""
connection = extract_connection(connection)
with open(filepath, 'rb') as f:
# Specify encoding for python 3 in order to be able to
# read files that has been created in python 2
options = {'encoding': 'latin1'} if six.PY3 else {}
data = pickle.load(f, **options)
load_dict(connection, data, ignore_missing, load_by)
def load_json(connection,
filepath,
ignore_missed=False,
load_by='names_or_order'):
"""
Load network parameters from JSON file.
Parameters
----------
{load_dict.connection}
filepath : str
Path to JSON file that will store network parameters.
{load_dict.ignore_missed}
{load_dict.load_by}
Raises
------
{load_dict.Raises}
Notes
-----
Install `ujson` library in order to speed up saving and
loading procedure.
Examples
--------
>>> from neupy import layers, storage
>>>
>>> connection = layers.Input(10) > layers.Softmax(3)
>>> storage.load_json(connection, '/path/to/parameters.json')
"""
connection = extract_connection(connection)
data = save_dict(connection)
with open(filepath, 'r') as f:
data = load_with_fastest_json_module(f)
load_dict(connection, data, ignore_missed, load_by)
def load_pickle(connection,
filepath,
ignore_missed=False,
load_by='names_or_order'):
"""
Load and set parameters for layers from the
specified filepath.
Parameters
----------
{load_dict.connection}
filepath : str
Path to pickle file that will store network parameters.
{load_dict.ignore_missed}
{load_dict.load_by}
Raises
------
{load_dict.Raises}
Examples
--------
>>> from neupy import layers, storage
>>>
>>> connection = layers.Input(10) > layers.Softmax(3)
>>> storage.load_pickle(connection, '/path/to/parameters.pickle')
"""
connection = extract_connection(connection)
with open(filepath, 'rb') as f:
if six.PY3:
# Specify encoding for python 3 in order to be able to
# read files that has been created in python 2
data = pickle.load(f, encoding='latin1') # skip coverage
else:
data = pickle.load(f) # skip coverage
load_dict(connection, data, ignore_missed, load_by)
def load_json(connection,
filepath,
ignore_missing=False,
load_by='names_or_order',
skip_validation=True):
"""
Load network parameters from JSON file.
Parameters
----------
{load_dict.connection}
filepath : str
Path to JSON file that will store network parameters.
{load_dict.ignore_missing}
{load_dict.load_by}
{load_dict.skip_validation}
Raises
------
{load_dict.Raises}
Examples
--------
>>> from neupy import layers, storage
>>>
>>> connection = layers.Input(10) > layers.Softmax(3)
>>> storage.load_json(connection, '/path/to/parameters.json')
"""
connection = extract_connection(connection)
data = save_dict(connection)
with open(filepath, 'r') as f:
data = json.load(f)
load_dict(connection, data, ignore_missing, load_by)
def mixture_of_experts(networks, gating_layer=None):
"""
Generates mixture of experts architecture from the set of
networks that has the same input and output shapes.
Mixture of experts learns to how to mix results from different
networks in order to get better performances. It adds gating layer
that using input data tries to figure out which of the networks
will make better contribution to the final result. The final result
mixes from all networks using different weights. The higher the weight
the larger contribution from the individual layer.
Parameters
----------
networks : list of connections or networks
These networks will be combine into mixture of experts.
Every network should have single 1D input layer and
single output layer. Another restriction is that all networks
should expect the same input and output layers.
gating_layer : None or layer
In case if value equal to `None` that the following layer
will be created.
.. code-block:: python
gating_layer = layers.Softmax(len(networks))
Output from the gating layer should be 1D and equal to
the number of networks.
Raises
------
ValueError
In case if there is some problem with input networks
or custom gating layer.
Returns
-------
connection
Mixture of experts network that combine all networks into
single one and adds gating layer to it.
Examples
--------
>>> from neupy import layers, algorithms, architectures
>>>
>>> network = architectures.mixture_of_experts([
... layers.join(
... layers.Input(10),
... layers.Relu(5),
... ),
... layers.join(
... layers.Input(10),
... layers.Relu(33),
... layers.Relu(5),
... ),
... layers.join(
... layers.Input(10),
... layers.Relu(12),
... layers.Relu(25),
... layers.Relu(5),
... ),
... ])
>>> network
10 -> [... 12 layers ...] -> 5
>>>
>>> gdnet = algorithms.Momentum(network, step=0.1)
"""
if not isinstance(networks, (list, tuple)):
raise ValueError("Networks should be specified as a list")
connections = []
for index, network in enumerate(networks):
connection = extract_connection(network)
check_if_connection_is_valid(connection, index)
connections.append(connection)
check_if_connections_compatible(connections)
first_connection = connections[0]
n_features = first_connection.input_shape[0]
n_layers_to_combine = len(connections)
if gating_layer is None:
gating_layer = layers.Softmax(n_layers_to_combine)
check_if_gating_layer_valid(gating_layer, n_layers_to_combine)
return layers.join(
layers.Input(n_features),
# Note: Gating network should be specified
# as a first parameter.
[gating_layer] + connections,
layers.GatedAverage(),
)
def layer_structure(connection, ignore_layers=None, filepath=None, show=True):
"""
Draw graphical representation of the layer connection
structure in form of directional graph.
Parameters
----------
connection : BaseLayer instance, BaseNetwork instance
ignore_layers : list or None
List of layer types that needs to be excluded
from the plot. Defaults to ``None``.
filepath : str or None
Path to the file that stores graph. ``None`` means
that file will be saved in temporary file.
Defaults to ``None``.
show : bool
``True`` opens PDF file. Defaults to ``True``.
Examples
--------
>>> from neupy import layers, plots
>>>
>>> connection = layers.Input(10) > layers.Sigmoid(1)
>>> plots.layer_structure(connection)
"""
connection = extract_connection(connection)
if ignore_layers is None:
ignore_layers = []
if filepath is None:
filepath = tempfile.mktemp()
ignore_layers = [ResidualConnection] + ignore_layers
forward_graph = connection.graph.forward_graph
forward_graph = exclude_layer_from_graph(forward_graph, ignore_layers)
digraph = graphviz.Digraph()
for layer in forward_graph.keys():
digraph.node(layer_uid(layer), str(layer))
output_id = 1
for from_layer, to_layers in forward_graph.items():
for to_layer in to_layers:
digraph.edge(layer_uid(from_layer),
layer_uid(to_layer),
label=format_label(from_layer.output_shape))
if not to_layers:
output = 'output-{}'.format(output_id)
digraph.node(output, 'Output #{}'.format(output_id))
digraph.edge(layer_uid(from_layer),
output,
label=" {}".format(from_layer.output_shape))
output_id += 1
digraph.render(filepath, view=show)
def save_dict(connection):
"""
Save network into the dictionary.
Parameters
----------
connection : network, list of layer or connection
Returns
-------
dict
Saved parameters and information about network in dictionary
using specific format. Learn more about the NeuPy's storage
format in the official documentation.
Examples
--------
>>> from neupy import layers, storage
>>>
>>> connection = layers.Input(10) > layers.Softmax(3)
>>> layers_data = storage.save_dict(connection)
>>>
>>> layers_data.keys()
['layers', 'graph', 'metadata']
"""
connection = extract_connection(connection)
data = {
'metadata': {
'language': 'python',
'library': 'neupy',
'version': neupy.__version__,
'created': strftime("%a, %d %b %Y %H:%M:%S %Z", gmtime()),
'theano_float': theano.config.floatX,
},
# Make it as a list in order to save the right order
# of paramters, otherwise it can be convert to the dictionary.
'graph': connection.graph.layer_names_only(),
'layers': [],
}
for layer in connection:
parameters = {}
configs = {}
for attrname, parameter in layer.parameters.items():
parameters[attrname] = {
'value': asfloat(parameter.get_value()),
'trainable': parameter.trainable,
}
for option_name in layer.options:
if option_name not in parameters:
configs[option_name] = getattr(layer, option_name)
data['layers'].append({
'class_name': layer.__class__.__name__,
'input_shape': layer.input_shape,
'output_shape': layer.output_shape,
'name': layer.name,
'parameters': parameters,
'configs': configs,
})
return data
def load_dict(connection, data, ignore_missed=False, load_by='names_or_order'):
"""
Load network connections from dictionary.
Parameters
----------
connection : network, list of layer or connection
data : dict
Dictionary that stores network parameters.
ignore_missed : bool
``False`` means that error will be triggered in case
if some of the layers doesn't have storage parameters
in the specified source. Defaults to ``False``.
load_by : {{``names``, ``order``, ``names_or_order``}}
Defines strategy that will be used during parameter loading
- ``names`` - Matches layers in the network with stored layer
using their names.
- ``order`` - Matches layers in the network with stored layer
using exect order of layers.
- ``names_or_order`` - Matches layers in the network with stored
layer trying to do it first using the same names and then
matching them sequentialy.
Defaults to ``names_or_order``.
Raises
------
ValueError
Happens in case if `ignore_missed=False` and there is no
parameters for some of the layers.
"""
if load_by not in ('names', 'order', 'names_or_order'):
raise ValueError(
"Invalid value for the `load_by` argument: {}. Should be "
"one of the following values: names, order, names_or_order."
"".format(load_by))
validate_data_structure(data)
connection = extract_connection(connection)
# We are only interested in layers that has parameters
layers = data['layers']
layers_data = [l for l in layers if l['parameters']]
layers_conn = [l for l in connection if l.parameters]
if not ignore_missed and len(layers_data) != len(layers_conn):
raise ParameterLoaderError(
"Couldn't load parameters from the dictionary. Connection "
"has {} layers with parameters whether stored data has {}"
"".format(len(layers_data), len(layers_conn)))
if load_by == 'names':
load_dict_by_names(layers_conn, layers_data, ignore_missed)
elif load_by == 'order':
load_dict_sequentially(layers_conn, layers_data)
else:
try:
# First we try to load parameters using there names as
# identifiers. Names are more reliable identifiers than
# order of layers in the network
load_dict_by_names(layers_conn, layers_data, ignore_missed)
except ParameterLoaderError:
# If we couldn't load data using layer names we will try to
# compare layers in sequence one by one. Even if names are
# different networks can be the same and order of parameters
# should also be the same
load_dict_sequentially(layers_conn, layers_data)
# We need to initalize connection, to make sure
# that each layer will generate shared variables
# and validate connections
connection.initialize()
