class BasePooling(BaseLayer):
""" Base class for the pooling layers.
Parameters
----------
size : tuple with 2 integers
Factor by which to downscale (vertical, horizontal).
(2,2) will halve the image in each dimension.
stride_size :
Stride size, which is the number of shifts over
rows/cols to get the next pool region. If stride_size is
None, it is considered equal to ds (no overlap on
pooling regions).
padding : tuple of two ints
(pad_h, pad_w), pad zeros to extend beyond four borders of
the images, pad_h is the size of the top and bottom margins,
and pad_w is the size of the left and right margins.
"""
size = TypedListProperty(required=True, element_type=int)
stride_size = StrideProperty(default=None)
padding = TypedListProperty(default=(0, 0), element_type=int, n_elements=2)
def __init__(self, size, **options):
options['size'] = size
super(BasePooling, self).__init__(**options)
def __repr__(self):
return '{name}({size})'.format(name=self.__class__.__name__,
size=self.size)
class Transpose(BaseLayer):
"""
Layer transposes input tensor. Permutes the dimensions according
to the ``perm`` parameter.
Parameters
----------
perm : tuple or list
A permutation of the dimensions of the input tensor.
{BaseLayer.name}
Methods
-------
{BaseLayer.Methods}
Attributes
----------
{BaseLayer.Attributes}
Examples
--------
>>> from neupy.layers import *
>>> network = Input((7, 11)) >> Transpose((0, 2, 1))
(?, 7, 11) -> [... 2 layers ...] -> (?, 11, 7)
"""
perm = TypedListProperty()
def __init__(self, perm, name=None):
super(Transpose, self).__init__(name=name)
self.perm = perm
def fail_if_shape_invalid(self, input_shape):
if input_shape and max(self.perm) >= input_shape.ndims:
raise LayerConnectionError(
"Cannot apply transpose operation to the "
"input. Permuntation: {}, Input shape: {}"
"".format(self.perm, input_shape))
def get_output_shape(self, input_shape):
input_shape = tf.TensorShape(input_shape)
self.fail_if_shape_invalid(input_shape)
if input_shape.ndims is None:
n_dims_expected = len(self.perm)
return tf.TensorShape([None] * n_dims_expected)
input_shape = np.array(input_shape.dims)
perm = list(self.perm)
return tf.TensorShape(input_shape[perm])
def output(self, input, **kwargs):
# Input value has batch dimension, but perm will never have it
# specified as (zero index), so we need to add it in order to
# fix batch dimesnion in place.
return tf.transpose(input, list(self.perm))
def __repr__(self):
return self._repr_arguments(self.perm, name=self.name)
class Convolution(ParameterBasedLayer):
""" Convolutional layer.
Parameters
----------
size : tuple of integers
Filter shape.
border_mode : {{'valid', 'full', 'half'}} or int or tuple with 2 int
Convolution border mode. Check Theano's `nnet.conv2d` doc.
stride_size : tuple with 1 or 2 integers or integer.
Stride size.
"""
size = TypedListProperty(required=True, element_type=int)
border_mode = BorderModeProperty(default='valid')
stride_size = StrideProperty(default=(1, 1))
def weight_shape(self):
return self.size
def bias_shape(self):
return self.size[:1]
def output(self, input_value):
bias = T.reshape(self.bias, (1, -1, 1, 1))
output = T.nnet.conv2d(input_value,
self.weight,
border_mode=self.border_mode,
subsample=self.stride_size)
return output + bias
class Transpose(BaseLayer):
"""
Transposes input. Permutes the dimensions according to ``perm``.
Parameters
----------
perm : tuple or list
A permutation of the dimensions of the input tensor. Layer cannot
transpose batch dimension and using ``0`` in the list of
permuted dimensions is not allowed.
{BaseLayer.Parameters}
Methods
-------
{BaseLayer.Methods}
Attributes
----------
{BaseLayer.Attributes}
Examples
--------
>>> from neupy.layers import *
>>> conn = Input((7, 11)) > Transpose((2, 1))
>>> conn.input_shape
(7, 11)
>>> conn.output_shape
(11, 7)
"""
perm = TypedListProperty()
def __init__(self, perm, **options):
if 0 in perm:
raise ValueError("Batch dimension has fixed position and 0 "
"index cannot be used.")
super(Transpose, self).__init__(perm=perm, **options)
def validate(self, input_shape):
if len(input_shape) < 2:
raise LayerConnectionError(
"Transpose expects input with at least 3 dimensions.")
@property
def output_shape(self):
# Input shape doesn't have information about the batch size and perm
# indeces require to have this dimension on zero's position.
input_shape = np.array(as_tuple(None, self.input_shape))
return as_tuple(input_shape[self.perm].tolist())
def output(self, input_value):
# Input value has batch dimension, but perm will never have it
# specified as (zero index), so we need to add it in order to
# fix batch dimesnion in place.
return tf.transpose(input_value, [0] + list(self.perm))
class Reshape(BaseLayer):
"""
Gives a new shape to an input value without changing
its data.
Parameters
----------
shape : tuple or list
New feature shape. ``None`` value means that feature
will be flatten in 1D vector. If you need to get the
output feature with more that 2 dimensions then you can
set up new feature shape using tuples. Defaults to ``None``.
Methods
-------
{BaseLayer.Methods}
Attributes
----------
{BaseLayer.Attributes}
"""
shape = TypedListProperty()
def __init__(self, shape=None, **options):
if shape is not None:
options['shape'] = shape
super(Reshape, self).__init__(**options)
@property
def output_shape(self):
if self.shape is not None:
return as_tuple(self.shape)
n_output_features = np.prod(self.input_shape)
return as_tuple(n_output_features)
def output(self, input_value):
"""
Reshape the feature space for the input value.
Parameters
----------
input_value : array-like or Theano variable
"""
n_samples = input_value.shape[0]
output_shape = as_tuple(n_samples, self.output_shape)
return T.reshape(input_value, output_shape)
class StepOutput(Output):
""" The behaviour for this layer is the same as for step function.
Parameters
----------
output_bounds : tuple
Value is must be a tuple which contains two elements where first one
identify lower output value and the second one - bigger. Defaults
to ``(0, 1)``.
critical_point : float
Critical point is set up step function bias. Value equal to this
point should be equal to the lower bound. Defaults to ``0``.
{Output.size}
"""
output_bounds = TypedListProperty(default=(0, 1))
critical_point = ProperFractionProperty(default=0)
def output(self, value):
lower_bound, upper_bound = self.output_bounds
return np.where(value <= self.critical_point, lower_bound, upper_bound)
class Reshape(BaseLayer):
""" Gives a new shape to an input value without changing
its data.
Parameters
----------
shape : tuple or list
New feature shape. ``None`` value means that feature
will be flatten in 1D vector. If you need to get the
output feature with more that 2 dimensions then you can
set up new feature shape using tuples. Defaults to ``None``.
"""
shape = TypedListProperty()
def __init__(self, shape=None, **options):
if shape is not None:
options['shape'] = shape
super(Reshape, self).__init__(**options)
def output(self, input_value):
""" Reshape the feature space for the input value.
Parameters
----------
input_value : array-like or Theano variable
"""
new_feature_shape = self.shape
input_shape = input_value.shape[0]
if new_feature_shape is None:
output_shape = input_value.shape[1:]
new_feature_shape = T.prod(output_shape)
output_shape = (input_shape, new_feature_shape)
else:
output_shape = (input_shape,) + new_feature_shape
return T.reshape(input_value, output_shape)
class BasePooling(BaseLayer):
""" Base class for the pooling layers.
Parameters
----------
size : tuple with 2 integers
Factor by which to downscale (vertical, horizontal).
(2,2) will halve the image in each dimension.
stride_size :
Stride size, which is the number of shifts over
rows/cols to get the next pool region. If stride_size is
None, it is considered equal to ds (no overlap on
pooling regions).
"""
size = TypedListProperty(required=True, element_type=int)
stride_size = StrideProperty(default=None)
def __init__(self, size, **options):
options['size'] = size
super(BasePooling, self).__init__(**options)
def __repr__(self):
return '{name}({size})'.format(name=self.__class__.__name__,
size=self.size)
class Convolution(ParameterBasedLayer):
"""
Convolutional layer.
Parameters
----------
size : tuple of int
Filter shape. In should be defined as a tuple with three integers
``(output channels, filter rows, filter columns)``.
border_mode : {{'valid', 'full', 'half'}} or int or tuple with 2 int
Convolution border mode. Check Theano's ``nnet.conv2d`` doc.
stride_size : tuple with 1 or 2 integers or integer.
Stride size.
{ParameterBasedLayer.weight}
{ParameterBasedLayer.bias}
Methods
-------
{ParameterBasedLayer.Methods}
Attributes
----------
{ParameterBasedLayer.Attributes}
"""
size = TypedListProperty(required=True, element_type=int)
border_mode = BorderModeProperty(default='valid')
stride_size = StrideProperty(default=(1, 1))
@property
def output_shape(self):
if self.input_shape is None:
return None
if len(self.input_shape) < 2:
raise ValueError(
"Convolutional layer expects an input shape with least 2 "
"dimensions, got {} with shape {}".format(
len(self.input_shape), self.input_shape))
border_mode = self.border_mode
n_kernels = self.size[0]
rows, cols = self.input_shape[-2:]
row_filter_size, col_filter_size = self.size[-2:]
row_stride, col_stride = self.stride_size
if isinstance(border_mode, tuple):
row_border_mode, col_border_mode = border_mode[-2:]
else:
row_border_mode, col_border_mode = border_mode, border_mode
output_rows = conv_output_shape(rows, row_filter_size, row_border_mode,
row_stride)
output_cols = conv_output_shape(cols, col_filter_size, col_border_mode,
col_stride)
return (n_kernels, output_rows, output_cols)
@property
def weight_shape(self):
n_channels = self.input_shape[0]
n_filters, n_rows, n_cols = self.size
return (n_filters, n_channels, n_rows, n_cols)
@property
def bias_shape(self):
return as_tuple(self.size[0])
def output(self, input_value):
bias = T.reshape(self.bias, (1, -1, 1, 1))
output = T.nnet.conv2d(input_value,
self.weight,
input_shape=as_tuple(None, self.input_shape),
filter_shape=self.weight_shape,
border_mode=self.border_mode,
subsample=self.stride_size)
return output + bias
class A(Configurable):
list_of_properties = TypedListProperty(element_type=(int, float),
n_elements=3)
class A(Configurable):
list_of_properties = TypedListProperty(element_type=str,
n_elements=3)
class BasePooling(BaseLayer):
"""
Base class for the pooling layers.
Parameters
----------
size : tuple with 2 integers
Factor by which to downscale ``(vertical, horizontal)``.
``(2, 2)`` will halve the image in each dimension.
stride : tuple or int.
Stride size, which is the number of shifts over
rows/cols to get the next pool region. If stride is
None, it is considered equal to ds (no overlap on
pooling regions).
padding : {{``valid``, ``same``}}
``(pad_h, pad_w)``, pad zeros to extend beyond four borders of
the images, pad_h is the size of the top and bottom margins,
and pad_w is the size of the left and right margins.
{BaseLayer.Parameters}
Methods
-------
{BaseLayer.Methods}
Attributes
----------
{BaseLayer.Attributes}
"""
size = TypedListProperty(required=True, element_type=int)
stride = Spatial2DProperty(allow_none=True)
padding = ChoiceProperty(choices=('SAME', 'VALID', 'same', 'valid'))
pooling_type = None
def __init__(self, size, stride=None, padding='valid', name=None):
super(BasePooling, self).__init__(name=name)
self.size = size
self.stride = stride
self.padding = padding
def fail_if_shape_invalid(self, input_shape):
if input_shape and input_shape.ndims != 4:
raise LayerConnectionError(
"Pooling layer expects an input with 4 "
"dimensions, got {} with shape {}. Layer: {}"
"".format(len(input_shape), input_shape, self))
def get_output_shape(self, input_shape):
input_shape = tf.TensorShape(input_shape)
if input_shape.ndims is None:
return tf.TensorShape((None, None, None, None))
self.fail_if_shape_invalid(input_shape)
n_samples, rows, cols, n_kernels = input_shape
row_filter_size, col_filter_size = self.size
stride = self.size if self.stride is None else self.stride
row_stride, col_stride = stride
output_rows = pooling_output_shape(
rows, row_filter_size, self.padding, row_stride)
output_cols = pooling_output_shape(
cols, col_filter_size, self.padding, col_stride)
# In python 2, we can get float number after rounding procedure
# and it might break processing in the subsequent layers.
return tf.TensorShape((n_samples, output_rows, output_cols, n_kernels))
def output(self, input_value, **kwargs):
return tf.nn.pool(
input_value,
self.size,
pooling_type=self.pooling_type,
padding=self.padding.upper(),
strides=self.stride or self.size,
data_format="NHWC")
def __repr__(self):
return self._repr_arguments(
self.size,
name=self.name,
stride=self.stride,
padding=self.padding,
)
class LVQ(BaseNetwork):
"""
Learning Vector Quantization (LVQ) algorithm.
Notes
-----
- Input data needs to be normalized, because LVQ uses
Euclidian distance to find clusters.
- Training error is just a ratio of miscassified
samples
Parameters
----------
n_inputs : int
Number of input units. It should be equal to the
number of features in the input data set.
n_subclasses : int, None
Defines total number of subclasses. Values should be greater
or equal to the number of classes. ``None`` will set up number
of subclasses equal to the number of classes. Defaults to ``None``
(or the same as ``n_classes``).
n_classes : int
Number of classes in the data set.
prototypes_per_class : list, None
Defines number of prototypes per each class. For instance,
if ``n_classes=3`` and ``n_subclasses=8`` then there are
can be 3 subclasses for the first class, 3 for the second one
and 2 for the third one (3 + 3 + 2 == 8). The following example
can be specified as ``prototypes_per_class=[3, 3, 2]``.
There are two rules that apply to this parameter:
1. ``sum(prototypes_per_class) == n_subclasses``
2. ``len(prototypes_per_class) == n_classes``
The ``None`` value will distribute approximately equal
number of subclasses per each class. It's approximately,
because in casses when ``n_subclasses % n_classes != 0``
there is no way to distribute equal number of subclasses
per each class.
Defaults to ``None``.
{BaseNetwork.step}
n_updates_to_stepdrop : int or None
If this options is not equal to ``None`` then after every
update LVQ reduces step size and do it until number of
applied updates would reach the ``n_updates_to_stepdrop``
value. The minimum possible step size defined in the
``minstep`` parameter.
Be aware that number of updates is not the same as number
of epochs. LVQ applies update after each propagated sample
through the network. Relations between this parameter and
maximum number of epochs is following
.. code-block:: python
n_updates_to_stepdrop = n_samples * n_max_epochs
If parameter equal to ``None`` then step size wouldn't be
reduced after each update.
Defaults to ``None``.
minstep : float
Step size would never be lower than this value. This
property useful only in case if ``n_updates_to_stepdrop``
is not ``None``. Defaults to ``1e-5``.
{BaseNetwork.show_epoch}
{BaseNetwork.shuffle_data}
{BaseNetwork.epoch_end_signal}
{BaseNetwork.train_end_signal}
{Verbose.verbose}
Methods
-------
{BaseSkeleton.predict}
{BaseSkeleton.fit}
"""
n_inputs = IntProperty(minval=1)
n_subclasses = IntProperty(minval=2, default=None, allow_none=True)
n_classes = IntProperty(minval=2)
prototypes_per_class = TypedListProperty(allow_none=True, default=None)
weight = Property(expected_type=(np.ndarray, init.Initializer),
allow_none=True, default=None)
n_updates_to_stepdrop = IntProperty(default=None, allow_none=True,
minval=1)
minstep = NumberProperty(minval=0, default=1e-5)
def __init__(self, **options):
self.initialized = False
super(LVQ, self).__init__(**options)
self.n_updates = 0
if self.n_subclasses is None:
self.n_subclasses = self.n_classes
if isinstance(self.weight, init.Initializer):
weight_shape = (self.n_inputs, self.n_subclasses)
self.weight = self.weight.sample(weight_shape)
if self.weight is not None:
self.initialized = True
if self.n_subclasses < self.n_classes:
raise ValueError("Number of subclasses should be greater "
"or equal to the number of classes. Network "
"was defined with {} subclasses and {} classes"
"".format(self.n_subclasses, self.n_classes))
if self.prototypes_per_class is None:
whole, reminder = divmod(self.n_subclasses, self.n_classes)
self.prototypes_per_class = [whole] * self.n_classes
if reminder:
# Since we have reminder left, it means that we cannot
# have an equal number of subclasses per each class,
# therefor we will add +1 to randomly selected class.
class_indeces = np.random.choice(self.n_classes, reminder,
replace=False)
for class_index in class_indeces:
self.prototypes_per_class[class_index] += 1
if len(self.prototypes_per_class) != self.n_classes:
raise ValueError("LVQ defined for classification problem that has "
"{} classes, but the `prototypes_per_class` "
"variable has defined data for {} classes."
"".format(self.n_classes,
len(self.prototypes_per_class)))
if sum(self.prototypes_per_class) != self.n_subclasses:
raise ValueError("Invalid distribution of subclasses for the "
"`prototypes_per_class` variable. Got total "
"of {} subclasses ({}) instead of {} expected"
"".format(sum(self.prototypes_per_class),
self.prototypes_per_class,
self.n_subclasses))
self.subclass_to_class = []
for class_id, n_prototypes in enumerate(self.prototypes_per_class):
self.subclass_to_class.extend([class_id] * n_prototypes)
@property
def training_step(self):
if self.n_updates_to_stepdrop is None:
return self.step
updates_ratio = (1 - self.n_updates / self.n_updates_to_stepdrop)
return self.minstep + (self.step - self.minstep) * updates_ratio
def predict(self, input_data):
if not self.initialized:
raise NotTrained("LVQ network hasn't been trained yet")
input_data = format_data(input_data)
subclass_to_class = self.subclass_to_class
weight = self.weight
predictions = []
for input_row in input_data:
output = euclid_distance(input_row, weight)
winner_subclass = int(output.argmin(axis=1))
predicted_class = subclass_to_class[winner_subclass]
predictions.append(predicted_class)
return np.array(predictions)
def train(self, input_train, target_train, *args, **kwargs):
input_train = format_data(input_train)
target_train = format_data(target_train)
n_input_samples = len(input_train)
if n_input_samples <= self.n_subclasses:
raise ValueError("Number of training input samples should be "
"greater than number of sublcasses. Training "
"method recived {} input samples."
"".format(n_input_samples))
if not self.initialized:
target_classes = sorted(np.unique(target_train).astype(np.int))
expected_classes = list(range(self.n_classes))
if target_classes != expected_classes:
raise ValueError("All classes should be integers from the "
"range [0, {}], but got the following "
"classes instead {}".format(
self.n_classes - 1, target_classes))
weights = []
iterator = zip(target_classes, self.prototypes_per_class)
for target_class, n_prototypes in iterator:
is_valid_class = (target_train[:, 0] == target_class)
is_valid_class = is_valid_class.astype('float64')
n_samples_per_class = sum(is_valid_class)
is_valid_class /= n_samples_per_class
if n_samples_per_class <= n_prototypes:
raise ValueError("Input data has {0} samples for class-{1}"
". Number of samples per specified "
"class-{1} should be greater than {2}."
"".format(n_samples_per_class,
target_class, n_prototypes))
class_weight_indeces = np.random.choice(
np.arange(n_input_samples), n_prototypes,
replace=False, p=is_valid_class)
class_weight = input_train[class_weight_indeces]
weights.extend(class_weight)
self.weight = np.array(weights)
self.initialized = True
super(LVQ, self).train(input_train, target_train, *args, **kwargs)
def train_epoch(self, input_train, target_train):
weight = self.weight
subclass_to_class = self.subclass_to_class
n_correct_predictions = 0
for input_row, target in zip(input_train, target_train):
step = self.training_step
output = euclid_distance(input_row, weight)
winner_subclass = int(output.argmin())
predicted_class = subclass_to_class[winner_subclass]
weight_update = input_row - weight[winner_subclass, :]
is_correct_prediction = (predicted_class == target)
if is_correct_prediction:
weight[winner_subclass, :] += step * weight_update
else:
weight[winner_subclass, :] -= step * weight_update
n_correct_predictions += is_correct_prediction
self.n_updates += 1
n_samples = len(input_train)
return 1 - n_correct_predictions / n_samples
class SOFM(Kohonen):
"""
Self-Organizing Feature Map (SOFM or SOM).
Notes
-----
- Training data samples should have normalized features.
Parameters
----------
{BaseAssociative.n_inputs}
n_outputs : int or None
Number of outputs. Parameter is optional in case if
``feature_grid`` was specified.
.. code-block:: python
if n_outputs is None:
n_outputs = np.prod(feature_grid)
learning_radius : int
Parameter defines radius within which we consider all
neurons as neighbours to the winning neuron. The bigger
the value the more neurons will be updated after each
iteration.
The ``0`` values means that we don't update
neighbour neurons.
Defaults to ``0``.
std : int, float
Parameters controls learning rate for each neighbour.
The further neighbour neuron from the winning neuron
the smaller that learning rate for it. Learning rate
scales based on the factors produced by the normal
distribution with center in the place of a winning
neuron and standard deviation specified as a parameter.
The learning rate for the winning neuron is always equal
to the value specified in the ``step`` parameter and for
neighbour neurons it's always lower.
The bigger the value for this parameter the bigger
learning rate for the neighbour neurons.
Defaults to ``1``.
features_grid : list, tuple, None
Feature grid defines shape of the output neurons.
The new shape should be compatible with the number
of outputs. It means that the following condition
should be true:
.. code-block:: python
np.prod(features_grid) == n_outputs
SOFM implementation supports n-dimensional grids.
For instance, in order to specify grid as cube instead of
the regular rectangular shape we can set up options as
the following:
.. code-block:: python
SOFM(
...
features_grid=(5, 5, 5),
...
)
Defaults to ``(n_outputs, 1)``.
grid_type : {{``rect``, ``hexagon``}}
Defines connection type in feature grid. Type defines
which neurons we will consider as closest to the winning
neuron during the training.
- ``rect`` - Connections between neurons will be organized
in hexagonal grid.
- ``hexagon`` - Connections between neurons will be organized
in hexagonal grid. It works only for 1d or 2d grids.
Defaults to ``rect``.
distance : {{``euclid``, ``dot_product``, ``cos``}}
Defines function that will be used to compute
closest weight to the input sample.
- ``dot_product``: Just a regular dot product between
data sample and network's weights
- ``euclid``: Euclidean distance between data sample
and network's weights
- ``cos``: Cosine distance between data sample and
network's weights
Defaults to ``euclid``.
reduce_radius_after : int or None
Every specified number of epochs ``learning_radius``
parameter will be reduced by ``1``. Process continues
until ``learning_radius`` equal to ``0``.
The ``None`` value disables parameter reduction
during the training.
Defaults to ``100``.
reduce_step_after : int or None
Defines reduction rate at which parameter ``step`` will
be reduced using the following formula:
.. code-block:: python
step = step / (1 + current_epoch / reduce_step_after)
The ``None`` value disables parameter reduction
during the training.
Defaults to ``100``.
reduce_std_after : int or None
Defines reduction rate at which parameter ``std`` will
be reduced using the following formula:
.. code-block:: python
std = std / (1 + current_epoch / reduce_std_after)
The ``None`` value disables parameter reduction
during the training.
Defaults to ``100``.
weight : array-like, Initializer or {{``init_pca``, ``sample_from_data``}}
Neural network weights.
Value defined manualy should have shape ``(n_inputs, n_outputs)``.
Also, it's possible to initialized weights base on the
training data. There are two options:
- ``sample_from_data`` - Before starting the training will
randomly take number of training samples equal to number
of expected outputs.
- ``init_pca`` - Before training starts SOFM will applies PCA
on a covariance matrix build from the training samples.
Weights will be generated based on the two eigenvectors
associated with the largest eigenvalues.
Defaults to :class:`Normal() <neupy.init.Normal>`.
{BaseNetwork.step}
{BaseNetwork.show_epoch}
{BaseNetwork.shuffle_data}
{BaseNetwork.signals}
{Verbose.verbose}
Methods
-------
init_weights(train_data)
Initialized weights based on the input data. It works only
for the `init_pca` and `sample_from_data` options. For other
cases it will throw an error.
{BaseSkeleton.predict}
{BaseAssociative.train}
{BaseSkeleton.fit}
Examples
--------
>>> import numpy as np
>>> from neupy import algorithms, utils
>>>
>>> utils.reproducible()
>>>
>>> data = np.array([
... [0.1961, 0.9806],
... [-0.1961, 0.9806],
... [-0.5812, -0.8137],
... [-0.8137, -0.5812],
... ])
>>>
>>> sofm = algorithms.SOFM(
... n_inputs=2,
... n_outputs=2,
... step=0.1,
... learning_radius=0
... )
>>> sofm.train(data, epochs=100)
>>> sofm.predict(data)
array([[0, 1],
[0, 1],
[1, 0],
[1, 0]])
"""
n_outputs = IntProperty(minval=1, allow_none=True, default=None)
weight = SOFMWeightParameter(default=init.Normal(),
choices={
'init_pca': linear_initialization,
'sample_from_data': sample_data,
})
features_grid = TypedListProperty(allow_none=True, default=None)
DistanceParameter = namedtuple('DistanceParameter', 'name func')
distance = ChoiceProperty(default='euclid',
choices={
'dot_product':
DistanceParameter(name='dot_product',
func=np.dot),
'euclid':
DistanceParameter(name='euclid',
func=neg_euclid_distance),
'cos':
DistanceParameter(name='cosine',
func=cosine_similarity),
})
GridTypeMethods = namedtuple('GridTypeMethods',
'name find_neighbours find_step_scaler')
grid_type = ChoiceProperty(
default='rect',
choices={
'rect':
GridTypeMethods(name='rectangle',
find_neighbours=find_neighbours_on_rect_grid,
find_step_scaler=find_step_scaler_on_rect_grid),
'hexagon':
GridTypeMethods(name='hexagon',
find_neighbours=find_neighbours_on_hexagon_grid,
find_step_scaler=find_step_scaler_on_hexagon_grid)
})
learning_radius = IntProperty(default=0, minval=0)
std = NumberProperty(minval=0, default=1)
reduce_radius_after = IntProperty(default=100, minval=1, allow_none=True)
reduce_std_after = IntProperty(default=100, minval=1, allow_none=True)
reduce_step_after = IntProperty(default=100, minval=1, allow_none=True)
def __init__(self, **options):
super(BaseAssociative, self).__init__(**options)
if self.n_outputs is None and self.features_grid is None:
raise ValueError("One of the following parameters has to be "
"specified: n_outputs, features_grid")
elif self.n_outputs is None:
self.n_outputs = np.prod(self.features_grid)
n_grid_elements = np.prod(self.features_grid)
invalid_feature_grid = (self.features_grid is not None
and n_grid_elements != self.n_outputs)
if invalid_feature_grid:
raise ValueError(
"Feature grid should contain the same number of elements "
"as in the output layer: {0}, but found: {1} (shape: {2})"
"".format(self.n_outputs, n_grid_elements, self.features_grid))
if self.features_grid is None:
self.features_grid = (self.n_outputs, 1)
if len(self.features_grid) > 2 and self.grid_type.name == 'hexagon':
raise ValueError("SOFM with hexagon grid type should have "
"one or two dimensional feature grid, but got "
"{}d instead (shape: {!r})".format(
len(self.features_grid), self.features_grid))
is_pca_init = (isinstance(options.get('weight'), six.string_types)
and options.get('weight') == 'init_pca')
self.initialized = False
if not callable(self.weight):
super(Kohonen, self).init_weights()
self.initialized = True
if self.distance.name == 'cosine':
self.weight /= np.linalg.norm(self.weight, axis=0)
elif is_pca_init and self.grid_type.name != 'rectangle':
raise WeightInitializationError(
"Cannot apply PCA weight initialization for non-rectangular "
"grid. Grid type: {}".format(self.grid_type.name))
def predict_raw(self, X):
X = format_data(X, is_feature1d=(self.n_inputs == 1))
if X.ndim != 2:
raise ValueError("Only 2D inputs are allowed")
n_samples = X.shape[0]
output = np.zeros((n_samples, self.n_outputs))
for i, input_row in enumerate(X):
output[i, :] = self.distance.func(input_row.reshape(1, -1),
self.weight)
return output
def update_indexes(self, layer_output):
neuron_winner = layer_output.argmax(axis=1).item(0)
winner_neuron_coords = np.unravel_index(neuron_winner,
self.features_grid)
learning_radius = self.learning_radius
step = self.step
std = self.std
if self.reduce_radius_after is not None:
learning_radius -= self.last_epoch // self.reduce_radius_after
learning_radius = max(0, learning_radius)
if self.reduce_step_after is not None:
step = decay_function(step, self.last_epoch,
self.reduce_step_after)
if self.reduce_std_after is not None:
std = decay_function(std, self.last_epoch, self.reduce_std_after)
methods = self.grid_type
output_grid = np.reshape(layer_output, self.features_grid)
output_with_neighbours = methods.find_neighbours(
grid=output_grid,
center=winner_neuron_coords,
radius=learning_radius)
step_scaler = methods.find_step_scaler(grid=output_grid,
center=winner_neuron_coords,
std=std)
index_y, = np.nonzero(output_with_neighbours.reshape(self.n_outputs))
step_scaler = step_scaler.reshape(self.n_outputs)
return index_y, step * step_scaler[index_y]
def init_weights(self, X_train):
if self.initialized:
raise WeightInitializationError(
"Weights have been already initialized")
weight_initializer = self.weight
self.weight = weight_initializer(X_train, self.features_grid)
self.initialized = True
if self.distance.name == 'cosine':
self.weight /= np.linalg.norm(self.weight, axis=0)
def train(self, X_train, epochs=100):
if not self.initialized:
self.init_weights(X_train)
super(SOFM, self).train(X_train, epochs=epochs)
def one_training_update(self, X_train, y_train=None):
step = self.step
predict = self.predict
update_indexes = self.update_indexes
error = 0
for input_row in X_train:
input_row = np.reshape(input_row, (1, input_row.size))
layer_output = predict(input_row)
index_y, step = update_indexes(layer_output)
distance = input_row.T - self.weight[:, index_y]
updated_weights = (self.weight[:, index_y] + step * distance)
if self.distance.name == 'cosine':
updated_weights /= np.linalg.norm(updated_weights, axis=0)
self.weight[:, index_y] = updated_weights
error += np.abs(distance).mean()
return error / len(X_train)
class DropBlock(Identity):
"""
DropBlock, a form of structured dropout, where units in a contiguous
region of a feature map are dropped together.
Parameters
----------
keep_proba : float
Fraction of the input units to keep. Value needs to be
between ``0`` and ``1``.
block_size : int or tuple
Size of the block to be dropped. Blocks that have squared shape can
be specified with a single integer value. For example, `block_size=5`
the same as `block_size=(5, 5)`.
{Identity.name}
Methods
-------
{Identity.Methods}
Attributes
----------
{Identity.Attributes}
See Also
--------
:layer:`Dropout` : Dropout layer.
References
----------
[1] Golnaz Ghiasi, Tsung-Yi Lin, Quoc V. Le. DropBlock: A regularization
method for convolutional networks, 2018.
Examples
--------
>>> from neupy.layers import *
>>> network = join(
... Input((28, 28, 1)),
...
... Convolution((3, 3, 16)) >> Relu(),
... DropBlock(keep_proba=0.1, block_size=5),
...
... Convolution((3, 3, 32)) >> Relu(),
... DropBlock(keep_proba=0.1, block_size=5),
... )
"""
keep_proba = ProperFractionProperty()
block_size = TypedListProperty(n_elements=2)
def __init__(self, keep_proba, block_size, name=None):
super(DropBlock, self).__init__(name=name)
if isinstance(block_size, int):
block_size = (block_size, block_size)
self.keep_proba = keep_proba
self.block_size = block_size
def get_output_shape(self, input_shape):
input_shape = tf.TensorShape(input_shape)
if input_shape and input_shape.ndims != 4:
raise LayerConnectionError(
"DropBlock layer expects input with 4 dimensions, got {} "
"with shape {}".format(len(input_shape), input_shape))
return input_shape
def output(self, input, training=False):
if not training:
return input
input = tf.convert_to_tensor(input, tf.float32)
input_shape = tf.shape(input)
block_height, block_width = self.block_size
height, width = input_shape[1], input_shape[2]
input_area = asfloat(width * height)
block_area = asfloat(block_width * block_height)
area = asfloat((width - block_width + 1) * (height - block_height + 1))
mask = bernoulli_sample(
mean=(1. - self.keep_proba) * input_area / (block_area * area),
shape=[
input_shape[0],
height - block_height + 1,
width - block_width + 1,
input_shape[3],
],
)
br_height = (block_height - 1) // 2
tl_height = (block_height - 1) - br_height
br_width = (block_width - 1) // 2
tl_width = (block_width - 1) - br_width
mask = tf.pad(mask, [
[0, 0],
[tl_height, br_height],
[tl_width, br_width],
[0, 0],
])
mask = tf.nn.max_pool(
mask,
[1, block_height, block_width, 1],
strides=[1, 1, 1, 1],
padding='SAME',
)
mask = tf.cast(1 - mask, tf.float32)
feature_normalizer = asfloat(tf.size(mask)) / tf.reduce_sum(mask)
return tf.multiply(input, mask) * feature_normalizer
class Input(BaseLayer):
"""
Layer defines network's input.
Parameters
----------
shape : int or tuple
Shape of the input features per sample. Batch
dimension has to be excluded from the shape.
{BaseLayer.name}
Methods
-------
{BaseLayer.Methods}
Attributes
----------
{BaseLayer.Attributes}
Examples
--------
Feedforward Neural Network (FNN)
In the example, input layer defines network that expects
2D inputs (matrices). In other words, input to the network
should be set of samples combined into matrix where each sample
has 10 dimensional vector associated with it.
>>> from neupy.layers import *
>>> network = Input(10) >> Relu(5) >> Softmax(3)
Convolutional Neural Network (CNN)
In the example, input layer specified that we expect multiple
28x28 image as an input and each image should have single
channel (images with no color).
>>> from neupy.layers import *
>>> network = join(
... Input((28, 28, 1)),
... Convolution((3, 3, 16)) >> Relu(),
... Convolution((3, 3, 16)) >> Relu(),
... Reshape()
... Softmax(10),
... )
"""
shape = TypedListProperty(element_type=(int, type(None)))
def __init__(self, shape, name=None):
super(Input, self).__init__(name=name)
if isinstance(shape, tf.TensorShape):
shape = tf_utils.shape_to_tuple(shape)
self.shape = as_tuple(shape)
@property
def input_shape(self):
batch_shape = tf.TensorShape([None])
return batch_shape.concatenate(self.shape)
def output(self, input, **kwargs):
return input
def get_output_shape(self, input_shape):
if not self.input_shape.is_compatible_with(input_shape):
raise LayerConnectionError(
"Input layer got unexpected input shape. "
"Received shape: {}, Expected shape: {}"
"".format(input_shape, self.input_shape)
)
return self.input_shape.merge_with(input_shape)
def __repr__(self):
return self._repr_arguments(
make_one_if_possible(self.shape),
name=self.name)
class BatchNorm(Identity):
"""
Batch normalization layer.
Parameters
----------
axes : tuple with ints or None
Axes along which normalization will be applied. The ``None``
value means that normalization will be applied over all axes
except the last one. In case of 4D tensor it will
be equal to ``(0, 1, 2)``. Defaults to ``None``.
epsilon : float
Epsilon is a positive constant that adds to the standard
deviation to prevent the division by zero.
Defaults to ``1e-5``.
alpha : float
Coefficient for the exponential moving average of
batch-wise means and standard deviations computed during
training; the closer to one, the more it will depend on
the last batches seen. Value needs to be between ``0`` and ``1``.
Defaults to ``0.1``.
gamma : array-like, Tensorfow variable, scalar or Initializer
Scale. Default initialization methods you can
find :ref:`here <init-methods>`.
Defaults to ``Constant(value=1)``.
beta : array-like, Tensorfow variable, scalar or Initializer
Offset. Default initialization methods you can
find :ref:`here <init-methods>`.
Defaults to ``Constant(value=0)``.
running_mean : array-like, Tensorfow variable, scalar or Initializer
Default initialization methods you can
find :ref:`here <init-methods>`.
Defaults to ``Constant(value=0)``.
running_inv_std : array-like, Tensorfow variable, scalar or Initializer
Default initialization methods you can
find :ref:`here <init-methods>`.
Defaults to ``Constant(value=1)``.
{Identity.name}
Methods
-------
{Identity.Methods}
Attributes
----------
{Identity.Attributes}
Examples
--------
Feedforward Neural Networks (FNN) with batch normalization after
activation function was applied.
>>> from neupy.layers import *
>>> network = join(
... Input(10),
... Relu(5) >> BatchNorm(),
... Relu(5) >> BatchNorm(),
... Sigmoid(1),
... )
Feedforward Neural Networks (FNN) with batch normalization before
activation function was applied.
>>> from neupy.layers import *
>>> network = join(
... Input(10),
... Linear(5) >> BatchNorm() >> Relu(),
... Linear(5) >> BatchNorm() >> Relu(),
... Sigmoid(1),
... )
Convolutional Neural Networks (CNN)
>>> from neupy.layers import *
>>> network = join(
... Input((28, 28, 1)),
... Convolution((3, 3, 16)) >> BatchNorm() >> Relu(),
... Convolution((3, 3, 16)) >> BatchNorm() >> Relu(),
... Reshape(),
... Softmax(10),
... )
References
----------
.. [1] Batch Normalization: Accelerating Deep Network Training
by Reducing Internal Covariate Shift,
http://arxiv.org/pdf/1502.03167v3.pdf
"""
axes = TypedListProperty(allow_none=True)
epsilon = NumberProperty(minval=0)
alpha = ProperFractionProperty()
beta = ParameterProperty()
gamma = ParameterProperty()
running_mean = ParameterProperty()
running_inv_std = ParameterProperty()
def __init__(self,
axes=None,
alpha=0.1,
beta=0,
gamma=1,
epsilon=1e-5,
running_mean=0,
running_inv_std=1,
name=None):
super(BatchNorm, self).__init__(name=name)
self.axes = axes
self.alpha = alpha
self.beta = beta
self.gamma = gamma
self.epsilon = epsilon
self.running_mean = running_mean
self.running_inv_std = running_inv_std
if axes is not None and len(set(axes)) != len(axes):
raise ValueError(
"Specified axes have to contain only unique values")
def create_variables(self, input_shape):
input_shape = tf.TensorShape(input_shape)
if input_shape.ndims is None:
raise WeightInitializationError(
"Cannot initialize variables for the batch normalization "
"layer, because input shape is undefined. Layer: {}"
"".format(self))
if self.axes is None:
# If ndims == 4 then axes = (0, 1, 2)
# If ndims == 2 then axes = (0,)
self.axes = tuple(range(input_shape.ndims - 1))
if any(axis >= input_shape.ndims for axis in self.axes):
raise LayerConnectionError(
"Batch normalization cannot be applied over one of "
"the axis, because input has only {} dimensions. Layer: {}"
"".format(input_shape.ndims, self))
parameter_shape = tuple([
input_shape[axis].value if axis not in self.axes else 1
for axis in range(input_shape.ndims)
])
if any(parameter is None for parameter in parameter_shape):
unknown_dim_index = parameter_shape.index(None)
raise WeightInitializationError(
"Cannot create variables for batch normalization, because "
"input has unknown dimension #{} (0-based indices). "
"Input shape: {}, Layer: {}".format(unknown_dim_index,
input_shape, self))
self.input_shape = input_shape
self.running_mean = self.variable(value=self.running_mean,
shape=parameter_shape,
name='running_mean',
trainable=False)
self.running_inv_std = self.variable(value=self.running_inv_std,
shape=parameter_shape,
name='running_inv_std',
trainable=False)
self.gamma = self.variable(value=self.gamma,
name='gamma',
shape=parameter_shape)
self.beta = self.variable(value=self.beta,
name='beta',
shape=parameter_shape)
def output(self, input, training=False):
input = tf.convert_to_tensor(input, dtype=tf.float32)
if not training:
mean = self.running_mean
inv_std = self.running_inv_std
else:
alpha = asfloat(self.alpha)
mean = tf.reduce_mean(
input,
self.axes,
keepdims=True,
name="mean",
)
variance = tf.reduce_mean(
tf.squared_difference(input, tf.stop_gradient(mean)),
self.axes,
keepdims=True,
name="variance",
)
inv_std = tf.rsqrt(variance + asfloat(self.epsilon))
tf.add_to_collection(
tf.GraphKeys.UPDATE_OPS,
self.running_inv_std.assign(
asfloat(1 - alpha) * self.running_inv_std +
alpha * inv_std))
tf.add_to_collection(
tf.GraphKeys.UPDATE_OPS,
self.running_mean.assign(
asfloat(1 - alpha) * self.running_mean + alpha * mean))
normalized_value = (input - mean) * inv_std
return self.gamma * normalized_value + self.beta
class Reshape(BaseLayer):
"""
Layer reshapes input tensor.
Parameters
----------
shape : tuple
New feature shape. If one dimension specified with the ``-1`` value
that this dimension will be computed from the total size that remains.
Defaults to ``-1``.
{BaseLayer.name}
Methods
-------
{BaseLayer.Methods}
Attributes
----------
{BaseLayer.Attributes}
Examples
--------
Covert 4D input to 2D
>>> from neupy.layers import *
>>> network = Input((2, 5, 5)) >> Reshape()
(?, 2, 5, 5) -> [... 2 layers ...] -> (?, 50)
Convert 3D to 4D
>>> from neupy.layers import *
>>> network = Input((5, 4)) >> Reshape((5, 2, 2))
(?, 5, 4) -> [... 2 layers ...] -> (?, 5, 2, 2)
"""
shape = TypedListProperty()
def __init__(self, shape=-1, name=None):
super(Reshape, self).__init__(name=name)
self.shape = as_tuple(shape)
if self.shape.count(-1) >= 2:
raise ValueError("Only single -1 value can be specified")
def get_output_shape(self, input_shape):
input_shape = tf.TensorShape(input_shape)
feature_shape = input_shape[1:]
missing_value = None
if -1 in self.shape and feature_shape.is_fully_defined():
known_shape_values = [val for val in self.shape if val != -1]
n_feature_values = np.prod(feature_shape.dims)
n_expected_values = np.prod(known_shape_values)
if n_feature_values % n_expected_values != 0:
raise ValueError(
"Input shape and specified shape are incompatible Shape: "
"{}, Input shape: {}".format(self.shape, input_shape))
missing_value = int(n_feature_values // n_expected_values)
n_sampes = input_shape[0]
new_feature_shape = [
missing_value if val == -1 else val for val in self.shape
]
return tf.TensorShape([n_sampes] + new_feature_shape)
def output(self, input, **kwargs):
"""
Reshape the feature space for the input value.
Parameters
----------
input : array-like or Tensorfow variable
"""
input = tf.convert_to_tensor(input, dtype=tf.float32)
input_shape = tf.shape(input)
n_samples = input_shape[0]
expected_shape = self.get_output_shape(input.shape)
feature_shape = expected_shape[1:]
if feature_shape.is_fully_defined():
# For cases when we have -1 in the shape and feature shape
# can be precomputed from the input we want to be explicit about
# expected output shape. Because of the unknown batch dimension
# it won't be possible for tensorflow to derive exact output
# shape from the -1
output_shape = as_tuple(n_samples, feature_shape.dims)
else:
output_shape = as_tuple(n_samples, self.shape)
return tf.reshape(input, output_shape)
def __repr__(self):
return self._repr_arguments(self.shape, name=self.name)
class BasePooling(BaseLayer):
"""
Base class for the pooling layers.
Parameters
----------
size : tuple with 2 integers
Factor by which to downscale (vertical, horizontal).
(2, 2) will halve the image in each dimension.
stride : tuple or int.
Stride size, which is the number of shifts over
rows/cols to get the next pool region. If stride is
None, it is considered equal to ds (no overlap on
pooling regions).
padding : tuple or int
(pad_h, pad_w), pad zeros to extend beyond four borders of
the images, pad_h is the size of the top and bottom margins,
and pad_w is the size of the left and right margins.
ignore_border : bool
When ``True``, ``(5, 5)`` input with size ``(2, 2)``
will generate a `(2, 2)` output. ``(3, 3)`` otherwise.
Defaults to ``True``.
{BaseLayer.Parameters}
Methods
-------
{BaseLayer.Methods}
Attributes
----------
{BaseLayer.Attributes}
"""
size = TypedListProperty(required=True, element_type=int)
stride = StrideProperty(default=None)
padding = PaddingProperty(default=0, element_type=int, n_elements=2)
ignore_border = Property(default=True, expected_type=bool)
def __init__(self, size, **options):
super(BasePooling, self).__init__(size=size, **options)
if not self.ignore_border and self.padding != (0, 0):
raise ValueError("Cannot set padding parameter equal to {} while "
"``ignore_border`` is equal to ``False``"
"".format(self.padding))
def validate(self, input_shape):
if len(input_shape) != 3:
raise LayerConnectionError("Pooling layer expects an input with 3 "
"dimensions, got {} with shape {}"
"".format(len(input_shape),
input_shape))
@property
def output_shape(self):
if self.input_shape is None:
return None
n_kernels, rows, cols = self.input_shape
row_filter_size, col_filter_size = self.size
stride = self.size if self.stride is None else self.stride
row_stride, col_stride = stride
row_padding, col_padding = self.padding
output_rows = pooling_output_shape(rows, row_filter_size, row_padding,
row_stride, self.ignore_border)
output_cols = pooling_output_shape(cols, col_filter_size, col_padding,
col_stride, self.ignore_border)
return (n_kernels, output_rows, output_cols)
def __repr__(self):
return '{name}({size})'.format(name=self.__class__.__name__,
size=self.size)
class Convolution(BaseLayer):
"""
Convolutional layer.
Parameters
----------
size : tuple of int
Filter shape. In should be defined as a tuple with three
integers ``(filter rows, filter columns, output channels)``.
padding : {{``same``, ``valid``}}, int, tuple
Zero padding for the input tensor.
- ``valid`` - Padding won't be added to the tensor. Result will be
the same as for ``padding=0``
- ``same`` - Padding will depend on the number of rows and columns
in the filter. This padding makes sure that image with the
``stride=1`` won't change its width and height. It's the same as
``padding=(filter rows // 2, filter columns // 2)``.
- Custom value for the padding can be specified as an integer, like
``padding=1`` or it can be specified as a tuple when different
dimensions have different padding values, for example
``padding=(2, 3)``.
Defaults to ``valid``.
stride : tuple with ints, int.
Stride size. Defaults to ``(1, 1)``
dilation : int, tuple
Rate for the filter upsampling. When ``dilation > 1`` layer will
become dilated convolution (or atrous convolution). Defaults to ``1``.
weight : array-like, Tensorfow variable, scalar or Initializer
Defines layer's weights. Shape of the weight will be equal to
``(filter rows, filter columns, input channels, output channels)``.
Default initialization methods you can find
:ref:`here <init-methods>`. Defaults to
:class:`HeNormal(gain=2) <neupy.init.HeNormal>`.
bias : 1D array-like, Tensorfow variable, scalar, Initializer or None
Defines layer's bias. Default initialization methods you can find
:ref:`here <init-methods>`. Defaults to
:class:`Constant(0) <neupy.init.Constant>`.
The ``None`` value excludes bias from the calculations and
do not add it into parameters list.
{BaseLayer.name}
Examples
--------
2D Convolution
>>> from neupy import layers
>>>
>>> layers.join(
... layers.Input((28, 28, 3)),
... layers.Convolution((3, 3, 16)),
... )
1D Convolution
>>> from neupy.layers import *
>>> network = join(
... Input((30, 10)),
... Reshape((30, 1, 10)), # convert 3D to 4D
... Convolution((3, 1, 16)),
... Reshape((-1, 16)) # convert 4D back to 3D
... )
>>> network
(?, 30, 10) -> [... 4 layers ...] -> (?, 28, 16)
Methods
-------
{BaseLayer.Methods}
Attributes
----------
{BaseLayer.Attributes}
"""
size = TypedListProperty(element_type=int, n_elements=3)
weight = ParameterProperty()
bias = ParameterProperty(allow_none=True)
padding = PaddingProperty()
stride = Spatial2DProperty()
dilation = Spatial2DProperty()
# We use gain=2 because it's suitable choice for relu non-linearity
# and relu is the most common non-linearity used for CNN.
def __init__(self, size, padding='valid', stride=1, dilation=1,
weight=init.HeNormal(gain=2), bias=0, name=None):
super(Convolution, self).__init__(name=name)
self.size = size
self.padding = padding
self.stride = stride
self.dilation = dilation
self.weight = weight
self.bias = bias
def fail_if_shape_invalid(self, input_shape):
if input_shape and input_shape.ndims != 4:
raise LayerConnectionError(
"Convolutional layer expects an input with 4 "
"dimensions, got {} with shape {}"
"".format(len(input_shape), input_shape))
def output_shape_per_dim(self, *args, **kwargs):
return conv_output_shape(*args, **kwargs)
def expected_output_shape(self, input_shape):
n_samples = input_shape[0]
row_filter_size, col_filter_size, n_kernels = self.size
row_stride, col_stride = self.stride
row_dilation, col_dilation = self.dilation
if isinstance(self.padding, (list, tuple)):
row_padding, col_padding = self.padding
else:
row_padding, col_padding = self.padding, self.padding
return (
n_samples,
self.output_shape_per_dim(
input_shape[1], row_filter_size,
row_padding, row_stride, row_dilation
),
self.output_shape_per_dim(
input_shape[2], col_filter_size,
col_padding, col_stride, col_dilation
),
n_kernels,
)
def get_output_shape(self, input_shape):
input_shape = tf.TensorShape(input_shape)
self.fail_if_shape_invalid(input_shape)
if input_shape.ndims is None:
n_samples = input_shape[0]
n_kernels = self.size[-1]
return tf.TensorShape((n_samples, None, None, n_kernels))
return tf.TensorShape(self.expected_output_shape(input_shape))
def create_variables(self, input_shape):
self.input_shape = input_shape
n_channels = input_shape[-1]
n_rows, n_cols, n_filters = self.size
# Compare to the regular convolution weights,
# transposed one has switched input and output channels.
self.weight = self.variable(
value=self.weight, name='weight',
shape=(n_rows, n_cols, n_channels, n_filters))
if self.bias is not None:
self.bias = self.variable(
value=self.bias, name='bias',
shape=as_tuple(n_filters))
def output(self, input, **kwargs):
input = tf.convert_to_tensor(input, tf.float32)
self.fail_if_shape_invalid(input.shape)
padding = self.padding
if not isinstance(padding, six.string_types):
height_pad, width_pad = padding
input = tf.pad(input, [
[0, 0],
[height_pad, height_pad],
[width_pad, width_pad],
[0, 0],
])
# VALID option will make sure that
# convolution won't use any padding.
padding = 'VALID'
output = tf.nn.convolution(
input,
self.weight,
padding=padding,
strides=self.stride,
dilation_rate=self.dilation,
data_format="NHWC",
)
if self.bias is not None:
bias = tf.reshape(self.bias, (1, 1, 1, -1))
output += bias
return output
def __repr__(self):
return self._repr_arguments(
self.size,
padding=self.padding,
stride=self.stride,
dilation=self.dilation,
weight=self.weight,
bias=self.bias,
name=self.name,
)
class BasePooling(BaseLayer):
"""
Base class for the pooling layers.
Parameters
----------
size : tuple with 2 integers
Factor by which to downscale (vertical, horizontal).
(2, 2) will halve the image in each dimension.
stride_size : tuple with 1 or 2 integers or integer.
Stride size, which is the number of shifts over
rows/cols to get the next pool region. If stride_size is
None, it is considered equal to ds (no overlap on
pooling regions).
padding : tuple of two ints
(pad_h, pad_w), pad zeros to extend beyond four borders of
the images, pad_h is the size of the top and bottom margins,
and pad_w is the size of the left and right margins.
Methods
-------
{BaseLayer.Methods}
Attributes
----------
{BaseLayer.Attributes}
"""
size = TypedListProperty(required=True, element_type=int)
stride_size = StrideProperty(default=None)
padding = TypedListProperty(default=(0, 0), element_type=int, n_elements=2)
def __init__(self, size, **options):
super(BasePooling, self).__init__(size=size, **options)
@property
def output_shape(self):
if self.input_shape is None:
return None
if len(self.input_shape) < 3:
raise ValueError(
"Convolutional layer expects an input shape with least 3 "
"dimensions, got {} with shape {}".format(
len(self.input_shape), self.input_shape))
n_kernels, rows, cols = self.input_shape[-3:]
row_filter_size, col_filter_size = self.size
stride_size = self.stride_size
if stride_size is None:
stride_size = self.size
row_stride, col_stride = stride_size
row_border_mode, col_border_mode = self.padding
output_rows = conv_output_shape(rows, row_filter_size, row_border_mode,
row_stride)
output_cols = conv_output_shape(cols, col_filter_size, col_border_mode,
col_stride)
return (n_kernels, output_rows, output_cols)
def __repr__(self):
return '{name}({size})'.format(name=self.__class__.__name__,
size=self.size)
class SOFM(Kohonen):
""" Self-Organizing Feature Map.
Parameters
----------
learning_radius : int
Learning radius.
features_grid : int
Learning radius.
transform : {{'linear', 'euclid', 'cos'}}
Indicate transformation operation related to the input layer.
The ``linear`` value mean that input data would be multiplied by
weights in typical way. The ``euclid`` method will identify the
closest weight vector to the input one. The ``cos`` made the same
as ``euclid``, but instead of euclid distance it uses cosine
similarity. Defaults to ``linear``.
{BaseAssociative.n_inputs}
{BaseAssociative.n_outputs}
{BaseAssociative.weight}
{BaseNetwork.step}
{BaseNetwork.show_epoch}
{BaseNetwork.shuffle_data}
{BaseNetwork.epoch_end_signal}
{BaseNetwork.train_end_signal}
{Verbose.verbose}
Methods
-------
{BaseSkeleton.predict}
{BaseAssociative.train}
{BaseSkeleton.fit}
"""
learning_radius = IntProperty(default=0, minval=0)
features_grid = TypedListProperty()
transform = ChoiceProperty(default='linear',
choices={
'linear': dot_product,
'euclid': neg_euclid_distance,
'cos': cosine_similarity,
})
def __init__(self, **options):
super(SOFM, self).__init__(**options)
invalid_feature_grid = (self.features_grid is not None
and mul(*self.features_grid) != self.n_outputs)
if invalid_feature_grid:
raise ValueError(
"Feature grid should contain the same number of elements as "
"in the output layer: {0}, but found: {1} ({2}x{3})"
"".format(self.n_outputs, mul(*self.features_grid),
self.features_grid[0], self.features_grid[1]))
def init_properties(self):
super(SOFM, self).init_properties()
if self.features_grid is None:
self.features_grid = (self.n_outputs, 1)
def predict_raw(self, input_data):
input_data = format_data(input_data)
output = np.zeros((input_data.shape[0], self.n_outputs))
for i, input_row in enumerate(input_data):
output[i, :] = self.transform(input_row.reshape(1, -1),
self.weight)
return output
def update_indexes(self, layer_output):
neuron_winner = layer_output.argmax(axis=1)
feature_bound = self.features_grid[1]
output_with_neightbours = neuron_neighbours(
np.reshape(layer_output, self.features_grid),
(neuron_winner // feature_bound, neuron_winner % feature_bound),
self.learning_radius)
index_y, _ = np.nonzero(
np.reshape(output_with_neightbours, (self.n_outputs, 1)))
return index_y
class PRelu(Linear):
"""
Layer with the parametrized ReLu used as an activation function.
Layer learns additional parameter ``alpha`` during the training.
It applies linear transformation when the ``n_units`` parameter
specified and parametrized relu function after the transformation.
When ``n_units`` is not specified, only parametrized relu function
will be applied to the input.
Parameters
----------
alpha_axes : int or tuple
Axes that will not include unique alpha parameter.
Single integer value defines the same as a tuple with one value.
Defaults to ``-1``.
alpha : array-like, Tensorfow variable, scalar or Initializer
Separate alpha parameter per each non-shared axis for the ReLu.
Scalar value means that each element in the tensor will be
equal to the specified value. Default initialization methods you
can find :ref:`here <init-methods>`.
Defaults to ``Constant(value=0.25)``.
{Linear.Parameters}
Methods
-------
{Linear.Methods}
Attributes
----------
{Linear.Attributes}
Examples
--------
Feedforward Neural Networks (FNN)
>>> from neupy.layers import *
>>> network = Input(10) >> PRelu(20) >> PRelu(1)
Convolutional Neural Networks (CNN)
>>> from neupy.layers import *
>>> network = join(
... Input((32, 32, 3)),
... Convolution((3, 3, 16)) >> PRelu(),
... Convolution((3, 3, 32)) >> PRelu(),
... Reshape(),
... Softmax(10),
... )
References
----------
.. [1] Delving Deep into Rectifiers: Surpassing Human-Level
Performance on ImageNet Classification.
https://arxiv.org/pdf/1502.01852v1.pdf
"""
alpha_axes = TypedListProperty()
alpha = ParameterProperty()
def __init__(self,
n_units=None,
alpha_axes=-1,
alpha=0.25,
weight=init.HeNormal(gain=2),
bias=0,
name=None):
self.alpha = alpha
self.alpha_axes = as_tuple(alpha_axes)
if 0 in self.alpha_axes:
raise ValueError("Cannot specify alpha for 0-axis")
super(PRelu, self).__init__(n_units=n_units,
weight=weight,
bias=bias,
name=name)
def get_output_shape(self, input_shape):
input_shape = tf.TensorShape(input_shape)
if input_shape and max(self.alpha_axes) >= input_shape.ndims:
max_axis_index = input_shape.ndims - 1
raise LayerConnectionError(
"Cannot specify alpha for the axis #{}. Maximum "
"available axis is {} (0-based indices)."
"".format(max(self.alpha_axes), max_axis_index))
return super(PRelu, self).get_output_shape(input_shape)
def create_variables(self, input_shape):
super(PRelu, self).create_variables(input_shape)
output_shape = self.get_output_shape(input_shape)
self.alpha = self.variable(
value=self.alpha,
name='alpha',
shape=[output_shape[axis] for axis in self.alpha_axes])
def activation_function(self, input):
input = tf.convert_to_tensor(input, dtype=tf.float32)
ndim = input.shape.ndims
dimensions = np.arange(ndim)
alpha_axes = dimensions[list(self.alpha_axes)]
alpha = tf_utils.dimshuffle(self.alpha, ndim, alpha_axes)
return tf.maximum(0.0, input) + alpha * tf.minimum(0.0, input)
def __repr__(self):
if self.n_units is None:
return self._repr_arguments(name=self.name,
alpha_axes=self.alpha_axes,
alpha=self.alpha)
return self._repr_arguments(self.n_units,
name=self.name,
alpha_axes=self.alpha_axes,
alpha=self.alpha,
weight=self.weight,
bias=self.bias)
class SOFM(Kohonen):
"""
Self-Organizing Feature Map (SOFM).
Parameters
----------
{BaseAssociative.n_inputs}
{BaseAssociative.n_outputs}
learning_radius : int
Learning radius.
features_grid : list, tuple, None
Feature grid defines shape of the output neurons.
The new shape should be compatible with the number
of outputs. Defaults to ``(n_outputs, 1)``.
transform : {{``linear``, ``euclid``, ``cos``}}
Indicate transformation operation related to the
input layer.
- The ``linear`` value mean that input data would be
multiplied by weights in typical way.
- The ``euclid`` method will identify the closest
weight vector to the input one.
- The ``cos`` transformation identifies cosine
similarity between input dataset and
network's weights.
Defaults to ``linear``.
{BaseAssociative.weight}
{BaseNetwork.step}
{BaseNetwork.show_epoch}
{BaseNetwork.shuffle_data}
{BaseNetwork.epoch_end_signal}
{BaseNetwork.train_end_signal}
{Verbose.verbose}
Methods
-------
{BaseSkeleton.predict}
{BaseAssociative.train}
{BaseSkeleton.fit}
Examples
--------
>>> import numpy as np
>>> from neupy import algorithms, environment
>>>
>>> environment.reproducible()
>>>
>>> data = np.array([
... [0.1961, 0.9806],
... [-0.1961, 0.9806],
... [-0.5812, -0.8137],
... [-0.8137, -0.5812],
... ])
>>>
>>> sofmnet = algorithms.SOFM(
... n_inputs=2,
... n_outputs=2,
... step=0.1,
... learning_radius=0,
... features_grid=(2, 1),
... )
>>> sofmnet.train(data, epochs=100)
>>> sofmnet.predict(data)
array([[0, 1],
[0, 1],
[1, 0],
[1, 0]])
"""
learning_radius = IntProperty(default=0, minval=0)
features_grid = TypedListProperty(allow_none=True, default=None)
transform = ChoiceProperty(default='linear',
choices={
'linear': np.dot,
'euclid': neg_euclid_distance,
'cos': cosine_similarity,
})
def __init__(self, **options):
super(SOFM, self).__init__(**options)
invalid_feature_grid = (self.features_grid is not None
and mul(*self.features_grid) != self.n_outputs)
if invalid_feature_grid:
raise ValueError(
"Feature grid should contain the same number of elements as "
"in the output layer: {0}, but found: {1} ({2}x{3})"
"".format(self.n_outputs, mul(*self.features_grid),
self.features_grid[0], self.features_grid[1]))
if self.features_grid is None:
self.features_grid = (self.n_outputs, 1)
def predict_raw(self, input_data):
input_data = format_data(input_data)
n_samples = input_data.shape[0]
output = np.zeros((n_samples, self.n_outputs))
for i, input_row in enumerate(input_data):
output[i, :] = self.transform(input_row.reshape(1, -1),
self.weight)
return output
def update_indexes(self, layer_output):
neuron_winner = layer_output.argmax(axis=1)
feature_bound = self.features_grid[1]
output_with_neightbours = neuron_neighbours(
np.reshape(layer_output, self.features_grid),
(neuron_winner // feature_bound, neuron_winner % feature_bound),
self.learning_radius)
index_y, _ = np.nonzero(
np.reshape(output_with_neightbours, (self.n_outputs, 1)))
return index_y
class Convolution(ParameterBasedLayer):
"""
Convolutional layer.
Parameters
----------
size : tuple of int
Filter shape. In should be defined as a tuple with three integers
``(output channels, filter rows, filter columns)``.
padding : {{``valid``, ``full``, ``half``}} or int or tuple with 2 int
Convolution border mode. Check Theano's ``nnet.conv2d`` doc.
Defaults to ``valid``.
stride : tuple with 1 or 2 integers or integer.
Stride size. Defaults to ``(1, 1)``
{ParameterBasedLayer.weight}
{ParameterBasedLayer.bias}
{BaseLayer.Parameters}
Examples
--------
2D Convolution
>>> from neupy import layers
>>>
>>> layers.join(
... layers.Input((3, 28, 28)),
... layers.Convolution((16, 3, 3)),
... )
1D Convolution
>>> from neupy import layers
>>>
>>> layers.join(
... layers.Input((10, 30)),
... layers.Reshape((10, 30, 1)),
... layers.Convolution((16, 3, 1)),
... )
Methods
-------
{ParameterBasedLayer.Methods}
Attributes
----------
{ParameterBasedLayer.Attributes}
"""
size = TypedListProperty(required=True, element_type=int)
padding = BorderModeProperty(default='valid')
stride = StrideProperty(default=(1, 1))
def validate(self, input_shape):
if len(input_shape) != 3:
raise LayerConnectionError(
"Convolutional layer expects an input with 3 "
"dimensions, got {} with shape {}"
"".format(len(input_shape), input_shape))
@property
def output_shape(self):
if self.input_shape is None:
return None
padding = self.padding
n_kernels = self.size[0]
rows, cols = self.input_shape[-2:]
row_filter_size, col_filter_size = self.size[-2:]
row_stride, col_stride = self.stride
if isinstance(padding, tuple):
row_padding, col_padding = padding[-2:]
else:
row_padding, col_padding = padding, padding
output_rows = conv_output_shape(rows, row_filter_size, row_padding,
row_stride)
output_cols = conv_output_shape(cols, col_filter_size, col_padding,
col_stride)
return (n_kernels, output_rows, output_cols)
@property
def weight_shape(self):
n_channels = self.input_shape[0]
n_filters, n_rows, n_cols = self.size
return (n_filters, n_channels, n_rows, n_cols)
@property
def bias_shape(self):
return as_tuple(self.size[0])
def output(self, input_value):
output = T.nnet.conv2d(input_value,
self.weight,
input_shape=as_tuple(None, self.input_shape),
filter_shape=self.weight_shape,
border_mode=self.padding,
subsample=self.stride)
if self.bias is not None:
bias = T.reshape(self.bias, (1, -1, 1, 1))
output += bias
return output
class Upscale(BaseLayer):
"""
Upscales input over two axis (height and width).
Parameters
----------
scale : int or tuple with two int
Scaling factor for the input value. In the tuple first
parameter identifies scale of the height and the second
one of the width.
{BaseLayer.name}
Methods
-------
{BaseLayer.Methods}
Attributes
----------
{BaseLayer.Attributes}
Examples
--------
>>> from neupy.layers import *
>>> network = Input((10, 10, 3)) >> Upscale((2, 2))
(?, 10, 10, 3) -> [... 2 layers ...] -> (?, 20, 20, 3)
"""
scale = TypedListProperty(n_elements=2)
def __init__(self, scale, name=None):
super(Upscale, self).__init__(name=name)
if isinstance(scale, int):
scale = as_tuple(scale, scale)
if any(element <= 0 for element in scale):
raise ValueError(
"Only positive integers are allowed for scale")
self.scale = scale
def fail_if_shape_invalid(self, input_shape):
if input_shape and input_shape.ndims != 4:
raise LayerConnectionError(
"Upscale layer should have an input value with 4 dimensions "
"(batch, height, width, channel), got input with {} "
"dimensions instead. Shape: {}"
"".format(input_shape.ndims, input_shape))
def get_output_shape(self, input_shape):
input_shape = tf.TensorShape(input_shape)
self.fail_if_shape_invalid(input_shape)
if input_shape.ndims is None:
return tf.TensorShape((None, None, None, None))
n_samples, height, width, channel = input_shape
height_scale, width_scale = self.scale
return tf.TensorShape([
n_samples,
height_scale * height,
width_scale * width,
channel,
])
def output(self, input_value, **kwargs):
input_value = tf.convert_to_tensor(input_value, dtype=tf.float32)
self.fail_if_shape_invalid(input_value.shape)
return tf_utils.repeat(input_value, as_tuple(1, self.scale, 1))
def __repr__(self):
return self._repr_arguments(self.scale, name=self.name)
class Convolution(ParameterBasedLayer):
"""
Convolutional layer.
Parameters
----------
size : tuple of int
Filter shape. In should be defined as a tuple with three
integers ``(filter rows, filter columns, output channels)``.
padding : {{``same``, ``valid``}}, int, tuple
Zero padding for the input tensor.
- ``valid`` - Padding won't be added to the tensor. Result will be
the same as for ``padding=0``
- ``same`` - Padding will depend on the number of rows and columns
in the filter. This padding makes sure that image with the
``stride=1`` won't change its width and height. It's the same as
``padding=(filter rows // 2, filter columns // 2)``.
- Custom value for the padding can be specified as an integer, like
``padding=1`` or it can be specified as a tuple when different
dimensions have different padding values, for example
``padding=(2, 3)``.
Defaults to ``valid``.
stride : tuple with ints, int.
Stride size. Defaults to ``(1, 1)``
dilation : int, tuple
Rate for the fiter upsampling. When ``dilation > 1`` layer will
become diated convolution (or atrous convolution). Defaults to ``1``.
weight : array-like, Tensorfow variable, scalar or Initializer
Defines layer's weights. Shape of the weight will be equal to
``(filter rows, filter columns, input channels, output channels)``.
Default initialization methods you can find
:ref:`here <init-methods>`. Defaults to
:class:`HeNormal(gain=2) <neupy.init.HeNormal>`.
{ParameterBasedLayer.bias}
{BaseLayer.Parameters}
Examples
--------
2D Convolution
>>> from neupy import layers
>>>
>>> layers.join(
... layers.Input((28, 28, 3)),
... layers.Convolution((3, 3, 16)),
... )
1D Convolution
>>> from neupy import layers
>>>
>>> layers.join(
... layers.Input((30, 10)),
... layers.Reshape((30, 1, 10)),
... layers.Convolution((3, 1, 16)),
... )
Methods
-------
{ParameterBasedLayer.Methods}
Attributes
----------
{ParameterBasedLayer.Attributes}
"""
# We use gain=2 because it's suitable choice for relu non-linearity
# and relu is the most common non-linearity used for CNN.
weight = ParameterProperty(default=init.HeNormal(gain=2))
size = TypedListProperty(required=True, element_type=int)
padding = PaddingProperty(default='valid')
stride = Spatial2DProperty(default=(1, 1))
dilation = Spatial2DProperty(default=1)
def validate(self, input_shape):
if input_shape and len(input_shape) != 3:
raise LayerConnectionError(
"Convolutional layer expects an input with 3 "
"dimensions, got {} with shape {}"
"".format(len(input_shape), input_shape))
def output_shape_per_dim(self, *args, **kwargs):
return conv_output_shape(*args, **kwargs)
def find_output_from_input_shape(self, input_shape):
padding = self.padding
rows, cols, _ = input_shape
row_filter_size, col_filter_size, n_kernels = self.size
row_stride, col_stride = self.stride
row_dilation, col_dilation = self.dilation or (1, 1)
if isinstance(padding, (list, tuple)):
row_padding, col_padding = padding
else:
row_padding, col_padding = padding, padding
output_rows = self.output_shape_per_dim(
rows, row_filter_size,
row_padding, row_stride, row_dilation,
)
output_cols = self.output_shape_per_dim(
cols, col_filter_size,
col_padding, col_stride, col_dilation,
)
return (output_rows, output_cols, n_kernels)
@property
def output_shape(self):
if self.input_shape is not None:
return self.find_output_from_input_shape(self.input_shape)
@property
def weight_shape(self):
n_channels = self.input_shape[-1]
n_rows, n_cols, n_filters = self.size
return (n_rows, n_cols, n_channels, n_filters)
@property
def bias_shape(self):
return as_tuple(self.size[-1])
def output(self, input_value):
padding = self.padding
if not isinstance(padding, six.string_types):
height_pad, weight_pad = padding
input_value = tf.pad(input_value, [
[0, 0],
[height_pad, height_pad],
[weight_pad, weight_pad],
[0, 0],
])
# VALID option will make sure that
# convolution won't use any padding.
padding = 'VALID'
output = tf.nn.convolution(
input_value,
self.weight,
padding=padding,
strides=self.stride,
dilation_rate=self.dilation,
data_format="NHWC"
)
if self.bias is not None:
bias = tf.reshape(self.bias, (1, 1, 1, -1))
output += bias
return output
class ParameterBasedLayer(BaseLayer):
""" Layer that creates weight and bias parameters.
Parameters
----------
size : int
Layer input size.
weight : 2D array-like or None
Define your layer weights. ``None`` means that your weights will be
generate randomly dependence on property ``init_method``.
``None`` by default.
bias : 1D array-like or None
Define your layer bias. ``None`` means that your weights will be
generate randomly dependence on property ``init_method``.
init_method : {{'bounded', 'normal', 'ortho', 'xavier_normal',\
'xavier_uniform', 'he_normal', 'he_uniform'}}
Weight initialization method. Defaults to ``xavier_normal``.
* ``normal`` will generate random weights from normal distribution \
with standard deviation equal to ``0.01``.
* ``bounded`` generate random weights from Uniform distribution.
* ``ortho`` generate random orthogonal matrix.
* ``xavier_normal`` generate random matrix from normal distrubtion \
where variance equal to :math:`\\frac{{2}}{{fan_{{in}} + \
fan_{{out}}}}`. Where :math:`fan_{{in}}` is a number of \
layer input units and :math:`fan_{{out}}` - number of layer \
output units.
* ``xavier_uniform`` generate random matrix from uniform \
distribution \ where :math:`w_{{ij}} \in \
[-\\sqrt{{\\frac{{6}}{{fan_{{in}} + fan_{{out}}}}}}, \
\\sqrt{{\\frac{{6}}{{fan_{{in}} + fan_{{out}}}}}}`].
* ``he_normal`` generate random matrix from normal distrubtion \
where variance equal to :math:`\\frac{{2}}{{fan_{{in}}}}`. \
Where :math:`fan_{{in}}` is a number of layer input units.
* ``he_uniform`` generate random matrix from uniformal \
distribution where :math:`w_{{ij}} \in [\
-\\sqrt{{\\frac{{6}}{{fan_{{in}}}}}}, \
\\sqrt{{\\frac{{6}}{{fan_{{in}}}}}}]`
bounds : tuple of two float
Available only for ``init_method`` equal to ``bounded``. Value
identify minimum and maximum possible value in random weights.
Defaults to ``(0, 1)``.
"""
size = IntProperty(minval=1)
weight = SharedArrayProperty(default=None)
bias = SharedArrayProperty(default=None)
bounds = TypedListProperty(default=(0, 1), element_type=(int, float))
init_method = ChoiceProperty(default=XAVIER_NORMAL,
choices=VALID_INIT_METHODS)
def __init__(self, size, **options):
if size is not None:
options['size'] = size
super(ParameterBasedLayer, self).__init__(**options)
def weight_shape(self):
output_size = self.relate_to_layer.size
return (self.size, output_size)
def bias_shape(self):
output_size = self.relate_to_layer.size
return (output_size,)
def initialize(self):
super(ParameterBasedLayer, self).initialize()
self.weight = create_shared_parameter(
value=self.weight,
name='weight_{}'.format(self.layer_id),
shape=self.weight_shape(),
bounds=self.bounds,
init_method=self.init_method,
)
self.bias = create_shared_parameter(
value=self.bias,
name='bias_{}'.format(self.layer_id),
shape=self.bias_shape(),
bounds=self.bounds,
init_method=self.init_method,
)
self.parameters = [self.weight, self.bias]
def __repr__(self):
classname = self.__class__.__name__
return '{name}({size})'.format(name=classname, size=self.size)
class Reshape(BaseLayer):
"""
Gives a new shape to an input value without changing
its data.
Parameters
----------
shape : tuple or list
New feature shape. ``None`` value means that feature
will be flatten in 1D vector. If you need to get the
output feature with more that 2 dimensions then you can
set up new feature shape using tuples. Defaults to ``None``.
{BaseLayer.Parameters}
Methods
-------
{BaseLayer.Methods}
Attributes
----------
{BaseLayer.Attributes}
Examples
--------
Covert 4D input to 2D
>>> from neupy import layers
>>>
>>> connection = layers.join(
... layers.Input((2, 5, 5)),
... layers.Reshape()
... )
>>>
>>> print("Input shape: {{}}".format(connection.input_shape))
Input shape: (2, 5, 5)
>>>
>>> print("Output shape: {{}}".format(connection.output_shape))
Output shape: (50,)
Convert 3D to 4D
>>> from neupy import layers
>>>
>>> connection = layers.join(
... layers.Input((5, 4)),
... layers.Reshape((5, 2, 2))
... )
>>>
>>> print("Input shape: {{}}".format(connection.input_shape))
Input shape: (5, 4)
>>>
>>> print("Output shape: {{}}".format(connection.output_shape))
Output shape: (5, 2, 2)
"""
shape = TypedListProperty()
def __init__(self, shape=None, **options):
if shape is not None:
options['shape'] = shape
super(Reshape, self).__init__(**options)
@property
def output_shape(self):
if self.shape is not None:
return as_tuple(self.shape)
n_output_features = np.prod(self.input_shape)
return as_tuple(n_output_features)
def output(self, input_value):
"""
Reshape the feature space for the input value.
Parameters
----------
input_value : array-like or Theano variable
"""
n_samples = input_value.shape[0]
output_shape = as_tuple(n_samples, self.output_shape)
return T.reshape(input_value, output_shape)
