class BasePooling(BaseLayer): """ Base class for the pooling layers. Parameters ---------- size : tuple with 2 integers Factor by which to downscale (vertical, horizontal). (2,2) will halve the image in each dimension. stride_size : Stride size, which is the number of shifts over rows/cols to get the next pool region. If stride_size is None, it is considered equal to ds (no overlap on pooling regions). padding : tuple of two ints (pad_h, pad_w), pad zeros to extend beyond four borders of the images, pad_h is the size of the top and bottom margins, and pad_w is the size of the left and right margins. """ size = TypedListProperty(required=True, element_type=int) stride_size = StrideProperty(default=None) padding = TypedListProperty(default=(0, 0), element_type=int, n_elements=2) def __init__(self, size, **options): options['size'] = size super(BasePooling, self).__init__(**options) def __repr__(self): return '{name}({size})'.format(name=self.__class__.__name__, size=self.size)
class Transpose(BaseLayer): """ Layer transposes input tensor. Permutes the dimensions according to the ``perm`` parameter. Parameters ---------- perm : tuple or list A permutation of the dimensions of the input tensor. {BaseLayer.name} Methods ------- {BaseLayer.Methods} Attributes ---------- {BaseLayer.Attributes} Examples -------- >>> from neupy.layers import * >>> network = Input((7, 11)) >> Transpose((0, 2, 1)) (?, 7, 11) -> [... 2 layers ...] -> (?, 11, 7) """ perm = TypedListProperty() def __init__(self, perm, name=None): super(Transpose, self).__init__(name=name) self.perm = perm def fail_if_shape_invalid(self, input_shape): if input_shape and max(self.perm) >= input_shape.ndims: raise LayerConnectionError( "Cannot apply transpose operation to the " "input. Permuntation: {}, Input shape: {}" "".format(self.perm, input_shape)) def get_output_shape(self, input_shape): input_shape = tf.TensorShape(input_shape) self.fail_if_shape_invalid(input_shape) if input_shape.ndims is None: n_dims_expected = len(self.perm) return tf.TensorShape([None] * n_dims_expected) input_shape = np.array(input_shape.dims) perm = list(self.perm) return tf.TensorShape(input_shape[perm]) def output(self, input, **kwargs): # Input value has batch dimension, but perm will never have it # specified as (zero index), so we need to add it in order to # fix batch dimesnion in place. return tf.transpose(input, list(self.perm)) def __repr__(self): return self._repr_arguments(self.perm, name=self.name)
class Convolution(ParameterBasedLayer): """ Convolutional layer. Parameters ---------- size : tuple of integers Filter shape. border_mode : {{'valid', 'full', 'half'}} or int or tuple with 2 int Convolution border mode. Check Theano's `nnet.conv2d` doc. stride_size : tuple with 1 or 2 integers or integer. Stride size. """ size = TypedListProperty(required=True, element_type=int) border_mode = BorderModeProperty(default='valid') stride_size = StrideProperty(default=(1, 1)) def weight_shape(self): return self.size def bias_shape(self): return self.size[:1] def output(self, input_value): bias = T.reshape(self.bias, (1, -1, 1, 1)) output = T.nnet.conv2d(input_value, self.weight, border_mode=self.border_mode, subsample=self.stride_size) return output + bias
class Transpose(BaseLayer): """ Transposes input. Permutes the dimensions according to ``perm``. Parameters ---------- perm : tuple or list A permutation of the dimensions of the input tensor. Layer cannot transpose batch dimension and using ``0`` in the list of permuted dimensions is not allowed. {BaseLayer.Parameters} Methods ------- {BaseLayer.Methods} Attributes ---------- {BaseLayer.Attributes} Examples -------- >>> from neupy.layers import * >>> conn = Input((7, 11)) > Transpose((2, 1)) >>> conn.input_shape (7, 11) >>> conn.output_shape (11, 7) """ perm = TypedListProperty() def __init__(self, perm, **options): if 0 in perm: raise ValueError("Batch dimension has fixed position and 0 " "index cannot be used.") super(Transpose, self).__init__(perm=perm, **options) def validate(self, input_shape): if len(input_shape) < 2: raise LayerConnectionError( "Transpose expects input with at least 3 dimensions.") @property def output_shape(self): # Input shape doesn't have information about the batch size and perm # indeces require to have this dimension on zero's position. input_shape = np.array(as_tuple(None, self.input_shape)) return as_tuple(input_shape[self.perm].tolist()) def output(self, input_value): # Input value has batch dimension, but perm will never have it # specified as (zero index), so we need to add it in order to # fix batch dimesnion in place. return tf.transpose(input_value, [0] + list(self.perm))
class Reshape(BaseLayer): """ Gives a new shape to an input value without changing its data. Parameters ---------- shape : tuple or list New feature shape. ``None`` value means that feature will be flatten in 1D vector. If you need to get the output feature with more that 2 dimensions then you can set up new feature shape using tuples. Defaults to ``None``. Methods ------- {BaseLayer.Methods} Attributes ---------- {BaseLayer.Attributes} """ shape = TypedListProperty() def __init__(self, shape=None, **options): if shape is not None: options['shape'] = shape super(Reshape, self).__init__(**options) @property def output_shape(self): if self.shape is not None: return as_tuple(self.shape) n_output_features = np.prod(self.input_shape) return as_tuple(n_output_features) def output(self, input_value): """ Reshape the feature space for the input value. Parameters ---------- input_value : array-like or Theano variable """ n_samples = input_value.shape[0] output_shape = as_tuple(n_samples, self.output_shape) return T.reshape(input_value, output_shape)
class StepOutput(Output): """ The behaviour for this layer is the same as for step function. Parameters ---------- output_bounds : tuple Value is must be a tuple which contains two elements where first one identify lower output value and the second one - bigger. Defaults to ``(0, 1)``. critical_point : float Critical point is set up step function bias. Value equal to this point should be equal to the lower bound. Defaults to ``0``. {Output.size} """ output_bounds = TypedListProperty(default=(0, 1)) critical_point = ProperFractionProperty(default=0) def output(self, value): lower_bound, upper_bound = self.output_bounds return np.where(value <= self.critical_point, lower_bound, upper_bound)
class Reshape(BaseLayer): """ Gives a new shape to an input value without changing its data. Parameters ---------- shape : tuple or list New feature shape. ``None`` value means that feature will be flatten in 1D vector. If you need to get the output feature with more that 2 dimensions then you can set up new feature shape using tuples. Defaults to ``None``. """ shape = TypedListProperty() def __init__(self, shape=None, **options): if shape is not None: options['shape'] = shape super(Reshape, self).__init__(**options) def output(self, input_value): """ Reshape the feature space for the input value. Parameters ---------- input_value : array-like or Theano variable """ new_feature_shape = self.shape input_shape = input_value.shape[0] if new_feature_shape is None: output_shape = input_value.shape[1:] new_feature_shape = T.prod(output_shape) output_shape = (input_shape, new_feature_shape) else: output_shape = (input_shape,) + new_feature_shape return T.reshape(input_value, output_shape)
class BasePooling(BaseLayer): """ Base class for the pooling layers. Parameters ---------- size : tuple with 2 integers Factor by which to downscale (vertical, horizontal). (2,2) will halve the image in each dimension. stride_size : Stride size, which is the number of shifts over rows/cols to get the next pool region. If stride_size is None, it is considered equal to ds (no overlap on pooling regions). """ size = TypedListProperty(required=True, element_type=int) stride_size = StrideProperty(default=None) def __init__(self, size, **options): options['size'] = size super(BasePooling, self).__init__(**options) def __repr__(self): return '{name}({size})'.format(name=self.__class__.__name__, size=self.size)
class Convolution(ParameterBasedLayer): """ Convolutional layer. Parameters ---------- size : tuple of int Filter shape. In should be defined as a tuple with three integers ``(output channels, filter rows, filter columns)``. border_mode : {{'valid', 'full', 'half'}} or int or tuple with 2 int Convolution border mode. Check Theano's ``nnet.conv2d`` doc. stride_size : tuple with 1 or 2 integers or integer. Stride size. {ParameterBasedLayer.weight} {ParameterBasedLayer.bias} Methods ------- {ParameterBasedLayer.Methods} Attributes ---------- {ParameterBasedLayer.Attributes} """ size = TypedListProperty(required=True, element_type=int) border_mode = BorderModeProperty(default='valid') stride_size = StrideProperty(default=(1, 1)) @property def output_shape(self): if self.input_shape is None: return None if len(self.input_shape) < 2: raise ValueError( "Convolutional layer expects an input shape with least 2 " "dimensions, got {} with shape {}".format( len(self.input_shape), self.input_shape)) border_mode = self.border_mode n_kernels = self.size[0] rows, cols = self.input_shape[-2:] row_filter_size, col_filter_size = self.size[-2:] row_stride, col_stride = self.stride_size if isinstance(border_mode, tuple): row_border_mode, col_border_mode = border_mode[-2:] else: row_border_mode, col_border_mode = border_mode, border_mode output_rows = conv_output_shape(rows, row_filter_size, row_border_mode, row_stride) output_cols = conv_output_shape(cols, col_filter_size, col_border_mode, col_stride) return (n_kernels, output_rows, output_cols) @property def weight_shape(self): n_channels = self.input_shape[0] n_filters, n_rows, n_cols = self.size return (n_filters, n_channels, n_rows, n_cols) @property def bias_shape(self): return as_tuple(self.size[0]) def output(self, input_value): bias = T.reshape(self.bias, (1, -1, 1, 1)) output = T.nnet.conv2d(input_value, self.weight, input_shape=as_tuple(None, self.input_shape), filter_shape=self.weight_shape, border_mode=self.border_mode, subsample=self.stride_size) return output + bias
class A(Configurable): list_of_properties = TypedListProperty(element_type=(int, float), n_elements=3)
class A(Configurable): list_of_properties = TypedListProperty(element_type=str, n_elements=3)
class BasePooling(BaseLayer): """ Base class for the pooling layers. Parameters ---------- size : tuple with 2 integers Factor by which to downscale ``(vertical, horizontal)``. ``(2, 2)`` will halve the image in each dimension. stride : tuple or int. Stride size, which is the number of shifts over rows/cols to get the next pool region. If stride is None, it is considered equal to ds (no overlap on pooling regions). padding : {{``valid``, ``same``}} ``(pad_h, pad_w)``, pad zeros to extend beyond four borders of the images, pad_h is the size of the top and bottom margins, and pad_w is the size of the left and right margins. {BaseLayer.Parameters} Methods ------- {BaseLayer.Methods} Attributes ---------- {BaseLayer.Attributes} """ size = TypedListProperty(required=True, element_type=int) stride = Spatial2DProperty(allow_none=True) padding = ChoiceProperty(choices=('SAME', 'VALID', 'same', 'valid')) pooling_type = None def __init__(self, size, stride=None, padding='valid', name=None): super(BasePooling, self).__init__(name=name) self.size = size self.stride = stride self.padding = padding def fail_if_shape_invalid(self, input_shape): if input_shape and input_shape.ndims != 4: raise LayerConnectionError( "Pooling layer expects an input with 4 " "dimensions, got {} with shape {}. Layer: {}" "".format(len(input_shape), input_shape, self)) def get_output_shape(self, input_shape): input_shape = tf.TensorShape(input_shape) if input_shape.ndims is None: return tf.TensorShape((None, None, None, None)) self.fail_if_shape_invalid(input_shape) n_samples, rows, cols, n_kernels = input_shape row_filter_size, col_filter_size = self.size stride = self.size if self.stride is None else self.stride row_stride, col_stride = stride output_rows = pooling_output_shape( rows, row_filter_size, self.padding, row_stride) output_cols = pooling_output_shape( cols, col_filter_size, self.padding, col_stride) # In python 2, we can get float number after rounding procedure # and it might break processing in the subsequent layers. return tf.TensorShape((n_samples, output_rows, output_cols, n_kernels)) def output(self, input_value, **kwargs): return tf.nn.pool( input_value, self.size, pooling_type=self.pooling_type, padding=self.padding.upper(), strides=self.stride or self.size, data_format="NHWC") def __repr__(self): return self._repr_arguments( self.size, name=self.name, stride=self.stride, padding=self.padding, )
class LVQ(BaseNetwork): """ Learning Vector Quantization (LVQ) algorithm. Notes ----- - Input data needs to be normalized, because LVQ uses Euclidian distance to find clusters. - Training error is just a ratio of miscassified samples Parameters ---------- n_inputs : int Number of input units. It should be equal to the number of features in the input data set. n_subclasses : int, None Defines total number of subclasses. Values should be greater or equal to the number of classes. ``None`` will set up number of subclasses equal to the number of classes. Defaults to ``None`` (or the same as ``n_classes``). n_classes : int Number of classes in the data set. prototypes_per_class : list, None Defines number of prototypes per each class. For instance, if ``n_classes=3`` and ``n_subclasses=8`` then there are can be 3 subclasses for the first class, 3 for the second one and 2 for the third one (3 + 3 + 2 == 8). The following example can be specified as ``prototypes_per_class=[3, 3, 2]``. There are two rules that apply to this parameter: 1. ``sum(prototypes_per_class) == n_subclasses`` 2. ``len(prototypes_per_class) == n_classes`` The ``None`` value will distribute approximately equal number of subclasses per each class. It's approximately, because in casses when ``n_subclasses % n_classes != 0`` there is no way to distribute equal number of subclasses per each class. Defaults to ``None``. {BaseNetwork.step} n_updates_to_stepdrop : int or None If this options is not equal to ``None`` then after every update LVQ reduces step size and do it until number of applied updates would reach the ``n_updates_to_stepdrop`` value. The minimum possible step size defined in the ``minstep`` parameter. Be aware that number of updates is not the same as number of epochs. LVQ applies update after each propagated sample through the network. Relations between this parameter and maximum number of epochs is following .. code-block:: python n_updates_to_stepdrop = n_samples * n_max_epochs If parameter equal to ``None`` then step size wouldn't be reduced after each update. Defaults to ``None``. minstep : float Step size would never be lower than this value. This property useful only in case if ``n_updates_to_stepdrop`` is not ``None``. Defaults to ``1e-5``. {BaseNetwork.show_epoch} {BaseNetwork.shuffle_data} {BaseNetwork.epoch_end_signal} {BaseNetwork.train_end_signal} {Verbose.verbose} Methods ------- {BaseSkeleton.predict} {BaseSkeleton.fit} """ n_inputs = IntProperty(minval=1) n_subclasses = IntProperty(minval=2, default=None, allow_none=True) n_classes = IntProperty(minval=2) prototypes_per_class = TypedListProperty(allow_none=True, default=None) weight = Property(expected_type=(np.ndarray, init.Initializer), allow_none=True, default=None) n_updates_to_stepdrop = IntProperty(default=None, allow_none=True, minval=1) minstep = NumberProperty(minval=0, default=1e-5) def __init__(self, **options): self.initialized = False super(LVQ, self).__init__(**options) self.n_updates = 0 if self.n_subclasses is None: self.n_subclasses = self.n_classes if isinstance(self.weight, init.Initializer): weight_shape = (self.n_inputs, self.n_subclasses) self.weight = self.weight.sample(weight_shape) if self.weight is not None: self.initialized = True if self.n_subclasses < self.n_classes: raise ValueError("Number of subclasses should be greater " "or equal to the number of classes. Network " "was defined with {} subclasses and {} classes" "".format(self.n_subclasses, self.n_classes)) if self.prototypes_per_class is None: whole, reminder = divmod(self.n_subclasses, self.n_classes) self.prototypes_per_class = [whole] * self.n_classes if reminder: # Since we have reminder left, it means that we cannot # have an equal number of subclasses per each class, # therefor we will add +1 to randomly selected class. class_indeces = np.random.choice(self.n_classes, reminder, replace=False) for class_index in class_indeces: self.prototypes_per_class[class_index] += 1 if len(self.prototypes_per_class) != self.n_classes: raise ValueError("LVQ defined for classification problem that has " "{} classes, but the `prototypes_per_class` " "variable has defined data for {} classes." "".format(self.n_classes, len(self.prototypes_per_class))) if sum(self.prototypes_per_class) != self.n_subclasses: raise ValueError("Invalid distribution of subclasses for the " "`prototypes_per_class` variable. Got total " "of {} subclasses ({}) instead of {} expected" "".format(sum(self.prototypes_per_class), self.prototypes_per_class, self.n_subclasses)) self.subclass_to_class = [] for class_id, n_prototypes in enumerate(self.prototypes_per_class): self.subclass_to_class.extend([class_id] * n_prototypes) @property def training_step(self): if self.n_updates_to_stepdrop is None: return self.step updates_ratio = (1 - self.n_updates / self.n_updates_to_stepdrop) return self.minstep + (self.step - self.minstep) * updates_ratio def predict(self, input_data): if not self.initialized: raise NotTrained("LVQ network hasn't been trained yet") input_data = format_data(input_data) subclass_to_class = self.subclass_to_class weight = self.weight predictions = [] for input_row in input_data: output = euclid_distance(input_row, weight) winner_subclass = int(output.argmin(axis=1)) predicted_class = subclass_to_class[winner_subclass] predictions.append(predicted_class) return np.array(predictions) def train(self, input_train, target_train, *args, **kwargs): input_train = format_data(input_train) target_train = format_data(target_train) n_input_samples = len(input_train) if n_input_samples <= self.n_subclasses: raise ValueError("Number of training input samples should be " "greater than number of sublcasses. Training " "method recived {} input samples." "".format(n_input_samples)) if not self.initialized: target_classes = sorted(np.unique(target_train).astype(np.int)) expected_classes = list(range(self.n_classes)) if target_classes != expected_classes: raise ValueError("All classes should be integers from the " "range [0, {}], but got the following " "classes instead {}".format( self.n_classes - 1, target_classes)) weights = [] iterator = zip(target_classes, self.prototypes_per_class) for target_class, n_prototypes in iterator: is_valid_class = (target_train[:, 0] == target_class) is_valid_class = is_valid_class.astype('float64') n_samples_per_class = sum(is_valid_class) is_valid_class /= n_samples_per_class if n_samples_per_class <= n_prototypes: raise ValueError("Input data has {0} samples for class-{1}" ". Number of samples per specified " "class-{1} should be greater than {2}." "".format(n_samples_per_class, target_class, n_prototypes)) class_weight_indeces = np.random.choice( np.arange(n_input_samples), n_prototypes, replace=False, p=is_valid_class) class_weight = input_train[class_weight_indeces] weights.extend(class_weight) self.weight = np.array(weights) self.initialized = True super(LVQ, self).train(input_train, target_train, *args, **kwargs) def train_epoch(self, input_train, target_train): weight = self.weight subclass_to_class = self.subclass_to_class n_correct_predictions = 0 for input_row, target in zip(input_train, target_train): step = self.training_step output = euclid_distance(input_row, weight) winner_subclass = int(output.argmin()) predicted_class = subclass_to_class[winner_subclass] weight_update = input_row - weight[winner_subclass, :] is_correct_prediction = (predicted_class == target) if is_correct_prediction: weight[winner_subclass, :] += step * weight_update else: weight[winner_subclass, :] -= step * weight_update n_correct_predictions += is_correct_prediction self.n_updates += 1 n_samples = len(input_train) return 1 - n_correct_predictions / n_samples
class SOFM(Kohonen): """ Self-Organizing Feature Map (SOFM or SOM). Notes ----- - Training data samples should have normalized features. Parameters ---------- {BaseAssociative.n_inputs} n_outputs : int or None Number of outputs. Parameter is optional in case if ``feature_grid`` was specified. .. code-block:: python if n_outputs is None: n_outputs = np.prod(feature_grid) learning_radius : int Parameter defines radius within which we consider all neurons as neighbours to the winning neuron. The bigger the value the more neurons will be updated after each iteration. The ``0`` values means that we don't update neighbour neurons. Defaults to ``0``. std : int, float Parameters controls learning rate for each neighbour. The further neighbour neuron from the winning neuron the smaller that learning rate for it. Learning rate scales based on the factors produced by the normal distribution with center in the place of a winning neuron and standard deviation specified as a parameter. The learning rate for the winning neuron is always equal to the value specified in the ``step`` parameter and for neighbour neurons it's always lower. The bigger the value for this parameter the bigger learning rate for the neighbour neurons. Defaults to ``1``. features_grid : list, tuple, None Feature grid defines shape of the output neurons. The new shape should be compatible with the number of outputs. It means that the following condition should be true: .. code-block:: python np.prod(features_grid) == n_outputs SOFM implementation supports n-dimensional grids. For instance, in order to specify grid as cube instead of the regular rectangular shape we can set up options as the following: .. code-block:: python SOFM( ... features_grid=(5, 5, 5), ... ) Defaults to ``(n_outputs, 1)``. grid_type : {{``rect``, ``hexagon``}} Defines connection type in feature grid. Type defines which neurons we will consider as closest to the winning neuron during the training. - ``rect`` - Connections between neurons will be organized in hexagonal grid. - ``hexagon`` - Connections between neurons will be organized in hexagonal grid. It works only for 1d or 2d grids. Defaults to ``rect``. distance : {{``euclid``, ``dot_product``, ``cos``}} Defines function that will be used to compute closest weight to the input sample. - ``dot_product``: Just a regular dot product between data sample and network's weights - ``euclid``: Euclidean distance between data sample and network's weights - ``cos``: Cosine distance between data sample and network's weights Defaults to ``euclid``. reduce_radius_after : int or None Every specified number of epochs ``learning_radius`` parameter will be reduced by ``1``. Process continues until ``learning_radius`` equal to ``0``. The ``None`` value disables parameter reduction during the training. Defaults to ``100``. reduce_step_after : int or None Defines reduction rate at which parameter ``step`` will be reduced using the following formula: .. code-block:: python step = step / (1 + current_epoch / reduce_step_after) The ``None`` value disables parameter reduction during the training. Defaults to ``100``. reduce_std_after : int or None Defines reduction rate at which parameter ``std`` will be reduced using the following formula: .. code-block:: python std = std / (1 + current_epoch / reduce_std_after) The ``None`` value disables parameter reduction during the training. Defaults to ``100``. weight : array-like, Initializer or {{``init_pca``, ``sample_from_data``}} Neural network weights. Value defined manualy should have shape ``(n_inputs, n_outputs)``. Also, it's possible to initialized weights base on the training data. There are two options: - ``sample_from_data`` - Before starting the training will randomly take number of training samples equal to number of expected outputs. - ``init_pca`` - Before training starts SOFM will applies PCA on a covariance matrix build from the training samples. Weights will be generated based on the two eigenvectors associated with the largest eigenvalues. Defaults to :class:`Normal() <neupy.init.Normal>`. {BaseNetwork.step} {BaseNetwork.show_epoch} {BaseNetwork.shuffle_data} {BaseNetwork.signals} {Verbose.verbose} Methods ------- init_weights(train_data) Initialized weights based on the input data. It works only for the `init_pca` and `sample_from_data` options. For other cases it will throw an error. {BaseSkeleton.predict} {BaseAssociative.train} {BaseSkeleton.fit} Examples -------- >>> import numpy as np >>> from neupy import algorithms, utils >>> >>> utils.reproducible() >>> >>> data = np.array([ ... [0.1961, 0.9806], ... [-0.1961, 0.9806], ... [-0.5812, -0.8137], ... [-0.8137, -0.5812], ... ]) >>> >>> sofm = algorithms.SOFM( ... n_inputs=2, ... n_outputs=2, ... step=0.1, ... learning_radius=0 ... ) >>> sofm.train(data, epochs=100) >>> sofm.predict(data) array([[0, 1], [0, 1], [1, 0], [1, 0]]) """ n_outputs = IntProperty(minval=1, allow_none=True, default=None) weight = SOFMWeightParameter(default=init.Normal(), choices={ 'init_pca': linear_initialization, 'sample_from_data': sample_data, }) features_grid = TypedListProperty(allow_none=True, default=None) DistanceParameter = namedtuple('DistanceParameter', 'name func') distance = ChoiceProperty(default='euclid', choices={ 'dot_product': DistanceParameter(name='dot_product', func=np.dot), 'euclid': DistanceParameter(name='euclid', func=neg_euclid_distance), 'cos': DistanceParameter(name='cosine', func=cosine_similarity), }) GridTypeMethods = namedtuple('GridTypeMethods', 'name find_neighbours find_step_scaler') grid_type = ChoiceProperty( default='rect', choices={ 'rect': GridTypeMethods(name='rectangle', find_neighbours=find_neighbours_on_rect_grid, find_step_scaler=find_step_scaler_on_rect_grid), 'hexagon': GridTypeMethods(name='hexagon', find_neighbours=find_neighbours_on_hexagon_grid, find_step_scaler=find_step_scaler_on_hexagon_grid) }) learning_radius = IntProperty(default=0, minval=0) std = NumberProperty(minval=0, default=1) reduce_radius_after = IntProperty(default=100, minval=1, allow_none=True) reduce_std_after = IntProperty(default=100, minval=1, allow_none=True) reduce_step_after = IntProperty(default=100, minval=1, allow_none=True) def __init__(self, **options): super(BaseAssociative, self).__init__(**options) if self.n_outputs is None and self.features_grid is None: raise ValueError("One of the following parameters has to be " "specified: n_outputs, features_grid") elif self.n_outputs is None: self.n_outputs = np.prod(self.features_grid) n_grid_elements = np.prod(self.features_grid) invalid_feature_grid = (self.features_grid is not None and n_grid_elements != self.n_outputs) if invalid_feature_grid: raise ValueError( "Feature grid should contain the same number of elements " "as in the output layer: {0}, but found: {1} (shape: {2})" "".format(self.n_outputs, n_grid_elements, self.features_grid)) if self.features_grid is None: self.features_grid = (self.n_outputs, 1) if len(self.features_grid) > 2 and self.grid_type.name == 'hexagon': raise ValueError("SOFM with hexagon grid type should have " "one or two dimensional feature grid, but got " "{}d instead (shape: {!r})".format( len(self.features_grid), self.features_grid)) is_pca_init = (isinstance(options.get('weight'), six.string_types) and options.get('weight') == 'init_pca') self.initialized = False if not callable(self.weight): super(Kohonen, self).init_weights() self.initialized = True if self.distance.name == 'cosine': self.weight /= np.linalg.norm(self.weight, axis=0) elif is_pca_init and self.grid_type.name != 'rectangle': raise WeightInitializationError( "Cannot apply PCA weight initialization for non-rectangular " "grid. Grid type: {}".format(self.grid_type.name)) def predict_raw(self, X): X = format_data(X, is_feature1d=(self.n_inputs == 1)) if X.ndim != 2: raise ValueError("Only 2D inputs are allowed") n_samples = X.shape[0] output = np.zeros((n_samples, self.n_outputs)) for i, input_row in enumerate(X): output[i, :] = self.distance.func(input_row.reshape(1, -1), self.weight) return output def update_indexes(self, layer_output): neuron_winner = layer_output.argmax(axis=1).item(0) winner_neuron_coords = np.unravel_index(neuron_winner, self.features_grid) learning_radius = self.learning_radius step = self.step std = self.std if self.reduce_radius_after is not None: learning_radius -= self.last_epoch // self.reduce_radius_after learning_radius = max(0, learning_radius) if self.reduce_step_after is not None: step = decay_function(step, self.last_epoch, self.reduce_step_after) if self.reduce_std_after is not None: std = decay_function(std, self.last_epoch, self.reduce_std_after) methods = self.grid_type output_grid = np.reshape(layer_output, self.features_grid) output_with_neighbours = methods.find_neighbours( grid=output_grid, center=winner_neuron_coords, radius=learning_radius) step_scaler = methods.find_step_scaler(grid=output_grid, center=winner_neuron_coords, std=std) index_y, = np.nonzero(output_with_neighbours.reshape(self.n_outputs)) step_scaler = step_scaler.reshape(self.n_outputs) return index_y, step * step_scaler[index_y] def init_weights(self, X_train): if self.initialized: raise WeightInitializationError( "Weights have been already initialized") weight_initializer = self.weight self.weight = weight_initializer(X_train, self.features_grid) self.initialized = True if self.distance.name == 'cosine': self.weight /= np.linalg.norm(self.weight, axis=0) def train(self, X_train, epochs=100): if not self.initialized: self.init_weights(X_train) super(SOFM, self).train(X_train, epochs=epochs) def one_training_update(self, X_train, y_train=None): step = self.step predict = self.predict update_indexes = self.update_indexes error = 0 for input_row in X_train: input_row = np.reshape(input_row, (1, input_row.size)) layer_output = predict(input_row) index_y, step = update_indexes(layer_output) distance = input_row.T - self.weight[:, index_y] updated_weights = (self.weight[:, index_y] + step * distance) if self.distance.name == 'cosine': updated_weights /= np.linalg.norm(updated_weights, axis=0) self.weight[:, index_y] = updated_weights error += np.abs(distance).mean() return error / len(X_train)
class DropBlock(Identity): """ DropBlock, a form of structured dropout, where units in a contiguous region of a feature map are dropped together. Parameters ---------- keep_proba : float Fraction of the input units to keep. Value needs to be between ``0`` and ``1``. block_size : int or tuple Size of the block to be dropped. Blocks that have squared shape can be specified with a single integer value. For example, `block_size=5` the same as `block_size=(5, 5)`. {Identity.name} Methods ------- {Identity.Methods} Attributes ---------- {Identity.Attributes} See Also -------- :layer:`Dropout` : Dropout layer. References ---------- [1] Golnaz Ghiasi, Tsung-Yi Lin, Quoc V. Le. DropBlock: A regularization method for convolutional networks, 2018. Examples -------- >>> from neupy.layers import * >>> network = join( ... Input((28, 28, 1)), ... ... Convolution((3, 3, 16)) >> Relu(), ... DropBlock(keep_proba=0.1, block_size=5), ... ... Convolution((3, 3, 32)) >> Relu(), ... DropBlock(keep_proba=0.1, block_size=5), ... ) """ keep_proba = ProperFractionProperty() block_size = TypedListProperty(n_elements=2) def __init__(self, keep_proba, block_size, name=None): super(DropBlock, self).__init__(name=name) if isinstance(block_size, int): block_size = (block_size, block_size) self.keep_proba = keep_proba self.block_size = block_size def get_output_shape(self, input_shape): input_shape = tf.TensorShape(input_shape) if input_shape and input_shape.ndims != 4: raise LayerConnectionError( "DropBlock layer expects input with 4 dimensions, got {} " "with shape {}".format(len(input_shape), input_shape)) return input_shape def output(self, input, training=False): if not training: return input input = tf.convert_to_tensor(input, tf.float32) input_shape = tf.shape(input) block_height, block_width = self.block_size height, width = input_shape[1], input_shape[2] input_area = asfloat(width * height) block_area = asfloat(block_width * block_height) area = asfloat((width - block_width + 1) * (height - block_height + 1)) mask = bernoulli_sample( mean=(1. - self.keep_proba) * input_area / (block_area * area), shape=[ input_shape[0], height - block_height + 1, width - block_width + 1, input_shape[3], ], ) br_height = (block_height - 1) // 2 tl_height = (block_height - 1) - br_height br_width = (block_width - 1) // 2 tl_width = (block_width - 1) - br_width mask = tf.pad(mask, [ [0, 0], [tl_height, br_height], [tl_width, br_width], [0, 0], ]) mask = tf.nn.max_pool( mask, [1, block_height, block_width, 1], strides=[1, 1, 1, 1], padding='SAME', ) mask = tf.cast(1 - mask, tf.float32) feature_normalizer = asfloat(tf.size(mask)) / tf.reduce_sum(mask) return tf.multiply(input, mask) * feature_normalizer
class Input(BaseLayer): """ Layer defines network's input. Parameters ---------- shape : int or tuple Shape of the input features per sample. Batch dimension has to be excluded from the shape. {BaseLayer.name} Methods ------- {BaseLayer.Methods} Attributes ---------- {BaseLayer.Attributes} Examples -------- Feedforward Neural Network (FNN) In the example, input layer defines network that expects 2D inputs (matrices). In other words, input to the network should be set of samples combined into matrix where each sample has 10 dimensional vector associated with it. >>> from neupy.layers import * >>> network = Input(10) >> Relu(5) >> Softmax(3) Convolutional Neural Network (CNN) In the example, input layer specified that we expect multiple 28x28 image as an input and each image should have single channel (images with no color). >>> from neupy.layers import * >>> network = join( ... Input((28, 28, 1)), ... Convolution((3, 3, 16)) >> Relu(), ... Convolution((3, 3, 16)) >> Relu(), ... Reshape() ... Softmax(10), ... ) """ shape = TypedListProperty(element_type=(int, type(None))) def __init__(self, shape, name=None): super(Input, self).__init__(name=name) if isinstance(shape, tf.TensorShape): shape = tf_utils.shape_to_tuple(shape) self.shape = as_tuple(shape) @property def input_shape(self): batch_shape = tf.TensorShape([None]) return batch_shape.concatenate(self.shape) def output(self, input, **kwargs): return input def get_output_shape(self, input_shape): if not self.input_shape.is_compatible_with(input_shape): raise LayerConnectionError( "Input layer got unexpected input shape. " "Received shape: {}, Expected shape: {}" "".format(input_shape, self.input_shape) ) return self.input_shape.merge_with(input_shape) def __repr__(self): return self._repr_arguments( make_one_if_possible(self.shape), name=self.name)
class BatchNorm(Identity): """ Batch normalization layer. Parameters ---------- axes : tuple with ints or None Axes along which normalization will be applied. The ``None`` value means that normalization will be applied over all axes except the last one. In case of 4D tensor it will be equal to ``(0, 1, 2)``. Defaults to ``None``. epsilon : float Epsilon is a positive constant that adds to the standard deviation to prevent the division by zero. Defaults to ``1e-5``. alpha : float Coefficient for the exponential moving average of batch-wise means and standard deviations computed during training; the closer to one, the more it will depend on the last batches seen. Value needs to be between ``0`` and ``1``. Defaults to ``0.1``. gamma : array-like, Tensorfow variable, scalar or Initializer Scale. Default initialization methods you can find :ref:`here <init-methods>`. Defaults to ``Constant(value=1)``. beta : array-like, Tensorfow variable, scalar or Initializer Offset. Default initialization methods you can find :ref:`here <init-methods>`. Defaults to ``Constant(value=0)``. running_mean : array-like, Tensorfow variable, scalar or Initializer Default initialization methods you can find :ref:`here <init-methods>`. Defaults to ``Constant(value=0)``. running_inv_std : array-like, Tensorfow variable, scalar or Initializer Default initialization methods you can find :ref:`here <init-methods>`. Defaults to ``Constant(value=1)``. {Identity.name} Methods ------- {Identity.Methods} Attributes ---------- {Identity.Attributes} Examples -------- Feedforward Neural Networks (FNN) with batch normalization after activation function was applied. >>> from neupy.layers import * >>> network = join( ... Input(10), ... Relu(5) >> BatchNorm(), ... Relu(5) >> BatchNorm(), ... Sigmoid(1), ... ) Feedforward Neural Networks (FNN) with batch normalization before activation function was applied. >>> from neupy.layers import * >>> network = join( ... Input(10), ... Linear(5) >> BatchNorm() >> Relu(), ... Linear(5) >> BatchNorm() >> Relu(), ... Sigmoid(1), ... ) Convolutional Neural Networks (CNN) >>> from neupy.layers import * >>> network = join( ... Input((28, 28, 1)), ... Convolution((3, 3, 16)) >> BatchNorm() >> Relu(), ... Convolution((3, 3, 16)) >> BatchNorm() >> Relu(), ... Reshape(), ... Softmax(10), ... ) References ---------- .. [1] Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift, http://arxiv.org/pdf/1502.03167v3.pdf """ axes = TypedListProperty(allow_none=True) epsilon = NumberProperty(minval=0) alpha = ProperFractionProperty() beta = ParameterProperty() gamma = ParameterProperty() running_mean = ParameterProperty() running_inv_std = ParameterProperty() def __init__(self, axes=None, alpha=0.1, beta=0, gamma=1, epsilon=1e-5, running_mean=0, running_inv_std=1, name=None): super(BatchNorm, self).__init__(name=name) self.axes = axes self.alpha = alpha self.beta = beta self.gamma = gamma self.epsilon = epsilon self.running_mean = running_mean self.running_inv_std = running_inv_std if axes is not None and len(set(axes)) != len(axes): raise ValueError( "Specified axes have to contain only unique values") def create_variables(self, input_shape): input_shape = tf.TensorShape(input_shape) if input_shape.ndims is None: raise WeightInitializationError( "Cannot initialize variables for the batch normalization " "layer, because input shape is undefined. Layer: {}" "".format(self)) if self.axes is None: # If ndims == 4 then axes = (0, 1, 2) # If ndims == 2 then axes = (0,) self.axes = tuple(range(input_shape.ndims - 1)) if any(axis >= input_shape.ndims for axis in self.axes): raise LayerConnectionError( "Batch normalization cannot be applied over one of " "the axis, because input has only {} dimensions. Layer: {}" "".format(input_shape.ndims, self)) parameter_shape = tuple([ input_shape[axis].value if axis not in self.axes else 1 for axis in range(input_shape.ndims) ]) if any(parameter is None for parameter in parameter_shape): unknown_dim_index = parameter_shape.index(None) raise WeightInitializationError( "Cannot create variables for batch normalization, because " "input has unknown dimension #{} (0-based indices). " "Input shape: {}, Layer: {}".format(unknown_dim_index, input_shape, self)) self.input_shape = input_shape self.running_mean = self.variable(value=self.running_mean, shape=parameter_shape, name='running_mean', trainable=False) self.running_inv_std = self.variable(value=self.running_inv_std, shape=parameter_shape, name='running_inv_std', trainable=False) self.gamma = self.variable(value=self.gamma, name='gamma', shape=parameter_shape) self.beta = self.variable(value=self.beta, name='beta', shape=parameter_shape) def output(self, input, training=False): input = tf.convert_to_tensor(input, dtype=tf.float32) if not training: mean = self.running_mean inv_std = self.running_inv_std else: alpha = asfloat(self.alpha) mean = tf.reduce_mean( input, self.axes, keepdims=True, name="mean", ) variance = tf.reduce_mean( tf.squared_difference(input, tf.stop_gradient(mean)), self.axes, keepdims=True, name="variance", ) inv_std = tf.rsqrt(variance + asfloat(self.epsilon)) tf.add_to_collection( tf.GraphKeys.UPDATE_OPS, self.running_inv_std.assign( asfloat(1 - alpha) * self.running_inv_std + alpha * inv_std)) tf.add_to_collection( tf.GraphKeys.UPDATE_OPS, self.running_mean.assign( asfloat(1 - alpha) * self.running_mean + alpha * mean)) normalized_value = (input - mean) * inv_std return self.gamma * normalized_value + self.beta
class Reshape(BaseLayer): """ Layer reshapes input tensor. Parameters ---------- shape : tuple New feature shape. If one dimension specified with the ``-1`` value that this dimension will be computed from the total size that remains. Defaults to ``-1``. {BaseLayer.name} Methods ------- {BaseLayer.Methods} Attributes ---------- {BaseLayer.Attributes} Examples -------- Covert 4D input to 2D >>> from neupy.layers import * >>> network = Input((2, 5, 5)) >> Reshape() (?, 2, 5, 5) -> [... 2 layers ...] -> (?, 50) Convert 3D to 4D >>> from neupy.layers import * >>> network = Input((5, 4)) >> Reshape((5, 2, 2)) (?, 5, 4) -> [... 2 layers ...] -> (?, 5, 2, 2) """ shape = TypedListProperty() def __init__(self, shape=-1, name=None): super(Reshape, self).__init__(name=name) self.shape = as_tuple(shape) if self.shape.count(-1) >= 2: raise ValueError("Only single -1 value can be specified") def get_output_shape(self, input_shape): input_shape = tf.TensorShape(input_shape) feature_shape = input_shape[1:] missing_value = None if -1 in self.shape and feature_shape.is_fully_defined(): known_shape_values = [val for val in self.shape if val != -1] n_feature_values = np.prod(feature_shape.dims) n_expected_values = np.prod(known_shape_values) if n_feature_values % n_expected_values != 0: raise ValueError( "Input shape and specified shape are incompatible Shape: " "{}, Input shape: {}".format(self.shape, input_shape)) missing_value = int(n_feature_values // n_expected_values) n_sampes = input_shape[0] new_feature_shape = [ missing_value if val == -1 else val for val in self.shape ] return tf.TensorShape([n_sampes] + new_feature_shape) def output(self, input, **kwargs): """ Reshape the feature space for the input value. Parameters ---------- input : array-like or Tensorfow variable """ input = tf.convert_to_tensor(input, dtype=tf.float32) input_shape = tf.shape(input) n_samples = input_shape[0] expected_shape = self.get_output_shape(input.shape) feature_shape = expected_shape[1:] if feature_shape.is_fully_defined(): # For cases when we have -1 in the shape and feature shape # can be precomputed from the input we want to be explicit about # expected output shape. Because of the unknown batch dimension # it won't be possible for tensorflow to derive exact output # shape from the -1 output_shape = as_tuple(n_samples, feature_shape.dims) else: output_shape = as_tuple(n_samples, self.shape) return tf.reshape(input, output_shape) def __repr__(self): return self._repr_arguments(self.shape, name=self.name)
class BasePooling(BaseLayer): """ Base class for the pooling layers. Parameters ---------- size : tuple with 2 integers Factor by which to downscale (vertical, horizontal). (2, 2) will halve the image in each dimension. stride : tuple or int. Stride size, which is the number of shifts over rows/cols to get the next pool region. If stride is None, it is considered equal to ds (no overlap on pooling regions). padding : tuple or int (pad_h, pad_w), pad zeros to extend beyond four borders of the images, pad_h is the size of the top and bottom margins, and pad_w is the size of the left and right margins. ignore_border : bool When ``True``, ``(5, 5)`` input with size ``(2, 2)`` will generate a `(2, 2)` output. ``(3, 3)`` otherwise. Defaults to ``True``. {BaseLayer.Parameters} Methods ------- {BaseLayer.Methods} Attributes ---------- {BaseLayer.Attributes} """ size = TypedListProperty(required=True, element_type=int) stride = StrideProperty(default=None) padding = PaddingProperty(default=0, element_type=int, n_elements=2) ignore_border = Property(default=True, expected_type=bool) def __init__(self, size, **options): super(BasePooling, self).__init__(size=size, **options) if not self.ignore_border and self.padding != (0, 0): raise ValueError("Cannot set padding parameter equal to {} while " "``ignore_border`` is equal to ``False``" "".format(self.padding)) def validate(self, input_shape): if len(input_shape) != 3: raise LayerConnectionError("Pooling layer expects an input with 3 " "dimensions, got {} with shape {}" "".format(len(input_shape), input_shape)) @property def output_shape(self): if self.input_shape is None: return None n_kernels, rows, cols = self.input_shape row_filter_size, col_filter_size = self.size stride = self.size if self.stride is None else self.stride row_stride, col_stride = stride row_padding, col_padding = self.padding output_rows = pooling_output_shape(rows, row_filter_size, row_padding, row_stride, self.ignore_border) output_cols = pooling_output_shape(cols, col_filter_size, col_padding, col_stride, self.ignore_border) return (n_kernels, output_rows, output_cols) def __repr__(self): return '{name}({size})'.format(name=self.__class__.__name__, size=self.size)
class Convolution(BaseLayer): """ Convolutional layer. Parameters ---------- size : tuple of int Filter shape. In should be defined as a tuple with three integers ``(filter rows, filter columns, output channels)``. padding : {{``same``, ``valid``}}, int, tuple Zero padding for the input tensor. - ``valid`` - Padding won't be added to the tensor. Result will be the same as for ``padding=0`` - ``same`` - Padding will depend on the number of rows and columns in the filter. This padding makes sure that image with the ``stride=1`` won't change its width and height. It's the same as ``padding=(filter rows // 2, filter columns // 2)``. - Custom value for the padding can be specified as an integer, like ``padding=1`` or it can be specified as a tuple when different dimensions have different padding values, for example ``padding=(2, 3)``. Defaults to ``valid``. stride : tuple with ints, int. Stride size. Defaults to ``(1, 1)`` dilation : int, tuple Rate for the filter upsampling. When ``dilation > 1`` layer will become dilated convolution (or atrous convolution). Defaults to ``1``. weight : array-like, Tensorfow variable, scalar or Initializer Defines layer's weights. Shape of the weight will be equal to ``(filter rows, filter columns, input channels, output channels)``. Default initialization methods you can find :ref:`here <init-methods>`. Defaults to :class:`HeNormal(gain=2) <neupy.init.HeNormal>`. bias : 1D array-like, Tensorfow variable, scalar, Initializer or None Defines layer's bias. Default initialization methods you can find :ref:`here <init-methods>`. Defaults to :class:`Constant(0) <neupy.init.Constant>`. The ``None`` value excludes bias from the calculations and do not add it into parameters list. {BaseLayer.name} Examples -------- 2D Convolution >>> from neupy import layers >>> >>> layers.join( ... layers.Input((28, 28, 3)), ... layers.Convolution((3, 3, 16)), ... ) 1D Convolution >>> from neupy.layers import * >>> network = join( ... Input((30, 10)), ... Reshape((30, 1, 10)), # convert 3D to 4D ... Convolution((3, 1, 16)), ... Reshape((-1, 16)) # convert 4D back to 3D ... ) >>> network (?, 30, 10) -> [... 4 layers ...] -> (?, 28, 16) Methods ------- {BaseLayer.Methods} Attributes ---------- {BaseLayer.Attributes} """ size = TypedListProperty(element_type=int, n_elements=3) weight = ParameterProperty() bias = ParameterProperty(allow_none=True) padding = PaddingProperty() stride = Spatial2DProperty() dilation = Spatial2DProperty() # We use gain=2 because it's suitable choice for relu non-linearity # and relu is the most common non-linearity used for CNN. def __init__(self, size, padding='valid', stride=1, dilation=1, weight=init.HeNormal(gain=2), bias=0, name=None): super(Convolution, self).__init__(name=name) self.size = size self.padding = padding self.stride = stride self.dilation = dilation self.weight = weight self.bias = bias def fail_if_shape_invalid(self, input_shape): if input_shape and input_shape.ndims != 4: raise LayerConnectionError( "Convolutional layer expects an input with 4 " "dimensions, got {} with shape {}" "".format(len(input_shape), input_shape)) def output_shape_per_dim(self, *args, **kwargs): return conv_output_shape(*args, **kwargs) def expected_output_shape(self, input_shape): n_samples = input_shape[0] row_filter_size, col_filter_size, n_kernels = self.size row_stride, col_stride = self.stride row_dilation, col_dilation = self.dilation if isinstance(self.padding, (list, tuple)): row_padding, col_padding = self.padding else: row_padding, col_padding = self.padding, self.padding return ( n_samples, self.output_shape_per_dim( input_shape[1], row_filter_size, row_padding, row_stride, row_dilation ), self.output_shape_per_dim( input_shape[2], col_filter_size, col_padding, col_stride, col_dilation ), n_kernels, ) def get_output_shape(self, input_shape): input_shape = tf.TensorShape(input_shape) self.fail_if_shape_invalid(input_shape) if input_shape.ndims is None: n_samples = input_shape[0] n_kernels = self.size[-1] return tf.TensorShape((n_samples, None, None, n_kernels)) return tf.TensorShape(self.expected_output_shape(input_shape)) def create_variables(self, input_shape): self.input_shape = input_shape n_channels = input_shape[-1] n_rows, n_cols, n_filters = self.size # Compare to the regular convolution weights, # transposed one has switched input and output channels. self.weight = self.variable( value=self.weight, name='weight', shape=(n_rows, n_cols, n_channels, n_filters)) if self.bias is not None: self.bias = self.variable( value=self.bias, name='bias', shape=as_tuple(n_filters)) def output(self, input, **kwargs): input = tf.convert_to_tensor(input, tf.float32) self.fail_if_shape_invalid(input.shape) padding = self.padding if not isinstance(padding, six.string_types): height_pad, width_pad = padding input = tf.pad(input, [ [0, 0], [height_pad, height_pad], [width_pad, width_pad], [0, 0], ]) # VALID option will make sure that # convolution won't use any padding. padding = 'VALID' output = tf.nn.convolution( input, self.weight, padding=padding, strides=self.stride, dilation_rate=self.dilation, data_format="NHWC", ) if self.bias is not None: bias = tf.reshape(self.bias, (1, 1, 1, -1)) output += bias return output def __repr__(self): return self._repr_arguments( self.size, padding=self.padding, stride=self.stride, dilation=self.dilation, weight=self.weight, bias=self.bias, name=self.name, )
class BasePooling(BaseLayer): """ Base class for the pooling layers. Parameters ---------- size : tuple with 2 integers Factor by which to downscale (vertical, horizontal). (2, 2) will halve the image in each dimension. stride_size : tuple with 1 or 2 integers or integer. Stride size, which is the number of shifts over rows/cols to get the next pool region. If stride_size is None, it is considered equal to ds (no overlap on pooling regions). padding : tuple of two ints (pad_h, pad_w), pad zeros to extend beyond four borders of the images, pad_h is the size of the top and bottom margins, and pad_w is the size of the left and right margins. Methods ------- {BaseLayer.Methods} Attributes ---------- {BaseLayer.Attributes} """ size = TypedListProperty(required=True, element_type=int) stride_size = StrideProperty(default=None) padding = TypedListProperty(default=(0, 0), element_type=int, n_elements=2) def __init__(self, size, **options): super(BasePooling, self).__init__(size=size, **options) @property def output_shape(self): if self.input_shape is None: return None if len(self.input_shape) < 3: raise ValueError( "Convolutional layer expects an input shape with least 3 " "dimensions, got {} with shape {}".format( len(self.input_shape), self.input_shape)) n_kernels, rows, cols = self.input_shape[-3:] row_filter_size, col_filter_size = self.size stride_size = self.stride_size if stride_size is None: stride_size = self.size row_stride, col_stride = stride_size row_border_mode, col_border_mode = self.padding output_rows = conv_output_shape(rows, row_filter_size, row_border_mode, row_stride) output_cols = conv_output_shape(cols, col_filter_size, col_border_mode, col_stride) return (n_kernels, output_rows, output_cols) def __repr__(self): return '{name}({size})'.format(name=self.__class__.__name__, size=self.size)
class SOFM(Kohonen): """ Self-Organizing Feature Map. Parameters ---------- learning_radius : int Learning radius. features_grid : int Learning radius. transform : {{'linear', 'euclid', 'cos'}} Indicate transformation operation related to the input layer. The ``linear`` value mean that input data would be multiplied by weights in typical way. The ``euclid`` method will identify the closest weight vector to the input one. The ``cos`` made the same as ``euclid``, but instead of euclid distance it uses cosine similarity. Defaults to ``linear``. {BaseAssociative.n_inputs} {BaseAssociative.n_outputs} {BaseAssociative.weight} {BaseNetwork.step} {BaseNetwork.show_epoch} {BaseNetwork.shuffle_data} {BaseNetwork.epoch_end_signal} {BaseNetwork.train_end_signal} {Verbose.verbose} Methods ------- {BaseSkeleton.predict} {BaseAssociative.train} {BaseSkeleton.fit} """ learning_radius = IntProperty(default=0, minval=0) features_grid = TypedListProperty() transform = ChoiceProperty(default='linear', choices={ 'linear': dot_product, 'euclid': neg_euclid_distance, 'cos': cosine_similarity, }) def __init__(self, **options): super(SOFM, self).__init__(**options) invalid_feature_grid = (self.features_grid is not None and mul(*self.features_grid) != self.n_outputs) if invalid_feature_grid: raise ValueError( "Feature grid should contain the same number of elements as " "in the output layer: {0}, but found: {1} ({2}x{3})" "".format(self.n_outputs, mul(*self.features_grid), self.features_grid[0], self.features_grid[1])) def init_properties(self): super(SOFM, self).init_properties() if self.features_grid is None: self.features_grid = (self.n_outputs, 1) def predict_raw(self, input_data): input_data = format_data(input_data) output = np.zeros((input_data.shape[0], self.n_outputs)) for i, input_row in enumerate(input_data): output[i, :] = self.transform(input_row.reshape(1, -1), self.weight) return output def update_indexes(self, layer_output): neuron_winner = layer_output.argmax(axis=1) feature_bound = self.features_grid[1] output_with_neightbours = neuron_neighbours( np.reshape(layer_output, self.features_grid), (neuron_winner // feature_bound, neuron_winner % feature_bound), self.learning_radius) index_y, _ = np.nonzero( np.reshape(output_with_neightbours, (self.n_outputs, 1))) return index_y
class PRelu(Linear): """ Layer with the parametrized ReLu used as an activation function. Layer learns additional parameter ``alpha`` during the training. It applies linear transformation when the ``n_units`` parameter specified and parametrized relu function after the transformation. When ``n_units`` is not specified, only parametrized relu function will be applied to the input. Parameters ---------- alpha_axes : int or tuple Axes that will not include unique alpha parameter. Single integer value defines the same as a tuple with one value. Defaults to ``-1``. alpha : array-like, Tensorfow variable, scalar or Initializer Separate alpha parameter per each non-shared axis for the ReLu. Scalar value means that each element in the tensor will be equal to the specified value. Default initialization methods you can find :ref:`here <init-methods>`. Defaults to ``Constant(value=0.25)``. {Linear.Parameters} Methods ------- {Linear.Methods} Attributes ---------- {Linear.Attributes} Examples -------- Feedforward Neural Networks (FNN) >>> from neupy.layers import * >>> network = Input(10) >> PRelu(20) >> PRelu(1) Convolutional Neural Networks (CNN) >>> from neupy.layers import * >>> network = join( ... Input((32, 32, 3)), ... Convolution((3, 3, 16)) >> PRelu(), ... Convolution((3, 3, 32)) >> PRelu(), ... Reshape(), ... Softmax(10), ... ) References ---------- .. [1] Delving Deep into Rectifiers: Surpassing Human-Level Performance on ImageNet Classification. https://arxiv.org/pdf/1502.01852v1.pdf """ alpha_axes = TypedListProperty() alpha = ParameterProperty() def __init__(self, n_units=None, alpha_axes=-1, alpha=0.25, weight=init.HeNormal(gain=2), bias=0, name=None): self.alpha = alpha self.alpha_axes = as_tuple(alpha_axes) if 0 in self.alpha_axes: raise ValueError("Cannot specify alpha for 0-axis") super(PRelu, self).__init__(n_units=n_units, weight=weight, bias=bias, name=name) def get_output_shape(self, input_shape): input_shape = tf.TensorShape(input_shape) if input_shape and max(self.alpha_axes) >= input_shape.ndims: max_axis_index = input_shape.ndims - 1 raise LayerConnectionError( "Cannot specify alpha for the axis #{}. Maximum " "available axis is {} (0-based indices)." "".format(max(self.alpha_axes), max_axis_index)) return super(PRelu, self).get_output_shape(input_shape) def create_variables(self, input_shape): super(PRelu, self).create_variables(input_shape) output_shape = self.get_output_shape(input_shape) self.alpha = self.variable( value=self.alpha, name='alpha', shape=[output_shape[axis] for axis in self.alpha_axes]) def activation_function(self, input): input = tf.convert_to_tensor(input, dtype=tf.float32) ndim = input.shape.ndims dimensions = np.arange(ndim) alpha_axes = dimensions[list(self.alpha_axes)] alpha = tf_utils.dimshuffle(self.alpha, ndim, alpha_axes) return tf.maximum(0.0, input) + alpha * tf.minimum(0.0, input) def __repr__(self): if self.n_units is None: return self._repr_arguments(name=self.name, alpha_axes=self.alpha_axes, alpha=self.alpha) return self._repr_arguments(self.n_units, name=self.name, alpha_axes=self.alpha_axes, alpha=self.alpha, weight=self.weight, bias=self.bias)
class SOFM(Kohonen): """ Self-Organizing Feature Map (SOFM). Parameters ---------- {BaseAssociative.n_inputs} {BaseAssociative.n_outputs} learning_radius : int Learning radius. features_grid : list, tuple, None Feature grid defines shape of the output neurons. The new shape should be compatible with the number of outputs. Defaults to ``(n_outputs, 1)``. transform : {{``linear``, ``euclid``, ``cos``}} Indicate transformation operation related to the input layer. - The ``linear`` value mean that input data would be multiplied by weights in typical way. - The ``euclid`` method will identify the closest weight vector to the input one. - The ``cos`` transformation identifies cosine similarity between input dataset and network's weights. Defaults to ``linear``. {BaseAssociative.weight} {BaseNetwork.step} {BaseNetwork.show_epoch} {BaseNetwork.shuffle_data} {BaseNetwork.epoch_end_signal} {BaseNetwork.train_end_signal} {Verbose.verbose} Methods ------- {BaseSkeleton.predict} {BaseAssociative.train} {BaseSkeleton.fit} Examples -------- >>> import numpy as np >>> from neupy import algorithms, environment >>> >>> environment.reproducible() >>> >>> data = np.array([ ... [0.1961, 0.9806], ... [-0.1961, 0.9806], ... [-0.5812, -0.8137], ... [-0.8137, -0.5812], ... ]) >>> >>> sofmnet = algorithms.SOFM( ... n_inputs=2, ... n_outputs=2, ... step=0.1, ... learning_radius=0, ... features_grid=(2, 1), ... ) >>> sofmnet.train(data, epochs=100) >>> sofmnet.predict(data) array([[0, 1], [0, 1], [1, 0], [1, 0]]) """ learning_radius = IntProperty(default=0, minval=0) features_grid = TypedListProperty(allow_none=True, default=None) transform = ChoiceProperty(default='linear', choices={ 'linear': np.dot, 'euclid': neg_euclid_distance, 'cos': cosine_similarity, }) def __init__(self, **options): super(SOFM, self).__init__(**options) invalid_feature_grid = (self.features_grid is not None and mul(*self.features_grid) != self.n_outputs) if invalid_feature_grid: raise ValueError( "Feature grid should contain the same number of elements as " "in the output layer: {0}, but found: {1} ({2}x{3})" "".format(self.n_outputs, mul(*self.features_grid), self.features_grid[0], self.features_grid[1])) if self.features_grid is None: self.features_grid = (self.n_outputs, 1) def predict_raw(self, input_data): input_data = format_data(input_data) n_samples = input_data.shape[0] output = np.zeros((n_samples, self.n_outputs)) for i, input_row in enumerate(input_data): output[i, :] = self.transform(input_row.reshape(1, -1), self.weight) return output def update_indexes(self, layer_output): neuron_winner = layer_output.argmax(axis=1) feature_bound = self.features_grid[1] output_with_neightbours = neuron_neighbours( np.reshape(layer_output, self.features_grid), (neuron_winner // feature_bound, neuron_winner % feature_bound), self.learning_radius) index_y, _ = np.nonzero( np.reshape(output_with_neightbours, (self.n_outputs, 1))) return index_y
class Convolution(ParameterBasedLayer): """ Convolutional layer. Parameters ---------- size : tuple of int Filter shape. In should be defined as a tuple with three integers ``(output channels, filter rows, filter columns)``. padding : {{``valid``, ``full``, ``half``}} or int or tuple with 2 int Convolution border mode. Check Theano's ``nnet.conv2d`` doc. Defaults to ``valid``. stride : tuple with 1 or 2 integers or integer. Stride size. Defaults to ``(1, 1)`` {ParameterBasedLayer.weight} {ParameterBasedLayer.bias} {BaseLayer.Parameters} Examples -------- 2D Convolution >>> from neupy import layers >>> >>> layers.join( ... layers.Input((3, 28, 28)), ... layers.Convolution((16, 3, 3)), ... ) 1D Convolution >>> from neupy import layers >>> >>> layers.join( ... layers.Input((10, 30)), ... layers.Reshape((10, 30, 1)), ... layers.Convolution((16, 3, 1)), ... ) Methods ------- {ParameterBasedLayer.Methods} Attributes ---------- {ParameterBasedLayer.Attributes} """ size = TypedListProperty(required=True, element_type=int) padding = BorderModeProperty(default='valid') stride = StrideProperty(default=(1, 1)) def validate(self, input_shape): if len(input_shape) != 3: raise LayerConnectionError( "Convolutional layer expects an input with 3 " "dimensions, got {} with shape {}" "".format(len(input_shape), input_shape)) @property def output_shape(self): if self.input_shape is None: return None padding = self.padding n_kernels = self.size[0] rows, cols = self.input_shape[-2:] row_filter_size, col_filter_size = self.size[-2:] row_stride, col_stride = self.stride if isinstance(padding, tuple): row_padding, col_padding = padding[-2:] else: row_padding, col_padding = padding, padding output_rows = conv_output_shape(rows, row_filter_size, row_padding, row_stride) output_cols = conv_output_shape(cols, col_filter_size, col_padding, col_stride) return (n_kernels, output_rows, output_cols) @property def weight_shape(self): n_channels = self.input_shape[0] n_filters, n_rows, n_cols = self.size return (n_filters, n_channels, n_rows, n_cols) @property def bias_shape(self): return as_tuple(self.size[0]) def output(self, input_value): output = T.nnet.conv2d(input_value, self.weight, input_shape=as_tuple(None, self.input_shape), filter_shape=self.weight_shape, border_mode=self.padding, subsample=self.stride) if self.bias is not None: bias = T.reshape(self.bias, (1, -1, 1, 1)) output += bias return output
class Upscale(BaseLayer): """ Upscales input over two axis (height and width). Parameters ---------- scale : int or tuple with two int Scaling factor for the input value. In the tuple first parameter identifies scale of the height and the second one of the width. {BaseLayer.name} Methods ------- {BaseLayer.Methods} Attributes ---------- {BaseLayer.Attributes} Examples -------- >>> from neupy.layers import * >>> network = Input((10, 10, 3)) >> Upscale((2, 2)) (?, 10, 10, 3) -> [... 2 layers ...] -> (?, 20, 20, 3) """ scale = TypedListProperty(n_elements=2) def __init__(self, scale, name=None): super(Upscale, self).__init__(name=name) if isinstance(scale, int): scale = as_tuple(scale, scale) if any(element <= 0 for element in scale): raise ValueError( "Only positive integers are allowed for scale") self.scale = scale def fail_if_shape_invalid(self, input_shape): if input_shape and input_shape.ndims != 4: raise LayerConnectionError( "Upscale layer should have an input value with 4 dimensions " "(batch, height, width, channel), got input with {} " "dimensions instead. Shape: {}" "".format(input_shape.ndims, input_shape)) def get_output_shape(self, input_shape): input_shape = tf.TensorShape(input_shape) self.fail_if_shape_invalid(input_shape) if input_shape.ndims is None: return tf.TensorShape((None, None, None, None)) n_samples, height, width, channel = input_shape height_scale, width_scale = self.scale return tf.TensorShape([ n_samples, height_scale * height, width_scale * width, channel, ]) def output(self, input_value, **kwargs): input_value = tf.convert_to_tensor(input_value, dtype=tf.float32) self.fail_if_shape_invalid(input_value.shape) return tf_utils.repeat(input_value, as_tuple(1, self.scale, 1)) def __repr__(self): return self._repr_arguments(self.scale, name=self.name)
class Convolution(ParameterBasedLayer): """ Convolutional layer. Parameters ---------- size : tuple of int Filter shape. In should be defined as a tuple with three integers ``(filter rows, filter columns, output channels)``. padding : {{``same``, ``valid``}}, int, tuple Zero padding for the input tensor. - ``valid`` - Padding won't be added to the tensor. Result will be the same as for ``padding=0`` - ``same`` - Padding will depend on the number of rows and columns in the filter. This padding makes sure that image with the ``stride=1`` won't change its width and height. It's the same as ``padding=(filter rows // 2, filter columns // 2)``. - Custom value for the padding can be specified as an integer, like ``padding=1`` or it can be specified as a tuple when different dimensions have different padding values, for example ``padding=(2, 3)``. Defaults to ``valid``. stride : tuple with ints, int. Stride size. Defaults to ``(1, 1)`` dilation : int, tuple Rate for the fiter upsampling. When ``dilation > 1`` layer will become diated convolution (or atrous convolution). Defaults to ``1``. weight : array-like, Tensorfow variable, scalar or Initializer Defines layer's weights. Shape of the weight will be equal to ``(filter rows, filter columns, input channels, output channels)``. Default initialization methods you can find :ref:`here <init-methods>`. Defaults to :class:`HeNormal(gain=2) <neupy.init.HeNormal>`. {ParameterBasedLayer.bias} {BaseLayer.Parameters} Examples -------- 2D Convolution >>> from neupy import layers >>> >>> layers.join( ... layers.Input((28, 28, 3)), ... layers.Convolution((3, 3, 16)), ... ) 1D Convolution >>> from neupy import layers >>> >>> layers.join( ... layers.Input((30, 10)), ... layers.Reshape((30, 1, 10)), ... layers.Convolution((3, 1, 16)), ... ) Methods ------- {ParameterBasedLayer.Methods} Attributes ---------- {ParameterBasedLayer.Attributes} """ # We use gain=2 because it's suitable choice for relu non-linearity # and relu is the most common non-linearity used for CNN. weight = ParameterProperty(default=init.HeNormal(gain=2)) size = TypedListProperty(required=True, element_type=int) padding = PaddingProperty(default='valid') stride = Spatial2DProperty(default=(1, 1)) dilation = Spatial2DProperty(default=1) def validate(self, input_shape): if input_shape and len(input_shape) != 3: raise LayerConnectionError( "Convolutional layer expects an input with 3 " "dimensions, got {} with shape {}" "".format(len(input_shape), input_shape)) def output_shape_per_dim(self, *args, **kwargs): return conv_output_shape(*args, **kwargs) def find_output_from_input_shape(self, input_shape): padding = self.padding rows, cols, _ = input_shape row_filter_size, col_filter_size, n_kernels = self.size row_stride, col_stride = self.stride row_dilation, col_dilation = self.dilation or (1, 1) if isinstance(padding, (list, tuple)): row_padding, col_padding = padding else: row_padding, col_padding = padding, padding output_rows = self.output_shape_per_dim( rows, row_filter_size, row_padding, row_stride, row_dilation, ) output_cols = self.output_shape_per_dim( cols, col_filter_size, col_padding, col_stride, col_dilation, ) return (output_rows, output_cols, n_kernels) @property def output_shape(self): if self.input_shape is not None: return self.find_output_from_input_shape(self.input_shape) @property def weight_shape(self): n_channels = self.input_shape[-1] n_rows, n_cols, n_filters = self.size return (n_rows, n_cols, n_channels, n_filters) @property def bias_shape(self): return as_tuple(self.size[-1]) def output(self, input_value): padding = self.padding if not isinstance(padding, six.string_types): height_pad, weight_pad = padding input_value = tf.pad(input_value, [ [0, 0], [height_pad, height_pad], [weight_pad, weight_pad], [0, 0], ]) # VALID option will make sure that # convolution won't use any padding. padding = 'VALID' output = tf.nn.convolution( input_value, self.weight, padding=padding, strides=self.stride, dilation_rate=self.dilation, data_format="NHWC" ) if self.bias is not None: bias = tf.reshape(self.bias, (1, 1, 1, -1)) output += bias return output
class ParameterBasedLayer(BaseLayer): """ Layer that creates weight and bias parameters. Parameters ---------- size : int Layer input size. weight : 2D array-like or None Define your layer weights. ``None`` means that your weights will be generate randomly dependence on property ``init_method``. ``None`` by default. bias : 1D array-like or None Define your layer bias. ``None`` means that your weights will be generate randomly dependence on property ``init_method``. init_method : {{'bounded', 'normal', 'ortho', 'xavier_normal',\ 'xavier_uniform', 'he_normal', 'he_uniform'}} Weight initialization method. Defaults to ``xavier_normal``. * ``normal`` will generate random weights from normal distribution \ with standard deviation equal to ``0.01``. * ``bounded`` generate random weights from Uniform distribution. * ``ortho`` generate random orthogonal matrix. * ``xavier_normal`` generate random matrix from normal distrubtion \ where variance equal to :math:`\\frac{{2}}{{fan_{{in}} + \ fan_{{out}}}}`. Where :math:`fan_{{in}}` is a number of \ layer input units and :math:`fan_{{out}}` - number of layer \ output units. * ``xavier_uniform`` generate random matrix from uniform \ distribution \ where :math:`w_{{ij}} \in \ [-\\sqrt{{\\frac{{6}}{{fan_{{in}} + fan_{{out}}}}}}, \ \\sqrt{{\\frac{{6}}{{fan_{{in}} + fan_{{out}}}}}}`]. * ``he_normal`` generate random matrix from normal distrubtion \ where variance equal to :math:`\\frac{{2}}{{fan_{{in}}}}`. \ Where :math:`fan_{{in}}` is a number of layer input units. * ``he_uniform`` generate random matrix from uniformal \ distribution where :math:`w_{{ij}} \in [\ -\\sqrt{{\\frac{{6}}{{fan_{{in}}}}}}, \ \\sqrt{{\\frac{{6}}{{fan_{{in}}}}}}]` bounds : tuple of two float Available only for ``init_method`` equal to ``bounded``. Value identify minimum and maximum possible value in random weights. Defaults to ``(0, 1)``. """ size = IntProperty(minval=1) weight = SharedArrayProperty(default=None) bias = SharedArrayProperty(default=None) bounds = TypedListProperty(default=(0, 1), element_type=(int, float)) init_method = ChoiceProperty(default=XAVIER_NORMAL, choices=VALID_INIT_METHODS) def __init__(self, size, **options): if size is not None: options['size'] = size super(ParameterBasedLayer, self).__init__(**options) def weight_shape(self): output_size = self.relate_to_layer.size return (self.size, output_size) def bias_shape(self): output_size = self.relate_to_layer.size return (output_size,) def initialize(self): super(ParameterBasedLayer, self).initialize() self.weight = create_shared_parameter( value=self.weight, name='weight_{}'.format(self.layer_id), shape=self.weight_shape(), bounds=self.bounds, init_method=self.init_method, ) self.bias = create_shared_parameter( value=self.bias, name='bias_{}'.format(self.layer_id), shape=self.bias_shape(), bounds=self.bounds, init_method=self.init_method, ) self.parameters = [self.weight, self.bias] def __repr__(self): classname = self.__class__.__name__ return '{name}({size})'.format(name=classname, size=self.size)
class Reshape(BaseLayer): """ Gives a new shape to an input value without changing its data. Parameters ---------- shape : tuple or list New feature shape. ``None`` value means that feature will be flatten in 1D vector. If you need to get the output feature with more that 2 dimensions then you can set up new feature shape using tuples. Defaults to ``None``. {BaseLayer.Parameters} Methods ------- {BaseLayer.Methods} Attributes ---------- {BaseLayer.Attributes} Examples -------- Covert 4D input to 2D >>> from neupy import layers >>> >>> connection = layers.join( ... layers.Input((2, 5, 5)), ... layers.Reshape() ... ) >>> >>> print("Input shape: {{}}".format(connection.input_shape)) Input shape: (2, 5, 5) >>> >>> print("Output shape: {{}}".format(connection.output_shape)) Output shape: (50,) Convert 3D to 4D >>> from neupy import layers >>> >>> connection = layers.join( ... layers.Input((5, 4)), ... layers.Reshape((5, 2, 2)) ... ) >>> >>> print("Input shape: {{}}".format(connection.input_shape)) Input shape: (5, 4) >>> >>> print("Output shape: {{}}".format(connection.output_shape)) Output shape: (5, 2, 2) """ shape = TypedListProperty() def __init__(self, shape=None, **options): if shape is not None: options['shape'] = shape super(Reshape, self).__init__(**options) @property def output_shape(self): if self.shape is not None: return as_tuple(self.shape) n_output_features = np.prod(self.input_shape) return as_tuple(n_output_features) def output(self, input_value): """ Reshape the feature space for the input value. Parameters ---------- input_value : array-like or Theano variable """ n_samples = input_value.shape[0] output_shape = as_tuple(n_samples, self.output_shape) return T.reshape(input_value, output_shape)