def test_prelu_param_updates(self): x_train, _, y_train, _ = simple_classification() prelu_layer1 = layers.PRelu(20, alpha=0.25) prelu_layer2 = layers.PRelu(1, alpha=0.25) gdnet = algorithms.GradientDescent([ layers.Input(10), prelu_layer1, prelu_layer2, ]) prelu1_alpha_before_training = prelu_layer1.alpha.get_value() prelu2_alpha_before_training = prelu_layer2.alpha.get_value() gdnet.train(x_train, y_train, epochs=10) prelu1_alpha_after_training = prelu_layer1.alpha.get_value() prelu2_alpha_after_training = prelu_layer2.alpha.get_value() self.assertTrue( all( np.not_equal( prelu1_alpha_before_training, prelu1_alpha_after_training, ))) self.assertTrue( all( np.not_equal( prelu2_alpha_before_training, prelu2_alpha_after_training, )))
def test_invalid_alpha_axes_parameter(self): prelu_layer = layers.PRelu(10, alpha_axes=2) with self.assertRaises(ValueError): # cannot specify 2-axis, because we only # have 0 and 1 axes (2D input) layers.Input(10) > prelu_layer with self.assertRaises(ValueError): # 0-axis is not allowed layers.PRelu(10, alpha_axes=0)
def test_invalid_alpha_axes_parameter(self): # there are can be specified axis 1, but not 2 prelu_layer = layers.PRelu(10, alpha_axes=2) connection = layers.Input(10) > prelu_layer with self.assertRaises(ValueError): prelu_layer.initialize() # cannot specify alpha per input sample prelu_layer = layers.PRelu(10, alpha_axes=0) connection = layers.Input(10) > prelu_layer with self.assertRaises(ValueError): prelu_layer.initialize()
def test_invalid_alpha_axes_parameter(self): network = layers.join( layers.PRelu(10, alpha_axes=2), layers.Relu(), ) with self.assertRaises(LayerConnectionError): # cannot specify 2-axis, because we only # have 0 and 1 axes (2D input) layers.join(layers.Input(10), network) with self.assertRaises(ValueError): # 0-axis is not allowed layers.PRelu(10, alpha_axes=0)
def test_prelu_alpha_init_constant_value(self): prelu_layer = layers.PRelu(10, alpha=0.25) prelu_layer.create_variables((None, 5)) alpha = self.eval(prelu_layer.alpha) self.assertEqual(alpha.shape, (10, )) np.testing.assert_array_almost_equal(alpha, np.ones(10) * 0.25)
def test_prelu_random_params(self): prelu_layer = layers.PRelu(10, alpha=None) connection = layers.Input(10) > prelu_layer prelu_layer.initialize() alpha = prelu_layer.alpha.get_value() self.assertEqual(10, np.unique(alpha).size)
def test_prelu_layer_param_dense(self): prelu_layer = layers.PRelu(10, alpha=0.25) layers.Input(10) > prelu_layer alpha = self.eval(prelu_layer.alpha) self.assertEqual(alpha.shape, (10,)) np.testing.assert_array_almost_equal(alpha, np.ones(10) * 0.25)
def test_prelu_layer_param_dense(self): prelu_layer = layers.PRelu(10, alpha=0.25) connection = layers.Input(10) > prelu_layer prelu_layer.initialize() alpha = prelu_layer.alpha.get_value() self.assertEqual(alpha.shape, (10, )) np.testing.assert_array_almost_equal(alpha, np.ones(10) * 0.25)
def test_prelu_output_by_dense_input(self): prelu_layer = layers.PRelu(1, alpha=0.25) layers.Input(1) > prelu_layer input_data = np.array([[10, 1, 0.1, 0, -0.1, -1]]).T expected_output = np.array([[10, 1, 0.1, 0, -0.025, -0.25]]).T actual_output = self.eval(prelu_layer.activation_function(input_data)) np.testing.assert_array_almost_equal(expected_output, actual_output)
def test_prelu_output_by_dense_input(self): prelu_layer = layers.PRelu(alpha=0.25) prelu_layer.create_variables((None, 1)) X = np.array([[10, 1, 0.1, 0, -0.1, -1]]).T expected_output = np.array([[10, 1, 0.1, 0, -0.025, -0.25]]).T actual_output = self.eval(prelu_layer.activation_function(X)) np.testing.assert_array_almost_equal(expected_output, actual_output)
def test_prelu_output_by_spatial_input(self): network = layers.join( layers.Input((10, 10, 3)), layers.Convolution((3, 3, 5)), layers.PRelu(alpha=0.25, alpha_axes=(1, 3)), ) X = asfloat(np.random.random((1, 10, 10, 3))) actual_output = self.eval(network.output(X)) self.assertEqual(actual_output.shape, (1, 8, 8, 5))
def test_prelu_layer_param_conv(self): input_layer = layers.Input((10, 10, 3)) conv_layer = layers.Convolution((3, 3, 5)) prelu_layer = layers.PRelu(alpha=0.25, alpha_axes=(1, 3)) input_layer > conv_layer > prelu_layer alpha = self.eval(prelu_layer.alpha) expected_alpha = np.ones((8, 5)) * 0.25 self.assertEqual(alpha.shape, (8, 5)) np.testing.assert_array_almost_equal(alpha, expected_alpha)
def test_prelu_layer_param_conv(self): network = layers.join( layers.Input((10, 10, 3)), layers.Convolution((3, 3, 5)), layers.PRelu(alpha=0.25, alpha_axes=(1, 3), name='prelu'), ) network.create_variables() alpha = self.eval(network.layer('prelu').alpha) expected_alpha = np.ones((8, 5)) * 0.25 self.assertEqual(alpha.shape, (8, 5)) np.testing.assert_array_almost_equal(alpha, expected_alpha)
def test_prelu_layer_param_conv(self): input_layer = layers.Input((3, 10, 10)) conv_layer = layers.Convolution((5, 3, 3)) prelu_layer = layers.PRelu(alpha=0.25, alpha_axes=(1, 3)) connection = input_layer > conv_layer > prelu_layer conv_layer.initialize() prelu_layer.initialize() alpha = prelu_layer.alpha.get_value() expected_alpha = np.ones((5, 8)) * 0.25 self.assertEqual(alpha.shape, (5, 8)) np.testing.assert_array_almost_equal(alpha, expected_alpha)
def test_prelu_output_by_spatial_input(self): input_data = asfloat(np.random.random((1, 10, 10, 3))) input_layer = layers.Input((10, 10, 3)) conv_layer = layers.Convolution((3, 3, 5)) prelu_layer = layers.PRelu(alpha=0.25, alpha_axes=(1, 3)) connection = input_layer > conv_layer > prelu_layer actual_output = input_data for layer in connection: actual_output = layer.output(actual_output) actual_output = self.eval(actual_output) self.assertEqual(actual_output.shape, (1, 8, 8, 5))
def test_prelu_variables(self): network = layers.join( layers.Input(2), layers.PRelu(3, name='prelu'), ) self.assertDictEqual(network.layer('prelu').variables, {}) network.create_variables() variables = network.layer('prelu').variables self.assertSequenceEqual(sorted(variables.keys()), ['alpha', 'bias', 'weight']) self.assertShapesEqual(variables['bias'].shape, (3, )) self.assertShapesEqual(variables['weight'].shape, (2, 3)) self.assertShapesEqual(variables['alpha'].shape, (3, ))
def model_network(self, algorithm='LevenbergMarquardt', model=None, opt=None): model = self.decode_model(model) if model is None: model = [ [1, 'hidden', 15, 'Linear'], [2, 'hidden', 10, 'Linear'], [3, 'output', self.output_classes, 'Elu'] ] # [Input(4), Elu(1)] # [Input(4), Elu(6), Elu(1)] EP: 100 layer_model = [layers.Input(self.input_features)] for layer in model: if layer[3] == 'Linear': layer_model.append(layers.Linear(layer[2])) if layer[3] == 'Relu': layer_model.append(layers.Relu(layer[2])) if layer[3] == 'Sigmoid': layer_model.append(layers.Sigmoid(layer[2])) if layer[3] == 'HardSigmoid': layer_model.append(layers.HardSigmoid(layer[2])) if layer[3] == 'Step': layer_model.append(layers.Step(layer[2])) if layer[3] == 'Tanh': layer_model.append(layers.Tanh(layer[2])) if layer[3] == 'Softplus': layer_model.append(layers.Softplus(layer[2])) if layer[3] == 'Softmax': layer_model.append(layers.Softmax(layer[2])) if layer[3] == 'Elu': layer_model.append(layers.Elu(layer[2])) if layer[3] == 'PRelu': layer_model.append(layers.PRelu(layer[2])) if layer[3] == 'LeakyRelu': layer_model.append(layers.LeakyRelu(layer[2])) print('Model warstw: ' + str(layer_model)) self.layers = layer_model self.select_algorithm(algorithm, options=opt)
mean = x_train.mean(axis=(0, 2, 3)).reshape((1, -1, 1, 1)) std = x_train.std(axis=(0, 2, 3)).reshape((1, -1, 1, 1)) x_train -= mean x_train /= std x_test -= mean x_test /= std target_scaler = OneHotEncoder() y_train = target_scaler.fit_transform(y_train.reshape((-1, 1))).todense() y_test = target_scaler.transform(y_test.reshape((-1, 1))).todense() network = algorithms.Adadelta( [ layers.Input((3, 32, 32)), layers.Convolution((64, 3, 3)) > layers.BatchNorm() > layers.PRelu(), layers.Convolution((64, 3, 3)) > layers.BatchNorm() > layers.PRelu(), layers.MaxPooling((2, 2)), layers.Convolution((128, 3, 3)) > layers.BatchNorm() > layers.PRelu(), layers.Convolution((128, 3, 3)) > layers.BatchNorm() > layers.PRelu(), layers.MaxPooling((2, 2)), layers.Reshape(), layers.Linear(1024) > layers.BatchNorm() > layers.PRelu(), layers.Linear(1024) > layers.BatchNorm() > layers.PRelu(), layers.Softmax(10), ], error='categorical_crossentropy', step=0.25, shuffle_data=True, batch_size=128, verbose=True,
def test_prelu_alpha_init_random_params(self): prelu_layer = layers.PRelu(10, alpha=init.XavierNormal()) prelu_layer.create_variables((None, 5)) alpha = self.eval(prelu_layer.alpha) self.assertEqual(10, np.unique(alpha).size)
def test_repr_with_size(self): self.assertEqual(str(layers.PRelu(10)), ("PRelu(10, alpha_axes=(-1,), alpha=Constant(0.25), " "weight=HeNormal(gain=2), bias=Constant(0), " "name='p-relu-1')"))
def test_prelu_random_params(self): prelu_layer = layers.PRelu(10, alpha=init.XavierNormal()) layers.Input(10) > prelu_layer alpha = self.eval(prelu_layer.alpha) self.assertEqual(10, np.unique(alpha).size)
verbose=True, step=0.1, momentum=0.99, shuffle_data=True, batch_size=64, error='binary_crossentropy', ) conv_autoencoder.architecture() conv_autoencoder.train(x_unlabeled_4d, x_unlabeled, x_labeled_4d, x_labeled, epochs=10) x_labeled_encoded = encoder.output(x_labeled_4d).eval() x_unlabeled_encoded = encoder.output(x_unlabeled_4d).eval() classifier_network = layers.join( layers.PRelu(512), layers.Dropout(0.25), layers.Softmax(10), ) encoder_classifier = algorithms.Adadelta( layers.Input(encoder.output_shape) > classifier_network, verbose=True, step=0.05, shuffle_data=True, batch_size=64, error='categorical_crossentropy', ) encoder_classifier.architecture() encoder_classifier.train(x_labeled_encoded, y_labeled, x_unlabeled_encoded, y_unlabeled, epochs=100)
def test_storage_save_dict(self): network = layers.join( layers.parallel([ layers.Input(2, name='input-1'), layers.PRelu(1, name='prelu') ], [ layers.Input(1, name='input-2'), layers.Sigmoid(4, name='sigmoid'), layers.BatchNorm(name='batch-norm'), ]), layers.Concatenate(name='concatenate'), layers.Softmax(3, name='softmax'), ) dict_network = storage.save_dict(network) expected_keys = ('metadata', 'layers', 'graph') self.assertItemsEqual(expected_keys, dict_network.keys()) expected_metadata_keys = ('created', 'language', 'library', 'version') actual_metadata_keys = dict_network['metadata'].keys() self.assertItemsEqual(expected_metadata_keys, actual_metadata_keys) self.assertEqual(len(dict_network['layers']), 7) expected_layers = [{ 'class_name': 'Input', 'configs': { 'name': 'input-1', 'shape': (2, ) }, 'name': 'input-1', }, { 'class_name': 'PRelu', 'configs': { 'alpha_axes': (-1, ), 'name': 'prelu', 'n_units': 1 }, 'name': 'prelu', }, { 'class_name': 'Input', 'configs': { 'name': 'input-2', 'shape': (1, ) }, 'name': 'input-2', }, { 'class_name': 'Sigmoid', 'configs': { 'name': 'sigmoid', 'n_units': 4 }, 'name': 'sigmoid', }, { 'class_name': 'BatchNorm', 'configs': { 'alpha': 0.1, 'axes': (0, ), 'epsilon': 1e-05, 'name': 'batch-norm' }, 'name': 'batch-norm', }, { 'class_name': 'Concatenate', 'configs': { 'axis': -1, 'name': 'concatenate' }, 'name': 'concatenate', }, { 'class_name': 'Softmax', 'configs': { 'name': 'softmax', 'n_units': 3 }, 'name': 'softmax', }] actual_layers = [] for i, layer in enumerate(dict_network['layers']): self.assertIn('parameters', layer, msg="Layer #" + str(i)) layer = copy.deepcopy(layer) del layer['parameters'] actual_layers.append(layer) self.assertEqual(actual_layers, expected_layers)
def test_repr_without_size(self): self.assertEqual( "PRelu(alpha_axes=(-1,), alpha=Constant(0.25), name='p-relu-1')", str(layers.PRelu()))