def test_upscale_layer_shape(self): Case = namedtuple("Case", "scale expected_shape") testcases = ( Case(scale=(2, 2), expected_shape=(None, 28, 28, 1)), Case(scale=(2, 1), expected_shape=(None, 28, 14, 1)), Case(scale=(1, 2), expected_shape=(None, 14, 28, 1)), Case(scale=(1, 1), expected_shape=(None, 14, 14, 1)), Case(scale=(1, 10), expected_shape=(None, 14, 140, 1)), ) for testcase in testcases: network = layers.join( layers.Input((14, 14, 1)), layers.Upscale(testcase.scale), ) self.assertShapesEqual(network.output_shape, testcase.expected_shape, msg="scale: {}".format(testcase.scale))
def test_dilated_convolution(self): network = layers.join( layers.Input((6, 6, 1)), layers.Convolution((3, 3, 1), dilation=2, weight=1, bias=None), ) input_value = asfloat(np.arange(36).reshape(1, 6, 6, 1)) actual_output = self.eval(network.output(input_value)) self.assertShapesEqual(actual_output.shape, (1, 2, 2, 1)) self.assertShapesEqual(actual_output.shape[1:], network.output_shape[1:]) actual_output = actual_output[0, :, :, 0] expected_output = np.array([ [126, 135], # every row value adds +1 per filter value (+9) [180, 189], # every col value adds +6 per filter value (+54) ]) np.testing.assert_array_almost_equal(actual_output, expected_output)
def test_tree_graph(self): l0 = layers.Input(1) l1 = layers.Sigmoid(10) l2 = layers.Sigmoid(20) l3 = layers.Sigmoid(30) l4 = layers.Sigmoid(40) l5 = layers.Sigmoid(50) l6 = layers.Sigmoid(60) # Tree Structure: # # l0 - l1 - l5 - l6 # \ # l2 - l4 # \ # -- l3 graph = LayerGraph() # Connection #1 graph.connect_layers(l0, l1) graph.connect_layers(l1, l5) graph.connect_layers(l5, l6) # Connection #2 graph.connect_layers(l1, l2) graph.connect_layers(l2, l3) # Connection #3 graph.connect_layers(l2, l4) for layer in graph.forward_graph: layer.initialize() subgraph = graph.subgraph_for_output(l6) self.assertEqual(1, len(subgraph.output_layers)) self.assertIs(l6, subgraph.output_layers[0]) self.assertEqual(1, len(subgraph.input_layers)) self.assertIs(l0, subgraph.input_layers[0]) text_input = asfloat(np.array([[1]])) expected_shapes = [(1, 30), (1, 40), (1, 60)] outputs = graph.propagate_forward(text_input) for output, expected_shape in zip(outputs, expected_shapes): output_value = self.eval(output) self.assertIn(output_value.shape, expected_shapes)
def test_errdiff(self): initial_step = 0.3 network = algorithms.GradientDescent( [ layers.Input(2), layers.Tanh(3), layers.Tanh(1), ], batch_size='all', step=initial_step, update_for_smaller_error=1.05, update_for_bigger_error=0.7, error_difference=1.04, addons=[algorithms.ErrDiffStepUpdate] ) network.train(xor_input_train, xor_target_train, epochs=200) self.assertNotEqual(self.eval(network.variables.step), initial_step) self.assertAlmostEqual(network.errors.last(), 0, places=4)
def test_quasi_newton_dfp(self): x_train, x_test, y_train, y_test = simple_classification() qnnet = algorithms.QuasiNewton( connection=[ layers.Input(10), layers.Sigmoid(30, weight=init.Orthogonal()), layers.Sigmoid(1, weight=init.Orthogonal()), ], shuffle_data=True, verbose=False, update_function='dfp', h0_scale=2, ) qnnet.train(x_train, y_train, x_test, y_test, epochs=10) result = qnnet.predict(x_test).round() roc_curve_score = metrics.roc_auc_score(result, y_test) self.assertAlmostEqual(0.92, roc_curve_score, places=2)
def test_max_pooling(self): X = asfloat( np.array([ [1, 2, 3, -1], [4, -6, 3, 1], [0, 0, 1, 0], [0, -1, 0, 0], ])).reshape(1, 4, 4, 1) expected_output = asfloat(np.array([ [4, 3], [0, 1], ])).reshape(1, 2, 2, 1) network = layers.join( layers.Input((4, 4, 1)), layers.MaxPooling((2, 2)), ) actual_output = self.eval(network.output(X)) np.testing.assert_array_almost_equal(actual_output, expected_output)
def test_quasi_newton_bfgs(self): x_train, x_test, y_train, y_test = simple_classification() qnnet = algorithms.QuasiNewton( connection=[ layers.Input(10), layers.Sigmoid(30, weight=init.Orthogonal()), layers.Sigmoid(1, weight=init.Orthogonal()), ], shuffle_data=True, show_epoch='20 times', update_function='bfgs', ) qnnet.train(x_train, y_train, x_test, y_test, epochs=50) result = qnnet.predict(x_test).round().astype(int) roc_curve_score = metrics.roc_auc_score(result, y_test) self.assertAlmostEqual(0.92, roc_curve_score, places=2)
def test_fail_many_to_many_connection(self): network_a = layers.join( layers.Input(10), layers.parallel( layers.Relu(5), layers.Relu(4), ), ) network_b = layers.join( layers.parallel( layers.Relu(5), layers.Relu(4), ), layers.Concatenate(), ) error_message = "Cannot make many to many connection between graphs" with self.assertRaisesRegexp(LayerConnectionError, error_message): layers.join(network_a, network_b)
def test_hessdiag(self): x_train, x_test, y_train, y_test = simple_classification() nw = algorithms.HessianDiagonal( connection=[ layers.Input(10), layers.Sigmoid(20, weight=init.Uniform(-1, 1), bias=init.Uniform(-1, 1)), layers.Sigmoid(1, weight=init.Uniform(-1, 1), bias=init.Uniform(-1, 1)), ], step=0.1, shuffle_data=False, verbose=False, min_eigval=0.01, ) nw.train(x_train / 2, y_train, epochs=10) self.assertAlmostEqual(0.10, nw.errors.last(), places=2)
def train_lstm(self, data, verbose=False, **lstm_options): x_train, x_test, y_train, y_test = data network = algorithms.RMSProp( [ layers.Input(self.n_time_steps), layers.Embedding(self.n_categories, 10), layers.LSTM(20, **lstm_options), layers.Sigmoid(1), ], step=0.05, verbose=verbose, batch_size=16, loss='binary_crossentropy', ) network.train(x_train, y_train, x_test, y_test, epochs=20) y_predicted = network.predict(x_test).round() accuracy = (y_predicted.T == y_test).mean() return accuracy
def run(self): self.prepareData() self.showData() # net = Net(1,8,2) lmnet = algorithms.LevenbergMarquardt([ layers.Input(1), layers.Sigmoid(8), layers.Sigmoid(2), ], verbose=True, shuffle_data=True) self.showNetData(lmnet, 'dumb') # net.Train(self.input, self.output) a = np.array(self.input) b = np.array(self.output) lmnet.fit(a, b, epochs=100) self.showNetData(lmnet, 'trained') plt.show()
def test_embedding_layer(self): weight = np.arange(10).reshape((5, 2)) input_layer = layers.Input(1) embedding_layer = layers.Embedding(5, 2, weight=weight) connection = layers.join(input_layer, embedding_layer) connection.initialize() input_vector = asfloat(np.array([[0, 1, 4]]).T) expected_output = np.array([ [[0, 1]], [[2, 3]], [[8, 9]], ]) actual_output = connection.output(input_vector).eval() self.assertEqual(embedding_layer.output_shape, (1, 2)) np.testing.assert_array_equal(expected_output, actual_output)
def test_conv_with_custom_int_padding(self): network = layers.join( layers.Input((5, 5, 1)), layers.Convolution((3, 3, 1), bias=0, weight=1, padding=2), ) x = asfloat(np.ones((1, 5, 5, 1))) expected_output = np.array([ [1, 2, 3, 3, 3, 2, 1], [2, 4, 6, 6, 6, 4, 2], [3, 6, 9, 9, 9, 6, 3], [3, 6, 9, 9, 9, 6, 3], [3, 6, 9, 9, 9, 6, 3], [2, 4, 6, 6, 6, 4, 2], [1, 2, 3, 3, 3, 2, 1], ]).reshape((1, 7, 7, 1)) actual_output = self.eval(network.output(x)) np.testing.assert_array_almost_equal(expected_output, actual_output)
def test_change_output_layer(self): network = layers.join( layers.Input(10, name='input-1'), layers.Relu(5, name='relu-1'), layers.Relu(1, name='relu-2'), ) self.assertShapesEqual(network.input_shape, (None, 10)) self.assertShapesEqual(network.output_shape, (None, 1)) self.assertEqual(len(network), 3) relu_1_network = network.end('relu-1') self.assertShapesEqual(relu_1_network.input_shape, (None, 10)) self.assertShapesEqual(relu_1_network.output_shape, (None, 5)) self.assertEqual(len(relu_1_network.layers), 2) x_test = asfloat(np.ones((7, 10))) y_predicted = self.eval(relu_1_network.output(x_test)) self.assertEqual(y_predicted.shape, (7, 5))
def test_cut_output_layers_in_sequence(self): network = layers.join( layers.Input(10, name='input-1'), layers.Relu(5, name='relu-1'), layers.Relu(1, name='relu-2'), ) self.assertShapesEqual(network.input_shape, (None, 10)) self.assertShapesEqual(network.output_shape, (None, 1)) self.assertEqual(len(network), 3) cutted_network = network.end('relu-1').end('input-1') self.assertShapesEqual(cutted_network.input_shape, (None, 10)) self.assertShapesEqual(cutted_network.output_shape, (None, 10)) self.assertEqual(len(cutted_network), 1) x_test = asfloat(np.ones((7, 10))) y_predicted = cutted_network.output(x_test) self.assertEqual(y_predicted.shape, (7, 10))
def test_storage_data_layer_compatibility(self): connection = layers.Input(2) > layers.Sigmoid(3, name='sigm') sigmoid = connection.layer('sigm') with self.assertRaises(ParameterLoaderError): validate_layer_compatibility(sigmoid, { 'name': 'sigm', 'class_name': 'Sigmoid', 'input_shape': (3,), # wrong input shape 'output_shape': (3,), 'configs': {}, 'parameters': { 'weight': {'trainable': True, 'value': np.ones((2, 3))}, 'bias': {'trainable': True, 'value': np.ones((3,))}, } }) with self.assertRaises(ParameterLoaderError): validate_layer_compatibility(sigmoid, { 'name': 'sigm', 'class_name': 'Sigmoid', 'input_shape': (2,), 'output_shape': (2,), # wrong output shape 'configs': {}, 'parameters': { 'weight': {'trainable': True, 'value': np.ones((2, 3))}, 'bias': {'trainable': True, 'value': np.ones((3,))}, } }) result = validate_layer_compatibility(sigmoid, { 'name': 'sigm', 'class_name': 'Sigmoid', 'input_shape': (2,), 'output_shape': (3,), 'configs': {}, 'parameters': { 'weight': {'trainable': True, 'value': np.ones((2, 3))}, 'bias': {'trainable': True, 'value': np.ones((3,))}, } }) self.assertIsNone(result)
def test_linear_search(self): methods = [ ('golden', 0.36517048), ('brent', 0.36848962), ] for method_name, valid_error in methods: np.random.seed(self.random_seed) dataset = datasets.load_boston() data, target = dataset.data, dataset.target data_scaler = preprocessing.MinMaxScaler() target_scaler = preprocessing.MinMaxScaler() x_train, x_test, y_train, y_test = train_test_split( data_scaler.fit_transform(data), target_scaler.fit_transform(target.reshape(-1, 1)), test_size=0.15) cgnet = algorithms.GradientDescent( connection=[ layers.Input(13), layers.Sigmoid(50), layers.Sigmoid(1), ], batch_size='all', show_epoch=1, verbose=False, search_method=method_name, tol=0.1, addons=[algorithms.LinearSearch], ) cgnet.train(x_train, y_train, x_test, y_test, epochs=10) y_predict = cgnet.predict(x_test) error = errors.rmsle( asfloat(target_scaler.inverse_transform(y_test)), asfloat(target_scaler.inverse_transform(y_predict)), ) error = self.eval(error) self.assertAlmostEqual(valid_error, error, places=5)
def test_custom_layer(self): class NewLayer(layers.BaseLayer): def __init__(self, *args, **kwargs): super(NewLayer, self).__init__(*args, **kwargs) self._input_shape = tf.TensorShape((None, None, None)) def create_variables(self, input_shape): self.input_shape = input_shape def output(self, input): return input new_layer = NewLayer() network = layers.join(layers.Input((10, 5)), new_layer) self.assertShapesEqual(network.output_shape, None) self.assertShapesEqual(new_layer.input_shape, (None, None, None)) network.create_variables() self.assertShapesEqual(network.output_shape, None) self.assertShapesEqual(new_layer.input_shape, (None, 10, 5))
def test_deconvolution_tuple_padding(self): network = layers.join( layers.Input((10, 10, 3)), layers.Convolution((3, 3, 7), padding=(9, 3)), layers.Deconvolution((3, 3, 4), padding=(9, 3)), ) shapes = network.output_shapes_per_layer shapes = {l: shape_to_tuple(s) for l, s in shapes.items()} self.assertSequenceEqual( shapes, { network.layers[0]: (None, 10, 10, 3), network.layers[1]: (None, 26, 14, 7), network.layers[2]: (None, 10, 10, 4), }) input_value = asfloat(np.random.random((1, 10, 10, 3))) actual_output = self.eval(network.output(input_value)) self.assertEqual(actual_output.shape, (1, 10, 10, 4))
def test_graph_repr(self): l1 = layers.Input(1) l2 = layers.Sigmoid(2) l3 = layers.Sigmoid(3) l4 = layers.Sigmoid(4) graph = LayerGraph() graph.connect_layers(l1, l2) graph.connect_layers(l2, l3) graph.connect_layers(l3, l4) expected_output = textwrap.dedent(""" [(Input(1), [Sigmoid(2)]), (Sigmoid(2), [Sigmoid(3)]), (Sigmoid(3), [Sigmoid(4)]), (Sigmoid(4), [])] """).strip() self.assertEqual(expected_output, repr(graph).strip())
def test_get_layer_by_name_from_connection(self): network = layers.join( layers.Input(10, name='input-1'), layers.Relu(8, name='relu-0'), layers.Relu(5, name='relu-1'), ) reul0 = network.layer('relu-0') self.assertShapesEqual(reul0.output_shape, (None, 8)) reul1 = network.layer('relu-1') self.assertShapesEqual(reul1.output_shape, (None, 5)) message = "Cannot find layer with name 'some-layer-name'" with self.assertRaisesRegexp(NameError, message): network.layer('some-layer-name') message = "Layer name expected to be a string" with self.assertRaisesRegexp(ValueError, message): network.layer(object)
def test_conv_shapes(self): border_modes = ['valid', 'full', 'half', 4, 5, (6, 3), (4, 4), (1, 1)] strides = [(1, 1), (2, 1), (2, 2)] x = asfloat(np.random.random((20, 2, 12, 11))) for stride, border_mode in product(strides, border_modes): input_layer = layers.Input((2, 12, 11)) conv_layer = layers.Convolution((5, 3, 4), border_mode=border_mode, stride_size=stride) connection = input_layer > conv_layer conv_layer.initialize() y = conv_layer.output(x).eval() actual_output_shape = as_tuple(y.shape[1:]) self.assertEqual(actual_output_shape, conv_layer.output_shape, msg='border_mode={}'.format(border_mode))
def train(X, Y): environment.reproducible() img_size = X.shape[1] network = algorithms.Momentum( [ layers.Input(img_size), layers.Relu(100), layers.Softmax(Y.shape[1]), ], error='categorical_crossentropy', step=0.01, verbose=True, shuffle_data=True, momentum=0.9, nesterov=True, ) network.architecture() network.train(X, Y, epochs=20) return network
def test_conv_shapes(self): paddings = ['valid', 'full', 'half', 4, 5, (6, 3), (4, 4), (1, 1)] strides = [(1, 1), (2, 1), (2, 2)] x = asfloat(np.random.random((20, 2, 12, 11))) for stride, padding in product(strides, paddings): input_layer = layers.Input((2, 12, 11)) conv_layer = layers.Convolution((5, 3, 4), padding=padding, stride=stride) input_layer > conv_layer conv_layer.initialize() y = conv_layer.output(x).eval() actual_output_shape = as_tuple(y.shape[1:]) self.assertEqual(actual_output_shape, conv_layer.output_shape, msg='padding={}'.format(padding))
def test_cut_along_lines_basic(self): network = algorithms.GradientDescent([ layers.Input(5), surgery.CutLine(), layers.Sigmoid(10), layers.Sigmoid(20), layers.Sigmoid(30), surgery.CutLine(), layers.Sigmoid(1), ]) for connection in (network, network.connection): _, interested_layers, _ = surgery.cut_along_lines(connection) cutted_shapes = [layer.output_shape for layer in interested_layers] self.assertEqual(as_tuple(*cutted_shapes), (10, 20, 30))
def test_conv_with_custom_int_padding(self): input_layer = layers.Input((5, 5, 1)) conv = layers.Convolution((3, 3, 1), bias=0, weight=1, padding=2) connection = input_layer > conv connection.initialize() x = asfloat(np.ones((1, 5, 5, 1))) expected_output = np.array([ [1, 2, 3, 3, 3, 2, 1], [2, 4, 6, 6, 6, 4, 2], [3, 6, 9, 9, 9, 6, 3], [3, 6, 9, 9, 9, 6, 3], [3, 6, 9, 9, 9, 6, 3], [2, 4, 6, 6, 6, 4, 2], [1, 2, 3, 3, 3, 2, 1], ]).reshape((1, 7, 7, 1)) actual_output = self.eval(connection.output(x)) np.testing.assert_array_almost_equal(expected_output, actual_output)
def test_change_input_layer(self): network = layers.join( layers.Input(10, name='input-1'), layers.Relu(5, name='relu-1'), layers.Relu(1, name='relu-2'), ) self.assertEqual(network.input_shape, (10, )) self.assertEqual(network.output_shape, (1, )) self.assertEqual(len(network), 3) relu_1_network = network.start('relu-1') self.assertEqual(relu_1_network.input_shape, (10, )) self.assertEqual(relu_1_network.output_shape, (1, )) self.assertEqual(len(relu_1_network), 2) predict = relu_1_network.compile() x_test = asfloat(np.ones((7, 10))) y_predicted = predict(x_test) self.assertEqual(y_predicted.shape, (7, 1))
def test_cut_input_layers_in_sequence(self): network = layers.join( layers.Input(10, name='input-1'), layers.Relu(5, name='relu-1'), layers.Relu(1, name='relu-2'), ) self.assertEqual(network.input_shape, (10, )) self.assertEqual(network.output_shape, (1, )) self.assertEqual(len(network), 3) cutted_network = network.start('relu-1').start('relu-2') self.assertEqual(cutted_network.input_shape, (5, )) self.assertEqual(cutted_network.output_shape, (1, )) self.assertEqual(len(cutted_network), 1) predict = cutted_network.compile() x_test = asfloat(np.ones((7, 5))) y_predicted = predict(x_test) self.assertEqual(y_predicted.shape, (7, 1))
def run_neural_net(): import_modules() dataset = datasets.load_boston() data, target = dataset.data, dataset.target data_scalar = preprocessing.MinMaxScaler() target_scalar = preprocessing.MinMaxScaler() data = data_scalar.fit_transform(data) target = target_scalar.fit_transform(target.reshape(-1, 1)) environment.reproducible() x_train, x_test, y_train, y_test = train_test_split(data, target, train_size=0.85) cgnet = algorithms.ConjugateGradient( connection=[ layers.Input(13), layers.Sigmoid(75), layers.Sigmoid(25), layers.Sigmoid(1), ], search_method='golden', show_epoch=1, verbose=True, addons=[algorithms.LinearSearch], ) cgnet.train(x_train, y_train, x_test, y_test, epochs=30) plots.error_plot(cgnet) y_predict = cgnet.predict(x_test).round(1) error = rmsle(target_scalar.invers_transform(y_test), \ target_scalar.invers_transform(y_predict)) return (error)
def ann_fs_fitness(solution): learning_rate, neuron = solution # fs_model = FeatureSelectionModel(X, Y) X_reduced = FeatureSelectionTransform(fs_model, X) xs, ys = BuildDataScale(X_reduced, Y) x_scale = DataScaleTransform(xs, X_reduced) y_scale = DataScaleTransform(ys, Y) layer = [ layers.Input(len(X_reduced[0])), layers.Sigmoid(neuron), layers.Sigmoid(5), ] net = ANNForecastBuild(layer, learning_rate) start_time = time.time() x_train, x_test, y_train, y_test = TrainANN(net, x_scale, y_scale, e=100) elapsed_time = time.time() - start_time print(time.strftime("%H:%M:%S", time.gmtime(elapsed_time))) #evaluation y_pred = net.predict(x_test) y_pred = DataScaleInverse(ys, y_pred) y_true = DataScaleInverse(ys, y_test) #Evaluation(y_true, y_pred) r2 = r2_score(y_true, y_pred) m_rmse = np.sqrt( mean_squared_error(y_true, y_pred, multioutput='raw_values')) rmse = np.sqrt(mean_squared_error(y_true, y_pred)) print('m_rmse : ' + str(m_rmse)) print('r2 : ' + str(r2)) print('rmse : ' + str(rmse)) print('lr : ' + str(learning_rate) + ', hidden neuron : ' + str(neuron)) output = rmse finished = output <= err_threshold #finished = output <= 0.01 fitness = 1 / output print(finished) print(fitness) return fitness, finished