def Inception_3(pooling): if pooling not in ('max', 'average'): raise ValueError("Invalid pooling option: {}".format(pooling)) if pooling == 'max': Pooling = layers.MaxPooling elif pooling == 'average': Pooling = partial(layers.AveragePooling, mode='exclude_padding') return layers.join( [[ ConvReluBN((320, 1, 1)), ], [ ConvReluBN((384, 1, 1)), [[ ConvReluBN((384, 1, 3), padding=(0, 1)), ], [ ConvReluBN((384, 3, 1), padding=(1, 0)), ]], ], [ ConvReluBN((448, 1, 1)), ConvReluBN((384, 3, 3), padding=1), [[ ConvReluBN((384, 1, 3), padding=(0, 1)), ], [ ConvReluBN((384, 3, 1), padding=(1, 0)), ]], ], [ Pooling((3, 3), stride=(1, 1), padding=1), ConvReluBN((192, 1, 1)), ]], layers.Concatenate(), )
def test_mixture_of_experts_problem_with_specific_network(self): with self.assertRaisesRegexp(ValueError, "specified as a list"): architectures.mixture_of_experts(*self.networks) with self.assertRaisesRegexp(ValueError, "has more than one input"): last_network = layers.join( layers.parallel( layers.Input(1), layers.Input(2), ), layers.Concatenate(), ) architectures.mixture_of_experts( networks=self.networks + [last_network]) with self.assertRaisesRegexp(ValueError, "has more than one output"): last_network = layers.join( layers.Input(1), layers.parallel( layers.Softmax(1), layers.Softmax(1), ), ) architectures.mixture_of_experts( networks=self.networks + [last_network]) error_message = ( "Each network from the mixture of experts has to " "process only 2-dimensional inputs. Network #2.+" "Input layer's shape: \(\?, 1, 1, 1\)" ) with self.assertRaisesRegexp(ValueError, error_message): last_network = layers.Input((1, 1, 1)) architectures.mixture_of_experts( networks=self.networks + [last_network])
def test_networks_with_complex_parallel_relations(self): input_layer = layers.Input((5, 5, 3)) network = layers.join( layers.parallel([ layers.Convolution((1, 1, 8)), ], [ layers.Convolution((1, 1, 4)), layers.parallel( layers.Convolution((1, 3, 2), padding='same'), layers.Convolution((3, 1, 2), padding='same'), ), ], [ layers.Convolution((1, 1, 8)), layers.Convolution((3, 3, 4), padding='same'), layers.parallel( layers.Convolution((1, 3, 2), padding='same'), layers.Convolution((3, 1, 2), padding='same'), ) ], [ layers.MaxPooling((3, 3), padding='same', stride=(1, 1)), layers.Convolution((1, 1, 8)), ]), layers.Concatenate(), ) self.assertShapesEqual(network.input_shape, [None, None, None, None]) self.assertShapesEqual(network.output_shape, (None, None, None, None)) # Connect them at the end, because we need to make # sure tha parallel networks defined without input shapes network = layers.join(input_layer, network) self.assertShapesEqual(network.output_shape, (None, 5, 5, 24))
def test_parallel_layer(self): input_layer = layers.Input((3, 8, 8)) parallel_layer = layers.join( [[ layers.Convolution((11, 5, 5)), ], [ layers.Convolution((10, 3, 3)), layers.Convolution((5, 3, 3)), ]], layers.Concatenate(), ) output_layer = layers.MaxPooling((2, 2)) conn = layers.join(input_layer, parallel_layer) output_connection = layers.join(conn, output_layer) x = T.tensor4() y = theano.function([x], conn.output(x)) x_tensor4 = asfloat(np.random.random((10, 3, 8, 8))) output = y(x_tensor4) self.assertEqual(output.shape, (10, 11 + 5, 4, 4)) output_function = theano.function([x], output_connection.output(x)) final_output = output_function(x_tensor4) self.assertEqual(final_output.shape, (10, 11 + 5, 2, 2))
def test_training_with_multiple_inputs(self): network = algorithms.GradientDescent( [ [ layers.Input(2) > layers.Sigmoid(3), layers.Input(3) > layers.Sigmoid(5), ], layers.Concatenate(), layers.Sigmoid(1), ], step=0.1, verbose=False, shuffle_data=True, ) x_train, x_test, y_train, y_test = simple_classification(n_samples=100, n_features=5) x_train_2, x_train_3 = x_train[:, :2], x_train[:, 2:] x_test_2, x_test_3 = x_test[:, :2], x_test[:, 2:] network.train([x_train_2, x_train_3], y_train, [x_test_2, x_test_3], y_test, epochs=100) error = network.validation_errors[-1] self.assertAlmostEqual(error, 0.14, places=2)
def test_graph_relations_in_format_of_layer_names(self): l1 = layers.Input(1, name='input') l2 = layers.Sigmoid(2, name='sigmoid-2') l3 = layers.Sigmoid(3, name='sigmoid-3') l4 = layers.Sigmoid(4, name='sigmoid-4') lc = layers.Concatenate(name='concat') graph = LayerGraph() graph.connect_layers(l1, l2) graph.connect_layers(l2, l3) graph.connect_layers(l3, l4) graph.connect_layers(l2, lc) graph.connect_layers(l3, lc) actual_graph = graph.layer_names_only() expected_graph = [ ('input', ['sigmoid-2']), ('sigmoid-2', ['sigmoid-3', 'concat']), ('sigmoid-3', ['sigmoid-4', 'concat']), ('sigmoid-4', []), ('concat', []), ] self.assertListEqual(actual_graph, expected_graph)
def Fire(s_1x1, e_1x1, e_3x3, name): return layers.join( layers.Convolution( (1, 1, s_1x1), padding='SAME', name=name + '/squeeze1x1' ), layers.Relu(), layers.parallel([ layers.Convolution( (1, 1, e_1x1), padding='SAME', name=name + '/expand1x1' ), layers.Relu(), ], [ layers.Convolution( (3, 3, e_3x3), padding='SAME', name=name + '/expand3x3' ), layers.Relu(), ]), layers.Concatenate(), )
def test_network_shape_multiple_inputs(self): in1 = layers.Input(10) in2 = layers.Input(20) conn = (in1 | in2) >> layers.Concatenate() self.assertShapesEqual(conn.input_shape, [(None, 10), (None, 20)]) self.assertShapesEqual(conn.output_shape, (None, 30))
def test_concatenate_init_error(self): input_layer_1 = layers.Input((3, 28, 28)) input_layer_2 = layers.Input((1, 28, 28)) concat_layer = layers.Concatenate(axis=2) layers.join(input_layer_1, concat_layer) with self.assertRaises(LayerConnectionError): layers.join(input_layer_2, concat_layer)
def test_inplace_parallel(self): network = layers.Input(10) network |= layers.Input(10) network >>= layers.Concatenate() self.assertEqual(len(network), 3) self.assertShapesEqual(network.input_shape, [(None, 10), (None, 10)]) self.assertShapesEqual(network.output_shape, (None, 20))
def test_concatenate_init_error(self): input_layer_1 = layers.Input((28, 28, 3)) input_layer_2 = layers.Input((28, 28, 1)) concat_layer = layers.Concatenate(axis=2) layers.join(input_layer_1, concat_layer) with self.assertRaisesRegexp(LayerConnectionError, "match over"): layers.join(input_layer_2, concat_layer)
def test_concatenate_basic(self): concat_layer = layers.Concatenate(axis=-1) x1_tensor4 = asfloat(np.random.random((1, 3, 4, 2))) x2_tensor4 = asfloat(np.random.random((1, 3, 4, 8))) output = self.eval(concat_layer.output(x1_tensor4, x2_tensor4)) self.assertEqual((1, 3, 4, 10), output.shape)
def test_parallel_with_joined_connections(self): # Should work without errors layers.join( [ layers.Convolution((11, 5, 5)) > layers.Relu(), layers.Convolution((10, 3, 3)) > layers.Relu(), ], layers.Concatenate() > layers.Relu(), )
def test_concatenate_init_error(self): inputs = layers.parallel( layers.Input((28, 28, 3)), layers.Input((28, 28, 1)), ) expected_message = "don't match over dimension #3" with self.assertRaisesRegexp(LayerConnectionError, expected_message): layers.join(inputs, layers.Concatenate(axis=2))
def test_concatenate_different_dim_number(self): inputs = layers.parallel( layers.Input((28, 28)), layers.Input((28, 28, 1)), ) expected_msg = "different number of dimensions" with self.assertRaisesRegexp(LayerConnectionError, expected_msg): layers.join(inputs, layers.Concatenate(axis=1))
def test_single_input_for_parallel_layers(self): left = layers.Input(10, name='input') > layers.Sigmoid(5) right = left > layers.Sigmoid(2) network = [left, right] > layers.Concatenate() self.assertEqual(len(network.input_layers), 1) input_layer = network.input_layers[0] self.assertEqual(input_layer.name, 'input')
def test_concatenate_different_dim_number(self): input_layer_1 = layers.Input((28, 28)) input_layer_2 = layers.Input((28, 28, 1)) concat_layer = layers.Concatenate(axis=1) layers.join(input_layer_1, concat_layer) expected_msg = "different number of dimensions" with self.assertRaisesRegexp(LayerConnectionError, expected_msg): layers.join(input_layer_2, concat_layer)
def test_network_representation_for_non_feedforward(self): input_layer = layers.Input(10) hidden_layer_1 = layers.Sigmoid(20) hidden_layer_2 = layers.Sigmoid(20) output_layer = layers.Concatenate() connection = layers.join(input_layer, hidden_layer_1, output_layer) connection = layers.join(input_layer, hidden_layer_2, output_layer) network = algorithms.GradientDescent(connection) self.assertIn("[... 4 layers ...]", str(network))
def test_parallel_layer_with_residual_connections(self): connection = layers.join( layers.Input((3, 8, 8)), [[ layers.Convolution((7, 1, 1)), layers.Relu() ], [ # Residual connection ]], layers.Concatenate(), ) self.assertEqual(connection.output_shape, (10, 8, 8))
def test_concatenate_basic(self): concat_layer = layers.Concatenate(axis=1) x1 = T.tensor4() x2 = T.tensor4() y = theano.function([x1, x2], concat_layer.output(x1, x2)) x1_tensor4 = asfloat(np.random.random((1, 2, 3, 4))) x2_tensor4 = asfloat(np.random.random((1, 8, 3, 4))) output = y(x1_tensor4, x2_tensor4) self.assertEqual((1, 10, 3, 4), output.shape)
def test_network_architecture_output_exception(self): input_layer = layers.Input(10) hidden_layer_1 = layers.Sigmoid(20) hidden_layer_2 = layers.Sigmoid(20) output_layer = layers.Concatenate() connection = layers.join(input_layer, hidden_layer_1, output_layer) connection = layers.join(input_layer, hidden_layer_2, output_layer) network = algorithms.GradientDescent(connection) with self.assertRaises(TypeError): network.architecture()
def test_is_sequential_partial_connection(self): connection_2 = layers.Input(10) > layers.Sigmoid(5) connection_31 = connection_2 > layers.Sigmoid(1) connection_32 = connection_2 > layers.Sigmoid(2) concatenate = layers.Concatenate() connection_4 = connection_31 > concatenate connection_4 = connection_32 > concatenate self.assertFalse(is_sequential(connection_4)) self.assertTrue(is_sequential(connection_31)) self.assertTrue(is_sequential(connection_32))
def test_saliency_map_invalid_n_inputs(self): new_network = layers.join( layers.parallel( layers.Input((28, 28, 3)), layers.Input((28, 28, 3)), ), layers.Concatenate(), self.network.start('conv'), ) message = ("Cannot build saliency map for the network that " "has more than one input layer.") with self.assertRaisesRegexp(InvalidConnection, message): plots.saliency_map(new_network, self.image)
def test_residual_networks(self): network = layers.join( layers.Input((5, 5, 3)), layers.parallel( layers.Identity(), layers.join( layers.Convolution((3, 3, 8), padding='same'), layers.Relu(), ), ), layers.Concatenate(), ) self.assertShapesEqual((None, 5, 5, 3), network.input_shape) self.assertShapesEqual((None, 5, 5, 11), network.output_shape)
def test_concat_with_late_inputs(self): network = layers.join( layers.parallel( layers.Relu(), layers.Relu(), ), layers.Concatenate(), ) self.assertShapesEqual(network.input_shape, [None, None]) self.assertShapesEqual(network.output_shape, None) network = layers.Input((10, 10, 3)) >> network self.assertShapesEqual(network.input_shape, (None, 10, 10, 3)) self.assertShapesEqual(network.output_shape, (None, 10, 10, 6))
def test_multi_inputs_propagation(self): network = layers.join( layers.parallel( layers.Input(10, name='input-1'), layers.Input(4, name='input-2'), ), layers.Concatenate(), ) x1 = asfloat(np.random.random((3, 10))) x2 = asfloat(np.random.random((3, 4))) out1 = self.eval(network.output(x1, x2)) out2 = self.eval(network.output({'input-2': x2, 'input-1': x1})) self.assertEqual((3, 14), out1.shape) np.testing.assert_array_almost_equal(out1, out2)
def test_concatenate_conv_layers(self): network = layers.join( layers.Input((28, 28, 3)), layers.parallel( layers.Convolution((5, 5, 7)), layers.join( layers.Convolution((3, 3, 1)), layers.Convolution((3, 3, 4)), ), ), layers.Concatenate(axis=-1)) self.assertShapesEqual((None, 24, 24, 11), network.output_shape) x_tensor4 = asfloat(np.random.random((5, 28, 28, 3))) actual_output = self.eval(network.output(x_tensor4)) self.assertEqual((5, 24, 24, 11), actual_output.shape)
def Inception_1(conv_filters): return layers.join( [[ ConvReluBN((conv_filters[0][0], 1, 1)), ], [ ConvReluBN((conv_filters[1][0], 1, 1)), ConvReluBN((conv_filters[1][1], 5, 5), padding=2), ], [ ConvReluBN((conv_filters[2][0], 1, 1)), ConvReluBN((conv_filters[2][1], 3, 3), padding=1), ConvReluBN((conv_filters[2][2], 3, 3), padding=1), ], [ layers.AveragePooling((3, 3), stride=(1, 1), padding=1, mode='exclude_padding'), ConvReluBN((conv_filters[3][0], 1, 1)), ]], layers.Concatenate(), )
def Fire(s_1x1, e_1x1, e_3x3, name): return layers.join( layers.Convolution((s_1x1, 1, 1), padding='half', name=name + '/squeeze1x1'), layers.Relu(), [[ layers.Convolution( (e_1x1, 1, 1), padding='half', name=name + '/expand1x1'), layers.Relu(), ], [ layers.Convolution( (e_3x3, 3, 3), padding='half', name=name + '/expand3x3'), layers.Relu(), ]], layers.Concatenate(), )
def test_concatenate_conv_layers(self): input_layer = layers.Input((28, 28, 3)) hidden_layer_1 = layers.Convolution((5, 5, 7)) hidden_layer_21 = layers.Convolution((3, 3, 1)) hidden_layer_22 = layers.Convolution((3, 3, 4)) concat_layer = layers.Concatenate(axis=-1) connection = layers.join(input_layer, hidden_layer_1, concat_layer) connection = layers.join(input_layer, hidden_layer_21, hidden_layer_22, concat_layer) connection.initialize() self.assertEqual((24, 24, 11), concat_layer.output_shape) x_tensor4 = asfloat(np.random.random((5, 28, 28, 3))) actual_output = self.eval(connection.output(x_tensor4)) self.assertEqual((5, 24, 24, 11), actual_output.shape)