def test_parallel_layer(self): input_layer = layers.Input((3, 8, 8)) parallel_layer = layers.join( [[ layers.Convolution((11, 5, 5)), ], [ layers.Convolution((10, 3, 3)), layers.Convolution((5, 3, 3)), ]], layers.Concatenate(), ) output_layer = layers.MaxPooling((2, 2)) conn = layers.join(input_layer, parallel_layer) output_connection = layers.join(conn, output_layer) x = T.tensor4() y = theano.function([x], conn.output(x)) x_tensor4 = asfloat(np.random.random((10, 3, 8, 8))) output = y(x_tensor4) self.assertEqual(output.shape, (10, 11 + 5, 4, 4)) output_function = theano.function([x], output_connection.output(x)) final_output = output_function(x_tensor4) self.assertEqual(final_output.shape, (10, 11 + 5, 2, 2))
def Fire(s_1x1, e_1x1, e_3x3, name): return layers.join( layers.Convolution( (1, 1, s_1x1), padding='SAME', name=name + '/squeeze1x1' ), layers.Relu(), layers.parallel([ layers.Convolution( (1, 1, e_1x1), padding='SAME', name=name + '/expand1x1' ), layers.Relu(), ], [ layers.Convolution( (3, 3, e_3x3), padding='SAME', name=name + '/expand3x3' ), layers.Relu(), ]), layers.Concatenate(), )
def test_subnetwork_in_conv_network(self): network = layers.join( layers.Input((28, 28, 1)), layers.Convolution((3, 3, 8)) >> layers.Relu(), layers.Convolution((3, 3, 8)) >> layers.Relu(), layers.MaxPooling((2, 2)), layers.Reshape(), layers.Softmax(1), ) self.assertEqual(8, len(network)) self.assertTrue(network.is_sequential()) self.assertShapesEqual(network.input_shape, (None, 28, 28, 1)) self.assertShapesEqual(network.output_shape, (None, 1)) expected_order = [ layers.Input, layers.Convolution, layers.Relu, layers.Convolution, layers.Relu, layers.MaxPooling, layers.Reshape, layers.Softmax, ] for actual_layer, expected_layer in zip(network, expected_order): self.assertIsInstance(actual_layer, expected_layer)
def squeezenet(): """ SqueezeNet network architecture with random parameters. Parameters can be loaded using ``neupy.storage`` module. SqueezeNet has roughly 1.2 million parameters. It is almost 50 times less than in AlexNet. Parameters can be stored as 5Mb file. Examples -------- >>> from neupy import architectures >>> squeezenet = architectures.squeezenet() >>> squeezenet (?, 227, 227, 3) -> [... 67 layers ...] -> (?, 1000) >>> >>> from neupy import algorithms >>> optimizer = algorithms.Momentum(squeezenet) See Also -------- :architecture:`vgg16` : VGG16 network :architecture:`vgg19` : VGG19 network :architecture:`resnet50` : ResNet50 network References ---------- SqueezeNet: AlexNet-level accuracy with 50x fewer parameters and <0.5MB model size https://arxiv.org/abs/1602.07360 """ return layers.join( layers.Input((227, 227, 3)), layers.Convolution((7, 7, 96), stride=(2, 2), padding='VALID', name='conv1'), layers.Relu(), layers.MaxPooling((3, 3), stride=(2, 2)), Fire(16, 64, 64, name='fire2'), Fire(16, 64, 64, name='fire3'), Fire(32, 128, 128, name='fire4'), layers.MaxPooling((2, 2)), Fire(32, 128, 128, name='fire5'), Fire(48, 192, 192, name='fire6'), Fire(48, 192, 192, name='fire7'), Fire(64, 256, 256, name='fire8'), layers.MaxPooling((2, 2)), Fire(64, 256, 256, name='fire9'), layers.Dropout(0.5), layers.Convolution((1, 1, 1000), name='conv10'), layers.GlobalPooling('avg'), layers.Reshape(), layers.Softmax(), )
def test_parallel_with_joined_connections(self): # Should work without errors layers.join( [ layers.Convolution((11, 5, 5)) > layers.Relu(), layers.Convolution((10, 3, 3)) > layers.Relu(), ], layers.Concatenate() > layers.Relu(), )
def test_networks_with_complex_parallel_relations(self): input_layer = layers.Input((5, 5, 3)) network = layers.join( layers.parallel([ layers.Convolution((1, 1, 8)), ], [ layers.Convolution((1, 1, 4)), layers.parallel( layers.Convolution((1, 3, 2), padding='same'), layers.Convolution((3, 1, 2), padding='same'), ), ], [ layers.Convolution((1, 1, 8)), layers.Convolution((3, 3, 4), padding='same'), layers.parallel( layers.Convolution((1, 3, 2), padding='same'), layers.Convolution((3, 1, 2), padding='same'), ) ], [ layers.MaxPooling((3, 3), padding='same', stride=(1, 1)), layers.Convolution((1, 1, 8)), ]), layers.Concatenate(), ) self.assertShapesEqual(network.input_shape, [None, None, None, None]) self.assertShapesEqual(network.output_shape, (None, None, None, None)) # Connect them at the end, because we need to make # sure tha parallel networks defined without input shapes network = layers.join(input_layer, network) self.assertShapesEqual(network.output_shape, (None, 5, 5, 24))
def test_conv_invalid_padding_exception(self): error_msg = "greater or equal to zero" with self.assertRaisesRegexp(ValueError, error_msg): layers.Convolution((1, 3, 3), padding=-1) error_msg = "Tuple .+ greater or equal to zero" with self.assertRaisesRegexp(ValueError, error_msg): layers.Convolution((1, 3, 3), padding=(2, -1)) with self.assertRaisesRegexp(ValueError, "invalid string value"): layers.Convolution((1, 3, 3), padding='NOT_SAME') with self.assertRaisesRegexp(ValueError, "contains two elements"): layers.Convolution((1, 3, 3), padding=(3, 3, 3))
def test_gated_average_layer_multi_dimensional_inputs(self): input_layer = layers.Input((5, 5, 1)) network = layers.join([ input_layer > layers.Reshape() > layers.Softmax(2), input_layer > layers.Convolution((2, 2, 3)), input_layer > layers.Convolution((2, 2, 3)), ], layers.GatedAverage()) self.assertEqual(network.input_shape, (5, 5, 1)) self.assertEqual(network.output_shape, (4, 4, 3)) random_input = asfloat(np.random.random((8, 5, 5, 1))) actual_output = self.eval(network.output(random_input)) self.assertEqual(actual_output.shape, (8, 4, 4, 3))
def test_invalid_arguments_exceptions(self): network = layers.join( layers.Input((3, 28, 28)), layers.Convolution((8, 3, 3), name='conv') > layers.Relu(), layers.Reshape(), layers.Softmax(10), ) image = np.ones((3, 28, 28)) with self.assertRaisesRegexp(ValueError, 'Invalid image shape'): plots.saliency_map(network, np.ones((28, 28))) with self.assertRaisesRegexp(ValueError, 'invalid value'): plots.saliency_map(network, image, mode='invalid-mode') with self.assertRaises(InvalidConnection): new_network = network > [ layers.Sigmoid(1), layers.Sigmoid(2) ] plots.saliency_map(new_network, image) with self.assertRaises(InvalidConnection): new_network = [ layers.Input((3, 28, 28)), layers.Input((3, 28, 28)) ] > network.start('conv') plots.saliency_map(new_network, image) with self.assertRaisesRegexp(InvalidConnection, 'invalid input shape'): plots.saliency_map(layers.Input(10) > layers.Relu(), image)
def test_invalid_border_mode(self): invalid_border_modes = ('invalid mode', -10, (10, -5)) for border_mode in invalid_border_modes: msg = "Input border mode: {}".format(border_mode) with self.assertRaises(ValueError, msg=msg): layers.Convolution((1, 2, 3), border_mode=border_mode)
def test_gated_average_layer_multi_dimensional_inputs(self): input_layer = layers.Input((1, 5, 5)) network = layers.join([ input_layer > layers.Reshape() > layers.Softmax(2), input_layer > layers.Convolution((3, 2, 2)), input_layer > layers.Convolution((3, 2, 2)), ], layers.GatedAverage()) self.assertEqual(network.input_shape, (1, 5, 5)) self.assertEqual(network.output_shape, (3, 4, 4)) predict = network.compile() random_input = asfloat(np.random.random((8, 1, 5, 5))) actual_output = predict(random_input) self.assertEqual(actual_output.shape, (8, 3, 4, 4))
def test_invalid_padding(self): invalid_paddings = ('invalid mode', -10, (10, -5)) for padding in invalid_paddings: msg = "Input border mode: {}".format(padding) with self.assertRaises(ValueError, msg=msg): layers.Convolution((1, 2, 3), padding=padding)
def test_convolution_repr(self): layer = layers.Convolution((3, 3, 10), name='conv') self.assertEqual( str(layer), ("Convolution((3, 3, 10), padding='VALID', stride=(1, 1), " "dilation=(1, 1), weight=HeNormal(gain=2), bias=Constant(0), " "name='conv')"))
def test_concatenate_conv_layers(self): network = layers.join( layers.Input((28, 28, 3)), layers.parallel( layers.Convolution((5, 5, 7)), layers.join( layers.Convolution((3, 3, 1)), layers.Convolution((3, 3, 4)), ), ), layers.Concatenate(axis=-1)) self.assertShapesEqual((None, 24, 24, 11), network.output_shape) x_tensor4 = asfloat(np.random.random((5, 28, 28, 3))) actual_output = self.eval(network.output(x_tensor4)) self.assertEqual((5, 24, 24, 11), actual_output.shape)
def test_conv_output_shape_when_input_unknown(self): block = layers.join( layers.Convolution((3, 3, 32)), layers.Relu(), layers.BatchNorm(), ) self.assertShapesEqual(block.input_shape, None) self.assertShapesEqual(block.output_shape, (None, None, None, 32))
def Fire(s_1x1, e_1x1, e_3x3, name): return layers.join( layers.Convolution((s_1x1, 1, 1), padding='half', name=name + '/squeeze1x1'), layers.Relu(), [[ layers.Convolution( (e_1x1, 1, 1), padding='half', name=name + '/expand1x1'), layers.Relu(), ], [ layers.Convolution( (e_3x3, 3, 3), padding='half', name=name + '/expand3x3'), layers.Relu(), ]], layers.Concatenate(), )
def test_connection_inside_connection_conv(self): connection = layers.join( layers.Input((28, 28, 1)), layers.Convolution((3, 3, 8)) > layers.Relu(), layers.Convolution((3, 3, 8)) > layers.Relu(), layers.MaxPooling((2, 2)), layers.Reshape(), layers.Softmax(1), ) self.assertEqual(8, len(connection)) expected_order = [ layers.Input, layers.Convolution, layers.Relu, layers.Convolution, layers.Relu, layers.MaxPooling, layers.Reshape, layers.Softmax ] for actual_layer, expected_layer in zip(connection, expected_order): self.assertIsInstance(actual_layer, expected_layer)
def test_connection_inside_connection_conv(self): connection = [ layers.Input((1, 28, 28)), layers.Convolution((8, 3, 3)) > layers.Relu(), layers.Convolution((8, 3, 3)) > layers.Relu(), layers.MaxPooling((2, 2)), layers.Reshape(), layers.Softmax(1), ] network = algorithms.GradientDescent(connection) self.assertEqual(8, len(network.layers)) self.assertIsInstance(network.layers[1], layers.Convolution) self.assertIsInstance(network.layers[2], layers.Relu) self.assertIsInstance(network.layers[3], layers.Convolution) self.assertIsInstance(network.layers[4], layers.Relu) self.assertIsInstance(network.layers[5], layers.MaxPooling)
def test_repeat_network(self): block = layers.join( layers.Convolution((3, 3, 32)), layers.Relu(), layers.BatchNorm(), ) network = layers.repeat(block, n=5) self.assertEqual(len(network), 15) self.assertShapesEqual(network.output_shape, (None, None, None, 32))
def test_global_pooling_late_shape_init(self): network = layers.join( layers.Convolution((3, 3, 12)), layers.GlobalPooling('max'), ) self.assertShapesEqual(network.output_shape, (None, None)) network = layers.join(layers.Input((10, 10, 1)), network) self.assertShapesEqual(network.output_shape, (None, 12))
def setUp(self): super(SaliencyMapTestCase, self).setUp() self.network = layers.join( layers.Input((28, 28, 3)), layers.Convolution((3, 3, 8), name='conv') >> layers.Relu(), layers.Reshape(), layers.Softmax(10), ) self.image = np.ones((28, 28, 3))
def test_gated_average_layer_multi_dimensional_inputs(self): network = layers.join( layers.Input((5, 5, 1)), layers.parallel( layers.Reshape() >> layers.Softmax(2), layers.Convolution((2, 2, 3)), layers.Convolution((2, 2, 3)), ), layers.GatedAverage(), ) self.assertShapesEqual(network.input_shape, (None, 5, 5, 1)) self.assertShapesEqual(network.output_shape, (None, 4, 4, 3)) random_input = asfloat(np.random.random((8, 5, 5, 1))) actual_output = self.eval(network.output(random_input)) self.assertEqual(actual_output.shape, (8, 4, 4, 3))
def test_concatenate_conv_layers(self): input_layer = layers.Input((28, 28, 3)) hidden_layer_1 = layers.Convolution((5, 5, 7)) hidden_layer_21 = layers.Convolution((3, 3, 1)) hidden_layer_22 = layers.Convolution((3, 3, 4)) concat_layer = layers.Concatenate(axis=-1) connection = layers.join(input_layer, hidden_layer_1, concat_layer) connection = layers.join(input_layer, hidden_layer_21, hidden_layer_22, concat_layer) connection.initialize() self.assertEqual((24, 24, 11), concat_layer.output_shape) x_tensor4 = asfloat(np.random.random((5, 28, 28, 3))) actual_output = self.eval(connection.output(x_tensor4)) self.assertEqual((5, 24, 24, 11), actual_output.shape)
def test_prelu_output_by_spatial_input(self): network = layers.join( layers.Input((10, 10, 3)), layers.Convolution((3, 3, 5)), layers.PRelu(alpha=0.25, alpha_axes=(1, 3)), ) X = asfloat(np.random.random((1, 10, 10, 3))) actual_output = self.eval(network.output(X)) self.assertEqual(actual_output.shape, (1, 8, 8, 5))
def test_convolution_params(self): inp = layers.Input((5, 5, 1)) conv = layers.Convolution((2, 2, 6)) # Propagate data through the network in # order to trigger initialization (inp >> conv).outputs self.assertEqual((2, 2, 1, 6), self.get_shape(conv.weight)) self.assertEqual((6,), self.get_shape(conv.bias))
def create_VIN(input_image_shape=(8, 8, 2), n_hidden_filters=150, n_state_filters=10, k=10): SamePadConvolution = partial(layers.Convolution, padding='SAME', bias=None) R = layers.join( layers.Input(input_image_shape, name='grid-input'), layers.Convolution((3, 3, n_hidden_filters), padding='SAME', weight=init.Normal(), bias=init.Normal()), SamePadConvolution((1, 1, 1), weight=init.Normal()), ) # Create shared weights q_weight = random_weight((3, 3, 1, n_state_filters)) fb_weight = random_weight((3, 3, 1, n_state_filters)) Q = R > SamePadConvolution((3, 3, n_state_filters), weight=q_weight) for i in range(k): V = Q > ChannelGlobalMaxPooling() Q = layers.join( # Convolve R and V separately and then add outputs together with # the Elementwise layer. This part of the code looks different # from the one that was used in the original VIN repo, but # it does the same operation. # # conv(x, w) == (conv(x1, w1) + conv(x2, w2)) # where, x = concat(x1, x2) # w = concat(w1, w2) # # See code sample from Github Gist: https://bit.ly/2zm3ntN [[ R, SamePadConvolution((3, 3, n_state_filters), weight=q_weight) ], [ V, SamePadConvolution((3, 3, n_state_filters), weight=fb_weight) ]], layers.Elementwise(merge_function=tf.add), ) input_state_1 = layers.Input(UNKNOWN, name='state-1-input') input_state_2 = layers.Input(UNKNOWN, name='state-2-input') # Select the conv-net channels at the state position (S1, S2) VIN = [Q, input_state_1, input_state_2] > SelectValueAtStatePosition() # Set up softmax layer that predicts actions base on (S1, S2) # position. Each action encodes specific direction: # N, S, E, W, NE, NW, SE, SW (in the same order) VIN = VIN > layers.Softmax(8, bias=None, weight=init.Normal()) return VIN
def test_convolution_params(self): weight_shape = (6, 1, 2, 2) bias_shape = (6, ) input_layer = layers.Input((1, 5, 5)) conv_layer = layers.Convolution((6, 2, 2)) connection = input_layer > conv_layer conv_layer.initialize() self.assertEqual(weight_shape, conv_layer.weight.get_value().shape) self.assertEqual(bias_shape, conv_layer.bias.get_value().shape)
def test_conv_without_bias(self): inp = layers.Input((5, 5, 1)) conv = layers.Convolution((3, 3, 1), bias=None, weight=1) network = inp >> conv network.outputs x = asfloat(np.ones((1, 5, 5, 1))) expected_output = 9 * np.ones((1, 3, 3, 1)) actual_output = self.eval(network.output(x)) np.testing.assert_array_almost_equal(expected_output, actual_output)
def test_conv_without_bias(self): input_layer = layers.Input((1, 5, 5)) conv = layers.Convolution((1, 3, 3), bias=None, weight=1) connection = input_layer > conv connection.initialize() x = asfloat(np.ones((1, 1, 5, 5))) expected_output = 9 * np.ones((1, 1, 3, 3)) actual_output = connection.output(x).eval() np.testing.assert_array_almost_equal(expected_output, actual_output)
def test_parallel_layer_with_residual_connections(self): connection = layers.join( layers.Input((3, 8, 8)), [[ layers.Convolution((7, 1, 1)), layers.Relu() ], [ # Residual connection ]], layers.Concatenate(), ) self.assertEqual(connection.output_shape, (10, 8, 8))