def test_parallel_layer(self):
input_layer = layers.Input((3, 8, 8))
parallel_layer = layers.join(
[[
layers.Convolution((11, 5, 5)),
], [
layers.Convolution((10, 3, 3)),
layers.Convolution((5, 3, 3)),
]],
layers.Concatenate(),
)
output_layer = layers.MaxPooling((2, 2))
conn = layers.join(input_layer, parallel_layer)
output_connection = layers.join(conn, output_layer)
x = T.tensor4()
y = theano.function([x], conn.output(x))
x_tensor4 = asfloat(np.random.random((10, 3, 8, 8)))
output = y(x_tensor4)
self.assertEqual(output.shape, (10, 11 + 5, 4, 4))
output_function = theano.function([x], output_connection.output(x))
final_output = output_function(x_tensor4)
self.assertEqual(final_output.shape, (10, 11 + 5, 2, 2))
def Fire(s_1x1, e_1x1, e_3x3, name):
return layers.join(
layers.Convolution(
(1, 1, s_1x1),
padding='SAME',
name=name + '/squeeze1x1'
),
layers.Relu(),
layers.parallel([
layers.Convolution(
(1, 1, e_1x1),
padding='SAME',
name=name + '/expand1x1'
),
layers.Relu(),
], [
layers.Convolution(
(3, 3, e_3x3),
padding='SAME',
name=name + '/expand3x3'
),
layers.Relu(),
]),
layers.Concatenate(),
)
def test_subnetwork_in_conv_network(self):
network = layers.join(
layers.Input((28, 28, 1)),
layers.Convolution((3, 3, 8)) >> layers.Relu(),
layers.Convolution((3, 3, 8)) >> layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Softmax(1),
)
self.assertEqual(8, len(network))
self.assertTrue(network.is_sequential())
self.assertShapesEqual(network.input_shape, (None, 28, 28, 1))
self.assertShapesEqual(network.output_shape, (None, 1))
expected_order = [
layers.Input,
layers.Convolution,
layers.Relu,
layers.Convolution,
layers.Relu,
layers.MaxPooling,
layers.Reshape,
layers.Softmax,
]
for actual_layer, expected_layer in zip(network, expected_order):
self.assertIsInstance(actual_layer, expected_layer)
def squeezenet():
"""
SqueezeNet network architecture with random parameters.
Parameters can be loaded using ``neupy.storage`` module.
SqueezeNet has roughly 1.2 million parameters. It is almost
50 times less than in AlexNet. Parameters can be stored as 5Mb
file.
Examples
--------
>>> from neupy import architectures
>>> squeezenet = architectures.squeezenet()
>>> squeezenet
(?, 227, 227, 3) -> [... 67 layers ...] -> (?, 1000)
>>>
>>> from neupy import algorithms
>>> optimizer = algorithms.Momentum(squeezenet)
See Also
--------
:architecture:`vgg16` : VGG16 network
:architecture:`vgg19` : VGG19 network
:architecture:`resnet50` : ResNet50 network
References
----------
SqueezeNet: AlexNet-level accuracy with 50x fewer parameters
and <0.5MB model size
https://arxiv.org/abs/1602.07360
"""
return layers.join(
layers.Input((227, 227, 3)),
layers.Convolution((7, 7, 96), stride=(2, 2),
padding='VALID', name='conv1'),
layers.Relu(),
layers.MaxPooling((3, 3), stride=(2, 2)),
Fire(16, 64, 64, name='fire2'),
Fire(16, 64, 64, name='fire3'),
Fire(32, 128, 128, name='fire4'),
layers.MaxPooling((2, 2)),
Fire(32, 128, 128, name='fire5'),
Fire(48, 192, 192, name='fire6'),
Fire(48, 192, 192, name='fire7'),
Fire(64, 256, 256, name='fire8'),
layers.MaxPooling((2, 2)),
Fire(64, 256, 256, name='fire9'),
layers.Dropout(0.5),
layers.Convolution((1, 1, 1000), name='conv10'),
layers.GlobalPooling('avg'),
layers.Reshape(),
layers.Softmax(),
)
def test_parallel_with_joined_connections(self):
# Should work without errors
layers.join(
[
layers.Convolution((11, 5, 5)) > layers.Relu(),
layers.Convolution((10, 3, 3)) > layers.Relu(),
],
layers.Concatenate() > layers.Relu(),
)
def test_networks_with_complex_parallel_relations(self):
input_layer = layers.Input((5, 5, 3))
network = layers.join(
layers.parallel([
layers.Convolution((1, 1, 8)),
], [
layers.Convolution((1, 1, 4)),
layers.parallel(
layers.Convolution((1, 3, 2), padding='same'),
layers.Convolution((3, 1, 2), padding='same'),
),
], [
layers.Convolution((1, 1, 8)),
layers.Convolution((3, 3, 4), padding='same'),
layers.parallel(
layers.Convolution((1, 3, 2), padding='same'),
layers.Convolution((3, 1, 2), padding='same'),
)
], [
layers.MaxPooling((3, 3), padding='same', stride=(1, 1)),
layers.Convolution((1, 1, 8)),
]),
layers.Concatenate(),
)
self.assertShapesEqual(network.input_shape, [None, None, None, None])
self.assertShapesEqual(network.output_shape, (None, None, None, None))
# Connect them at the end, because we need to make
# sure tha parallel networks defined without input shapes
network = layers.join(input_layer, network)
self.assertShapesEqual(network.output_shape, (None, 5, 5, 24))
def test_conv_invalid_padding_exception(self):
error_msg = "greater or equal to zero"
with self.assertRaisesRegexp(ValueError, error_msg):
layers.Convolution((1, 3, 3), padding=-1)
error_msg = "Tuple .+ greater or equal to zero"
with self.assertRaisesRegexp(ValueError, error_msg):
layers.Convolution((1, 3, 3), padding=(2, -1))
with self.assertRaisesRegexp(ValueError, "invalid string value"):
layers.Convolution((1, 3, 3), padding='NOT_SAME')
with self.assertRaisesRegexp(ValueError, "contains two elements"):
layers.Convolution((1, 3, 3), padding=(3, 3, 3))
def test_gated_average_layer_multi_dimensional_inputs(self):
input_layer = layers.Input((5, 5, 1))
network = layers.join([
input_layer > layers.Reshape() > layers.Softmax(2),
input_layer > layers.Convolution((2, 2, 3)),
input_layer > layers.Convolution((2, 2, 3)),
], layers.GatedAverage())
self.assertEqual(network.input_shape, (5, 5, 1))
self.assertEqual(network.output_shape, (4, 4, 3))
random_input = asfloat(np.random.random((8, 5, 5, 1)))
actual_output = self.eval(network.output(random_input))
self.assertEqual(actual_output.shape, (8, 4, 4, 3))
def test_invalid_arguments_exceptions(self):
network = layers.join(
layers.Input((3, 28, 28)),
layers.Convolution((8, 3, 3), name='conv') > layers.Relu(),
layers.Reshape(),
layers.Softmax(10),
)
image = np.ones((3, 28, 28))
with self.assertRaisesRegexp(ValueError, 'Invalid image shape'):
plots.saliency_map(network, np.ones((28, 28)))
with self.assertRaisesRegexp(ValueError, 'invalid value'):
plots.saliency_map(network, image, mode='invalid-mode')
with self.assertRaises(InvalidConnection):
new_network = network > [
layers.Sigmoid(1), layers.Sigmoid(2)
]
plots.saliency_map(new_network, image)
with self.assertRaises(InvalidConnection):
new_network = [
layers.Input((3, 28, 28)), layers.Input((3, 28, 28))
] > network.start('conv')
plots.saliency_map(new_network, image)
with self.assertRaisesRegexp(InvalidConnection, 'invalid input shape'):
plots.saliency_map(layers.Input(10) > layers.Relu(), image)
def test_invalid_border_mode(self):
invalid_border_modes = ('invalid mode', -10, (10, -5))
for border_mode in invalid_border_modes:
msg = "Input border mode: {}".format(border_mode)
with self.assertRaises(ValueError, msg=msg):
layers.Convolution((1, 2, 3), border_mode=border_mode)
def test_gated_average_layer_multi_dimensional_inputs(self):
input_layer = layers.Input((1, 5, 5))
network = layers.join([
input_layer > layers.Reshape() > layers.Softmax(2),
input_layer > layers.Convolution((3, 2, 2)),
input_layer > layers.Convolution((3, 2, 2)),
], layers.GatedAverage())
self.assertEqual(network.input_shape, (1, 5, 5))
self.assertEqual(network.output_shape, (3, 4, 4))
predict = network.compile()
random_input = asfloat(np.random.random((8, 1, 5, 5)))
actual_output = predict(random_input)
self.assertEqual(actual_output.shape, (8, 3, 4, 4))
def test_invalid_padding(self):
invalid_paddings = ('invalid mode', -10, (10, -5))
for padding in invalid_paddings:
msg = "Input border mode: {}".format(padding)
with self.assertRaises(ValueError, msg=msg):
layers.Convolution((1, 2, 3), padding=padding)
def test_convolution_repr(self):
layer = layers.Convolution((3, 3, 10), name='conv')
self.assertEqual(
str(layer),
("Convolution((3, 3, 10), padding='VALID', stride=(1, 1), "
"dilation=(1, 1), weight=HeNormal(gain=2), bias=Constant(0), "
"name='conv')"))
def test_concatenate_conv_layers(self):
network = layers.join(
layers.Input((28, 28, 3)),
layers.parallel(
layers.Convolution((5, 5, 7)),
layers.join(
layers.Convolution((3, 3, 1)),
layers.Convolution((3, 3, 4)),
),
), layers.Concatenate(axis=-1))
self.assertShapesEqual((None, 24, 24, 11), network.output_shape)
x_tensor4 = asfloat(np.random.random((5, 28, 28, 3)))
actual_output = self.eval(network.output(x_tensor4))
self.assertEqual((5, 24, 24, 11), actual_output.shape)
def test_conv_output_shape_when_input_unknown(self):
block = layers.join(
layers.Convolution((3, 3, 32)),
layers.Relu(),
layers.BatchNorm(),
)
self.assertShapesEqual(block.input_shape, None)
self.assertShapesEqual(block.output_shape, (None, None, None, 32))
def Fire(s_1x1, e_1x1, e_3x3, name):
return layers.join(
layers.Convolution((s_1x1, 1, 1),
padding='half',
name=name + '/squeeze1x1'),
layers.Relu(),
[[
layers.Convolution(
(e_1x1, 1, 1), padding='half', name=name + '/expand1x1'),
layers.Relu(),
],
[
layers.Convolution(
(e_3x3, 3, 3), padding='half', name=name + '/expand3x3'),
layers.Relu(),
]],
layers.Concatenate(),
)
def test_connection_inside_connection_conv(self):
connection = layers.join(
layers.Input((28, 28, 1)),
layers.Convolution((3, 3, 8)) > layers.Relu(),
layers.Convolution((3, 3, 8)) > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Softmax(1),
)
self.assertEqual(8, len(connection))
expected_order = [
layers.Input, layers.Convolution, layers.Relu, layers.Convolution,
layers.Relu, layers.MaxPooling, layers.Reshape, layers.Softmax
]
for actual_layer, expected_layer in zip(connection, expected_order):
self.assertIsInstance(actual_layer, expected_layer)
def test_connection_inside_connection_conv(self):
connection = [
layers.Input((1, 28, 28)),
layers.Convolution((8, 3, 3)) > layers.Relu(),
layers.Convolution((8, 3, 3)) > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Softmax(1),
]
network = algorithms.GradientDescent(connection)
self.assertEqual(8, len(network.layers))
self.assertIsInstance(network.layers[1], layers.Convolution)
self.assertIsInstance(network.layers[2], layers.Relu)
self.assertIsInstance(network.layers[3], layers.Convolution)
self.assertIsInstance(network.layers[4], layers.Relu)
self.assertIsInstance(network.layers[5], layers.MaxPooling)
def test_repeat_network(self):
block = layers.join(
layers.Convolution((3, 3, 32)),
layers.Relu(),
layers.BatchNorm(),
)
network = layers.repeat(block, n=5)
self.assertEqual(len(network), 15)
self.assertShapesEqual(network.output_shape, (None, None, None, 32))
def test_global_pooling_late_shape_init(self):
network = layers.join(
layers.Convolution((3, 3, 12)),
layers.GlobalPooling('max'),
)
self.assertShapesEqual(network.output_shape, (None, None))
network = layers.join(layers.Input((10, 10, 1)), network)
self.assertShapesEqual(network.output_shape, (None, 12))
def setUp(self):
super(SaliencyMapTestCase, self).setUp()
self.network = layers.join(
layers.Input((28, 28, 3)),
layers.Convolution((3, 3, 8), name='conv') >> layers.Relu(),
layers.Reshape(),
layers.Softmax(10),
)
self.image = np.ones((28, 28, 3))
def test_gated_average_layer_multi_dimensional_inputs(self):
network = layers.join(
layers.Input((5, 5, 1)),
layers.parallel(
layers.Reshape() >> layers.Softmax(2),
layers.Convolution((2, 2, 3)),
layers.Convolution((2, 2, 3)),
),
layers.GatedAverage(),
)
self.assertShapesEqual(network.input_shape, (None, 5, 5, 1))
self.assertShapesEqual(network.output_shape, (None, 4, 4, 3))
random_input = asfloat(np.random.random((8, 5, 5, 1)))
actual_output = self.eval(network.output(random_input))
self.assertEqual(actual_output.shape, (8, 4, 4, 3))
def test_concatenate_conv_layers(self):
input_layer = layers.Input((28, 28, 3))
hidden_layer_1 = layers.Convolution((5, 5, 7))
hidden_layer_21 = layers.Convolution((3, 3, 1))
hidden_layer_22 = layers.Convolution((3, 3, 4))
concat_layer = layers.Concatenate(axis=-1)
connection = layers.join(input_layer, hidden_layer_1, concat_layer)
connection = layers.join(input_layer, hidden_layer_21, hidden_layer_22,
concat_layer)
connection.initialize()
self.assertEqual((24, 24, 11), concat_layer.output_shape)
x_tensor4 = asfloat(np.random.random((5, 28, 28, 3)))
actual_output = self.eval(connection.output(x_tensor4))
self.assertEqual((5, 24, 24, 11), actual_output.shape)
def test_prelu_output_by_spatial_input(self):
network = layers.join(
layers.Input((10, 10, 3)),
layers.Convolution((3, 3, 5)),
layers.PRelu(alpha=0.25, alpha_axes=(1, 3)),
)
X = asfloat(np.random.random((1, 10, 10, 3)))
actual_output = self.eval(network.output(X))
self.assertEqual(actual_output.shape, (1, 8, 8, 5))
def test_convolution_params(self):
inp = layers.Input((5, 5, 1))
conv = layers.Convolution((2, 2, 6))
# Propagate data through the network in
# order to trigger initialization
(inp >> conv).outputs
self.assertEqual((2, 2, 1, 6), self.get_shape(conv.weight))
self.assertEqual((6,), self.get_shape(conv.bias))
def create_VIN(input_image_shape=(8, 8, 2), n_hidden_filters=150,
n_state_filters=10, k=10):
SamePadConvolution = partial(layers.Convolution, padding='SAME', bias=None)
R = layers.join(
layers.Input(input_image_shape, name='grid-input'),
layers.Convolution((3, 3, n_hidden_filters),
padding='SAME',
weight=init.Normal(),
bias=init.Normal()),
SamePadConvolution((1, 1, 1), weight=init.Normal()),
)
# Create shared weights
q_weight = random_weight((3, 3, 1, n_state_filters))
fb_weight = random_weight((3, 3, 1, n_state_filters))
Q = R > SamePadConvolution((3, 3, n_state_filters), weight=q_weight)
for i in range(k):
V = Q > ChannelGlobalMaxPooling()
Q = layers.join(
# Convolve R and V separately and then add outputs together with
# the Elementwise layer. This part of the code looks different
# from the one that was used in the original VIN repo, but
# it does the same operation.
#
# conv(x, w) == (conv(x1, w1) + conv(x2, w2))
# where, x = concat(x1, x2)
# w = concat(w1, w2)
#
# See code sample from Github Gist: https://bit.ly/2zm3ntN
[[
R,
SamePadConvolution((3, 3, n_state_filters), weight=q_weight)
], [
V,
SamePadConvolution((3, 3, n_state_filters), weight=fb_weight)
]],
layers.Elementwise(merge_function=tf.add),
)
input_state_1 = layers.Input(UNKNOWN, name='state-1-input')
input_state_2 = layers.Input(UNKNOWN, name='state-2-input')
# Select the conv-net channels at the state position (S1, S2)
VIN = [Q, input_state_1, input_state_2] > SelectValueAtStatePosition()
# Set up softmax layer that predicts actions base on (S1, S2)
# position. Each action encodes specific direction:
# N, S, E, W, NE, NW, SE, SW (in the same order)
VIN = VIN > layers.Softmax(8, bias=None, weight=init.Normal())
return VIN
def test_convolution_params(self):
weight_shape = (6, 1, 2, 2)
bias_shape = (6, )
input_layer = layers.Input((1, 5, 5))
conv_layer = layers.Convolution((6, 2, 2))
connection = input_layer > conv_layer
conv_layer.initialize()
self.assertEqual(weight_shape, conv_layer.weight.get_value().shape)
self.assertEqual(bias_shape, conv_layer.bias.get_value().shape)
def test_conv_without_bias(self):
inp = layers.Input((5, 5, 1))
conv = layers.Convolution((3, 3, 1), bias=None, weight=1)
network = inp >> conv
network.outputs
x = asfloat(np.ones((1, 5, 5, 1)))
expected_output = 9 * np.ones((1, 3, 3, 1))
actual_output = self.eval(network.output(x))
np.testing.assert_array_almost_equal(expected_output, actual_output)
def test_conv_without_bias(self):
input_layer = layers.Input((1, 5, 5))
conv = layers.Convolution((1, 3, 3), bias=None, weight=1)
connection = input_layer > conv
connection.initialize()
x = asfloat(np.ones((1, 1, 5, 5)))
expected_output = 9 * np.ones((1, 1, 3, 3))
actual_output = connection.output(x).eval()
np.testing.assert_array_almost_equal(expected_output, actual_output)
def test_parallel_layer_with_residual_connections(self):
connection = layers.join(
layers.Input((3, 8, 8)),
[[
layers.Convolution((7, 1, 1)),
layers.Relu()
], [
# Residual connection
]],
layers.Concatenate(),
)
self.assertEqual(connection.output_shape, (10, 8, 8))
