def squeezenet():
"""
SqueezeNet network architecture with random parameters.
Parameters can be loaded using ``neupy.storage`` module.
SqueezeNet has roughly 1.2 million parameters. It is almost
50 times less than in AlexNet. Parameters can be stored as 5Mb
file.
Examples
--------
>>> from neupy import architectures
>>> squeezenet = architectures.squeezenet()
>>> squeezenet
(?, 227, 227, 3) -> [... 67 layers ...] -> (?, 1000)
>>>
>>> from neupy import algorithms
>>> optimizer = algorithms.Momentum(squeezenet)
See Also
--------
:architecture:`vgg16` : VGG16 network
:architecture:`vgg19` : VGG19 network
:architecture:`resnet50` : ResNet50 network
References
----------
SqueezeNet: AlexNet-level accuracy with 50x fewer parameters
and <0.5MB model size
https://arxiv.org/abs/1602.07360
"""
return layers.join(
layers.Input((227, 227, 3)),
layers.Convolution((7, 7, 96), stride=(2, 2),
padding='VALID', name='conv1'),
layers.Relu(),
layers.MaxPooling((3, 3), stride=(2, 2)),
Fire(16, 64, 64, name='fire2'),
Fire(16, 64, 64, name='fire3'),
Fire(32, 128, 128, name='fire4'),
layers.MaxPooling((2, 2)),
Fire(32, 128, 128, name='fire5'),
Fire(48, 192, 192, name='fire6'),
Fire(48, 192, 192, name='fire7'),
Fire(64, 256, 256, name='fire8'),
layers.MaxPooling((2, 2)),
Fire(64, 256, 256, name='fire9'),
layers.Dropout(0.5),
layers.Convolution((1, 1, 1000), name='conv10'),
layers.GlobalPooling('avg'),
layers.Reshape(),
layers.Softmax(),
)
def network():
HalfPadConvolution = partial(layers.Convolution, padding='half')
return layers.join(
layers.Input((3, 224, 224)),
HalfPadConvolution((20, 5, 5), name='conv1_1') > layers.Relu(),
HalfPadConvolution((20, 5, 5), name='conv1_2') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((60, 5, 5), name='conv2_1') > layers.Relu(),
HalfPadConvolution((60, 5, 5), name='conv2_2') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((120, 5, 5), name='conv3_1') > layers.Relu(),
HalfPadConvolution((120, 5, 5), name='conv3_2') > layers.Relu(),
HalfPadConvolution((150, 5, 5), name='conv3_3') > layers.Relu(),
HalfPadConvolution((150, 5, 5), name='conv3_4') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((128, 5, 5), name='conv4_1') > layers.Relu(),
HalfPadConvolution((128, 5, 5), name='conv4_2') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv4_3') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv4_4') > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Linear(2000, name='dense_1') > layers.Relu(),
layers.Dropout(0.5),
layers.Linear(1000, name='dense_2') > layers.Relu(),
layers.Dropout(0.5),
layers.Linear(1000, name='dense_3') > layers.Softmax(),
)
def test_pooling_invalid_connections_exceptions(self):
# Invalid input shape
input_layer = layers.Input(10)
max_pool_layer = layers.MaxPooling((2, 2))
with self.assertRaises(LayerConnectionError):
layers.join(input_layer, max_pool_layer)
# Invalid combination of parameters
with self.assertRaises(ValueError):
layers.MaxPooling((2, 2), ignore_border=False, padding=1)
def test_pooling_invalid_connections_exceptions(self):
# Invalid input shape
input_layer = layers.Input(10)
max_pool_layer = layers.MaxPooling((2, 2))
with self.assertRaises(LayerConnectionError):
layers.join(input_layer, max_pool_layer)
with self.assertRaises(ValueError):
layers.MaxPooling((2, 2), padding='TEST')
with self.assertRaises(ValueError):
layers.MaxPooling((2, 2), padding=1)
def test_pooling_repr(self):
layer = layers.MaxPooling((2, 2))
self.assertEqual(
str(layer),
("MaxPooling((2, 2), stride=None, padding='valid', "
"name='max-pooling-1')"),
)
layer = layers.MaxPooling((2, 2), stride=(3, 3))
self.assertEqual(
str(layer),
("MaxPooling((2, 2), stride=(3, 3), padding='valid', "
"name='max-pooling-2')"),
)
def test_networks_with_complex_parallel_relations(self):
input_layer = layers.Input((5, 5, 3))
network = layers.join(
layers.parallel([
layers.Convolution((1, 1, 8)),
], [
layers.Convolution((1, 1, 4)),
layers.parallel(
layers.Convolution((1, 3, 2), padding='same'),
layers.Convolution((3, 1, 2), padding='same'),
),
], [
layers.Convolution((1, 1, 8)),
layers.Convolution((3, 3, 4), padding='same'),
layers.parallel(
layers.Convolution((1, 3, 2), padding='same'),
layers.Convolution((3, 1, 2), padding='same'),
)
], [
layers.MaxPooling((3, 3), padding='same', stride=(1, 1)),
layers.Convolution((1, 1, 8)),
]),
layers.Concatenate(),
)
self.assertShapesEqual(network.input_shape, [None, None, None, None])
self.assertShapesEqual(network.output_shape, (None, None, None, None))
# Connect them at the end, because we need to make
# sure tha parallel networks defined without input shapes
network = layers.join(input_layer, network)
self.assertShapesEqual(network.output_shape, (None, 5, 5, 24))
def test_subnetwork_in_conv_network(self):
network = layers.join(
layers.Input((28, 28, 1)),
layers.Convolution((3, 3, 8)) >> layers.Relu(),
layers.Convolution((3, 3, 8)) >> layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Softmax(1),
)
self.assertEqual(8, len(network))
self.assertTrue(network.is_sequential())
self.assertShapesEqual(network.input_shape, (None, 28, 28, 1))
self.assertShapesEqual(network.output_shape, (None, 1))
expected_order = [
layers.Input,
layers.Convolution,
layers.Relu,
layers.Convolution,
layers.Relu,
layers.MaxPooling,
layers.Reshape,
layers.Softmax,
]
for actual_layer, expected_layer in zip(network, expected_order):
self.assertIsInstance(actual_layer, expected_layer)
def test_parallel_layer(self):
input_layer = layers.Input((3, 8, 8))
parallel_layer = layers.join(
[[
layers.Convolution((11, 5, 5)),
], [
layers.Convolution((10, 3, 3)),
layers.Convolution((5, 3, 3)),
]],
layers.Concatenate(),
)
output_layer = layers.MaxPooling((2, 2))
conn = layers.join(input_layer, parallel_layer)
output_connection = layers.join(conn, output_layer)
x = T.tensor4()
y = theano.function([x], conn.output(x))
x_tensor4 = asfloat(np.random.random((10, 3, 8, 8)))
output = y(x_tensor4)
self.assertEqual(output.shape, (10, 11 + 5, 4, 4))
output_function = theano.function([x], output_connection.output(x))
final_output = output_function(x_tensor4)
self.assertEqual(final_output.shape, (10, 11 + 5, 2, 2))
def test_pooling_stride_int(self):
network = layers.join(
layers.Input((28, 28, 1)),
layers.MaxPooling((2, 2), stride=1, padding='same'),
)
self.assertShapesEqual(
network.input_shape,
network.output_shape)
def test_max_pooling(self):
input_data = asfloat(
np.array([
[1, 2, 3, -1],
[4, -6, 3, 1],
[0, 0, 1, 0],
[0, -1, 0, 0],
])).reshape(1, 4, 4, 1)
expected_output = asfloat(np.array([
[4, 3],
[0, 1],
])).reshape(1, 2, 2, 1)
max_pool_layer = layers.MaxPooling((2, 2))
actual_output = self.eval(max_pool_layer.output(input_data))
np.testing.assert_array_almost_equal(actual_output, expected_output)
def test_max_pooling(self):
input_data = theano.shared(
asfloat(
np.array([
[1, 2, 3, -1],
[4, -6, 3, 1],
[0, 0, 1, 0],
[0, -1, 0, 0],
])))
expected_output = asfloat(np.array([
[4, 3],
[0, 1],
]))
max_pool_layer = layers.MaxPooling((2, 2))
actual_output = max_pool_layer.output(input_data).eval()
np.testing.assert_array_almost_equal(actual_output, expected_output)
def test_connection_inside_connection_conv(self):
connection = [
layers.Input((1, 28, 28)),
layers.Convolution((8, 3, 3)) > layers.Relu(),
layers.Convolution((8, 3, 3)) > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Softmax(1),
]
network = algorithms.GradientDescent(connection)
self.assertEqual(8, len(network.layers))
self.assertIsInstance(network.layers[1], layers.Convolution)
self.assertIsInstance(network.layers[2], layers.Relu)
self.assertIsInstance(network.layers[3], layers.Convolution)
self.assertIsInstance(network.layers[4], layers.Relu)
self.assertIsInstance(network.layers[5], layers.MaxPooling)
def test_connection_inside_connection_conv(self):
connection = layers.join(
layers.Input((28, 28, 1)),
layers.Convolution((3, 3, 8)) > layers.Relu(),
layers.Convolution((3, 3, 8)) > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Softmax(1),
)
self.assertEqual(8, len(connection))
expected_order = [
layers.Input, layers.Convolution, layers.Relu, layers.Convolution,
layers.Relu, layers.MaxPooling, layers.Reshape, layers.Softmax
]
for actual_layer, expected_layer in zip(connection, expected_order):
self.assertIsInstance(actual_layer, expected_layer)
def test_max_pooling(self):
X = asfloat(np.array([
[1, 2, 3, -1],
[4, -6, 3, 1],
[0, 0, 1, 0],
[0, -1, 0, 0],
])).reshape(1, 4, 4, 1)
expected_output = asfloat(np.array([
[4, 3],
[0, 1],
])).reshape(1, 2, 2, 1)
network = layers.join(
layers.Input((4, 4, 1)),
layers.MaxPooling((2, 2)),
)
actual_output = self.eval(network.output(X))
np.testing.assert_array_almost_equal(actual_output, expected_output)
def Inception(nfilters):
return layers.join(
[[
layers.MaxPooling((3, 3), stride=1, padding='SAME'),
layers.Convolution((1, 1, nfilters[0])),
layers.Relu(),
], [
layers.Convolution((1, 1, nfilters[1])),
layers.Relu(),
], [
layers.Convolution((1, 1, nfilters[2])),
layers.Relu(),
layers.Convolution((3, 3, nfilters[3]), padding='SAME'),
layers.Relu(),
], [
layers.Convolution((1, 1, nfilters[4])),
layers.Relu(),
layers.Convolution((5, 5, nfilters[5]), padding='SAME'),
layers.Relu(),
]],
layers.Concatenate(),
)
def test_parallel_layer(self):
input_layer = layers.Input((8, 8, 3))
parallel_layer = layers.join(
[[
layers.Convolution((5, 5, 11)),
], [
layers.Convolution((3, 3, 10)),
layers.Convolution((3, 3, 5)),
]],
layers.Concatenate(),
)
output_layer = layers.MaxPooling((2, 2))
conn = layers.join(input_layer, parallel_layer)
output_connection = layers.join(conn, output_layer)
x_tensor4 = asfloat(np.random.random((10, 8, 8, 3)))
output = self.eval(conn.output(x_tensor4))
self.assertEqual(output.shape, (10, 4, 4, 11 + 5))
final_output = output_connection.predict(x_tensor4)
self.assertEqual(final_output.shape, (10, 2, 2, 11 + 5))
def test_inline_network_with_parallel_network(self):
left_branch = layers.join(
layers.Convolution((3, 3, 32)),
layers.Relu(),
layers.MaxPooling((2, 2)),
)
right_branch = layers.join(
layers.Convolution((7, 7, 16)),
layers.Relu(),
)
input_layer = layers.Input((10, 10, 3))
concat = layers.Concatenate()
network_concat = input_layer > (left_branch | right_branch) > concat
network = network_concat > layers.Reshape() > layers.Softmax()
self.assertShapesEqual(network_concat.input_shape, (None, 10, 10, 3))
self.assertShapesEqual(network_concat.output_shape, (None, 4, 4, 48))
self.assertShapesEqual(network.input_shape, (None, 10, 10, 3))
self.assertShapesEqual(network.output_shape, (None, 768))
def test_inline_connection_with_parallel_connection(self):
left_branch = layers.join(
layers.Convolution((32, 3, 3)),
layers.Relu(),
layers.MaxPooling((2, 2)),
)
right_branch = layers.join(
layers.Convolution((16, 7, 7)),
layers.Relu(),
)
input_layer = layers.Input((3, 10, 10))
concat = layers.Concatenate()
network_concat = input_layer > [left_branch, right_branch] > concat
network = network_concat > layers.Reshape() > layers.Softmax()
self.assertEqual(network_concat.input_shape, (3, 10, 10))
self.assertEqual(network_concat.output_shape, (48, 4, 4))
self.assertEqual(network.input_shape, (3, 10, 10))
self.assertEqual(network.output_shape, (768,))
def test_connections_with_complex_parallel_relations(self):
input_layer = layers.Input((3, 5, 5))
connection = layers.join(
[[
layers.Convolution((8, 1, 1)),
],
[
layers.Convolution((4, 1, 1)),
[[
layers.Convolution((2, 1, 3), padding=(0, 1)),
], [
layers.Convolution((2, 3, 1), padding=(1, 0)),
]],
],
[
layers.Convolution((8, 1, 1)),
layers.Convolution((4, 3, 3), padding=1),
[[
layers.Convolution((2, 1, 3), padding=(0, 1)),
], [
layers.Convolution((2, 3, 1), padding=(1, 0)),
]],
],
[
layers.MaxPooling((3, 3), stride=(1, 1), padding=1),
layers.Convolution((8, 1, 1)),
]],
layers.Concatenate(),
)
self.assertEqual(connection.input_shape, [None, None, None, None])
# Connect them at the end, because we need to make
# sure tha parallel connections defined without
# input shapes
connection = input_layer > connection
self.assertEqual((24, 5, 5), connection.output_shape)
def Inception(nfilters):
return layers.join(
[[
layers.MaxPooling((3, 3), stride=1, padding=(1, 1)),
layers.Convolution((nfilters[0], 1, 1)),
layers.Relu(),
], [
layers.Convolution((nfilters[1], 1, 1)),
layers.Relu(),
],
[
layers.Convolution((nfilters[2], 1, 1)),
layers.Relu(),
layers.Convolution((nfilters[3], 3, 3), padding='half'),
layers.Relu(),
],
[
layers.Convolution((nfilters[4], 1, 1)),
layers.Relu(),
layers.Convolution((nfilters[5], 5, 5), padding='half'),
layers.Relu(),
]],
layers.Concatenate(),
)
],
[
layers.Convolution((nfilters[4], 1, 1)),
layers.Relu(),
layers.Convolution((nfilters[5], 5, 5), padding='half'),
layers.Relu(),
]],
layers.Concatenate(),
)
googlenet = layers.join(
layers.Input((3, None, None)),
layers.Convolution((64, 7, 7), padding='half', stride=2),
layers.Relu(),
layers.MaxPooling((3, 3), stride=2),
layers.LocalResponseNorm(alpha=0.00002, k=1),
layers.Convolution((64, 1, 1)) > layers.Relu(),
layers.Convolution((192, 3, 3), padding='half') > layers.Relu(),
layers.LocalResponseNorm(alpha=0.00002, k=1),
layers.MaxPooling((3, 3), stride=2),
Inception((32, 64, 96, 128, 16, 32)),
Inception((64, 128, 128, 192, 32, 96)),
layers.MaxPooling((3, 3), stride=2),
Inception((64, 192, 96, 208, 16, 48)),
Inception((64, 160, 112, 224, 24, 64)),
Inception((64, 128, 128, 256, 24, 64)),
Inception((64, 112, 144, 288, 32, 64)),
Inception((128, 256, 160, 320, 32, 128)),
layers.MaxPooling((3, 3), stride=2),
Inception((128, 256, 160, 320, 32, 128)),
data = mnist.data / 255.
data = data - data.mean(axis=0)
n_samples = data.shape[0]
data = data.reshape((n_samples, 1, 28, 28))
x_train, x_test, y_train, y_test = cross_validation.train_test_split(
data.astype(np.float32), target.astype(np.float32), train_size=(6 / 7.))
network = algorithms.Adadelta(
[
layers.Convolution((32, 1, 3, 3)),
layers.Relu(),
layers.Convolution((48, 32, 3, 3)),
layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Dropout(0.2),
layers.Reshape(),
layers.Relu(6912),
layers.Dropout(0.3),
layers.Softmax(200),
layers.ArgmaxOutput(10),
],
error='categorical_crossentropy',
step=1.0,
verbose=True,
shuffle_data=True,
epochs_step_minimizator=8,
addons=[algorithms.SimpleStepMinimization],
)
network.architecture()
def vgg16():
"""
VGG16 network architecture with random parameters. Parameters
can be loaded using ``neupy.storage`` module.
Originally VGG16 was built in order to solve image classification
problem. It was used in the ImageNet competition. The goal of the
competition is to build a model that classifies image into one of
the 1,000 categories. Categories include animals, objects, transports
and so on.
VGG16 has roughly 138 million parameters.
Examples
--------
>>> from neupy import architectures
>>> vgg16 = architectures.vgg16()
>>> vgg16
(?, 224, 224, 3) -> [... 41 layers ...] -> (?, 1000)
>>>
>>> from neupy import algorithms
>>> optimizer = algorithms.Momentum(vgg16, verbose=True)
See Also
--------
:architecture:`vgg19` : VGG19 network
:architecture:`squeezenet` : SqueezeNet network
:architecture:`resnet50` : ResNet50 network
References
----------
Very Deep Convolutional Networks for Large-Scale Image Recognition.
https://arxiv.org/abs/1409.1556
"""
SamePadConv = layers.Convolution.define(padding='SAME')
return layers.join(
layers.Input((224, 224, 3)),
SamePadConv((3, 3, 64), name='conv1_1') >> layers.Relu(),
SamePadConv((3, 3, 64), name='conv1_2') >> layers.Relu(),
layers.MaxPooling((2, 2)),
SamePadConv((3, 3, 128), name='conv2_1') >> layers.Relu(),
SamePadConv((3, 3, 128), name='conv2_2') >> layers.Relu(),
layers.MaxPooling((2, 2)),
SamePadConv((3, 3, 256), name='conv3_1') >> layers.Relu(),
SamePadConv((3, 3, 256), name='conv3_2') >> layers.Relu(),
SamePadConv((3, 3, 256), name='conv3_3') >> layers.Relu(),
layers.MaxPooling((2, 2)),
SamePadConv((3, 3, 512), name='conv4_1') >> layers.Relu(),
SamePadConv((3, 3, 512), name='conv4_2') >> layers.Relu(),
SamePadConv((3, 3, 512), name='conv4_3') >> layers.Relu(),
layers.MaxPooling((2, 2)),
SamePadConv((3, 3, 512), name='conv5_1') >> layers.Relu(),
SamePadConv((3, 3, 512), name='conv5_2') >> layers.Relu(),
SamePadConv((3, 3, 512), name='conv5_3') >> layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Linear(4096, name='dense_1') >> layers.Relu(),
layers.Dropout(0.5),
layers.Linear(4096, name='dense_2') >> layers.Relu(),
layers.Dropout(0.5),
layers.Linear(1000, name='dense_3') >> layers.Softmax(),
)
def resnet50(input_shape=(224, 224, 3), include_global_pool=True,
in_out_ratio=32):
"""
ResNet50 network architecture with random parameters. Parameters
can be loaded using ``neupy.storage`` module.
ResNet50 has roughly 25.5 million parameters.
Parameters
----------
input_shape : tuple
Network's input shape. Defaults to ``(224, 224, 3)``.
include_global_pool : bool
Specifies if returned output should include global pooling
layer. Defaults to ``True``.
in_out_ratio : {4, 8, 16, 32}
Every layer that applies strides reduces height and width per every
image. There are 5 of these layers in Resnet and at the end each
dimensions gets reduced by ``32``. For example, 224x224 image
will be reduced to 7x7 image patches. This parameter specifies
what level of reduction we want to obtain after we've propagated
network through all the convolution layers.
Notes
-----
Because of the global pooling layer, ResNet50 can be applied to
the images with variable sizes. The only limitation is that image
size should be bigger than 32x32, otherwise network won't be able
to apply all transformations to the image.
Examples
--------
ResNet-50 for ImageNet classification
>>> from neupy import architectures, algorithms
>>>
>>> resnet = architectures.resnet50()
>>> resnet
(?, 224, 224, 3) -> [... 187 layers ...] -> (?, 1000)
>>>
>>> optimizer = algorithms.Momentum(resnet50)
ResNet-50 for custom classification task
>>> from neupy import architectures
>>> resnet = architectures.resnet50(include_global_pool=False)
>>> resnet
(?, 224, 224, 3) -> [... 185 layers ...] -> (?, 7, 7, 2048)
>>>
>>> from neupy.layers import *
>>> resnet = resnet >> GlobalPooling('avg') >> Softmax(21)
(?, 224, 224, 3) -> [... 187 layers ...] -> (?, 21)
ResNet-50 for image segmentation
>>> from neupy import architectures
>>> resnet = architectures.resnet50(
... include_global_pool=False,
... in_out_ratio=8,
... )
>>> resnet
(?, 224, 224, 3) -> [... 185 layers ...] -> (?, 28, 28, 2048)
See Also
--------
:architecture:`vgg16` : VGG16 network
:architecture:`squeezenet` : SqueezeNet network
:architecture:`resnet50` : ResNet-50 network
References
----------
Deep Residual Learning for Image Recognition.
https://arxiv.org/abs/1512.03385
"""
in_out_configs = {
4: {'strides': [1, 1, 1], 'rates': [2, 4, 8]},
8: {'strides': [2, 1, 1], 'rates': [1, 2, 4]},
16: {'strides': [2, 2, 1], 'rates': [1, 1, 2]},
32: {'strides': [2, 2, 2], 'rates': [1, 1, 1]},
}
if in_out_ratio not in in_out_configs:
raise ValueError(
"Expected one of the folowing in_out_ratio values: {}, got "
"{} instead.".format(in_out_configs.keys(), in_out_ratio))
strides = in_out_configs[in_out_ratio]['strides']
rates = in_out_configs[in_out_ratio]['rates']
resnet = layers.join(
layers.Input(input_shape),
# Convolutional layer reduces image's height and width by a factor
# of 2 (because of the stride)
# from (3, 224, 224) to (64, 112, 112)
layers.Convolution(
(7, 7, 64), stride=2, bias=None,
padding='same', name='conv1'
),
layers.BatchNorm(name='bn_conv1'),
layers.Relu(),
# Stride equal two 2 reduces image size by a factor of two
# from (64, 112, 112) to (64, 56, 56)
layers.MaxPooling((3, 3), stride=2, padding="same"),
# The branch option applies extra convolution x+ batch
# normalization transformations to the residual
ResidualUnit(64, name='2a', has_branch=True),
ResidualUnit(64, name='2b'),
ResidualUnit(64, name='2c'),
# When stride=2 reduces width and hight by factor of 2
ResidualUnit(128, stride=strides[0], name='3a', has_branch=True),
ResidualUnit(128, rate=rates[0], name='3b'),
ResidualUnit(128, rate=rates[0], name='3c'),
ResidualUnit(128, rate=rates[0], name='3d'),
# When stride=2 reduces width and hight by factor of 2
ResidualUnit(256, rate=rates[0], name='4a',
stride=strides[1], has_branch=True),
ResidualUnit(256, rate=rates[1], name='4b'),
ResidualUnit(256, rate=rates[1], name='4c'),
ResidualUnit(256, rate=rates[1], name='4d'),
ResidualUnit(256, rate=rates[1], name='4e'),
ResidualUnit(256, rate=rates[1], name='4f'),
# When stride=2 reduces width and hight by factor of 2
ResidualUnit(512, rate=rates[1], name='5a',
stride=strides[2], has_branch=True),
ResidualUnit(512, rate=rates[2], name='5b'),
ResidualUnit(512, rate=rates[2], name='5c'),
)
if include_global_pool:
resnet = layers.join(
resnet,
# Since the final residual unit has 2048 output filters, global
# pooling will replace every output image with single average
# value. Despite input image size, output from this layer always
# will be a vector with 2048 values.
layers.GlobalPooling('avg'),
layers.Softmax(1000, name='fc1000'),
)
return resnet
bias=None),
layers.BatchNorm(),
)
return layers.join(
[main_branch, residual_branch],
layers.Elementwise() > layers.Relu(),
)
resnet50 = layers.join(
layers.Input((3, 224, 224)),
layers.Convolution((64, 7, 7), stride=2, padding=3),
layers.BatchNorm(),
layers.Relu(),
layers.MaxPooling((3, 3), stride=(2, 2), ignore_border=False),
ResidualUnit(64, 256, stride=1, has_branch=True),
ResidualUnit(64, 256, stride=1),
ResidualUnit(64, 256, stride=1),
ResidualUnit(128, 512, stride=2, has_branch=True),
ResidualUnit(128, 512, stride=1),
ResidualUnit(128, 512, stride=1),
ResidualUnit(128, 512, stride=1),
ResidualUnit(256, 1024, stride=2, has_branch=True),
ResidualUnit(256, 1024, stride=1),
ResidualUnit(256, 1024, stride=1),
ResidualUnit(256, 1024, stride=1),
ResidualUnit(256, 1024, stride=1),
ResidualUnit(256, 1024, stride=1),
ResidualUnit(512, 2048, stride=2, has_branch=True),
ResidualUnit(512, 2048, stride=1),
def vgg19():
"""
VGG19 network architecture with random parameters. Parameters
can be loaded using ``neupy.storage`` module.
Originally VGG19 was built in order to solve image classification
problem. It was used in the ImageNet competition. The goal of the
competition is to build a model that classifies image into one of
the 1,000 categories. Categories include animals, objects, transports
and so on.
VGG19 has roughly 143 million parameters.
Examples
--------
>>> from neupy import architectures
>>> vgg19 = architectures.vgg19()
>>> vgg19
(3, 224, 224) -> [... 44 layers ...] -> 1000
>>>
>>> from neupy import algorithms
>>> network = algorithms.Momentum(vgg19)
See Also
--------
:architecture:`vgg16` : VGG16 network
:architecture:`squeezenet` : SqueezeNet network
:architecture:`alexnet` : AlexNet network
:architecture:`resnet50` : ResNet50 network
References
----------
Very Deep Convolutional Networks for Large-Scale Image Recognition.
https://arxiv.org/abs/1409.1556
"""
HalfPadConvolution = partial(layers.Convolution, padding='half')
return layers.join(
layers.Input((3, 224, 224)),
HalfPadConvolution((64, 3, 3), name='conv1_1') > layers.Relu(),
HalfPadConvolution((64, 3, 3), name='conv1_2') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((128, 3, 3), name='conv2_1') > layers.Relu(),
HalfPadConvolution((128, 3, 3), name='conv2_2') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((256, 3, 3), name='conv3_1') > layers.Relu(),
HalfPadConvolution((256, 3, 3), name='conv3_2') > layers.Relu(),
HalfPadConvolution((256, 3, 3), name='conv3_3') > layers.Relu(),
HalfPadConvolution((256, 3, 3), name='conv3_4') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((512, 3, 3), name='conv4_1') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv4_2') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv4_3') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv4_4') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((512, 3, 3), name='conv5_1') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv5_2') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv5_3') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv5_4') > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Linear(4096, name='dense_1') > layers.Relu(),
layers.Dropout(0.5),
layers.Linear(4096, name='dense_2') > layers.Relu(),
layers.Dropout(0.5),
layers.Linear(1000, name='dense_3') > layers.Softmax(),
)
return input_value[:, :, :, self.from_channel:self.to_channel]
def __repr__(self):
return "{}({}, {})".format(
self.__class__.__name__,
self.from_channel,
self.to_channel)
alexnet = layers.join(
layers.Input((227, 227, 3)),
layers.Convolution((11, 11, 96), stride=(4, 4), name='conv_1'),
layers.Relu(),
layers.MaxPooling((3, 3), stride=(2, 2)),
layers.LocalResponseNorm(),
[[
SliceChannels(0, 48),
layers.Convolution((5, 5, 128), padding='SAME', name='conv_2_1'),
layers.Relu(),
], [
SliceChannels(48, 96),
layers.Convolution((5, 5, 128), padding='SAME', name='conv_2_2'),
layers.Relu(),
]],
layers.Concatenate(),
layers.MaxPooling((3, 3), stride=(2, 2)),
layers.LocalResponseNorm(),
def test_pooling_stride_int(self):
max_pool_layer = layers.MaxPooling((2, 2), stride=1)
self.assertEqual(max_pool_layer.input_shape,
max_pool_layer.output_shape)
def test_pooling_repr(self):
layer = layers.MaxPooling((2, 2))
self.assertEqual("MaxPooling((2, 2))", str(layer))
def alexnet():
"""
AlexNet network architecture with random parameters. Parameters
can be loaded using ``neupy.storage`` module.
Originally AlexNet was built in order to solve image classification
problem. It was used in the ImageNet competition. The goal of the
competition is to build a model that classifies image into one of
the 1,000 categories. Categories include animals, objects, transports
and so on.
AlexNet has roughly 61 million parameters.
Examples
--------
>>> from neupy import architectures
>>> alexnet = architectures.alexnet()
>>> alexnet
(3, 227, 227) -> [... 37 layers ...] -> 1000
>>>
>>> from neupy import algorithms
>>> network = algorithms.Momentum(alexnet)
See Also
--------
:architecture:`vgg16` : VGG16 network
:architecture:`vgg19` : VGG19 network
:architecture:`squeezenet` : SqueezeNet network
:architecture:`resnet50` : ResNet50 network
References
----------
ImageNet Classification with Deep Convolutional Neural Networks
https://goo.gl/479oZZ
"""
return layers.join(
layers.Input((3, 227, 227)),
layers.Convolution((96, 11, 11), stride=(4, 4), name='conv_1'),
layers.Relu(),
layers.MaxPooling((3, 3), stride=(2, 2)),
layers.LocalResponseNorm(),
[[
SliceChannels(0, 48),
layers.Convolution((128, 5, 5), padding=2, name='conv_2_1'),
layers.Relu(),
],
[
SliceChannels(48, 96),
layers.Convolution((128, 5, 5), padding=2, name='conv_2_2'),
layers.Relu(),
]],
layers.Concatenate(),
layers.MaxPooling((3, 3), stride=(2, 2)),
layers.LocalResponseNorm(),
layers.Convolution((384, 3, 3), padding=1, name='conv_3'),
layers.Relu(),
[[
SliceChannels(0, 192),
layers.Convolution((192, 3, 3), padding=1, name='conv_4_1'),
layers.Relu(),
],
[
SliceChannels(192, 384),
layers.Convolution((192, 3, 3), padding=1, name='conv_4_2'),
layers.Relu(),
]],
layers.Concatenate(),
[[
SliceChannels(0, 192),
layers.Convolution((128, 3, 3), padding=1, name='conv_5_1'),
layers.Relu(),
],
[
SliceChannels(192, 384),
layers.Convolution((128, 3, 3), padding=1, name='conv_5_2'),
layers.Relu(),
]],
layers.Concatenate(),
layers.MaxPooling((3, 3), stride=(2, 2)),
layers.Reshape(),
layers.Relu(4096, name='dense_1') > layers.Dropout(0.5),
layers.Relu(4096, name='dense_2') > layers.Dropout(0.5),
layers.Softmax(1000, name='dense_3'),
)
