def network():
HalfPadConvolution = partial(layers.Convolution, padding='half')
return layers.join(
layers.Input((3, 224, 224)),
HalfPadConvolution((20, 5, 5), name='conv1_1') > layers.Relu(),
HalfPadConvolution((20, 5, 5), name='conv1_2') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((60, 5, 5), name='conv2_1') > layers.Relu(),
HalfPadConvolution((60, 5, 5), name='conv2_2') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((120, 5, 5), name='conv3_1') > layers.Relu(),
HalfPadConvolution((120, 5, 5), name='conv3_2') > layers.Relu(),
HalfPadConvolution((150, 5, 5), name='conv3_3') > layers.Relu(),
HalfPadConvolution((150, 5, 5), name='conv3_4') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((128, 5, 5), name='conv4_1') > layers.Relu(),
HalfPadConvolution((128, 5, 5), name='conv4_2') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv4_3') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv4_4') > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Linear(2000, name='dense_1') > layers.Relu(),
layers.Dropout(0.5),
layers.Linear(1000, name='dense_2') > layers.Relu(),
layers.Dropout(0.5),
layers.Linear(1000, name='dense_3') > layers.Softmax(),
)
def test_dropout_disable_training_state(self):
test_input = np.ones((50, 20))
dropout_layer = layers.Dropout(proba=0.5)
with dropout_layer.disable_training_state():
layer_output = dropout_layer.output(test_input)
np.testing.assert_array_equal(layer_output, test_input)
def squeezenet():
"""
SqueezeNet network architecture with random parameters.
Parameters can be loaded using ``neupy.storage`` module.
SqueezeNet has roughly 1.2 million parameters. It is almost
50 times less than in AlexNet. Parameters can be stored as 5Mb
file.
Examples
--------
>>> from neupy import architectures
>>> squeezenet = architectures.squeezenet()
>>> squeezenet
(?, 227, 227, 3) -> [... 67 layers ...] -> (?, 1000)
>>>
>>> from neupy import algorithms
>>> optimizer = algorithms.Momentum(squeezenet)
See Also
--------
:architecture:`vgg16` : VGG16 network
:architecture:`vgg19` : VGG19 network
:architecture:`resnet50` : ResNet50 network
References
----------
SqueezeNet: AlexNet-level accuracy with 50x fewer parameters
and <0.5MB model size
https://arxiv.org/abs/1602.07360
"""
return layers.join(
layers.Input((227, 227, 3)),
layers.Convolution((7, 7, 96), stride=(2, 2),
padding='VALID', name='conv1'),
layers.Relu(),
layers.MaxPooling((3, 3), stride=(2, 2)),
Fire(16, 64, 64, name='fire2'),
Fire(16, 64, 64, name='fire3'),
Fire(32, 128, 128, name='fire4'),
layers.MaxPooling((2, 2)),
Fire(32, 128, 128, name='fire5'),
Fire(48, 192, 192, name='fire6'),
Fire(48, 192, 192, name='fire7'),
Fire(64, 256, 256, name='fire8'),
layers.MaxPooling((2, 2)),
Fire(64, 256, 256, name='fire9'),
layers.Dropout(0.5),
layers.Convolution((1, 1, 1000), name='conv10'),
layers.GlobalPooling('avg'),
layers.Reshape(),
layers.Softmax(),
)
def test_dropout_layer(self):
test_input = np.ones((50, 20))
dropout_layer = layers.Dropout(proba=0.5)
layer_output = self.eval(dropout_layer.output(test_input))
self.assertGreater(layer_output.sum(), 900)
self.assertLess(layer_output.sum(), 1100)
self.assertTrue(
np.all(np.bitwise_or(layer_output == 0, layer_output == 2)))
def test_bfgs(self):
x_train, x_test, y_train, y_test = simple_classification()
qnnet = algorithms.QuasiNewton(
connection=[
layers.Input(10),
layers.Dropout(0.1),
layers.Sigmoid(30, weight=init.Orthogonal()),
layers.Sigmoid(1, weight=init.Orthogonal()),
],
shuffle_data=True,
show_epoch='20 times',
verbose=False,
)
qnnet.train(x_train, y_train, x_test, y_test, epochs=20)
result = qnnet.predict(x_test).round().astype(int)
roc_curve_score = metrics.roc_auc_score(result, y_test)
self.assertAlmostEqual(0.92, roc_curve_score, places=2)
def train_network(parameters):
print("Parameters:")
pprint(parameters)
print()
step = parameters['step']
batch_size = int(parameters['batch_size'])
proba = parameters['dropout']
activation_layer = parameters['act_func_type']
layer_sizes = [int(n) for n in parameters['layers']['n_units_layer']]
network = layers.Input(784)
for layer_size in layer_sizes:
network = network > activation_layer(layer_size)
network = network > layers.Dropout(proba) > layers.Softmax(10)
mnet = algorithms.RMSProp(
network,
batch_size=batch_size,
step=step,
error='categorical_crossentropy',
shuffle_data=True,
epoch_end_signal=on_epoch_end,
)
mnet.train(x_train, y_train, epochs=50)
score = mnet.prediction_error(x_test, y_test)
y_predicted = mnet.predict(x_test).argmax(axis=1)
accuracy = metrics.accuracy_score(y_test.argmax(axis=1), y_predicted)
print("Final score: {}".format(score))
print("Accuracy: {:.2%}".format(accuracy))
return score
def test_dropout_repr(self):
layer = layers.Dropout(0.5)
self.assertEqual("Dropout(proba=0.5)", str(layer))
zca = preprocessing.ZCA()
zca.train(x_train)
x_train = zca.transform(x_train)
x_test = zca.transform(x_test)
target_scaler = OneHotEncoder()
y_train = target_scaler.fit_transform(y_train.reshape((-1, 1))).todense()
y_test = target_scaler.transform(y_test.reshape((-1, 1))).todense()
network = algorithms.Adadelta([
layers.Convolution((64, 3, 3, 3), border_mode='full'),
layers.Relu(),
layers.Convolution((64, 64, 3, 3)),
layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Dropout(proba=0.25),
layers.Convolution((128, 64, 3, 3), border_mode='full'),
layers.Relu(),
layers.Convolution((128, 128, 3, 3)),
layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Dropout(proba=0.25),
layers.Convolution((256, 128, 3, 3), border_mode='full'),
layers.Relu(),
layers.Convolution((256, 256, 3, 3)),
layers.Relu(),
layers.Convolution((256, 256, 1, 1)),
layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Dropout(proba=0.25),
layers.Reshape(),
step=0.1,
momentum=0.99,
shuffle_data=True,
batch_size=64,
error='binary_crossentropy',
)
conv_autoencoder.architecture()
conv_autoencoder.train(x_unlabeled_4d, x_unlabeled,
x_labeled_4d, x_labeled, epochs=10)
x_labeled_encoded = encoder.output(x_labeled_4d).eval()
x_unlabeled_encoded = encoder.output(x_unlabeled_4d).eval()
classifier_network = layers.join(
layers.PRelu(512),
layers.Dropout(0.25),
layers.Softmax(10),
)
encoder_classifier = algorithms.Adadelta(
layers.Input(encoder.output_shape) > classifier_network,
verbose=True,
step=0.05,
shuffle_data=True,
batch_size=64,
error='categorical_crossentropy',
)
encoder_classifier.architecture()
encoder_classifier.train(x_labeled_encoded, y_labeled,
x_unlabeled_encoded, y_unlabeled, epochs=100)
description='Trains network to play CartPole game', )
parser.add_argument(
'-p',
'--pretrained',
dest='use_pretrained',
action='store_true',
help='load pretrained network from file and play without training',
)
args = parser.parse_args()
network = algorithms.RMSProp(
[
layers.Input(4),
layers.Relu(64),
layers.Relu(48),
layers.Relu(32),
layers.Relu(64) > layers.Dropout(0.2),
# Expecting two different actions:
# 1. Move left
# 2. Move right
layers.Linear(2),
],
step=0.0005,
error='rmse',
batch_size='full',
verbose=False)
env = gym.make('CartPole-v0')
env.seed(0) # To make results reproducible for the gym
memory_size = 1000 # Number of samples stored in the memory
step = parameters['step']
batch_size = int(parameters['batch_size'])
proba = parameters['dropout']
activation_layer = parameters['act_func_type']
layer_sizes = [int(n) for n in parameters['layers']['n_units_layer']]
from neupy import layers
network = layers.Input(784)
for layer_size in layer_sizes:
network = network > activation_layer(layer_size)
network = network > layers.Dropout(proba) > layers.Softmax(10)
from neupy import algorithms
from neupy.exceptions import StopTraining
def on_epoch_end(network):
if network.errors.last() > 10:
raise StopTraining("Training was interrupted. Error is to high.")
mnet = algorithms.RMSProp(
network,
batch_size=batch_size,
step=step,
error='categorical_crossentropy',
shuffle_data=True,
def vgg19():
"""
VGG19 network architecture with random parameters. Parameters
can be loaded using ``neupy.storage`` module.
Originally VGG19 was built in order to solve image classification
problem. It was used in the ImageNet competition. The goal of the
competition is to build a model that classifies image into one of
the 1,000 categories. Categories include animals, objects, transports
and so on.
VGG19 has roughly 143 million parameters.
Examples
--------
>>> from neupy import architectures
>>> vgg19 = architectures.vgg19()
>>> vgg19
(3, 224, 224) -> [... 44 layers ...] -> 1000
>>>
>>> from neupy import algorithms
>>> network = algorithms.Momentum(vgg19)
See Also
--------
:architecture:`vgg16` : VGG16 network
:architecture:`squeezenet` : SqueezeNet network
:architecture:`alexnet` : AlexNet network
:architecture:`resnet50` : ResNet50 network
References
----------
Very Deep Convolutional Networks for Large-Scale Image Recognition.
https://arxiv.org/abs/1409.1556
"""
HalfPadConvolution = partial(layers.Convolution, padding='half')
return layers.join(
layers.Input((3, 224, 224)),
HalfPadConvolution((64, 3, 3), name='conv1_1') > layers.Relu(),
HalfPadConvolution((64, 3, 3), name='conv1_2') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((128, 3, 3), name='conv2_1') > layers.Relu(),
HalfPadConvolution((128, 3, 3), name='conv2_2') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((256, 3, 3), name='conv3_1') > layers.Relu(),
HalfPadConvolution((256, 3, 3), name='conv3_2') > layers.Relu(),
HalfPadConvolution((256, 3, 3), name='conv3_3') > layers.Relu(),
HalfPadConvolution((256, 3, 3), name='conv3_4') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((512, 3, 3), name='conv4_1') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv4_2') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv4_3') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv4_4') > layers.Relu(),
layers.MaxPooling((2, 2)),
HalfPadConvolution((512, 3, 3), name='conv5_1') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv5_2') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv5_3') > layers.Relu(),
HalfPadConvolution((512, 3, 3), name='conv5_4') > layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Linear(4096, name='dense_1') > layers.Relu(),
layers.Dropout(0.5),
layers.Linear(4096, name='dense_2') > layers.Relu(),
layers.Dropout(0.5),
layers.Linear(1000, name='dense_3') > layers.Softmax(),
)
[[
SliceChannels(0, 192),
layers.Convolution((3, 3, 192), padding='SAME', name='conv_4_1'),
layers.Relu(),
], [
SliceChannels(192, 384),
layers.Convolution((3, 3, 192), padding='SAME', name='conv_4_2'),
layers.Relu(),
]],
layers.Concatenate(),
[[
SliceChannels(0, 192),
layers.Convolution((3, 3, 128), padding='SAME', name='conv_5_1'),
layers.Relu(),
], [
SliceChannels(192, 384),
layers.Convolution((3, 3, 128), padding='SAME', name='conv_5_2'),
layers.Relu(),
]],
layers.Concatenate(),
layers.MaxPooling((3, 3), stride=(2, 2)),
layers.Reshape(),
layers.Relu(4096, name='dense_1') > layers.Dropout(0.5),
layers.Relu(4096, name='dense_2') > layers.Dropout(0.5),
layers.Softmax(1000, name='dense_3'),
)
plots.network_structure(alexnet)
def alexnet():
"""
AlexNet network architecture with random parameters. Parameters
can be loaded using ``neupy.storage`` module.
Originally AlexNet was built in order to solve image classification
problem. It was used in the ImageNet competition. The goal of the
competition is to build a model that classifies image into one of
the 1,000 categories. Categories include animals, objects, transports
and so on.
AlexNet has roughly 61 million parameters.
Examples
--------
>>> from neupy import architectures
>>> alexnet = architectures.alexnet()
>>> alexnet
(3, 227, 227) -> [... 37 layers ...] -> 1000
>>>
>>> from neupy import algorithms
>>> network = algorithms.Momentum(alexnet)
See Also
--------
:architecture:`vgg16` : VGG16 network
:architecture:`vgg19` : VGG19 network
:architecture:`squeezenet` : SqueezeNet network
:architecture:`resnet50` : ResNet50 network
References
----------
ImageNet Classification with Deep Convolutional Neural Networks
https://goo.gl/479oZZ
"""
return layers.join(
layers.Input((3, 227, 227)),
layers.Convolution((96, 11, 11), stride=(4, 4), name='conv_1'),
layers.Relu(),
layers.MaxPooling((3, 3), stride=(2, 2)),
layers.LocalResponseNorm(),
[[
SliceChannels(0, 48),
layers.Convolution((128, 5, 5), padding=2, name='conv_2_1'),
layers.Relu(),
],
[
SliceChannels(48, 96),
layers.Convolution((128, 5, 5), padding=2, name='conv_2_2'),
layers.Relu(),
]],
layers.Concatenate(),
layers.MaxPooling((3, 3), stride=(2, 2)),
layers.LocalResponseNorm(),
layers.Convolution((384, 3, 3), padding=1, name='conv_3'),
layers.Relu(),
[[
SliceChannels(0, 192),
layers.Convolution((192, 3, 3), padding=1, name='conv_4_1'),
layers.Relu(),
],
[
SliceChannels(192, 384),
layers.Convolution((192, 3, 3), padding=1, name='conv_4_2'),
layers.Relu(),
]],
layers.Concatenate(),
[[
SliceChannels(0, 192),
layers.Convolution((128, 3, 3), padding=1, name='conv_5_1'),
layers.Relu(),
],
[
SliceChannels(192, 384),
layers.Convolution((128, 3, 3), padding=1, name='conv_5_2'),
layers.Relu(),
]],
layers.Concatenate(),
layers.MaxPooling((3, 3), stride=(2, 2)),
layers.Reshape(),
layers.Relu(4096, name='dense_1') > layers.Dropout(0.5),
layers.Relu(4096, name='dense_2') > layers.Dropout(0.5),
layers.Softmax(1000, name='dense_3'),
)
layers.Embedding(n_unique_categories, 4),
# Reshape (batch_size, 3, 4) to (batch_size, 12)
layers.Reshape(),
],
[
# 17 numerical inputs
layers.Input(17),
]
],
# Concatenate (batch_size, 12) and (batch_size, 17)
# into one matrix with shape (batch_size, 29)
layers.Concatenate(),
layers.Relu(128),
layers.Relu(32) > layers.Dropout(0.5),
layers.Sigmoid(1)
],
step=0.2,
verbose=True,
momentum=0.9,
nesterov=True,
error='binary_crossentropy',
# Applied max-norm regularizer to prevent overfitting.
# Maximum possible norm for any weight is specified by
# the `max_norm` parameter.
addons=[algorithms.MaxNormRegularization],
max_norm=10,
)
def vgg16():
"""
VGG16 network architecture with random parameters. Parameters
can be loaded using ``neupy.storage`` module.
Originally VGG16 was built in order to solve image classification
problem. It was used in the ImageNet competition. The goal of the
competition is to build a model that classifies image into one of
the 1,000 categories. Categories include animals, objects, transports
and so on.
VGG16 has roughly 138 million parameters.
Examples
--------
>>> from neupy import architectures
>>> vgg16 = architectures.vgg16()
>>> vgg16
(?, 224, 224, 3) -> [... 41 layers ...] -> (?, 1000)
>>>
>>> from neupy import algorithms
>>> optimizer = algorithms.Momentum(vgg16, verbose=True)
See Also
--------
:architecture:`vgg19` : VGG19 network
:architecture:`squeezenet` : SqueezeNet network
:architecture:`resnet50` : ResNet50 network
References
----------
Very Deep Convolutional Networks for Large-Scale Image Recognition.
https://arxiv.org/abs/1409.1556
"""
SamePadConv = layers.Convolution.define(padding='SAME')
return layers.join(
layers.Input((224, 224, 3)),
SamePadConv((3, 3, 64), name='conv1_1') >> layers.Relu(),
SamePadConv((3, 3, 64), name='conv1_2') >> layers.Relu(),
layers.MaxPooling((2, 2)),
SamePadConv((3, 3, 128), name='conv2_1') >> layers.Relu(),
SamePadConv((3, 3, 128), name='conv2_2') >> layers.Relu(),
layers.MaxPooling((2, 2)),
SamePadConv((3, 3, 256), name='conv3_1') >> layers.Relu(),
SamePadConv((3, 3, 256), name='conv3_2') >> layers.Relu(),
SamePadConv((3, 3, 256), name='conv3_3') >> layers.Relu(),
layers.MaxPooling((2, 2)),
SamePadConv((3, 3, 512), name='conv4_1') >> layers.Relu(),
SamePadConv((3, 3, 512), name='conv4_2') >> layers.Relu(),
SamePadConv((3, 3, 512), name='conv4_3') >> layers.Relu(),
layers.MaxPooling((2, 2)),
SamePadConv((3, 3, 512), name='conv5_1') >> layers.Relu(),
SamePadConv((3, 3, 512), name='conv5_2') >> layers.Relu(),
SamePadConv((3, 3, 512), name='conv5_3') >> layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Reshape(),
layers.Linear(4096, name='dense_1') >> layers.Relu(),
layers.Dropout(0.5),
layers.Linear(4096, name='dense_2') >> layers.Relu(),
layers.Dropout(0.5),
layers.Linear(1000, name='dense_3') >> layers.Softmax(),
)
data = data - data.mean(axis=0)
n_samples = data.shape[0]
data = data.reshape((n_samples, 1, 28, 28))
x_train, x_test, y_train, y_test = cross_validation.train_test_split(
data.astype(np.float32), target.astype(np.float32), train_size=(6 / 7.))
network = algorithms.Adadelta(
[
layers.Convolution((32, 1, 3, 3)),
layers.Relu(),
layers.Convolution((48, 32, 3, 3)),
layers.Relu(),
layers.MaxPooling((2, 2)),
layers.Dropout(0.2),
layers.Reshape(),
layers.Relu(6912),
layers.Dropout(0.3),
layers.Softmax(200),
layers.ArgmaxOutput(10),
],
error='categorical_crossentropy',
step=1.0,
verbose=True,
shuffle_data=True,
epochs_step_minimizator=8,
addons=[algorithms.SimpleStepMinimization],
)
network.architecture()
network.train(x_train, y_train, x_test, y_test, epochs=6)
stride=(2, 2),
padding='valid',
name='conv1'),
layers.Relu(),
layers.MaxPooling((3, 3), stride=(2, 2)),
Fire(16, 64, 64, name='fire2'),
Fire(16, 64, 64, name='fire3'),
Fire(32, 128, 128, name='fire4'),
layers.MaxPooling((2, 2)),
Fire(32, 128, 128, name='fire5'),
Fire(48, 192, 192, name='fire6'),
Fire(48, 192, 192, name='fire7'),
Fire(64, 256, 256, name='fire8'),
layers.MaxPooling((2, 2)),
Fire(64, 256, 256, name='fire9'),
layers.Dropout(0.5),
layers.Convolution((1000, 1, 1), padding='valid', name='conv10'),
layers.GlobalPooling(function=T.mean),
layers.Reshape(),
layers.Softmax(),
)
if not os.path.exists(SQUEEZENET_WEIGHTS_FILE):
download_file(url=("http://srv70.putdrive.com/putstorage/DownloadFileHash/"
"6B0A15B43A5A4A5QQWE2304100EWQS/squeezenet.pickle"),
filepath=SQUEEZENET_WEIGHTS_FILE,
description='Downloading weights')
storage.load(squeezenet, SQUEEZENET_WEIGHTS_FILE)
monkey_image = load_image(os.path.join(CURRENT_DIR, 'images',
