def network(): HalfPadConvolution = partial(layers.Convolution, padding='half') return layers.join( layers.Input((3, 224, 224)), HalfPadConvolution((20, 5, 5), name='conv1_1') > layers.Relu(), HalfPadConvolution((20, 5, 5), name='conv1_2') > layers.Relu(), layers.MaxPooling((2, 2)), HalfPadConvolution((60, 5, 5), name='conv2_1') > layers.Relu(), HalfPadConvolution((60, 5, 5), name='conv2_2') > layers.Relu(), layers.MaxPooling((2, 2)), HalfPadConvolution((120, 5, 5), name='conv3_1') > layers.Relu(), HalfPadConvolution((120, 5, 5), name='conv3_2') > layers.Relu(), HalfPadConvolution((150, 5, 5), name='conv3_3') > layers.Relu(), HalfPadConvolution((150, 5, 5), name='conv3_4') > layers.Relu(), layers.MaxPooling((2, 2)), HalfPadConvolution((128, 5, 5), name='conv4_1') > layers.Relu(), HalfPadConvolution((128, 5, 5), name='conv4_2') > layers.Relu(), HalfPadConvolution((512, 3, 3), name='conv4_3') > layers.Relu(), HalfPadConvolution((512, 3, 3), name='conv4_4') > layers.Relu(), layers.MaxPooling((2, 2)), layers.Reshape(), layers.Linear(2000, name='dense_1') > layers.Relu(), layers.Dropout(0.5), layers.Linear(1000, name='dense_2') > layers.Relu(), layers.Dropout(0.5), layers.Linear(1000, name='dense_3') > layers.Softmax(), )
def test_dropout_disable_training_state(self): test_input = np.ones((50, 20)) dropout_layer = layers.Dropout(proba=0.5) with dropout_layer.disable_training_state(): layer_output = dropout_layer.output(test_input) np.testing.assert_array_equal(layer_output, test_input)
def squeezenet(): """ SqueezeNet network architecture with random parameters. Parameters can be loaded using ``neupy.storage`` module. SqueezeNet has roughly 1.2 million parameters. It is almost 50 times less than in AlexNet. Parameters can be stored as 5Mb file. Examples -------- >>> from neupy import architectures >>> squeezenet = architectures.squeezenet() >>> squeezenet (?, 227, 227, 3) -> [... 67 layers ...] -> (?, 1000) >>> >>> from neupy import algorithms >>> optimizer = algorithms.Momentum(squeezenet) See Also -------- :architecture:`vgg16` : VGG16 network :architecture:`vgg19` : VGG19 network :architecture:`resnet50` : ResNet50 network References ---------- SqueezeNet: AlexNet-level accuracy with 50x fewer parameters and <0.5MB model size https://arxiv.org/abs/1602.07360 """ return layers.join( layers.Input((227, 227, 3)), layers.Convolution((7, 7, 96), stride=(2, 2), padding='VALID', name='conv1'), layers.Relu(), layers.MaxPooling((3, 3), stride=(2, 2)), Fire(16, 64, 64, name='fire2'), Fire(16, 64, 64, name='fire3'), Fire(32, 128, 128, name='fire4'), layers.MaxPooling((2, 2)), Fire(32, 128, 128, name='fire5'), Fire(48, 192, 192, name='fire6'), Fire(48, 192, 192, name='fire7'), Fire(64, 256, 256, name='fire8'), layers.MaxPooling((2, 2)), Fire(64, 256, 256, name='fire9'), layers.Dropout(0.5), layers.Convolution((1, 1, 1000), name='conv10'), layers.GlobalPooling('avg'), layers.Reshape(), layers.Softmax(), )
def test_dropout_layer(self): test_input = np.ones((50, 20)) dropout_layer = layers.Dropout(proba=0.5) layer_output = self.eval(dropout_layer.output(test_input)) self.assertGreater(layer_output.sum(), 900) self.assertLess(layer_output.sum(), 1100) self.assertTrue( np.all(np.bitwise_or(layer_output == 0, layer_output == 2)))
def test_bfgs(self): x_train, x_test, y_train, y_test = simple_classification() qnnet = algorithms.QuasiNewton( connection=[ layers.Input(10), layers.Dropout(0.1), layers.Sigmoid(30, weight=init.Orthogonal()), layers.Sigmoid(1, weight=init.Orthogonal()), ], shuffle_data=True, show_epoch='20 times', verbose=False, ) qnnet.train(x_train, y_train, x_test, y_test, epochs=20) result = qnnet.predict(x_test).round().astype(int) roc_curve_score = metrics.roc_auc_score(result, y_test) self.assertAlmostEqual(0.92, roc_curve_score, places=2)
def train_network(parameters): print("Parameters:") pprint(parameters) print() step = parameters['step'] batch_size = int(parameters['batch_size']) proba = parameters['dropout'] activation_layer = parameters['act_func_type'] layer_sizes = [int(n) for n in parameters['layers']['n_units_layer']] network = layers.Input(784) for layer_size in layer_sizes: network = network > activation_layer(layer_size) network = network > layers.Dropout(proba) > layers.Softmax(10) mnet = algorithms.RMSProp( network, batch_size=batch_size, step=step, error='categorical_crossentropy', shuffle_data=True, epoch_end_signal=on_epoch_end, ) mnet.train(x_train, y_train, epochs=50) score = mnet.prediction_error(x_test, y_test) y_predicted = mnet.predict(x_test).argmax(axis=1) accuracy = metrics.accuracy_score(y_test.argmax(axis=1), y_predicted) print("Final score: {}".format(score)) print("Accuracy: {:.2%}".format(accuracy)) return score
def test_dropout_repr(self): layer = layers.Dropout(0.5) self.assertEqual("Dropout(proba=0.5)", str(layer))
zca = preprocessing.ZCA() zca.train(x_train) x_train = zca.transform(x_train) x_test = zca.transform(x_test) target_scaler = OneHotEncoder() y_train = target_scaler.fit_transform(y_train.reshape((-1, 1))).todense() y_test = target_scaler.transform(y_test.reshape((-1, 1))).todense() network = algorithms.Adadelta([ layers.Convolution((64, 3, 3, 3), border_mode='full'), layers.Relu(), layers.Convolution((64, 64, 3, 3)), layers.Relu(), layers.MaxPooling((2, 2)), layers.Dropout(proba=0.25), layers.Convolution((128, 64, 3, 3), border_mode='full'), layers.Relu(), layers.Convolution((128, 128, 3, 3)), layers.Relu(), layers.MaxPooling((2, 2)), layers.Dropout(proba=0.25), layers.Convolution((256, 128, 3, 3), border_mode='full'), layers.Relu(), layers.Convolution((256, 256, 3, 3)), layers.Relu(), layers.Convolution((256, 256, 1, 1)), layers.Relu(), layers.MaxPooling((2, 2)), layers.Dropout(proba=0.25), layers.Reshape(),
step=0.1, momentum=0.99, shuffle_data=True, batch_size=64, error='binary_crossentropy', ) conv_autoencoder.architecture() conv_autoencoder.train(x_unlabeled_4d, x_unlabeled, x_labeled_4d, x_labeled, epochs=10) x_labeled_encoded = encoder.output(x_labeled_4d).eval() x_unlabeled_encoded = encoder.output(x_unlabeled_4d).eval() classifier_network = layers.join( layers.PRelu(512), layers.Dropout(0.25), layers.Softmax(10), ) encoder_classifier = algorithms.Adadelta( layers.Input(encoder.output_shape) > classifier_network, verbose=True, step=0.05, shuffle_data=True, batch_size=64, error='categorical_crossentropy', ) encoder_classifier.architecture() encoder_classifier.train(x_labeled_encoded, y_labeled, x_unlabeled_encoded, y_unlabeled, epochs=100)
description='Trains network to play CartPole game', ) parser.add_argument( '-p', '--pretrained', dest='use_pretrained', action='store_true', help='load pretrained network from file and play without training', ) args = parser.parse_args() network = algorithms.RMSProp( [ layers.Input(4), layers.Relu(64), layers.Relu(48), layers.Relu(32), layers.Relu(64) > layers.Dropout(0.2), # Expecting two different actions: # 1. Move left # 2. Move right layers.Linear(2), ], step=0.0005, error='rmse', batch_size='full', verbose=False) env = gym.make('CartPole-v0') env.seed(0) # To make results reproducible for the gym memory_size = 1000 # Number of samples stored in the memory
step = parameters['step'] batch_size = int(parameters['batch_size']) proba = parameters['dropout'] activation_layer = parameters['act_func_type'] layer_sizes = [int(n) for n in parameters['layers']['n_units_layer']] from neupy import layers network = layers.Input(784) for layer_size in layer_sizes: network = network > activation_layer(layer_size) network = network > layers.Dropout(proba) > layers.Softmax(10) from neupy import algorithms from neupy.exceptions import StopTraining def on_epoch_end(network): if network.errors.last() > 10: raise StopTraining("Training was interrupted. Error is to high.") mnet = algorithms.RMSProp( network, batch_size=batch_size, step=step, error='categorical_crossentropy', shuffle_data=True,
def vgg19(): """ VGG19 network architecture with random parameters. Parameters can be loaded using ``neupy.storage`` module. Originally VGG19 was built in order to solve image classification problem. It was used in the ImageNet competition. The goal of the competition is to build a model that classifies image into one of the 1,000 categories. Categories include animals, objects, transports and so on. VGG19 has roughly 143 million parameters. Examples -------- >>> from neupy import architectures >>> vgg19 = architectures.vgg19() >>> vgg19 (3, 224, 224) -> [... 44 layers ...] -> 1000 >>> >>> from neupy import algorithms >>> network = algorithms.Momentum(vgg19) See Also -------- :architecture:`vgg16` : VGG16 network :architecture:`squeezenet` : SqueezeNet network :architecture:`alexnet` : AlexNet network :architecture:`resnet50` : ResNet50 network References ---------- Very Deep Convolutional Networks for Large-Scale Image Recognition. https://arxiv.org/abs/1409.1556 """ HalfPadConvolution = partial(layers.Convolution, padding='half') return layers.join( layers.Input((3, 224, 224)), HalfPadConvolution((64, 3, 3), name='conv1_1') > layers.Relu(), HalfPadConvolution((64, 3, 3), name='conv1_2') > layers.Relu(), layers.MaxPooling((2, 2)), HalfPadConvolution((128, 3, 3), name='conv2_1') > layers.Relu(), HalfPadConvolution((128, 3, 3), name='conv2_2') > layers.Relu(), layers.MaxPooling((2, 2)), HalfPadConvolution((256, 3, 3), name='conv3_1') > layers.Relu(), HalfPadConvolution((256, 3, 3), name='conv3_2') > layers.Relu(), HalfPadConvolution((256, 3, 3), name='conv3_3') > layers.Relu(), HalfPadConvolution((256, 3, 3), name='conv3_4') > layers.Relu(), layers.MaxPooling((2, 2)), HalfPadConvolution((512, 3, 3), name='conv4_1') > layers.Relu(), HalfPadConvolution((512, 3, 3), name='conv4_2') > layers.Relu(), HalfPadConvolution((512, 3, 3), name='conv4_3') > layers.Relu(), HalfPadConvolution((512, 3, 3), name='conv4_4') > layers.Relu(), layers.MaxPooling((2, 2)), HalfPadConvolution((512, 3, 3), name='conv5_1') > layers.Relu(), HalfPadConvolution((512, 3, 3), name='conv5_2') > layers.Relu(), HalfPadConvolution((512, 3, 3), name='conv5_3') > layers.Relu(), HalfPadConvolution((512, 3, 3), name='conv5_4') > layers.Relu(), layers.MaxPooling((2, 2)), layers.Reshape(), layers.Linear(4096, name='dense_1') > layers.Relu(), layers.Dropout(0.5), layers.Linear(4096, name='dense_2') > layers.Relu(), layers.Dropout(0.5), layers.Linear(1000, name='dense_3') > layers.Softmax(), )
[[ SliceChannels(0, 192), layers.Convolution((3, 3, 192), padding='SAME', name='conv_4_1'), layers.Relu(), ], [ SliceChannels(192, 384), layers.Convolution((3, 3, 192), padding='SAME', name='conv_4_2'), layers.Relu(), ]], layers.Concatenate(), [[ SliceChannels(0, 192), layers.Convolution((3, 3, 128), padding='SAME', name='conv_5_1'), layers.Relu(), ], [ SliceChannels(192, 384), layers.Convolution((3, 3, 128), padding='SAME', name='conv_5_2'), layers.Relu(), ]], layers.Concatenate(), layers.MaxPooling((3, 3), stride=(2, 2)), layers.Reshape(), layers.Relu(4096, name='dense_1') > layers.Dropout(0.5), layers.Relu(4096, name='dense_2') > layers.Dropout(0.5), layers.Softmax(1000, name='dense_3'), ) plots.network_structure(alexnet)
def alexnet(): """ AlexNet network architecture with random parameters. Parameters can be loaded using ``neupy.storage`` module. Originally AlexNet was built in order to solve image classification problem. It was used in the ImageNet competition. The goal of the competition is to build a model that classifies image into one of the 1,000 categories. Categories include animals, objects, transports and so on. AlexNet has roughly 61 million parameters. Examples -------- >>> from neupy import architectures >>> alexnet = architectures.alexnet() >>> alexnet (3, 227, 227) -> [... 37 layers ...] -> 1000 >>> >>> from neupy import algorithms >>> network = algorithms.Momentum(alexnet) See Also -------- :architecture:`vgg16` : VGG16 network :architecture:`vgg19` : VGG19 network :architecture:`squeezenet` : SqueezeNet network :architecture:`resnet50` : ResNet50 network References ---------- ImageNet Classification with Deep Convolutional Neural Networks https://goo.gl/479oZZ """ return layers.join( layers.Input((3, 227, 227)), layers.Convolution((96, 11, 11), stride=(4, 4), name='conv_1'), layers.Relu(), layers.MaxPooling((3, 3), stride=(2, 2)), layers.LocalResponseNorm(), [[ SliceChannels(0, 48), layers.Convolution((128, 5, 5), padding=2, name='conv_2_1'), layers.Relu(), ], [ SliceChannels(48, 96), layers.Convolution((128, 5, 5), padding=2, name='conv_2_2'), layers.Relu(), ]], layers.Concatenate(), layers.MaxPooling((3, 3), stride=(2, 2)), layers.LocalResponseNorm(), layers.Convolution((384, 3, 3), padding=1, name='conv_3'), layers.Relu(), [[ SliceChannels(0, 192), layers.Convolution((192, 3, 3), padding=1, name='conv_4_1'), layers.Relu(), ], [ SliceChannels(192, 384), layers.Convolution((192, 3, 3), padding=1, name='conv_4_2'), layers.Relu(), ]], layers.Concatenate(), [[ SliceChannels(0, 192), layers.Convolution((128, 3, 3), padding=1, name='conv_5_1'), layers.Relu(), ], [ SliceChannels(192, 384), layers.Convolution((128, 3, 3), padding=1, name='conv_5_2'), layers.Relu(), ]], layers.Concatenate(), layers.MaxPooling((3, 3), stride=(2, 2)), layers.Reshape(), layers.Relu(4096, name='dense_1') > layers.Dropout(0.5), layers.Relu(4096, name='dense_2') > layers.Dropout(0.5), layers.Softmax(1000, name='dense_3'), )
layers.Embedding(n_unique_categories, 4), # Reshape (batch_size, 3, 4) to (batch_size, 12) layers.Reshape(), ], [ # 17 numerical inputs layers.Input(17), ] ], # Concatenate (batch_size, 12) and (batch_size, 17) # into one matrix with shape (batch_size, 29) layers.Concatenate(), layers.Relu(128), layers.Relu(32) > layers.Dropout(0.5), layers.Sigmoid(1) ], step=0.2, verbose=True, momentum=0.9, nesterov=True, error='binary_crossentropy', # Applied max-norm regularizer to prevent overfitting. # Maximum possible norm for any weight is specified by # the `max_norm` parameter. addons=[algorithms.MaxNormRegularization], max_norm=10, )
def vgg16(): """ VGG16 network architecture with random parameters. Parameters can be loaded using ``neupy.storage`` module. Originally VGG16 was built in order to solve image classification problem. It was used in the ImageNet competition. The goal of the competition is to build a model that classifies image into one of the 1,000 categories. Categories include animals, objects, transports and so on. VGG16 has roughly 138 million parameters. Examples -------- >>> from neupy import architectures >>> vgg16 = architectures.vgg16() >>> vgg16 (?, 224, 224, 3) -> [... 41 layers ...] -> (?, 1000) >>> >>> from neupy import algorithms >>> optimizer = algorithms.Momentum(vgg16, verbose=True) See Also -------- :architecture:`vgg19` : VGG19 network :architecture:`squeezenet` : SqueezeNet network :architecture:`resnet50` : ResNet50 network References ---------- Very Deep Convolutional Networks for Large-Scale Image Recognition. https://arxiv.org/abs/1409.1556 """ SamePadConv = layers.Convolution.define(padding='SAME') return layers.join( layers.Input((224, 224, 3)), SamePadConv((3, 3, 64), name='conv1_1') >> layers.Relu(), SamePadConv((3, 3, 64), name='conv1_2') >> layers.Relu(), layers.MaxPooling((2, 2)), SamePadConv((3, 3, 128), name='conv2_1') >> layers.Relu(), SamePadConv((3, 3, 128), name='conv2_2') >> layers.Relu(), layers.MaxPooling((2, 2)), SamePadConv((3, 3, 256), name='conv3_1') >> layers.Relu(), SamePadConv((3, 3, 256), name='conv3_2') >> layers.Relu(), SamePadConv((3, 3, 256), name='conv3_3') >> layers.Relu(), layers.MaxPooling((2, 2)), SamePadConv((3, 3, 512), name='conv4_1') >> layers.Relu(), SamePadConv((3, 3, 512), name='conv4_2') >> layers.Relu(), SamePadConv((3, 3, 512), name='conv4_3') >> layers.Relu(), layers.MaxPooling((2, 2)), SamePadConv((3, 3, 512), name='conv5_1') >> layers.Relu(), SamePadConv((3, 3, 512), name='conv5_2') >> layers.Relu(), SamePadConv((3, 3, 512), name='conv5_3') >> layers.Relu(), layers.MaxPooling((2, 2)), layers.Reshape(), layers.Linear(4096, name='dense_1') >> layers.Relu(), layers.Dropout(0.5), layers.Linear(4096, name='dense_2') >> layers.Relu(), layers.Dropout(0.5), layers.Linear(1000, name='dense_3') >> layers.Softmax(), )
data = data - data.mean(axis=0) n_samples = data.shape[0] data = data.reshape((n_samples, 1, 28, 28)) x_train, x_test, y_train, y_test = cross_validation.train_test_split( data.astype(np.float32), target.astype(np.float32), train_size=(6 / 7.)) network = algorithms.Adadelta( [ layers.Convolution((32, 1, 3, 3)), layers.Relu(), layers.Convolution((48, 32, 3, 3)), layers.Relu(), layers.MaxPooling((2, 2)), layers.Dropout(0.2), layers.Reshape(), layers.Relu(6912), layers.Dropout(0.3), layers.Softmax(200), layers.ArgmaxOutput(10), ], error='categorical_crossentropy', step=1.0, verbose=True, shuffle_data=True, epochs_step_minimizator=8, addons=[algorithms.SimpleStepMinimization], ) network.architecture() network.train(x_train, y_train, x_test, y_test, epochs=6)
stride=(2, 2), padding='valid', name='conv1'), layers.Relu(), layers.MaxPooling((3, 3), stride=(2, 2)), Fire(16, 64, 64, name='fire2'), Fire(16, 64, 64, name='fire3'), Fire(32, 128, 128, name='fire4'), layers.MaxPooling((2, 2)), Fire(32, 128, 128, name='fire5'), Fire(48, 192, 192, name='fire6'), Fire(48, 192, 192, name='fire7'), Fire(64, 256, 256, name='fire8'), layers.MaxPooling((2, 2)), Fire(64, 256, 256, name='fire9'), layers.Dropout(0.5), layers.Convolution((1000, 1, 1), padding='valid', name='conv10'), layers.GlobalPooling(function=T.mean), layers.Reshape(), layers.Softmax(), ) if not os.path.exists(SQUEEZENET_WEIGHTS_FILE): download_file(url=("http://srv70.putdrive.com/putstorage/DownloadFileHash/" "6B0A15B43A5A4A5QQWE2304100EWQS/squeezenet.pickle"), filepath=SQUEEZENET_WEIGHTS_FILE, description='Downloading weights') storage.load(squeezenet, SQUEEZENET_WEIGHTS_FILE) monkey_image = load_image(os.path.join(CURRENT_DIR, 'images',