def create_model(base_model_file, input_features, num_classes, dropout_rate=0.5, freeze_weights=False): # Load the pretrained classification net and find nodes base_model = load_model(base_model_file) feature_node = find_by_name(base_model, 'features') beforePooling_node = find_by_name(base_model, "z.x.x.r") #graph.plot(base_model, filename="base_model.pdf") # Write graph visualization # Clone model until right before the pooling layer, ie. until including z.x.x.r modelCloned = combine([beforePooling_node.owner]).clone( CloneMethod.freeze if freeze_weights else CloneMethod.clone, {feature_node: placeholder(name='features')}) # Center the input around zero and set model input. # Do this early, to avoid CNTK bug with wrongly estimated layer shapes feat_norm = input_features - constant(114) model = modelCloned(feat_norm) # Pool over all spatial dimensions and add dropout layer avgPool = GlobalAveragePooling(name="poolingLayer")(model) if dropout_rate > 0: avgPoolDrop = Dropout(dropout_rate)(avgPool) else: avgPoolDrop = avgPool # Add new dense layer for class prediction finalModel = Dense(num_classes, activation=None, name="prediction")(avgPoolDrop) return finalModel
def ffnet(optimizer): inputs = 2 outputs = 2 layers = 2 hidden_dimension = 50 # input variables denoting the features and label data features = C.input_variable((inputs), np.float32) label = C.input_variable((outputs), np.float32) # Instantiate the feedforward classification model my_model = Sequential([ Dense(hidden_dimension, activation=C.sigmoid, init=C.glorot_uniform(seed=98052)), Dense(outputs, init=C.glorot_uniform(seed=98052)) ]) z = my_model(features) ce = C.cross_entropy_with_softmax(z, label) pe = C.classification_error(z, label) # Instantiate the trainer object to drive the model training lr_per_minibatch = learning_rate_schedule(0.125, UnitType.minibatch) progress_printer = ProgressPrinter(0) trainer = C.Trainer(z, (ce, pe), [optimizer(z.parameters, lr_per_minibatch)], progress_printer) # Get minibatches of training data and perform model training minibatch_size = 25 num_minibatches_to_train = 63 for i in range(num_minibatches_to_train): train_features, labels = generate_random_data(minibatch_size, inputs, outputs) # Specify the mapping of input variables in the model to actual minibatch data to be trained with trainer.train_minibatch({features: train_features, label: labels}) test_features, test_labels = generate_random_data(minibatch_size, inputs, outputs) avg_error = trainer.test_minibatch({ features: test_features, label: test_labels }) print(' error rate on an unseen minibatch: {}'.format(avg_error)) return z.parameters
def create_model(X): # defines few stacked GRUs l1 = Recurrence(step_function=GRU(shape=20))(X) l2 = Recurrence(step_function=GRU(shape=20))(l1) l3 = Dense(shape=1)(l2) return l3
def Blocked_Dense(dim, activation=None): dense = Dense(dim, activation=activation) @C.layers.BlockFunction('blocked_dense', 'blocked_dense') def func(x): return dense(x) return func
def create_shallow_model(input, out_dims): convolutional_layer_1_1 = Convolution((7,7), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(input) convolutional_layer_1_2 = Convolution((25,25), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(convolutional_layer_1_1) pooling_layer_1 = MaxPooling((25,25), strides=(5,5))(convolutional_layer_1_2 ) convolutional_layer_2_1 = Convolution((3,3), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(pooling_layer_1) pooling_layer_2 = MaxPooling((2,2), strides=(2,2))(convolutional_layer_2_1) fully_connected_layer_1 = Dense(512, init=glorot_uniform())(pooling_layer_2) fully_connected_layer_2 = Dense(128, init=glorot_uniform())(fully_connected_layer_1) dropout_layer = Dropout(0.5)(fully_connected_layer_2) output_layer = Dense(out_dims, init=glorot_uniform(), activation=None)(dropout_layer) return output_layer
def _setup_test_model(self): inputs = input_variable(shape=(1, ), dtype=np.float32) outputs = input_variable(shape=(1, ), dtype=np.float32) q = Dense(1, activation=None)(inputs) loss = squared_error(q, outputs) return {'inputs': inputs, 'outputs': outputs, 'f': q, 'loss': loss}
def create_vgg16(feature_var, num_classes, dropout=0.9): with default_options(activation=None, pad=True, bias=True): z = Sequential([ # we separate Convolution and ReLU to name the output for feature # extraction (usually before ReLU) For( range(2), lambda i: [ Convolution2D((3, 3), 64, name='conv1_{}'.format(i)), Activation(activation=relu, name='relu1_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool1'), For( range(2), lambda i: [ Convolution2D((3, 3), 128, name='conv2_{}'.format(i)), Activation(activation=relu, name='relu2_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool2'), For( range(3), lambda i: [ Convolution2D((3, 3), 256, name='conv3_{}'.format(i)), Activation(activation=relu, name='relu3_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool3'), For( range(3), lambda i: [ Convolution2D((3, 3), 512, name='conv4_{}'.format(i)), Activation(activation=relu, name='relu4_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool4'), For( range(3), lambda i: [ Convolution2D((3, 3), 512, name='conv5_{}'.format(i)), Activation(activation=relu, name='relu5_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool5'), Dense(4096, name='fc6'), Activation(activation=relu, name='relu6'), Dropout(dropout, name='drop6'), Dense(4096, name='fc7'), Activation(activation=relu, name='relu7'), Dropout(dropout, name='drop7'), Dense(num_classes, name='fc8') ])(feature_var) return z
def _setup_test_model(self, *args, **kwargs): inputs = placeholder(shape=(1, )) outputs = input_variable(shape=(1, ), dtype=np.float32) q = Dense(1, activation=None)(inputs) loss = cross_entropy_with_softmax(q, outputs) return {'inputs': inputs, 'outputs': outputs, 'f': q, 'loss': loss}
def create_basic_model(input, out_dims): # convolutional_layer_1 = Convolution((3,3), 64, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(input) # pooling_layer_1 = MaxPooling((2,2), strides=(1,1))(convolutional_layer_1 ) convolutional_layer_2 = Convolution((5,5), 64, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(input) pooling_layer_2 = MaxPooling((2,2), strides=(1,1))(convolutional_layer_2) convolutional_layer_3 = Convolution((9,9), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(pooling_layer_2) pooling_layer_3 = MaxPooling((3,3), strides=(2,2))(convolutional_layer_3) ## fully_connected_layer = Dense(256, init=glorot_uniform())(pooling_layer_3) dropout_layer = Dropout(0.5)(fully_connected_layer) output_layer = Dense(out_dims, init=glorot_uniform(), activation=None)(dropout_layer) return output_layer
def LSTM_sequence_classifier_net(input, num_output_classes, embedding_dim, LSTM_dim, cell_dim): lstm_classifier = Sequential([ Embedding(embedding_dim), Recurrence(LSTM(LSTM_dim, cell_dim)), sequence.last, Dense(num_output_classes) ]) return lstm_classifier(input)
def test_htk_deserializers(): mbsize = 640 epoch_size = 1000 * mbsize lr = [0.001] feature_dim = 33 num_classes = 132 context = 2 os.chdir(data_path) features_file = "glob_0000.scp" labels_file = "glob_0000.mlf" label_mapping_file = "state.list" fd = HTKFeatureDeserializer( StreamDefs(amazing_features=StreamDef( shape=feature_dim, context=(context, context), scp=features_file))) ld = HTKMLFDeserializer( label_mapping_file, StreamDefs( awesome_labels=StreamDef(shape=num_classes, mlf=labels_file))) reader = MinibatchSource([fd, ld]) features = C.input_variable(((2 * context + 1) * feature_dim)) labels = C.input_variable((num_classes)) model = Sequential( [For(range(3), lambda: Recurrence(LSTM(256))), Dense(num_classes)]) z = model(features) ce = C.cross_entropy_with_softmax(z, labels) errs = C.classification_error(z, labels) learner = C.adam_sgd(z.parameters, lr=C.learning_rate_schedule(lr, C.UnitType.sample, epoch_size), momentum=C.momentum_as_time_constant_schedule(1000), low_memory=True, gradient_clipping_threshold_per_sample=15, gradient_clipping_with_truncation=True) trainer = C.Trainer(z, (ce, errs), learner) input_map = { features: reader.streams.amazing_features, labels: reader.streams.awesome_labels } pp = C.ProgressPrinter(freq=0) # just run and verify it doesn't crash for i in range(3): mb_data = reader.next_minibatch(mbsize, input_map=input_map) trainer.train_minibatch(mb_data) pp.update_with_trainer(trainer, with_metric=True) assert True os.chdir(abs_path)
def ffnet(learner, trainer=None): inputs = 5 outputs = 3 layers = 2 hidden_dimension = 3 if trainer is None: # input variables denoting the features and label data features = C.input_variable((inputs), np.float32) label = C.input_variable((outputs), np.float32) # Instantiate the feedforward classification model my_model = Sequential ([ Dense(hidden_dimension, activation=C.sigmoid, init=C.glorot_uniform(seed=98052)), Dense(outputs, init=C.glorot_uniform(seed=98052))]) z = my_model(features) ce = C.cross_entropy_with_softmax(z, label) pe = C.classification_error(z, label) # Instantiate the trainer object to drive the model training progress_printer = ProgressPrinter(0) trainer = C.Trainer(z, (ce, pe), [learner(z.parameters)], [progress_printer]) else: features = trainer.loss_function.arguments[0] label = trainer.loss_function.arguments[1] # Get minibatches of training data and perform model training minibatch_size = 25 num_minibatches_to_train = 100 aggregate_loss = 0.0 for i in range(num_minibatches_to_train): train_features, labels = generate_random_data(minibatch_size, inputs, outputs) # Specify the mapping of input variables in the model to actual minibatch data to be trained with trainer.train_minibatch({features : train_features, label : labels}) sample_count = trainer.previous_minibatch_sample_count aggregate_loss += trainer.previous_minibatch_loss_average * sample_count last_avg_error = aggregate_loss / trainer.total_number_of_samples_seen test_features, test_labels = generate_random_data(minibatch_size, inputs, outputs) avg_error = trainer.test_minibatch({features : test_features, label : test_labels}) print(' error rate on an unseen minibatch: {}'.format(avg_error)) return last_avg_error, avg_error, trainer
def test_layers_dense(device_id): y = input(2) dat = np.array([[-1., 1.]], dtype=np.float32) #################################################### # Test 1: no activation #################################################### p = Dense(2, activation=None, name='foo')(y) res = p(y).eval({y: dat}) npout = np.matrix(dat[0]) * p.foo.W.value + p.foo.b.value np.testing.assert_array_equal(res, npout, err_msg='Error in dense layer') #################################################### # Test 2: with activation #################################################### p = Dense(2, activation=sigmoid, name='foo')(y) res = p(y).eval({y: dat}) def _sigmoid(x): return 1./(1 + np.exp(-x)) npout = _sigmoid(np.matrix(dat[0]) * p.foo.W.value + p.foo.b.value) np.testing.assert_array_almost_equal(res, npout, decimal=7, err_msg='Error in dense layer with sigmoid') #################################################### # Test 3: 2-dense layer #################################################### p = Dense(3, activation=None, name='foo')(y) q = Dense(3, activation=None, name='bar')(p) res = q(y).eval({y: dat}) npout1 = np.matrix(dat[0]) * p.foo.W.value + p.foo.b.value npout = npout1 * q.bar.W.value + q.bar.b.value np.testing.assert_array_almost_equal(res, npout, decimal=7, err_msg='Error in 2-dense layer') #################################################### # Test 4: Failing configuration #################################################### with pytest.raises(ValueError): Dense(2, input_rank=1, map_rank=1) # input_rank and map_rank can be specified at the same time
def create_vgg9_model(input, num_classes): with default_options(activation=relu): model = Sequential([ LayerStack( 3, lambda i: [ Convolution((3, 3), [64, 96, 128][i], init=glorot_uniform(), pad=True), Convolution((3, 3), [64, 96, 128][i], init=glorot_uniform(), pad=True), MaxPooling((3, 3), strides=(2, 2)) ]), LayerStack(2, lambda: [Dense(1024, init=glorot_uniform())]), Dense(num_classes, init=glorot_uniform(), activation=None) ]) return model(input)
def ffnet(): input_dim = 2 num_output_classes = 2 num_hidden_layers = 2 hidden_layers_dim = 50 # Input variables denoting the features and label data feature = input_variable((input_dim), np.float32) label = input_variable((num_output_classes), np.float32) netout = Sequential([ For(range(num_hidden_layers), lambda i: Dense(hidden_layers_dim, activation=sigmoid)), Dense(num_output_classes) ])(feature) ce = cross_entropy_with_softmax(netout, label) pe = classification_error(netout, label) lr_per_minibatch = learning_parameter_schedule(0.5) # Instantiate the trainer object to drive the model training learner = sgd(netout.parameters, lr=lr_per_minibatch) progress_printer = ProgressPrinter(128) trainer = Trainer(netout, (ce, pe), learner, progress_printer) # Get minibatches of training data and perform model training minibatch_size = 25 for i in range(1024): features, labels = generate_random_data(minibatch_size, input_dim, num_output_classes) # Specify the mapping of input variables in the model to actual # minibatch data to be trained with trainer.train_minibatch({feature: features, label: labels}) trainer.summarize_training_progress() test_features, test_labels = generate_random_data(minibatch_size, input_dim, num_output_classes) avg_error = trainer.test_minibatch({ feature: test_features, label: test_labels }) return avg_error
def create_advanced_model(input, out_dims): with default_options(activation=relu): model = Sequential([ For(range(2), lambda i: [ # lambda with one parameter Convolution((3,3), [32,64][i], pad=True), # depth depends on i Convolution((5,5), [32,64][i], pad=True), Convolution((9,9), [32,64][i], pad=True), MaxPooling((3,3), strides=(2,2)) ]), For(range(2), lambda : [ # lambda without parameter Dense(512), Dropout(0.5) ]), Dense(out_dims, activation=None) ]) output_layer=model(input) return output_layer
def create_model(output_dim): return Sequential([ For( range(num_layers), lambda: Sequential([ Stabilizer(), Recurrence(LSTM(hidden_dim), go_backwards=False) ])), Dense(output_dim) ])
def ffnet(): inputs = 2 outputs = 2 layers = 2 hidden_dimension = 50 # input variables denoting the features and label data features = C.input_variable((inputs), np.float32) label = C.input_variable((outputs), np.float32) # Instantiate the feedforward classification model my_model = Sequential ([ Dense(hidden_dimension, activation=C.sigmoid), Dense(outputs)]) z = my_model(features) ce = C.cross_entropy_with_softmax(z, label) pe = C.classification_error(z, label) # Instantiate the trainer object to drive the model training lr_per_minibatch = C.learning_parameter_schedule(0.125) progress_printer = ProgressPrinter(0) trainer = C.Trainer(z, (ce, pe), [sgd(z.parameters, lr=lr_per_minibatch)], [progress_printer]) # Get minibatches of training data and perform model training minibatch_size = 25 num_minibatches_to_train = 1024 aggregate_loss = 0.0 for i in range(num_minibatches_to_train): train_features, labels = generate_random_data(minibatch_size, inputs, outputs) # Specify the mapping of input variables in the model to actual minibatch data to be trained with trainer.train_minibatch({features : train_features, label : labels}) sample_count = trainer.previous_minibatch_sample_count aggregate_loss += trainer.previous_minibatch_loss_average * sample_count last_avg_error = aggregate_loss / trainer.total_number_of_samples_seen test_features, test_labels = generate_random_data(minibatch_size, inputs, outputs) avg_error = trainer.test_minibatch({features : test_features, label : test_labels}) print(' error rate on an unseen minibatch: {}'.format(avg_error)) return last_avg_error, avg_error
def create_symbol(): # Weight initialiser from uniform distribution # Activation (unless states) is None with cntk.layers.default_options(init=cntk.glorot_uniform(), activation=cntk.relu): x = Convolution2D(filter_shape=(3, 3), num_filters=50, pad=True)(features) x = Convolution2D(filter_shape=(3, 3), num_filters=50, pad=True)(x) x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x) x = Dropout(0.25)(x) x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x) x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x) x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x) x = Dropout(0.25)(x) x = Dense(512)(x) x = Dropout(0.5)(x) x = Dense(N_CLASSES, activation=None)(x) return x
def test_dense_layer(self): """Test a model with a single CNTK Dense layer against the equivalent ELL predictor. This verifies that the import functions reshape and reorder values appropriately and that the equivalent ELL layer produces comparable output """ # Create a Dense CNTK layer with no bias or activation denseLayer = Dense(5, bias=False) x = input((2, 3, 4)) # Input order for CNTK is channels, rows, columns cntkModel = denseLayer(x) # Create a test set of weights to use for both CNTK and ELL layers # CNTK has these in channels, rows, columns, [output shape] order weightValues = np.arange(120, dtype=np.float_).reshape(2, 3, 4, 5) # Set the weights denseLayer.parameters[0].value = weightValues # create an ELL Tensor from the cntk weights, which re-orders the # weights and produces an appropriately dimensioned tensor weightTensor = cntk_converters.\ get_tensor_from_cntk_dense_weight_parameter( denseLayer.parameters[0]) # Create the equivalent ELL predictor layerParameters = ell.neural.LayerParameters( # Input order for ELL is rows, columns, channels ell.math.TensorShape(3, 4, 2), ell.neural.NoPadding(), ell.math.TensorShape(1, 1, 5), ell.neural.NoPadding(), ell.nodes.PortType.smallReal) layer = ell.neural.FullyConnectedLayer(layerParameters, weightTensor) predictor = ell.neural.NeuralNetworkPredictor([layer]) # Get the results for both inputValues = np.arange(24, dtype=np.float32).reshape(2, 3, 4) orderedInputValues = cntk_converters.get_vector_from_cntk_array( inputValues) cntkResults = cntkModel(inputValues) orderedCntkResults = cntk_converters.get_vector_from_cntk_array( cntkResults) ellResults = predictor.Predict(orderedInputValues) # Compare the results np.testing.assert_array_equal( orderedCntkResults, ellResults, 'results for Dense layer do not match!') # now run same over ELL compiled model self.verify_compiled(predictor, orderedInputValues, orderedCntkResults, "dense", "test")
def create_model(): ''' Creates the model to train :return: Returns the last output of a sequential model using LSTMs ''' return Sequential([ For(range(NUMBER_LAYERS), lambda: Sequential([Recurrence(LSTM(HIDDEN_LAYER_DIMENSIONS))])), sequence.last, Dense(NUM_OUTPUT_CLASSES) ])
def test_failing_for_constructor(): with pytest.raises((ValueError, TypeError)): network = For(range(3), Dense(5)) class MyFunction: def __call__(self, x): return Dense(x) with pytest.raises((ValueError, TypeError)): network = For(range(3), MyFunction()) with pytest.raises((ValueError, TypeError)): network = For(range(3), MyFunction()(5))
def test_for_constructor_layer(layers_count, dense_units): x = input(4) network = For(range(layers_count), lambda i: Dense(dense_units)) expected_num_of_parameters = 2 * layers_count assert len(network.parameters) == expected_num_of_parameters res = network(x) expected_output_shape = (dense_units, ) assert res.shape == expected_output_shape
def bn_inception_model(input, labelDim, bnTimeConst): # 224 x 224 x 3 conv1 = conv_bn_relu_layer(input, 64, (7, 7), (2, 2), True, bnTimeConst) # 112 x 112 x 64 pool1 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=True)(conv1) # 56 x 56 x 64 conv2a = conv_bn_relu_layer(pool1, 64, (1, 1), (1, 1), True, bnTimeConst) # 56 x 56 x 64 conv2b = conv_bn_relu_layer(conv2a, 192, (3, 3), (1, 1), True, bnTimeConst) # 56 x 56 x 192 pool2 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=True)(conv2b) # Inception Blocks # 28 x 28 x 192 inception3a = inception_block_with_avgpool(pool2, 64, 64, 64, 64, 96, 32, bnTimeConst) # 28 x 28 x 256 inception3b = inception_block_with_avgpool(inception3a, 64, 64, 96, 64, 96, 64, bnTimeConst) # 28 x 28 x 320 inception3c = inception_block_pass_through(inception3b, 0, 128, 160, 64, 96, 0, bnTimeConst) # 14 x 14 x 576 inception4a = inception_block_with_avgpool(inception3c, 224, 64, 96, 96, 128, 128, bnTimeConst) # 14 x 14 x 576 inception4b = inception_block_with_avgpool(inception4a, 192, 96, 128, 96, 128, 128, bnTimeConst) # 14 x 14 x 576 inception4c = inception_block_with_avgpool(inception4b, 160, 128, 160, 128, 160, 128, bnTimeConst) # 14 x 14 x 576 inception4d = inception_block_with_avgpool(inception4c, 96, 128, 192, 160, 192, 128, bnTimeConst) # 14 x 14 x 576 inception4e = inception_block_pass_through(inception4d, 0, 128, 192, 192, 256, 0, bnTimeConst) # 7 x 7 x 1024 inception5a = inception_block_with_avgpool(inception4e, 352, 192, 320, 160, 224, 128, bnTimeConst) # 7 x 7 x 1024 inception5b = inception_block_with_maxpool(inception5a, 352, 192, 320, 192, 224, 128, bnTimeConst) # Global Average # 7 x 7 x 1024 pool3 = AveragePooling(filter_shape=(7, 7))(inception5b) # 1 x 1 x 1024 z = Dense(labelDim, init=he_normal())(pool3) return z
def modify_model(features, n_classes): loaded_model = load_model(model_file) feature_node = find_by_name(loaded_model, 'features') last_node = find_by_name(loaded_model, 'h2_d') all_layers = combine([last_node.owner ]).clone(CloneMethod.freeze, {feature_node: placeholder()}) feat_norm = features - Constant(114) fc_out = all_layers(feat_norm) z = Dense(num_classes)(fc_out) return (z)
def create_network(): input_var = cntk.sequence.input_variable((num_channels, frame_height, frame_width), name='input_var') target_var = cntk.input_variable((num_classes,), is_sparse=True, name='target_var') with cntk.layers.default_options(enable_self_stabilization=True): model = Sequential([ resnet_model(cntk.placeholder()), Label('resnet'), Dense(hidden_dim, name='cnn_fc'), cntk.layers.Stabilizer(), bidirectional_recurrence(LSTM(hidden_dim // 2), LSTM(hidden_dim // 2)), cntk.sequence.last, BatchNormalization(), Dense(num_classes) ])(input_var) return { 'input': input_var, 'target': target_var, 'model': model, 'loss': cntk.cross_entropy_with_softmax(model, target_var), 'metric': cntk.classification_error(model, target_var) }
def bn_inceptionv2_cifar_model(input, labelDim, bnTimeConst): # 32 x 32 x 3 conv1a = conv_bn_relu_layer(input, 32, (3,3), (1,1), True, bnTimeConst) # 32 x 32 x 32 conv1b = conv_bn_relu_layer(conv1a, 32, (3,3), (1,1), True, bnTimeConst) # 32 x 32 x 32 conv1c = conv_bn_relu_layer(conv1b, 32, (3,3), (1,1), True, bnTimeConst) # 32 x 32 x 64 conv2a = conv_bn_relu_layer(conv1c, 64, (3,3), (1,1), True, bnTimeConst) # 32 x 32 x 64 conv2b = conv_bn_relu_layer(conv2a, 64, (3,3), (1,1), True, bnTimeConst) # 32 x 32 x 64 conv2c = conv_bn_relu_layer(conv2b, 64, (3,3), (1,1), True, bnTimeConst) # Inception Blocks # 32 x 32 x 64 inception3a = inception_block1(conv2b, 32, 32, 32, 32, 48, 16, bnTimeConst) # 32 x 32 x 128 inception3b = inception_block1(inception3a, 0, 64, 80, 32, 48, 0, bnTimeConst) # 32 x 32 x 128 inception3c = inception_block1(inception3b, 0, 64, 80, 32, 48, 0, bnTimeConst) # 32 x 32 x 128 pool1 = AveragePooling(filter_shape=(3,3), strides = (2,2), pad = True)(inception3c) # 16 x 16 x 256 inception4a = inception_block2(inception3c, 96, 48, 64, 48, 64, 64, bnTimeConst) # 16 x 16 x 256 inception4b = inception_block2(inception4a, 96, 48, 64, 48, 64, 64, bnTimeConst) # 16 x 16 x 256 inception4c = inception_block2(inception4b, 96, 48, 64, 48, 64, 64, bnTimeConst) # 16 x 16 x 288 inception4d = inception_block2(inception4c, 48, 64, 96, 80, 96, 64, bnTimeConst) # 16 x 16 x 288 inception4e = inception_block2(inception4d, 0, 128, 192, 192, 256, 0, bnTimeConst) # 16 x 16 x 288 pool2 = AveragePooling(filter_shape=(3,3), strides = (2,2), pad = True)(inception4e) # Inception Blocks # 8 x 8 x 512 inception5a = inception_block1(pool2, 176, 96, 160, 96, 112, 64, bnTimeConst) # 8 x 8 x 512 inception5b = inception_block1(inception5a, 176, 96, 160, 96, 112, 64, bnTimeConst) # Global Average # 8 x 8 x 512 pool3 = AveragePooling(filter_shape=(8,8))(inception5a) # 1 x 1 x 512 z = Dense(labelDim, init=he_normal())(pool3) return z
def test_sequential_clique_with_layers(input_elements, expected): x = C.input_variable(input_elements) np_data = np.arange(input_elements, dtype=np.float32) unit_dense = Dense(input_elements, activation=None, init=1) seq_clique = SequentialClique([unit_dense, unit_dense, unit_dense])(x) assert seq_clique.shape == x.shape res = seq_clique.eval(np_data) assert res[0].shape == (input_elements, ) assert np.unique(res[0])[0] == expected
def create_convolutional_neural_network(self, input_vars, out_dims, dropout_prob=0.0): convolutional_layer_1 = Convolution((5, 5), 32, strides=1, activation=cntk.ops.relu, pad=True)(input_vars) pooling_layer_1 = MaxPooling((2, 2), strides=(2, 2), pad=True)(convolutional_layer_1) convolutional_layer_2 = Convolution((5, 5), 64, strides=1, activation=cntk.ops.relu, pad=True)(pooling_layer_1) pooling_layer_2 = MaxPooling((2, 2), strides=(2, 2), pad=True)(convolutional_layer_2) fully_connected_layer = Dense( 1024, activation=cntk.ops.relu)(pooling_layer_2) dropout_layer = Dropout(dropout_prob)(fully_connected_layer) output_layer = Dense(out_dims, activation=None)(dropout_layer) return output_layer
def inceptionv1_cifar_model2(input, labelDim, bnTimeConst): # 32 x 32 x 3 conv1 = conv_bn_relu_layer(input, 32, (3, 3), (1, 1), True, bnTimeConst) # 32 x 32 x 32 conv2 = conv_bn_relu_layer(conv1, 32, (3, 3), (1, 1), True, bnTimeConst) # Inception Blocks # 32 x 32 x 64 inception3a = inception_block_with_maxpool(conv2, 32, 32, 32, 32, 48, 16, bnTimeConst) # 32 x 32 x 128 inception3b = inception_block_with_maxpool(inception3a, 32, 32, 32, 32, 48, 16, bnTimeConst) maxpool1 = MaxPooling((3, 3), strides=(2, 2), pad=True)(inception3b) # 16 x 16 x 128 inception4a = inception_block_with_maxpool(maxpool1, 96, 48, 64, 48, 64, 64, bnTimeConst) # 16 x 16 x 288 inception4b = inception_block_with_maxpool(inception4a, 96, 48, 64, 48, 64, 64, bnTimeConst) # 16 x 16 x 288 inception4c = inception_block_with_maxpool(inception4b, 96, 48, 64, 48, 64, 64, bnTimeConst) # 16 x 16 x 288 inception4d = inception_block_with_maxpool(inception4c, 96, 48, 64, 48, 64, 64, bnTimeConst) # 16 x 16 x 288 inception4e = inception_block_with_maxpool(inception4d, 96, 48, 64, 48, 64, 64, bnTimeConst) maxpool2 = MaxPooling((3, 3), strides=(2, 2), pad=True)(inception4e) # 8 x 8 x 288 inception5a = inception_block_with_maxpool(inception4e, 176, 96, 160, 96, 112, 64, bnTimeConst) # 8 x 8 x 512 inception5b = inception_block_with_maxpool(inception5a, 176, 96, 160, 96, 112, 64, bnTimeConst) # Global Average # 8 x 8 x 512 pool1 = AveragePooling(filter_shape=(8, 8))(inception5b) # 1 x 1 x 512 z = Dense(labelDim, init=he_normal())(pool1) return z
def test_large_model_serialization_float(tmpdir): import os; from cntk.layers import Recurrence, LSTM, Dense type_size = np.dtype(np.float32).itemsize two_gb = 2**31 size = (2097152 + 4, 256, 512, 4096) assert size[0] * size[1] * type_size > two_gb device = C.device.cpu() i = C.sequence.input(size[0]) w = C.Parameter((size[0], size[1]), init=C.uniform(3.0, seed=12345), device=device) e = C.times(i, w) h_fwd = Recurrence(LSTM(size[2]))(e) h_bwd = Recurrence(LSTM(size[2]), go_backwards=True)(e) h_last_fwd = C.sequence.last(h_fwd) h_first_bwd = C.sequence.first(h_bwd) t = C.splice(h_last_fwd, h_first_bwd) z1 = Dense(size[2], activation=C.relu)(t) z = Dense(2, activation=None)(z1) filename = str(tmpdir / 'test_large_model_serialization_float.out') z.save(filename) assert os.path.getsize(filename) > two_gb y = C.Function.load(filename, device=device) assert (len(z.parameters) == len(y.parameters)) for param_pair in zip(z.parameters, y.parameters): assert param_pair[0].shape == param_pair[1].shape assert np.allclose(param_pair[0].value, param_pair[1].value)