def create_basic_model(input, out_dims): convolutional_layer_1 = Convolution((5, 5), 16, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(input) pooling_layer_1 = MaxPooling((2, 2), strides=(1, 1))(convolutional_layer_1) convolutional_layer_2 = Convolution((5, 5), 16, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(pooling_layer_1) pooling_layer_2 = MaxPooling((3, 3), strides=(2, 2))(convolutional_layer_2) # convolutional_layer_3 = Convolution((9, 9), 16, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(pooling_layer_2) pooling_layer_3 = MaxPooling((3, 3), strides=(2, 2))(convolutional_layer_3) fully_connected_layer = Dense(256, init=glorot_uniform())(pooling_layer_3) dropout_layer = Dropout(0.5)(fully_connected_layer) output_layer = Dense(out_dims, init=glorot_uniform(), activation=None)(dropout_layer) return output_layer
def pre_block(input, bnTimeConst): # 32 x 32 x 3 conv1a = conv_bn_relu_layer(input, 32, (3, 3), (1, 1), True, bnTimeConst) # 32 x 32 x 32 conv1b = conv_bn_relu_layer(conv1a, 32, (3, 3), (1, 1), True, bnTimeConst) # 32 x 32 x 32 conv1c = conv_bn_relu_layer(conv1b, 64, (3, 3), (1, 1), True, bnTimeConst) c1 = MaxPooling((3, 3), strides=(1, 1), pad=True)(conv1c) c2 = conv_bn_relu_layer(conv1c, 96, (3, 3), (1, 1), True, bnTimeConst) d = splice(c1, c2, axis=0) e1 = conv_bn_relu_layer(d, 64, (1, 1), (1, 1), True, bnTimeConst) e2 = conv_bn_relu_layer(e1, 96, (3, 3), (1, 1), True, bnTimeConst) f1 = conv_bn_relu_layer(d, 64, (1, 1), (1, 1), True, bnTimeConst) f2 = conv_bn_relu_layer(f1, 64, (3, 1), (1, 1), True, bnTimeConst) f3 = conv_bn_relu_layer(f2, 64, (1, 3), (1, 1), True, bnTimeConst) f4 = conv_bn_relu_layer(f3, 96, (3, 3), (1, 1), True, bnTimeConst) g = splice(e2, f4, axis=0) h1 = conv_bn_relu_layer(g, 128, (3, 3), (1, 1), True, bnTimeConst) i1 = MaxPooling((3, 3), strides=(1, 1), pad=True)(g) out = splice(h1, i1, axis=0) return out
def pre_block(input): # 32 x 32 x 3 conv1a = conv_bn_relu(input, (3, 3), 16, (1, 1)) # 32 x 32 x 32 conv1b = conv_bn_relu(conv1a, (3, 3), 16, (1, 1)) # 32 x 32 x 32 conv1c = conv_bn_relu(conv1b, (3, 3), 16, (1, 1)) c1 = MaxPooling((3, 3), strides=(1, 1), pad=True)(conv1c) c2 = conv_bn_relu(conv1c, (3, 3), 16, (1, 1)) d = splice(c1, c2, axis=0) # 32 x 32 x 32 e1 = conv_bn_relu(d, (1, 1), 32, (1, 1)) e2 = conv_bn_relu(e1, (3, 3), 32, (1, 1)) f1 = conv_bn_relu(d, (1, 1), 32, (1, 1)) f2 = conv_bn_relu(f1, (3, 1), 32, (1, 1)) f3 = conv_bn_relu(f2, (1, 3), 32, (1, 1)) f4 = conv_bn_relu(f3, (3, 3), 32, (1, 1)) g = splice(e2, f4, axis=0) # 32 x 32 x 64 h1 = conv_bn_relu(g, (3, 3), 64, (1, 1)) i1 = MaxPooling((3, 3), strides=(1, 1), pad=True)(g) out = splice(h1, i1, axis=0) # 32 x 32 x 128 return out
def __call__(self, num_classes=10, act_type=relu, mdl_conv1a_nf=40, mdl_conv1b_nf=60, mdl_conv2a_nf=50, mdl_conv2b_nf=75, mdl_fc1_nh=75, mdl_drop2a_p=0.033, mdl_drop2b_p=0.097, mdl_drop3_p=0.412, **kwargs): input_var = input_variable((1, self.img_h, self.img_w), np.float32) label_var = input_variable((self.n_dim), np.float32) conv1a = Convolution(filter_shape=(3, 3), num_filters=int(mdl_conv1a_nf), activation=act_type, init=glorot_uniform(), pad=True, name='conv1a')(input_var) conv1b = Convolution(filter_shape=(3, 3), num_filters=int(mdl_conv1b_nf), activation=act_type, init=glorot_uniform(), pad=True, name='conv1b')(conv1a) pool1 = MaxPooling(filter_shape=(2, 2), strides=(2, 2), name='pool1')(conv1b) conv2a = Convolution(filter_shape=(3, 3), num_filters=int(mdl_conv2a_nf), activation=act_type, init=glorot_uniform(), pad=True, name='conv2a')(pool1) drop2a = Dropout(prob=mdl_drop2a_p, name="drop2a")(conv2a) conv2b = Convolution(filter_shape=(3, 3), num_filters=int(mdl_conv2b_nf), activation=act_type, init=glorot_uniform(), pad=True, name='conv2b')(drop2a) drop2b = Dropout(prob=mdl_drop2a_p, name="drop2a")(conv2b) pool2 = MaxPooling(filter_shape=(2, 2), strides=(2, 2), name='pool2')(drop2b) fc1 = Dense(shape=int(mdl_fc1_nh), init=glorot_uniform(), activation=act_type, name='fc1')(pool2) drop3 = Dropout(prob=mdl_drop3_p, name="drop3")(fc1) #fc2 = Dense(shape=num_classes, init=glorot_uniform(), activation=softmax, name='fc2')(drop3) fc2 = Dense(shape=num_classes, init=glorot_uniform(), activation=None, name='fc2')(drop3) return input_var, label_var, fc2
def create_model(self): mean_removed_features = minus(self.input, constant(114), name='mean_removed_input') with default_options(activation=None, pad=True, bias=True): self.model = Sequential([ Convolution2D((11, 11), 96, init=normal(0.01), pad=False, name='conv1'), Activation(activation=relu, name='relu1'), self.__local_response_normalization(1.0, 2, 0.0001, 0.75, name='norm1'), MaxPooling((3, 3), (2, 2), name='pool1'), Convolution2D((5, 5), 192, init=normal(0.01), init_bias=0.1, name='conv2'), Activation(activation=relu, name='relu2'), self.__local_response_normalization(1.0, 2, 0.0001, 0.75, name='norm2'), MaxPooling((3, 3), (2, 2), name='pool2'), Convolution2D((3, 3), 384, init=normal(0.01), name='conv3'), Activation(activation=relu, name='relu3'), Convolution2D((3, 3), 384, init=normal(0.01), init_bias=0.1, name='conv4'), Activation(activation=relu, name='relu4'), Convolution2D((3, 3), 256, init=normal(0.01), init_bias=0.1, name='conv5'), Activation(activation=relu, name='relu5'), MaxPooling((3, 3), (2, 2), name='pool5'), Dense(4096, init=normal(0.005), init_bias=0.1, name='fc6'), Activation(activation=relu, name='relu6'), Dropout(0.5, name='drop6'), Dense(4096, init=normal(0.005), init_bias=0.1, name='fc7'), Activation(activation=relu, name='relu7'), Dropout(0.5, name='drop7'), Dense(self.number_labels, init=normal(0.01), name='fc8') ])(mean_removed_features)
def create_alexnet(): # Input variables denoting the features and label data feature_var = input_variable((num_channels, image_height, image_width)) label_var = input_variable((num_classes)) # apply model to input # remove mean value input = minus(feature_var, constant(114), name='mean_removed_input') with default_options(activation=None, pad=True, bias=True): z = Sequential([ # we separate Convolution and ReLU to name the output for feature extraction (usually before ReLU) Convolution2D((11,11), 96, init=normal(0.01), pad=False, strides=(4,4), name='conv1'), Activation(activation=relu, name='relu1'), LocalResponseNormalization(1.0, 2, 0.0001, 0.75, name='norm1'), MaxPooling((3,3), (2,2), name='pool1'), Convolution2D((5,5), 192, init=normal(0.01), init_bias=0.1, name='conv2'), Activation(activation=relu, name='relu2'), LocalResponseNormalization(1.0, 2, 0.0001, 0.75, name='norm2'), MaxPooling((3,3), (2,2), name='pool2'), Convolution2D((3,3), 384, init=normal(0.01), name='conv3'), Activation(activation=relu, name='relu3'), Convolution2D((3,3), 384, init=normal(0.01), init_bias=0.1, name='conv4'), Activation(activation=relu, name='relu4'), Convolution2D((3,3), 256, init=normal(0.01), init_bias=0.1, name='conv5'), Activation(activation=relu, name='relu5'), MaxPooling((3,3), (2,2), name='pool5'), Dense(4096, init=normal(0.005), init_bias=0.1, name='fc6'), Activation(activation=relu, name='relu6'), Dropout(0.5, name='drop6'), Dense(4096, init=normal(0.005), init_bias=0.1, name='fc7'), Activation(activation=relu, name='relu7'), Dropout(0.5, name='drop7'), Dense(num_classes, init=normal(0.01), name='fc8') ])(input) # loss and metric ce = cross_entropy_with_softmax(z, label_var) pe = classification_error(z, label_var) pe5 = classification_error(z, label_var, topN=5) log_number_of_parameters(z) ; print() return { 'feature': feature_var, 'label': label_var, 'ce' : ce, 'pe' : pe, 'pe5': pe5, 'output': z }
def bn_inception_model(input, labelDim, bnTimeConst): # 224 x 224 x 3 conv1 = conv_bn_relu_layer(input, 64, (7, 7), (2, 2), True, bnTimeConst) # 112 x 112 x 64 pool1 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=True)(conv1) # 56 x 56 x 64 conv2a = conv_bn_relu_layer(pool1, 64, (1, 1), (1, 1), True, bnTimeConst) # 56 x 56 x 64 conv2b = conv_bn_relu_layer(conv2a, 192, (3, 3), (1, 1), True, bnTimeConst) # 56 x 56 x 192 pool2 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=True)(conv2b) # Inception Blocks # 28 x 28 x 192 inception3a = inception_block_with_avgpool(pool2, 64, 64, 64, 64, 96, 32, bnTimeConst) # 28 x 28 x 256 inception3b = inception_block_with_avgpool(inception3a, 64, 64, 96, 64, 96, 64, bnTimeConst) # 28 x 28 x 320 inception3c = inception_block_pass_through(inception3b, 0, 128, 160, 64, 96, 0, bnTimeConst) # 14 x 14 x 576 inception4a = inception_block_with_avgpool(inception3c, 224, 64, 96, 96, 128, 128, bnTimeConst) # 14 x 14 x 576 inception4b = inception_block_with_avgpool(inception4a, 192, 96, 128, 96, 128, 128, bnTimeConst) # 14 x 14 x 576 inception4c = inception_block_with_avgpool(inception4b, 160, 128, 160, 128, 160, 128, bnTimeConst) # 14 x 14 x 576 inception4d = inception_block_with_avgpool(inception4c, 96, 128, 192, 160, 192, 128, bnTimeConst) # 14 x 14 x 576 inception4e = inception_block_pass_through(inception4d, 0, 128, 192, 192, 256, 0, bnTimeConst) # 7 x 7 x 1024 inception5a = inception_block_with_avgpool(inception4e, 352, 192, 320, 160, 224, 128, bnTimeConst) # 7 x 7 x 1024 inception5b = inception_block_with_maxpool(inception5a, 352, 192, 320, 192, 224, 128, bnTimeConst) # Global Average # 7 x 7 x 1024 pool3 = AveragePooling(filter_shape=(7, 7))(inception5b) # 1 x 1 x 1024 z = Dense(labelDim, init=he_normal())(pool3) return z
def inceptionv1_cifar_model2(input, labelDim, bnTimeConst): # 32 x 32 x 3 conv1 = conv_bn_relu_layer(input, 32, (3, 3), (1, 1), True, bnTimeConst) # 32 x 32 x 32 conv2 = conv_bn_relu_layer(conv1, 32, (3, 3), (1, 1), True, bnTimeConst) # Inception Blocks # 32 x 32 x 64 inception3a = inception_block_with_maxpool(conv2, 32, 32, 32, 32, 48, 16, bnTimeConst) # 32 x 32 x 128 inception3b = inception_block_with_maxpool(inception3a, 32, 32, 32, 32, 48, 16, bnTimeConst) maxpool1 = MaxPooling((3, 3), strides=(2, 2), pad=True)(inception3b) # 16 x 16 x 128 inception4a = inception_block_with_maxpool(maxpool1, 96, 48, 64, 48, 64, 64, bnTimeConst) # 16 x 16 x 288 inception4b = inception_block_with_maxpool(inception4a, 96, 48, 64, 48, 64, 64, bnTimeConst) # 16 x 16 x 288 inception4c = inception_block_with_maxpool(inception4b, 96, 48, 64, 48, 64, 64, bnTimeConst) # 16 x 16 x 288 inception4d = inception_block_with_maxpool(inception4c, 96, 48, 64, 48, 64, 64, bnTimeConst) # 16 x 16 x 288 inception4e = inception_block_with_maxpool(inception4d, 96, 48, 64, 48, 64, 64, bnTimeConst) maxpool2 = MaxPooling((3, 3), strides=(2, 2), pad=True)(inception4e) # 8 x 8 x 288 inception5a = inception_block_with_maxpool(inception4e, 176, 96, 160, 96, 112, 64, bnTimeConst) # 8 x 8 x 512 inception5b = inception_block_with_maxpool(inception5a, 176, 96, 160, 96, 112, 64, bnTimeConst) # Global Average # 8 x 8 x 512 pool1 = AveragePooling(filter_shape=(8, 8))(inception5b) # 1 x 1 x 512 z = Dense(labelDim, init=he_normal())(pool1) return z
def create_convolutional_neural_network(input_vars, out_dims): convolutional_layer_1 = Convolution((5, 5), 32, strides=1, activation=cntk.ops.relu, pad=True, init=glorot_normal(), init_bias=0.1) pooling_layer_1 = MaxPooling((2, 2), strides=(2, 2), pad=True) convolutional_layer_2 = Convolution((5, 5), 64, strides=1, activation=cntk.ops.relu, pad=True, init=glorot_normal(), init_bias=0.1) pooling_layer_2 = MaxPooling((2, 2), strides=(2, 2), pad=True) convolutional_layer_3 = Convolution((5, 5), 128, strides=1, activation=cntk.ops.relu, pad=True, init=glorot_normal(), init_bias=0.1) pooling_layer_3 = MaxPooling((2, 2), strides=(2, 2), pad=True) fully_connected_layer = Dense(1024, activation=cntk.ops.relu, init=glorot_normal(), init_bias=0.1) output_layer = Dense(out_dims, activation=None, init=glorot_normal(), init_bias=0.1) model = Sequential([ convolutional_layer_1, pooling_layer_1, convolutional_layer_2, pooling_layer_2, #convolutional_layer_3, pooling_layer_3, fully_connected_layer, output_layer ])(input_vars) return model
def create_imagenet_model_bottleneck(input, num_stack_layers, num_classes, stride1x1, stride3x3): c_map = [64, 128, 256, 512, 1024, 2048] # conv1 and max pooling conv1 = conv_bn_relu(input, (7, 7), c_map[0], strides=(2, 2)) pool1 = MaxPooling((3, 3), strides=(2, 2), pad=True)(conv1) # conv2_x r2_1 = resnet_bottleneck_inc(pool1, c_map[2], c_map[0], (1, 1), (1, 1)) r2_2 = resnet_bottleneck_stack(r2_1, num_stack_layers[0], c_map[2], c_map[0]) # conv3_x r3_1 = resnet_bottleneck_inc(r2_2, c_map[3], c_map[1], stride1x1, stride3x3) r3_2 = resnet_bottleneck_stack(r3_1, num_stack_layers[1], c_map[3], c_map[1]) # conv4_x r4_1 = resnet_bottleneck_inc(r3_2, c_map[4], c_map[2], stride1x1, stride3x3) r4_2 = resnet_bottleneck_stack(r4_1, num_stack_layers[2], c_map[4], c_map[2]) # conv5_x r5_1 = resnet_bottleneck_inc(r4_2, c_map[5], c_map[3], stride1x1, stride3x3) r5_2 = resnet_bottleneck_stack(r5_1, num_stack_layers[3], c_map[5], c_map[3]) # Global average pooling and output pool = AveragePooling(filter_shape=(7, 7), name='final_avg_pooling')(r5_2) z = Dense(num_classes, init=C.normal(0.01))(pool) return z
def create_convolutional_neural_network(input_vars, out_dims, dropout_prob=0.0): convolutional_layer_1 = Convolution((5, 5), 32, strides=1, activation=cntk.ops.relu, pad=True, init=gaussian(), init_bias=0.1)(input_vars) pooling_layer_1 = MaxPooling((2, 2), strides=(2, 2), pad=True)(convolutional_layer_1) convolutional_layer_2 = Convolution((5, 5), 64, strides=1, activation=cntk.ops.relu, pad=True, init=gaussian(), init_bias=0.1)(pooling_layer_1) pooling_layer_2 = MaxPooling((2, 2), strides=(2, 2), pad=True)(convolutional_layer_2) convolutional_layer_3 = Convolution((5, 5), 128, strides=1, activation=cntk.ops.relu, pad=True, init=gaussian(), init_bias=0.1)(pooling_layer_2) pooling_layer_3 = MaxPooling((2, 2), strides=(2, 2), pad=True)(convolutional_layer_3) fully_connected_layer = Dense(1024, activation=cntk.ops.relu, init=gaussian(), init_bias=0.1)(pooling_layer_3) dropout_layer = Dropout(dropout_prob)(fully_connected_layer) output_layer = Dense(out_dims, activation=None, init=gaussian(), init_bias=0.1)(dropout_layer) return output_layer
def test_layers_conv_pool_unpool_deconv(): pass inC, inH, inW = 1,4,4 y = input((inC,inH, inW)) cMap = 1 zero_pad = True conv_init = 1 filter_shape = (2,2) pooling_strides = (2,2) dat = np.arange(0,16, dtype=np.float32).reshape(1,1,4,4) conv = Convolution(filter_shape, cMap, pad=zero_pad, init=conv_init,activation=None)(y) pool = MaxPooling(filter_shape, pooling_strides)(conv) unpool = MaxUnpooling(filter_shape, pooling_strides)(pool, conv) z = ConvolutionTranspose(filter_shape, cMap, init=conv_init, pad=zero_pad)(unpool) assert z.shape == y.shape res = z(dat) expected_res = np.asarray([[30, 64, 34], [76, 160, 84], [46, 96, 50]], np.float32) np.testing.assert_array_almost_equal(res[0][0][1:,1:], expected_res, decimal=6, err_msg="Wrong values in conv/pooling/unpooling/conv_transposed")
def test_depth_first_search_blocks(depth, prefix_count): from cntk.layers import Sequential, Convolution, MaxPooling, Dense from cntk.default_options import default_options def Blocked_Dense(dim, activation=None): dense = Dense(dim, activation=activation) @C.layers.BlockFunction('blocked_dense', 'blocked_dense') def func(x): return dense(x) return func with default_options(activation=C.relu): image_to_vec = Sequential([ Convolution((5, 5), 32, pad=True), MaxPooling((3, 3), strides=(2, 2)), Dense(10, activation=None), Blocked_Dense(10) ]) in1 = C.input_variable(shape=(3, 256, 256), name='image') img = image_to_vec(in1) found = C.logging.graph.depth_first_search(img, lambda x: True, depth=depth) found_str = [str(v) for v in found] assert len(found) == sum(prefix_count.values()) for prefix, count in prefix_count.items(): assert sum(f.startswith(prefix) for f in found_str) == count
def inception_block_with_maxpool(input, num1x1, num3x3r, num3x3, num3x3dblr, num3x3dbl, numPool, bnTimeConst): # 1x1 branch1x1 = conv_bn_relu_layer(input, num1x1, (1, 1), (1, 1), True, bnTimeConst) # 1x1 -> 3x3 branch3x3_reduce = conv_bn_relu_layer(input, num3x3r, (1, 1), (1, 1), True, bnTimeConst) branch3x3 = conv_bn_relu_layer(branch3x3_reduce, num3x3, (3, 3), (1, 1), True, bnTimeConst) # 1x1 -> 3x3 -> 3x3 branch3x3dbl_reduce = conv_bn_relu_layer(input, num3x3dblr, (1, 1), (1, 1), True, bnTimeConst) branch3x3dbl_conv = conv_bn_relu_layer(branch3x3dbl_reduce, num3x3dbl, (3, 3), (1, 1), True, bnTimeConst) branch3x3dbl = conv_bn_relu_layer(branch3x3dbl_conv, num3x3dbl, (3, 3), (1, 1), True, bnTimeConst) # max pooling -> 1x1 branchPool_maxpool = MaxPooling((3, 3), strides=(1, 1), pad=True)(input) branchPool = conv_bn_relu_layer(branchPool_maxpool, numPool, (1, 1), (1, 1), True, bnTimeConst) out = splice(branch1x1, branch3x3, branch3x3dbl, branchPool, axis=0) return out
def create_shallow_model(input, out_dims): convolutional_layer_1_1 = Convolution((7,7), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(input) convolutional_layer_1_2 = Convolution((25,25), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(convolutional_layer_1_1) pooling_layer_1 = MaxPooling((25,25), strides=(5,5))(convolutional_layer_1_2 ) convolutional_layer_2_1 = Convolution((3,3), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(pooling_layer_1) pooling_layer_2 = MaxPooling((2,2), strides=(2,2))(convolutional_layer_2_1) fully_connected_layer_1 = Dense(512, init=glorot_uniform())(pooling_layer_2) fully_connected_layer_2 = Dense(128, init=glorot_uniform())(fully_connected_layer_1) dropout_layer = Dropout(0.5)(fully_connected_layer_2) output_layer = Dense(out_dims, init=glorot_uniform(), activation=None)(dropout_layer) return output_layer
def create_vgg16(feature_var, num_classes, dropout=0.9): with default_options(activation=None, pad=True, bias=True): z = Sequential([ # we separate Convolution and ReLU to name the output for feature # extraction (usually before ReLU) For( range(2), lambda i: [ Convolution2D((3, 3), 64, name='conv1_{}'.format(i)), Activation(activation=relu, name='relu1_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool1'), For( range(2), lambda i: [ Convolution2D((3, 3), 128, name='conv2_{}'.format(i)), Activation(activation=relu, name='relu2_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool2'), For( range(3), lambda i: [ Convolution2D((3, 3), 256, name='conv3_{}'.format(i)), Activation(activation=relu, name='relu3_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool3'), For( range(3), lambda i: [ Convolution2D((3, 3), 512, name='conv4_{}'.format(i)), Activation(activation=relu, name='relu4_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool4'), For( range(3), lambda i: [ Convolution2D((3, 3), 512, name='conv5_{}'.format(i)), Activation(activation=relu, name='relu5_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool5'), Dense(4096, name='fc6'), Activation(activation=relu, name='relu6'), Dropout(dropout, name='drop6'), Dense(4096, name='fc7'), Activation(activation=relu, name='relu7'), Dropout(dropout, name='drop7'), Dense(num_classes, name='fc8') ])(feature_var) return z
def pre_block(input): c1 = conv_bn_relu(input, (3,3), 32, (1,1)) c2 = conv_bn_relu(c1, (3,3), 32, (1,1)) c3 = conv_bn_relu(c2, (3,3), 64, (1,1)) c4 = MaxPooling(filter_shape = (3,3), strides = (1,1), pad = True, name = 'pool')(c3) c5 = conv_bn_relu(c4, (1,1), 80, (1,1)) c6 = conv_bn_relu(c5, (3,3), 128, (1,1)) c7 = conv_bn_relu(c6, (3,3), 128, (1,1)) return c7
def create_symbol(): # Weight initialiser from uniform distribution # Activation (unless states) is None with cntk.layers.default_options(init=cntk.glorot_uniform(), activation=cntk.relu): x = Convolution2D(filter_shape=(3, 3), num_filters=50, pad=True)(features) x = Convolution2D(filter_shape=(3, 3), num_filters=50, pad=True)(x) x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x) x = Dropout(0.25)(x) x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x) x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x) x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x) x = Dropout(0.25)(x) x = Dense(512)(x) x = Dropout(0.5)(x) x = Dense(N_CLASSES, activation=None)(x) return x
def reduction_A(input, b1, c1, c2, c3, bnTimeConst): A1 = MaxPooling((3, 3), strides=(2, 2), pad=True)(input) B1 = conv_bn_relu_layer(input, b1, (3, 3), (2, 2), True, bnTimeConst) C1 = conv_bn_relu_layer(input, c1, (1, 1), (1, 1), True, bnTimeConst) C2 = conv_bn_relu_layer(C1, c2, (3, 3), (1, 1), True, bnTimeConst) C3 = conv_bn_relu_layer(C2, c3, (3, 3), (2, 2), True, bnTimeConst) out = splice(A1, B1, C3, axis=0) return out
def reduction_A(input, b1, c1, c2, c3): A1 = MaxPooling((3, 3), strides=(2, 2), pad=True)(input) B1 = conv_bn_relu(input, (3, 3), b1, (2, 2)) C1 = conv_bn_relu(input, (1, 1), c1, (1, 1)) C2 = conv_bn_relu(C1, (3, 3), c2, (1, 1)) C3 = conv_bn_relu(C2, (3, 3), c3, (2, 2)) out = splice(A1, B1, C3, axis=0) return out
def create_convnet_cifar10_model(num_classes): with default_options(activation=relu, pad=True): return Sequential([ For( range(2), lambda: [ Convolution2D((3, 3), 64), Convolution2D((3, 3), 64), MaxPooling((3, 3), strides=2) ]), For(range(2), lambda i: [Dense([256, 128][i]), Dropout(0.5)]), Dense(num_classes, activation=None) ])
def create_model(feature_dimensions, classes): with default_options(activation=relu, init=glorot_uniform()): model = Sequential([ For( range(3), lambda i: [ Convolution((5, 5), [32, 32, 64][i], pad=True), BatchNormalization(map_rank=1), MaxPooling((3, 3), strides=(2, 2)) ]), Dense(64), BatchNormalization(map_rank=1), Dense(len(classes), activation=None) ]) return model(feature_dimensions)
def create_vgg9_model(input, num_classes): with default_options(activation=relu): model = Sequential([ LayerStack(3, lambda i: [ Convolution((3,3), [64,96,128][i], init=glorot_uniform(), pad=True), Convolution((3,3), [64,96,128][i], init=glorot_uniform(), pad=True), MaxPooling((3,3), strides=(2,2)) ]), LayerStack(2, lambda : [ Dense(1024, init=glorot_uniform()) ]), Dense(num_classes, init=glorot_uniform(), activation=None) ]) return model(input)
def inception_block_2(input, num3x3, num3x3dbl, bnTimeConst): # 3x3 Convolution branch3x3 = conv_bn_relu_layer(input, num3x3, (3,3), (2,2), True, bnTimeConst) # Double 3x3 Convolution branch3x3dbl_1 = conv_bn_relu_layer(input, num3x3dbl[0], (1,1), (1,1), True, bnTimeConst) branch3x3dbl_2 = conv_bn_relu_layer(branch3x3dbl_1, num3x3dbl[1], (3,3), (1,1), True, bnTimeConst) branch3x3dbl = conv_bn_relu_layer(branch3x3dbl_2, num3x3dbl[2], (3,3), (2,2), True, bnTimeConst) # Max Pooling branchPool = MaxPooling((3,3), strides=(2,2), pad=True)(input) out = splice(branch3x3, branch3x3dbl, branchPool, axis=0) return out
def reduction_B(input, b1, b2, c1, c2, d1, d2, d3): A1 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=True)(input) B1 = conv_bn_relu(input, (1, 1), b1, (1, 1)) B2 = conv_bn_relu(B1, (3, 3), b2, (2, 2)) C1 = conv_bn_relu(input, (1, 1), c1, (1, 1)) C2 = conv_bn_relu(C1, (3, 3), c2, (2, 2)) D1 = conv_bn_relu(input, (1, 1), d1, (1, 1)) D2 = conv_bn_relu(D1, (3, 3), d2, (1, 1)) D3 = conv_bn_relu(D2, (3, 3), d3, (2, 2)) out = splice(A1, B2, C2, D3, axis=0) return out
def inception_block_pass_through(input, num1x1, num3x3r, num3x3, num3x3dblr, num3x3dbl, numPool, bnTimeConst): # 3x3 Convolution branch3x3_reduce = conv_bn_relu_layer(input, num3x3r, (1,1), (1,1), True, bnTimeConst) branch3x3 = conv_bn_relu_layer(branch3x3_reduce, num3x3, (3,3), (2,2), True, bnTimeConst) # Double 3x3 Convolution branch3x3dbl_reduce = conv_bn_relu_layer(input, num3x3dblr, (1,1), (1,1), True, bnTimeConst) branch3x3dbl_conv = conv_bn_relu_layer(branch3x3dbl_reduce, num3x3dbl, (3,3), (1,1), True, bnTimeConst) branch3x3dbl = conv_bn_relu_layer(branch3x3dbl_conv, num3x3dbl, (3,3), (2,2), True, bnTimeConst) # Max Pooling branchPool = MaxPooling((3,3), strides=(2,2), pad=True)(input) out = splice(branch3x3, branch3x3dbl, branchPool, axis=0) return out
def inception_block_4(input, num3x3, num7x7_3x3, bnTimeConst): # 3x3 Convolution branch3x3_1 = conv_bn_relu_layer(input, num3x3[0], (1,1), (1,1), True, bnTimeConst) branch3x3 = conv_bn_relu_layer(branch3x3_1, num3x3[1], (3,3), (2,2), False, bnTimeConst) # 7x7 3x3 Convolution branch7x7_3x3_1 = conv_bn_relu_layer(input, num7x7_3x3[0], (1,1), (1,1), True, bnTimeConst) branch7x7_3x3_2 = conv_bn_relu_layer(branch7x7_3x3_1, num7x7_3x3[1], (1,7), (1,1), True, bnTimeConst) branch7x7_3x3_3 = conv_bn_relu_layer(branch7x7_3x3_2, num7x7_3x3[2], (7,1), (1,1), True, bnTimeConst) branch7x7_3x3 = conv_bn_relu_layer(branch7x7_3x3_3, num7x7_3x3[3], (3,3), (2,2), False, bnTimeConst) # Max Pooling branchPool = MaxPooling((3,3), strides=(2,2), pad=False)(input) out = splice(branch3x3, branch7x7_3x3, branchPool, axis=0) return out
def create_advanced_model(input, out_dims): with default_options(activation=relu): model = Sequential([ For(range(2), lambda i: [ # lambda with one parameter Convolution((3,3), [32,64][i], pad=True), # depth depends on i Convolution((5,5), [32,64][i], pad=True), Convolution((9,9), [32,64][i], pad=True), MaxPooling((3,3), strides=(2,2)) ]), For(range(2), lambda : [ # lambda without parameter Dense(512), Dropout(0.5) ]), Dense(out_dims, activation=None) ]) output_layer=model(input) return output_layer
def test_depth_first_search_blocks(depth, prefix_count): from cntk.layers import Sequential, Convolution, MaxPooling, Dense from cntk.default_options import default_options with default_options(activation=relu): image_to_vec = Sequential([ Convolution((5, 5), 32, pad=True), MaxPooling((3, 3), strides=(2, 2)), Dense(10, activation=None) ]) in1 = input(shape=(3, 256, 256), name='image') img = image_to_vec(in1) found = depth_first_search(img, lambda x: True, depth=depth) found_str = [str(v) for v in found] assert len(found) == sum(prefix_count.values()) for prefix, count in prefix_count.items(): assert sum(f.startswith(prefix) for f in found_str) == count
def create_resnet34(input, num_classes): c_map = [64, 128, 256, 512] num_layers = [3, 3, 5, 2] conv = conv_bn_relu(input, (7, 7), c_map[0], (2, 2)) maxPool = MaxPooling((3, 3), (2, 2), pad=True)(conv) r1 = resnet_basic_stack(maxPool, num_layers[0], c_map[0]) r2_1 = resnet_basic_inc(r1, c_map[1]) r2_2 = resnet_basic_stack(r2_1, num_layers[1], c_map[1]) r3_1 = resnet_basic_inc(r2_2, c_map[2]) r3_2 = resnet_basic_stack(r3_1, num_layers[2], c_map[2]) r4_1 = resnet_basic_inc(r3_2, c_map[3]) r4_2 = resnet_basic_stack(r4_1, num_layers[3], c_map[3]) # Global average pooling and output pool = AveragePooling(filter_shape=(7, 7))(r4_2) z = Dense(num_classes)(pool) return z