def create_model(input_dim): row = sequence.input_variable(shape=input_dim) col = sequence.input_variable(shape=input_dim) rowh = Sequential([Embedding(opt.embed), Stabilizer(), Dropout(opt.dropout)])(row) colh = Sequential([Embedding(opt.embed), Stabilizer(), Dropout(opt.dropout)])(col) x = C.splice(rowh, colh, axis=-1) x = lightlstm(opt.embed, opt.nhid)(x) x = For(range(opt.layer-1), lambda: lightlstm(opt.nhid, opt.nhid))(x) rowh = C.slice(x, -1, opt.nhid * 0, opt.nhid * 1) colh = C.slice(x, -1, opt.nhid * 1, opt.nhid * 2) row_predict = Sequential([Dropout(opt.dropout), Dense(input_dim)])(rowh) col_predict = Sequential([Dropout(opt.dropout), Dense(input_dim)])(colh) # variable : row label and col label row_label = sequence.input_variable(shape=input_dim) col_label = sequence.input_variable(shape=input_dim) model = C.combine([row_predict, col_predict]) return {'row': row, 'col': col, 'row_label': row_label, 'col_label': col_label, 'model': model}
def __call__(self, num_classes=10, act_type=relu, mdl_conv1a_nf=40, mdl_conv1b_nf=60, mdl_conv2a_nf=50, mdl_conv2b_nf=75, mdl_fc1_nh=75, mdl_drop2a_p=0.033, mdl_drop2b_p=0.097, mdl_drop3_p=0.412, **kwargs): input_var = input_variable((1, self.img_h, self.img_w), np.float32) label_var = input_variable((self.n_dim), np.float32) conv1a = Convolution(filter_shape=(3, 3), num_filters=int(mdl_conv1a_nf), activation=act_type, init=glorot_uniform(), pad=True, name='conv1a')(input_var) conv1b = Convolution(filter_shape=(3, 3), num_filters=int(mdl_conv1b_nf), activation=act_type, init=glorot_uniform(), pad=True, name='conv1b')(conv1a) pool1 = MaxPooling(filter_shape=(2, 2), strides=(2, 2), name='pool1')(conv1b) conv2a = Convolution(filter_shape=(3, 3), num_filters=int(mdl_conv2a_nf), activation=act_type, init=glorot_uniform(), pad=True, name='conv2a')(pool1) drop2a = Dropout(prob=mdl_drop2a_p, name="drop2a")(conv2a) conv2b = Convolution(filter_shape=(3, 3), num_filters=int(mdl_conv2b_nf), activation=act_type, init=glorot_uniform(), pad=True, name='conv2b')(drop2a) drop2b = Dropout(prob=mdl_drop2a_p, name="drop2a")(conv2b) pool2 = MaxPooling(filter_shape=(2, 2), strides=(2, 2), name='pool2')(drop2b) fc1 = Dense(shape=int(mdl_fc1_nh), init=glorot_uniform(), activation=act_type, name='fc1')(pool2) drop3 = Dropout(prob=mdl_drop3_p, name="drop3")(fc1) #fc2 = Dense(shape=num_classes, init=glorot_uniform(), activation=softmax, name='fc2')(drop3) fc2 = Dense(shape=num_classes, init=glorot_uniform(), activation=None, name='fc2')(drop3) return input_var, label_var, fc2
def create_model(self): mean_removed_features = minus(self.input, constant(114), name='mean_removed_input') with default_options(activation=None, pad=True, bias=True): self.model = Sequential([ Convolution2D((11, 11), 96, init=normal(0.01), pad=False, name='conv1'), Activation(activation=relu, name='relu1'), self.__local_response_normalization(1.0, 2, 0.0001, 0.75, name='norm1'), MaxPooling((3, 3), (2, 2), name='pool1'), Convolution2D((5, 5), 192, init=normal(0.01), init_bias=0.1, name='conv2'), Activation(activation=relu, name='relu2'), self.__local_response_normalization(1.0, 2, 0.0001, 0.75, name='norm2'), MaxPooling((3, 3), (2, 2), name='pool2'), Convolution2D((3, 3), 384, init=normal(0.01), name='conv3'), Activation(activation=relu, name='relu3'), Convolution2D((3, 3), 384, init=normal(0.01), init_bias=0.1, name='conv4'), Activation(activation=relu, name='relu4'), Convolution2D((3, 3), 256, init=normal(0.01), init_bias=0.1, name='conv5'), Activation(activation=relu, name='relu5'), MaxPooling((3, 3), (2, 2), name='pool5'), Dense(4096, init=normal(0.005), init_bias=0.1, name='fc6'), Activation(activation=relu, name='relu6'), Dropout(0.5, name='drop6'), Dense(4096, init=normal(0.005), init_bias=0.1, name='fc7'), Activation(activation=relu, name='relu7'), Dropout(0.5, name='drop7'), Dense(self.number_labels, init=normal(0.01), name='fc8') ])(mean_removed_features)
def create_alexnet(): # Input variables denoting the features and label data feature_var = input_variable((num_channels, image_height, image_width)) label_var = input_variable((num_classes)) # apply model to input # remove mean value input = minus(feature_var, constant(114), name='mean_removed_input') with default_options(activation=None, pad=True, bias=True): z = Sequential([ # we separate Convolution and ReLU to name the output for feature extraction (usually before ReLU) Convolution2D((11,11), 96, init=normal(0.01), pad=False, strides=(4,4), name='conv1'), Activation(activation=relu, name='relu1'), LocalResponseNormalization(1.0, 2, 0.0001, 0.75, name='norm1'), MaxPooling((3,3), (2,2), name='pool1'), Convolution2D((5,5), 192, init=normal(0.01), init_bias=0.1, name='conv2'), Activation(activation=relu, name='relu2'), LocalResponseNormalization(1.0, 2, 0.0001, 0.75, name='norm2'), MaxPooling((3,3), (2,2), name='pool2'), Convolution2D((3,3), 384, init=normal(0.01), name='conv3'), Activation(activation=relu, name='relu3'), Convolution2D((3,3), 384, init=normal(0.01), init_bias=0.1, name='conv4'), Activation(activation=relu, name='relu4'), Convolution2D((3,3), 256, init=normal(0.01), init_bias=0.1, name='conv5'), Activation(activation=relu, name='relu5'), MaxPooling((3,3), (2,2), name='pool5'), Dense(4096, init=normal(0.005), init_bias=0.1, name='fc6'), Activation(activation=relu, name='relu6'), Dropout(0.5, name='drop6'), Dense(4096, init=normal(0.005), init_bias=0.1, name='fc7'), Activation(activation=relu, name='relu7'), Dropout(0.5, name='drop7'), Dense(num_classes, init=normal(0.01), name='fc8') ])(input) # loss and metric ce = cross_entropy_with_softmax(z, label_var) pe = classification_error(z, label_var) pe5 = classification_error(z, label_var, topN=5) log_number_of_parameters(z) ; print() return { 'feature': feature_var, 'label': label_var, 'ce' : ce, 'pe' : pe, 'pe5': pe5, 'output': z }
def create_model(base_model_file, input_features, num_classes, dropout_rate=0.5, freeze_weights=False): # Load the pretrained classification net and find nodes base_model = load_model(base_model_file) feature_node = find_by_name(base_model, 'features') beforePooling_node = find_by_name(base_model, "z.x.x.r") #graph.plot(base_model, filename="base_model.pdf") # Write graph visualization # Clone model until right before the pooling layer, ie. until including z.x.x.r modelCloned = combine([beforePooling_node.owner]).clone( CloneMethod.freeze if freeze_weights else CloneMethod.clone, {feature_node: placeholder(name='features')}) # Center the input around zero and set model input. # Do this early, to avoid CNTK bug with wrongly estimated layer shapes feat_norm = input_features - constant(114) model = modelCloned(feat_norm) # Pool over all spatial dimensions and add dropout layer avgPool = GlobalAveragePooling(name="poolingLayer")(model) if dropout_rate > 0: avgPoolDrop = Dropout(dropout_rate)(avgPool) else: avgPoolDrop = avgPool # Add new dense layer for class prediction finalModel = Dense(num_classes, activation=None, name="prediction")(avgPoolDrop) return finalModel
def create_basic_model(input, out_dims): convolutional_layer_1 = Convolution((5, 5), 16, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(input) pooling_layer_1 = MaxPooling((2, 2), strides=(1, 1))(convolutional_layer_1) convolutional_layer_2 = Convolution((5, 5), 16, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(pooling_layer_1) pooling_layer_2 = MaxPooling((3, 3), strides=(2, 2))(convolutional_layer_2) # convolutional_layer_3 = Convolution((9, 9), 16, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(pooling_layer_2) pooling_layer_3 = MaxPooling((3, 3), strides=(2, 2))(convolutional_layer_3) fully_connected_layer = Dense(256, init=glorot_uniform())(pooling_layer_3) dropout_layer = Dropout(0.5)(fully_connected_layer) output_layer = Dense(out_dims, init=glorot_uniform(), activation=None)(dropout_layer) return output_layer
def create_vgg16(feature_var, num_classes, dropout=0.9): with default_options(activation=None, pad=True, bias=True): z = Sequential([ # we separate Convolution and ReLU to name the output for feature # extraction (usually before ReLU) For( range(2), lambda i: [ Convolution2D((3, 3), 64, name='conv1_{}'.format(i)), Activation(activation=relu, name='relu1_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool1'), For( range(2), lambda i: [ Convolution2D((3, 3), 128, name='conv2_{}'.format(i)), Activation(activation=relu, name='relu2_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool2'), For( range(3), lambda i: [ Convolution2D((3, 3), 256, name='conv3_{}'.format(i)), Activation(activation=relu, name='relu3_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool3'), For( range(3), lambda i: [ Convolution2D((3, 3), 512, name='conv4_{}'.format(i)), Activation(activation=relu, name='relu4_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool4'), For( range(3), lambda i: [ Convolution2D((3, 3), 512, name='conv5_{}'.format(i)), Activation(activation=relu, name='relu5_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool5'), Dense(4096, name='fc6'), Activation(activation=relu, name='relu6'), Dropout(dropout, name='drop6'), Dense(4096, name='fc7'), Activation(activation=relu, name='relu7'), Dropout(dropout, name='drop7'), Dense(num_classes, name='fc8') ])(feature_var) return z
def pix2pix_generator(h): with C.layers.default_options(init=C.normal(0.02), pad=True, bias=False, map_rank=1, use_cntk_engine=True): h_enc1 = C.leaky_relu(Convolution2D((4, 4), 64, strides=2, bias=True)(h), alpha=0.2) h_enc2 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 128, strides=2)(h_enc1)), alpha=0.2) h_enc3 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 256, strides=2)(h_enc2)), alpha=0.2) h_enc4 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=2)(h_enc3)), alpha=0.2) h_enc5 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=2)(h_enc4)), alpha=0.2) h_enc6 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=2)(h_enc5)), alpha=0.2) h_enc7 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=1)(h_enc6)), alpha=0.2) h_enc8 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=1)(h_enc7)), alpha=0.2) h_dec8 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D( (4, 4), 512, strides=1, pad=True, output_shape=(img_height // 64, img_width // 64))(h_enc8))) h_dec8 = C.splice(h_dec8, h_enc8, axis=0) h_dec8 = C.relu(h_dec8) h_dec7 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D( (4, 4), 512, strides=1, pad=True, output_shape=(img_height // 64, img_width // 64))(h_dec8))) h_dec7 = C.splice(h_dec7, h_enc7, axis=0) h_dec7 = C.relu(h_dec7) h_dec6 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D( (4, 4), 512, strides=1, pad=True, output_shape=(img_height // 64, img_width // 64))(h_dec7))) h_dec6 = C.splice(h_dec6, h_enc6, axis=0) h_dec6 = C.relu(h_dec6) h_dec5 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D( (4, 4), 512, strides=2, pad=True, output_shape=(img_height // 32, img_width // 32))(h_dec6))) h_dec5 = C.splice(h_dec5, h_enc5, axis=0) h_dec5 = C.relu(h_dec5) h_dec4 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D( (4, 4), 512, strides=2, pad=True, output_shape=(img_height // 16, img_width // 16))(h_dec5))) h_dec4 = C.splice(h_dec4, h_enc4, axis=0) h_dec4 = C.relu(h_dec4) h_dec3 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D( (4, 4), 256, strides=2, pad=True, output_shape=(img_height // 8, img_width // 8))(h_dec4))) h_dec3 = C.splice(h_dec3, h_enc3, axis=0) h_dec3 = C.relu(h_dec3) h_dec2 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D( (4, 4), 128, strides=2, pad=True, output_shape=(img_height // 4, img_width // 4))(h_dec3))) h_dec2 = C.splice(h_dec2, h_enc2, axis=0) h_dec2 = C.relu(h_dec2) h_dec1 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D( (4, 4), 64, strides=2, pad=True, output_shape=(img_height // 2, img_width // 2))(h_dec2))) h_dec1 = C.splice(h_dec1, h_enc1, axis=0) h_dec1 = C.relu(h_dec1) h = ConvolutionTranspose2D((4, 4), 3, activation=C.tanh, strides=2, pad=True, bias=True, output_shape=(img_height, img_width))(h_dec1) return h
def resnet_drop(input, num_filters): c1 = conv_bn_relu(input, (3,3), num_filters) c2 = conv_bn(c1, (3,3), num_filters, bn_init_scale = 1) b1 = Dropout(0.5)(input) p = b1 + c2 return relu(p)
def create_symbol(): # Weight initialiser from uniform distribution # Activation (unless states) is None with cntk.layers.default_options(init=cntk.glorot_uniform(), activation=cntk.relu): x = Convolution2D(filter_shape=(3, 3), num_filters=50, pad=True)(features) x = Convolution2D(filter_shape=(3, 3), num_filters=50, pad=True)(x) x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x) x = Dropout(0.25)(x) x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x) x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x) x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x) x = Dropout(0.25)(x) x = Dense(512)(x) x = Dropout(0.5)(x) x = Dense(N_CLASSES, activation=None)(x) return x
def resnet_model(layer_input): layer1 = resnet_basic_stack(layer_input, 2, (3, 3), 19, (1, 1), prefix='conv1') layer1 = MaxPooling((3, 3), strides=(2, 2), name='pool1')(layer1) layer1 = Dropout(0.3, name='drop1')(layer1) layer2 = resnet_basic_inc(layer1, (3, 3), 8, (2, 2), prefix='conv21') layer2 = resnet_basic_stack(layer2, 1, (3, 3), 8, (1, 1), prefix='conv22') layer2 = Dropout(0.3, name='drop2')(layer2) layer3 = resnet_basic_inc(layer2, (3, 3), 16, (2, 2), prefix='conv31') layer3 = resnet_basic_stack(layer3, 1, (3, 3), 16, (1, 1), prefix='conv32') layer3 = Dropout(0.3, name='drop3')(layer3) layer4 = resnet_basic_inc(layer3, (3, 3), 32, (2, 2), prefix='conv41') layer4 = resnet_basic_stack(layer4, 1, (3, 3), 32, (1, 1), prefix='conv42') layer4 = Dropout(0.3, name='drop4')(layer4) return layer4
def create_convnet_cifar10_model(num_classes): with default_options(activation=relu, pad=True): return Sequential([ For( range(2), lambda: [ Convolution2D((3, 3), 64), Convolution2D((3, 3), 64), MaxPooling((3, 3), strides=2) ]), For(range(2), lambda i: [Dense([256, 128][i]), Dropout(0.5)]), Dense(num_classes, activation=None) ])
def test_layers_dropout(): dat = np.array([[1., 1., 1., 1.]], dtype=np.float32) y = input(4) p = Dense(1, activation=None, name='foo')(y) z = Dropout(0.75, name='bar')(p) res = z(y).eval({y: dat}) expected_res = np.sum(p.foo.W.value) np.testing.assert_array_almost_equal(res, expected_res, decimal=7, \ err_msg="Error in dropout computation") z = Dropout(keep_prob=0.25, name='bar')(p) res = z(y).eval({y: dat}) np.testing.assert_array_almost_equal(res, expected_res, decimal=7, \ err_msg="Error in dropout computation with keep_prob") with pytest.raises(ValueError): z = Dropout(keep_prob=-1.5, name='bar')(p) with pytest.raises(ValueError): z = Dropout(1.5, name='bar')(p)
def create_convolutional_neural_network(input_vars, out_dims, dropout_prob=0.0): convolutional_layer_1 = Convolution((5, 5), 32, strides=1, activation=cntk.ops.relu, pad=True, init=gaussian(), init_bias=0.1)(input_vars) pooling_layer_1 = MaxPooling((2, 2), strides=(2, 2), pad=True)(convolutional_layer_1) convolutional_layer_2 = Convolution((5, 5), 64, strides=1, activation=cntk.ops.relu, pad=True, init=gaussian(), init_bias=0.1)(pooling_layer_1) pooling_layer_2 = MaxPooling((2, 2), strides=(2, 2), pad=True)(convolutional_layer_2) convolutional_layer_3 = Convolution((5, 5), 128, strides=1, activation=cntk.ops.relu, pad=True, init=gaussian(), init_bias=0.1)(pooling_layer_2) pooling_layer_3 = MaxPooling((2, 2), strides=(2, 2), pad=True)(convolutional_layer_3) fully_connected_layer = Dense(1024, activation=cntk.ops.relu, init=gaussian(), init_bias=0.1)(pooling_layer_3) dropout_layer = Dropout(dropout_prob)(fully_connected_layer) output_layer = Dense(out_dims, activation=None, init=gaussian(), init_bias=0.1)(dropout_layer) return output_layer
def create_shallow_model(input, out_dims): convolutional_layer_1_1 = Convolution((7,7), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(input) convolutional_layer_1_2 = Convolution((25,25), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(convolutional_layer_1_1) pooling_layer_1 = MaxPooling((25,25), strides=(5,5))(convolutional_layer_1_2 ) convolutional_layer_2_1 = Convolution((3,3), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(pooling_layer_1) pooling_layer_2 = MaxPooling((2,2), strides=(2,2))(convolutional_layer_2_1) fully_connected_layer_1 = Dense(512, init=glorot_uniform())(pooling_layer_2) fully_connected_layer_2 = Dense(128, init=glorot_uniform())(fully_connected_layer_1) dropout_layer = Dropout(0.5)(fully_connected_layer_2) output_layer = Dense(out_dims, init=glorot_uniform(), activation=None)(dropout_layer) return output_layer
def create_advanced_model(input, out_dims): with default_options(activation=relu): model = Sequential([ For(range(2), lambda i: [ # lambda with one parameter Convolution((3,3), [32,64][i], pad=True), # depth depends on i Convolution((5,5), [32,64][i], pad=True), Convolution((9,9), [32,64][i], pad=True), MaxPooling((3,3), strides=(2,2)) ]), For(range(2), lambda : [ # lambda without parameter Dense(512), Dropout(0.5) ]), Dense(out_dims, activation=None) ]) output_layer=model(input) return output_layer
def create_model(base_model_file, input_features, params): num_classes = params['num_classes'] dropout_rate = params['dropout_rate'] freeze_weights = params['freeze_weights'] # Load the pretrained classification net and find nodes base_model = load_model(base_model_file) log = logging.getLogger("neuralnets1.utils.create_model") log.info('Loaded base model - %s with layers:' % base_model_file) node_outputs = get_node_outputs(base_model) [log.info('%s , %s' % (layer.name, layer.shape)) for layer in node_outputs] graph.plot(base_model, filename="base_model.pdf") # Write graph visualization feature_node = find_by_name(base_model, 'features') beforePooling_node = find_by_name(base_model, "z.x.x.r") # Clone model until right before the pooling layer, ie. until including z.x.x.r modelCloned = combine([beforePooling_node.owner]).clone( CloneMethod.freeze if freeze_weights else CloneMethod.clone, {feature_node: placeholder(name='features')}) # Center the input around zero and set model input. # Do this early, to avoid CNTK bug with wrongly estimated layer shapes feat_norm = input_features - constant(114) model = modelCloned(feat_norm) # Add pool layer avgPool = GlobalAveragePooling(name="poolingLayer")( model) # assign name to the layer and add to the model # Add drop out layer if dropout_rate > 0: avgPoolDrop = Dropout(dropout_rate)( avgPool ) # add drop out layer with specified drop out rate and add it to the model else: avgPoolDrop = avgPool # Add new dense layer for class prediction finalModel = Dense(num_classes, activation=None, name="Dense")(avgPoolDrop) return finalModel
def inception_v3_cifar_model(input, labelDim, bnTimeConst): # 32 x 32 x 3 conv1 = conv_bn_relu_layer(input, 32, (3, 3), (1, 1), True, bnTimeConst) # 32 x 32 x 32 conv2 = conv_bn_relu_layer(conv1, 32, (3, 3), (1, 1), True, bnTimeConst) # 32 x 32 x 32 conv3 = conv_bn_relu_layer(conv2, 64, (3, 3), (1, 1), True, bnTimeConst) # 32 x 32 x 64 conv4 = conv_bn_relu_layer(conv3, 80, (1, 1), (1, 1), True, bnTimeConst) # 32 x 32 x 80 conv5 = conv_bn_relu_layer(conv4, 128, (3, 3), (1, 1), True, bnTimeConst) # 32 x 32 x 128 pool1 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=True)(conv5) # # Inception Blocks # 16 x 16 x 128 mixed1 = inception_block_1(pool1, 32, [32, 48], [48, 64, 64], 32, bnTimeConst) # 16 x 16 x 160 mixed2 = inception_block_1(mixed1, 32, [32, 48], [48, 64, 64], 64, bnTimeConst) # 16 x 16 x 160 mixed3 = inception_block_1(mixed2, 32, [32, 48], [48, 64, 64], 64, bnTimeConst) # 16 x 16 x 160 #mixed4 = inception_block_2(mixed3, 32, [48, 64, 64], bnTimeConst) mixed4 = inception_block_pass_through(mixed3, 0, 64, 80, 32, 48, 0, bnTimeConst) # 8 x 8 x 256 mixed5 = inception_block_3(mixed4, 192, [48, 64, 64], [128, 128, 128, 128, 192], 192, bnTimeConst) # 8 x 8 x mixed6 = inception_block_3(mixed5, 192, [160, 160, 192], [160, 160, 160, 160, 192], 192, bnTimeConst) # 8 x 8 x 768 mixed7 = inception_block_3(mixed6, 192, [160, 160, 192], [160, 160, 160, 160, 192], 192, bnTimeConst) # 8 x 8 x 768 mixed8 = inception_block_3(mixed7, 192, [192, 192, 192], [192, 192, 192, 192, 192], 192, bnTimeConst) # 8 x 8 x 768 mixed9 = inception_block_3(mixed8, 192, [192, 192, 192], [192, 192, 192, 192, 192], 192, bnTimeConst) # 8 x 8 x 1280 mixed10 = inception_block_5(mixed9, 320, [384, 384, 384], [448, 384, 384, 384], 192, bnTimeConst) # 8 x 8 x 2048 mixed11 = inception_block_5(mixed10, 320, [384, 384, 384], [448, 384, 384, 384], 192, bnTimeConst) # 8 x 8 x 2048 # # Prediction # pool3 = AveragePooling(filter_shape=(8, 8))(mixed11) # 1 x 1 x 2048 drop = Dropout(dropout_rate=0.2)(pool3) # 1 x 1 x 2048 z = Dense(labelDim, init=he_normal())(drop) # # Auxiliary # # 8 x 8 x 768 auxPool = AveragePooling(filter_shape=(3, 3), strides=(1, 1), pad=True)(mixed8) # 5 x 5 x 768 auxConv1 = conv_bn_relu_layer(auxPool, 128, (1, 1), (1, 1), True, bnTimeConst) # 5 x 5 x 128 auxConv2 = conv_bn_relu_layer(auxConv1, 256, (3, 3), (1, 1), True, bnTimeConst) # 1 x 1 x 768 aux = Dense(labelDim, init=he_normal())(auxConv2) return {'z': z, 'aux': aux}
input_folder_path = "src/ml/data/autcar_training" output_folder_path = "src/ml/data/autcar_training_balanced" image_width = 224 image_height = 168 trainer = Trainer(deeplearning_framework="cntk", image_height=image_height, image_width=image_width) trainer.create_balanced_dataset(input_folder_path, output_folder_path=output_folder_path) model = Sequential([ Convolution2D(filter_shape=(5,5), num_filters=32, strides=(1,1), pad=True, name="first_conv"), BatchNormalization(map_rank=1), Activation(relu), MaxPooling(filter_shape=(3,3), strides=(2,2), name="first_max"), Convolution2D(filter_shape=(3,3), num_filters=48, strides=(1,1), pad=True, name="second_conv"), BatchNormalization(map_rank=1), Activation(relu), MaxPooling(filter_shape=(3,3), strides=(2,2), name="second_max"), Convolution2D(filter_shape=(3,3), num_filters=64, strides=(1,1), pad=True, name="third_conv"), BatchNormalization(map_rank=1), Activation(relu), MaxPooling(filter_shape=(3,3), strides=(2,2), name="third_max"), Convolution2D(filter_shape=(5,5), num_filters=32, strides=(1,1), pad=True, name="fourth_conv"), BatchNormalization(map_rank=1), Activation(relu), Dense(100, activation=relu), Dropout(0.1), Dense(12, activation=softmax) ]) trainer.train(output_folder_path, model, epochs=4, output_model_path="driver_cntk.onnx") trainer.test("driver_cntk.onnx", output_folder_path+"/test_map.txt")
def create_vgg19(): # Input variables denoting the features and label data feature_var = input((num_channels, image_height, image_width)) label_var = input((num_classes)) # apply model to input # remove mean value input = minus(feature_var, constant([[[104]], [[117]], [[124]]]), name='mean_removed_input') with default_options(activation=None, pad=True, bias=True): z = Sequential([ # we separate Convolution and ReLU to name the output for feature extraction (usually before ReLU) For( range(2), lambda i: [ Convolution2D((3, 3), 64, name='conv1_{}'.format(i)), Activation(activation=relu, name='relu1_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool1'), For( range(2), lambda i: [ Convolution2D((3, 3), 128, name='conv2_{}'.format(i)), Activation(activation=relu, name='relu2_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool2'), For( range(4), lambda i: [ Convolution2D((3, 3), 256, name='conv3_{}'.format(i)), Activation(activation=relu, name='relu3_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool3'), For( range(4), lambda i: [ Convolution2D((3, 3), 512, name='conv4_{}'.format(i)), Activation(activation=relu, name='relu4_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool4'), For( range(4), lambda i: [ Convolution2D((3, 3), 512, name='conv5_{}'.format(i)), Activation(activation=relu, name='relu5_{}'.format(i)), ]), MaxPooling((2, 2), (2, 2), name='pool5'), Dense(4096, name='fc6'), Activation(activation=relu, name='relu6'), Dropout(0.5, name='drop6'), Dense(4096, name='fc7'), Activation(activation=relu, name='relu7'), Dropout(0.5, name='drop7'), Dense(num_classes, name='fc8') ])(input) # loss and metric ce = cross_entropy_with_softmax(z, label_var) pe = classification_error(z, label_var) pe5 = classification_error(z, label_var, topN=5) log_number_of_parameters(z) print() return { 'feature': feature_var, 'label': label_var, 'ce': ce, 'pe': pe, 'pe5': pe5, 'output': z }
def VGG16(num_classes: int): """ for image classification """ layer1 = Conv2DMaxPool(2, conv_filter_shape=(3, 3), pool_filter_shape=(2, 2), conv_num_filters=64, activation=C.relu, conv_pad=True, pool_strides=(2, 2), name_prefix='layer1') layer2 = Conv2DMaxPool(2, conv_filter_shape=(3, 3), pool_filter_shape=(2, 2), conv_num_filters=128, activation=C.relu, conv_pad=True, pool_strides=(2, 2), name_prefix='layer2') layer3 = Conv2DMaxPool(2, conv_filter_shape=(3, 3), pool_filter_shape=(2, 2), conv_num_filters=256, activation=C.relu, conv_pad=True, pool_strides=(2, 2), name_prefix='layer3') layer4 = Conv2DMaxPool(3, conv_filter_shape=(3, 3), pool_filter_shape=(2, 2), conv_num_filters=512, activation=C.relu, conv_pad=True, pool_strides=(2, 2), name_prefix='layer4') layer5 = Conv2DMaxPool(3, conv_filter_shape=(3, 3), pool_filter_shape=(2, 2), conv_num_filters=512, activation=C.relu, conv_pad=True, pool_strides=(2, 2), name_prefix='layer5') dense1 = Dense(4096, activation=C.relu, name='layer6') dropout1 = Dropout(0.5) dense2 = Dense(4096, activation=C.relu, name='layer7') dropout2 = Dropout(0.5) dense3 = Dense(num_classes, activation=C.relu, name='layer8') def model(x): x = layer1(x) x = layer2(x) x = layer3(x) x = layer4(x) x = layer5(x) x = dropout1(dense1(x)) x = dropout2(dense2(x)) x = dense3(x) return x return model
def create_model_ext(input, ext_values, out_dims): # in VGG style #https://www.cs.toronto.edu/~frossard/post/vgg16/ convolutional_layer_1_1 = Convolution((3, 3), 16, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(input) convolutional_layer_1_2 = Convolution( (5, 5), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(convolutional_layer_1_1) pooling_layer_1 = MaxPooling((2, 2), strides=(2, 2))(convolutional_layer_1_2) convolutional_layer_2_1 = Convolution((3, 3), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(pooling_layer_1) convolutional_layer_2_2 = Convolution( (7, 7), 64, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(convolutional_layer_2_1) pooling_layer_2 = MaxPooling((2, 2), strides=(1, 1))(convolutional_layer_2_2) convolutional_layer_3_1 = Convolution((3, 3), 64, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(pooling_layer_2) convolutional_layer_3_2 = Convolution( (7, 7), 96, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(convolutional_layer_3_1) pooling_layer_3 = MaxPooling((2, 2), strides=(1, 1))(convolutional_layer_3_2) convolutional_layer_4_1 = Convolution((3, 3), 96, init=glorot_uniform(), activation=relu, pad=True, strides=(1, 1))(pooling_layer_3) pooling_layer_4 = MaxPooling((2, 2), strides=(1, 1))(convolutional_layer_4_1) ## fully_connected_layer_1 = Dense(512, init=glorot_uniform())(pooling_layer_4) dropout_layer_1 = Dropout(0.5)(fully_connected_layer_1) fully_connected_with_extra_values = splice(dropout_layer_1, ext_values, axis=0) fully_connected_layer_2 = Dense( 256, init=glorot_uniform())(fully_connected_with_extra_values) fully_connected_layer_3 = Dense( 128, init=glorot_uniform())(fully_connected_layer_2) dropout_layer_2 = Dropout(0.5)(fully_connected_layer_3) output_layer = Dense(out_dims, init=glorot_uniform(), activation=None)(dropout_layer_2) return output_layer
def inception_v3_model(input, labelDim, dropRate, bnTimeConst): # 299 x 299 x 3 conv1 = conv_bn_relu_layer(input, 32, (3, 3), (2, 2), False, bnTimeConst) # 149 x 149 x 32 conv2 = conv_bn_relu_layer(conv1, 32, (3, 3), (1, 1), False, bnTimeConst) # 147 x 147 x 32 conv3 = conv_bn_relu_layer(conv2, 64, (3, 3), (1, 1), True, bnTimeConst) # 147 x 147 x 64 pool1 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=False)(conv3) # 73 x 73 x 64 conv4 = conv_bn_relu_layer(pool1, 80, (1, 1), (1, 1), False, bnTimeConst) # 73 x 73 x 80 conv5 = conv_bn_relu_layer(conv4, 192, (3, 3), (1, 1), False, bnTimeConst) # 71 x 71 x 192 pool2 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=False)(conv5) # 35 x 35 x 192 # # Inception Blocks # mixed1 = inception_block_1(pool2, 64, [48, 64], [64, 96, 96], 32, bnTimeConst) # 35 x 35 x 256 mixed2 = inception_block_1(mixed1, 64, [48, 64], [64, 96, 96], 64, bnTimeConst) # 35 x 35 x 288 mixed3 = inception_block_1(mixed2, 64, [48, 64], [64, 96, 96], 64, bnTimeConst) # 35 x 35 x 288 mixed4 = inception_block_2(mixed3, 384, [64, 96, 96], bnTimeConst) # 17 x 17 x 768 mixed5 = inception_block_3(mixed4, 192, [128, 128, 192], [128, 128, 128, 128, 192], 192, bnTimeConst) # 17 x 17 x 768 mixed6 = inception_block_3(mixed5, 192, [160, 160, 192], [160, 160, 160, 160, 192], 192, bnTimeConst) # 17 x 17 x 768 mixed7 = inception_block_3(mixed6, 192, [160, 160, 192], [160, 160, 160, 160, 192], 192, bnTimeConst) # 17 x 17 x 768 mixed8 = inception_block_3(mixed7, 192, [192, 192, 192], [192, 192, 192, 192, 192], 192, bnTimeConst) # 17 x 17 x 768 mixed9 = inception_block_4(mixed8, [192, 320], [192, 192, 192, 192], bnTimeConst) # 8 x 8 x 1280 mixed10 = inception_block_5(mixed9, 320, [384, 384, 384], [448, 384, 384, 384], 192, bnTimeConst) # 8 x 8 x 2048 mixed11 = inception_block_5(mixed10, 320, [384, 384, 384], [448, 384, 384, 384], 192, bnTimeConst) # 8 x 8 x 2048 # # Prediction # pool3 = AveragePooling(filter_shape=(8, 8), pad=False)(mixed11) # 1 x 1 x 2048 drop = Dropout(dropout_rate=dropRate)(pool3) # 1 x 1 x 2048 z = Dense(labelDim, init=he_normal())(drop) # # Auxiliary # # 17 x 17 x 768 auxPool = AveragePooling(filter_shape=(5, 5), strides=(3, 3), pad=False)(mixed8) # 5 x 5 x 768 auxConv1 = conv_bn_relu_layer(auxPool, 128, (1, 1), (1, 1), True, bnTimeConst) # 5 x 5 x 128 auxConv2 = conv_bn_relu_layer(auxConv1, 768, (5, 5), (1, 1), False, bnTimeConst) # 1 x 1 x 768 aux = Dense(labelDim, init=he_normal())(auxConv2) return {'z': z, 'aux': aux}