def create_model(input_dim):
row = sequence.input_variable(shape=input_dim)
col = sequence.input_variable(shape=input_dim)
rowh = Sequential([Embedding(opt.embed), Stabilizer(), Dropout(opt.dropout)])(row)
colh = Sequential([Embedding(opt.embed), Stabilizer(), Dropout(opt.dropout)])(col)
x = C.splice(rowh, colh, axis=-1)
x = lightlstm(opt.embed, opt.nhid)(x)
x = For(range(opt.layer-1), lambda: lightlstm(opt.nhid, opt.nhid))(x)
rowh = C.slice(x, -1, opt.nhid * 0, opt.nhid * 1)
colh = C.slice(x, -1, opt.nhid * 1, opt.nhid * 2)
row_predict = Sequential([Dropout(opt.dropout), Dense(input_dim)])(rowh)
col_predict = Sequential([Dropout(opt.dropout), Dense(input_dim)])(colh)
# variable : row label and col label
row_label = sequence.input_variable(shape=input_dim)
col_label = sequence.input_variable(shape=input_dim)
model = C.combine([row_predict, col_predict])
return {'row': row,
'col': col,
'row_label': row_label,
'col_label': col_label,
'model': model}
def __call__(self,
num_classes=10,
act_type=relu,
mdl_conv1a_nf=40,
mdl_conv1b_nf=60,
mdl_conv2a_nf=50,
mdl_conv2b_nf=75,
mdl_fc1_nh=75,
mdl_drop2a_p=0.033,
mdl_drop2b_p=0.097,
mdl_drop3_p=0.412,
**kwargs):
input_var = input_variable((1, self.img_h, self.img_w), np.float32)
label_var = input_variable((self.n_dim), np.float32)
conv1a = Convolution(filter_shape=(3, 3), num_filters=int(mdl_conv1a_nf), activation=act_type, init=glorot_uniform(), pad=True, name='conv1a')(input_var)
conv1b = Convolution(filter_shape=(3, 3), num_filters=int(mdl_conv1b_nf), activation=act_type, init=glorot_uniform(), pad=True, name='conv1b')(conv1a)
pool1 = MaxPooling(filter_shape=(2, 2), strides=(2, 2), name='pool1')(conv1b)
conv2a = Convolution(filter_shape=(3, 3), num_filters=int(mdl_conv2a_nf), activation=act_type, init=glorot_uniform(), pad=True, name='conv2a')(pool1)
drop2a = Dropout(prob=mdl_drop2a_p, name="drop2a")(conv2a)
conv2b = Convolution(filter_shape=(3, 3), num_filters=int(mdl_conv2b_nf), activation=act_type, init=glorot_uniform(), pad=True, name='conv2b')(drop2a)
drop2b = Dropout(prob=mdl_drop2a_p, name="drop2a")(conv2b)
pool2 = MaxPooling(filter_shape=(2, 2), strides=(2, 2), name='pool2')(drop2b)
fc1 = Dense(shape=int(mdl_fc1_nh), init=glorot_uniform(), activation=act_type, name='fc1')(pool2)
drop3 = Dropout(prob=mdl_drop3_p, name="drop3")(fc1)
#fc2 = Dense(shape=num_classes, init=glorot_uniform(), activation=softmax, name='fc2')(drop3)
fc2 = Dense(shape=num_classes, init=glorot_uniform(), activation=None, name='fc2')(drop3)
return input_var, label_var, fc2
def create_model(self):
mean_removed_features = minus(self.input,
constant(114),
name='mean_removed_input')
with default_options(activation=None, pad=True, bias=True):
self.model = Sequential([
Convolution2D((11, 11),
96,
init=normal(0.01),
pad=False,
name='conv1'),
Activation(activation=relu, name='relu1'),
self.__local_response_normalization(1.0,
2,
0.0001,
0.75,
name='norm1'),
MaxPooling((3, 3), (2, 2), name='pool1'),
Convolution2D((5, 5),
192,
init=normal(0.01),
init_bias=0.1,
name='conv2'),
Activation(activation=relu, name='relu2'),
self.__local_response_normalization(1.0,
2,
0.0001,
0.75,
name='norm2'),
MaxPooling((3, 3), (2, 2), name='pool2'),
Convolution2D((3, 3), 384, init=normal(0.01), name='conv3'),
Activation(activation=relu, name='relu3'),
Convolution2D((3, 3),
384,
init=normal(0.01),
init_bias=0.1,
name='conv4'),
Activation(activation=relu, name='relu4'),
Convolution2D((3, 3),
256,
init=normal(0.01),
init_bias=0.1,
name='conv5'),
Activation(activation=relu, name='relu5'),
MaxPooling((3, 3), (2, 2), name='pool5'),
Dense(4096, init=normal(0.005), init_bias=0.1, name='fc6'),
Activation(activation=relu, name='relu6'),
Dropout(0.5, name='drop6'),
Dense(4096, init=normal(0.005), init_bias=0.1, name='fc7'),
Activation(activation=relu, name='relu7'),
Dropout(0.5, name='drop7'),
Dense(self.number_labels, init=normal(0.01), name='fc8')
])(mean_removed_features)
def create_alexnet():
# Input variables denoting the features and label data
feature_var = input_variable((num_channels, image_height, image_width))
label_var = input_variable((num_classes))
# apply model to input
# remove mean value
input = minus(feature_var, constant(114), name='mean_removed_input')
with default_options(activation=None, pad=True, bias=True):
z = Sequential([
# we separate Convolution and ReLU to name the output for feature extraction (usually before ReLU)
Convolution2D((11,11), 96, init=normal(0.01), pad=False, strides=(4,4), name='conv1'),
Activation(activation=relu, name='relu1'),
LocalResponseNormalization(1.0, 2, 0.0001, 0.75, name='norm1'),
MaxPooling((3,3), (2,2), name='pool1'),
Convolution2D((5,5), 192, init=normal(0.01), init_bias=0.1, name='conv2'),
Activation(activation=relu, name='relu2'),
LocalResponseNormalization(1.0, 2, 0.0001, 0.75, name='norm2'),
MaxPooling((3,3), (2,2), name='pool2'),
Convolution2D((3,3), 384, init=normal(0.01), name='conv3'),
Activation(activation=relu, name='relu3'),
Convolution2D((3,3), 384, init=normal(0.01), init_bias=0.1, name='conv4'),
Activation(activation=relu, name='relu4'),
Convolution2D((3,3), 256, init=normal(0.01), init_bias=0.1, name='conv5'),
Activation(activation=relu, name='relu5'),
MaxPooling((3,3), (2,2), name='pool5'),
Dense(4096, init=normal(0.005), init_bias=0.1, name='fc6'),
Activation(activation=relu, name='relu6'),
Dropout(0.5, name='drop6'),
Dense(4096, init=normal(0.005), init_bias=0.1, name='fc7'),
Activation(activation=relu, name='relu7'),
Dropout(0.5, name='drop7'),
Dense(num_classes, init=normal(0.01), name='fc8')
])(input)
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
pe5 = classification_error(z, label_var, topN=5)
log_number_of_parameters(z) ; print()
return {
'feature': feature_var,
'label': label_var,
'ce' : ce,
'pe' : pe,
'pe5': pe5,
'output': z
}
def create_model(base_model_file,
input_features,
num_classes,
dropout_rate=0.5,
freeze_weights=False):
# Load the pretrained classification net and find nodes
base_model = load_model(base_model_file)
feature_node = find_by_name(base_model, 'features')
beforePooling_node = find_by_name(base_model, "z.x.x.r")
#graph.plot(base_model, filename="base_model.pdf") # Write graph visualization
# Clone model until right before the pooling layer, ie. until including z.x.x.r
modelCloned = combine([beforePooling_node.owner]).clone(
CloneMethod.freeze if freeze_weights else CloneMethod.clone,
{feature_node: placeholder(name='features')})
# Center the input around zero and set model input.
# Do this early, to avoid CNTK bug with wrongly estimated layer shapes
feat_norm = input_features - constant(114)
model = modelCloned(feat_norm)
# Pool over all spatial dimensions and add dropout layer
avgPool = GlobalAveragePooling(name="poolingLayer")(model)
if dropout_rate > 0:
avgPoolDrop = Dropout(dropout_rate)(avgPool)
else:
avgPoolDrop = avgPool
# Add new dense layer for class prediction
finalModel = Dense(num_classes, activation=None,
name="prediction")(avgPoolDrop)
return finalModel
def create_basic_model(input, out_dims):
convolutional_layer_1 = Convolution((5, 5),
16,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(input)
pooling_layer_1 = MaxPooling((2, 2), strides=(1, 1))(convolutional_layer_1)
convolutional_layer_2 = Convolution((5, 5),
16,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(pooling_layer_1)
pooling_layer_2 = MaxPooling((3, 3), strides=(2, 2))(convolutional_layer_2)
#
convolutional_layer_3 = Convolution((9, 9),
16,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(pooling_layer_2)
pooling_layer_3 = MaxPooling((3, 3), strides=(2, 2))(convolutional_layer_3)
fully_connected_layer = Dense(256, init=glorot_uniform())(pooling_layer_3)
dropout_layer = Dropout(0.5)(fully_connected_layer)
output_layer = Dense(out_dims, init=glorot_uniform(),
activation=None)(dropout_layer)
return output_layer
def create_vgg16(feature_var, num_classes, dropout=0.9):
with default_options(activation=None, pad=True, bias=True):
z = Sequential([
# we separate Convolution and ReLU to name the output for feature
# extraction (usually before ReLU)
For(
range(2), lambda i: [
Convolution2D((3, 3), 64, name='conv1_{}'.format(i)),
Activation(activation=relu, name='relu1_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool1'),
For(
range(2), lambda i: [
Convolution2D((3, 3), 128, name='conv2_{}'.format(i)),
Activation(activation=relu, name='relu2_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool2'),
For(
range(3), lambda i: [
Convolution2D((3, 3), 256, name='conv3_{}'.format(i)),
Activation(activation=relu, name='relu3_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool3'),
For(
range(3), lambda i: [
Convolution2D((3, 3), 512, name='conv4_{}'.format(i)),
Activation(activation=relu, name='relu4_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool4'),
For(
range(3), lambda i: [
Convolution2D((3, 3), 512, name='conv5_{}'.format(i)),
Activation(activation=relu, name='relu5_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool5'),
Dense(4096, name='fc6'),
Activation(activation=relu, name='relu6'),
Dropout(dropout, name='drop6'),
Dense(4096, name='fc7'),
Activation(activation=relu, name='relu7'),
Dropout(dropout, name='drop7'),
Dense(num_classes, name='fc8')
])(feature_var)
return z
def pix2pix_generator(h):
with C.layers.default_options(init=C.normal(0.02), pad=True, bias=False, map_rank=1, use_cntk_engine=True):
h_enc1 = C.leaky_relu(Convolution2D((4, 4), 64, strides=2, bias=True)(h), alpha=0.2)
h_enc2 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 128, strides=2)(h_enc1)), alpha=0.2)
h_enc3 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 256, strides=2)(h_enc2)), alpha=0.2)
h_enc4 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=2)(h_enc3)), alpha=0.2)
h_enc5 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=2)(h_enc4)), alpha=0.2)
h_enc6 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=2)(h_enc5)), alpha=0.2)
h_enc7 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=1)(h_enc6)), alpha=0.2)
h_enc8 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=1)(h_enc7)), alpha=0.2)
h_dec8 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 512, strides=1, pad=True, output_shape=(img_height // 64, img_width // 64))(h_enc8)))
h_dec8 = C.splice(h_dec8, h_enc8, axis=0)
h_dec8 = C.relu(h_dec8)
h_dec7 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 512, strides=1, pad=True, output_shape=(img_height // 64, img_width // 64))(h_dec8)))
h_dec7 = C.splice(h_dec7, h_enc7, axis=0)
h_dec7 = C.relu(h_dec7)
h_dec6 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 512, strides=1, pad=True, output_shape=(img_height // 64, img_width // 64))(h_dec7)))
h_dec6 = C.splice(h_dec6, h_enc6, axis=0)
h_dec6 = C.relu(h_dec6)
h_dec5 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 512, strides=2, pad=True, output_shape=(img_height // 32, img_width // 32))(h_dec6)))
h_dec5 = C.splice(h_dec5, h_enc5, axis=0)
h_dec5 = C.relu(h_dec5)
h_dec4 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 512, strides=2, pad=True, output_shape=(img_height // 16, img_width // 16))(h_dec5)))
h_dec4 = C.splice(h_dec4, h_enc4, axis=0)
h_dec4 = C.relu(h_dec4)
h_dec3 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 256, strides=2, pad=True, output_shape=(img_height // 8, img_width // 8))(h_dec4)))
h_dec3 = C.splice(h_dec3, h_enc3, axis=0)
h_dec3 = C.relu(h_dec3)
h_dec2 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 128, strides=2, pad=True, output_shape=(img_height // 4, img_width // 4))(h_dec3)))
h_dec2 = C.splice(h_dec2, h_enc2, axis=0)
h_dec2 = C.relu(h_dec2)
h_dec1 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 64, strides=2, pad=True, output_shape=(img_height // 2, img_width // 2))(h_dec2)))
h_dec1 = C.splice(h_dec1, h_enc1, axis=0)
h_dec1 = C.relu(h_dec1)
h = ConvolutionTranspose2D((4, 4), 3, activation=C.tanh, strides=2, pad=True, bias=True,
output_shape=(img_height, img_width))(h_dec1)
return h
def resnet_drop(input, num_filters):
c1 = conv_bn_relu(input, (3,3), num_filters)
c2 = conv_bn(c1, (3,3), num_filters, bn_init_scale = 1)
b1 = Dropout(0.5)(input)
p = b1 + c2
return relu(p)
def create_symbol():
# Weight initialiser from uniform distribution
# Activation (unless states) is None
with cntk.layers.default_options(init=cntk.glorot_uniform(),
activation=cntk.relu):
x = Convolution2D(filter_shape=(3, 3), num_filters=50,
pad=True)(features)
x = Convolution2D(filter_shape=(3, 3), num_filters=50, pad=True)(x)
x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x)
x = Dropout(0.25)(x)
x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x)
x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x)
x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x)
x = Dropout(0.25)(x)
x = Dense(512)(x)
x = Dropout(0.5)(x)
x = Dense(N_CLASSES, activation=None)(x)
return x
def resnet_model(layer_input):
layer1 = resnet_basic_stack(layer_input,
2, (3, 3),
19, (1, 1),
prefix='conv1')
layer1 = MaxPooling((3, 3), strides=(2, 2), name='pool1')(layer1)
layer1 = Dropout(0.3, name='drop1')(layer1)
layer2 = resnet_basic_inc(layer1, (3, 3), 8, (2, 2), prefix='conv21')
layer2 = resnet_basic_stack(layer2, 1, (3, 3), 8, (1, 1), prefix='conv22')
layer2 = Dropout(0.3, name='drop2')(layer2)
layer3 = resnet_basic_inc(layer2, (3, 3), 16, (2, 2), prefix='conv31')
layer3 = resnet_basic_stack(layer3, 1, (3, 3), 16, (1, 1), prefix='conv32')
layer3 = Dropout(0.3, name='drop3')(layer3)
layer4 = resnet_basic_inc(layer3, (3, 3), 32, (2, 2), prefix='conv41')
layer4 = resnet_basic_stack(layer4, 1, (3, 3), 32, (1, 1), prefix='conv42')
layer4 = Dropout(0.3, name='drop4')(layer4)
return layer4
def create_convnet_cifar10_model(num_classes):
with default_options(activation=relu, pad=True):
return Sequential([
For(
range(2), lambda: [
Convolution2D((3, 3), 64),
Convolution2D((3, 3), 64),
MaxPooling((3, 3), strides=2)
]),
For(range(2), lambda i: [Dense([256, 128][i]),
Dropout(0.5)]),
Dense(num_classes, activation=None)
])
def test_layers_dropout():
dat = np.array([[1., 1., 1., 1.]], dtype=np.float32)
y = input(4)
p = Dense(1, activation=None, name='foo')(y)
z = Dropout(0.75, name='bar')(p)
res = z(y).eval({y: dat})
expected_res = np.sum(p.foo.W.value)
np.testing.assert_array_almost_equal(res, expected_res, decimal=7, \
err_msg="Error in dropout computation")
z = Dropout(keep_prob=0.25, name='bar')(p)
res = z(y).eval({y: dat})
np.testing.assert_array_almost_equal(res, expected_res, decimal=7, \
err_msg="Error in dropout computation with keep_prob")
with pytest.raises(ValueError):
z = Dropout(keep_prob=-1.5, name='bar')(p)
with pytest.raises(ValueError):
z = Dropout(1.5, name='bar')(p)
def create_convolutional_neural_network(input_vars, out_dims, dropout_prob=0.0):
convolutional_layer_1 = Convolution((5, 5), 32, strides=1, activation=cntk.ops.relu, pad=True, init=gaussian(), init_bias=0.1)(input_vars)
pooling_layer_1 = MaxPooling((2, 2), strides=(2, 2), pad=True)(convolutional_layer_1)
convolutional_layer_2 = Convolution((5, 5), 64, strides=1, activation=cntk.ops.relu, pad=True, init=gaussian(), init_bias=0.1)(pooling_layer_1)
pooling_layer_2 = MaxPooling((2, 2), strides=(2, 2), pad=True)(convolutional_layer_2)
convolutional_layer_3 = Convolution((5, 5), 128, strides=1, activation=cntk.ops.relu, pad=True, init=gaussian(), init_bias=0.1)(pooling_layer_2)
pooling_layer_3 = MaxPooling((2, 2), strides=(2, 2), pad=True)(convolutional_layer_3)
fully_connected_layer = Dense(1024, activation=cntk.ops.relu, init=gaussian(), init_bias=0.1)(pooling_layer_3)
dropout_layer = Dropout(dropout_prob)(fully_connected_layer)
output_layer = Dense(out_dims, activation=None, init=gaussian(), init_bias=0.1)(dropout_layer)
return output_layer
def create_shallow_model(input, out_dims):
convolutional_layer_1_1 = Convolution((7,7), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(input)
convolutional_layer_1_2 = Convolution((25,25), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(convolutional_layer_1_1)
pooling_layer_1 = MaxPooling((25,25), strides=(5,5))(convolutional_layer_1_2 )
convolutional_layer_2_1 = Convolution((3,3), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(pooling_layer_1)
pooling_layer_2 = MaxPooling((2,2), strides=(2,2))(convolutional_layer_2_1)
fully_connected_layer_1 = Dense(512, init=glorot_uniform())(pooling_layer_2)
fully_connected_layer_2 = Dense(128, init=glorot_uniform())(fully_connected_layer_1)
dropout_layer = Dropout(0.5)(fully_connected_layer_2)
output_layer = Dense(out_dims, init=glorot_uniform(), activation=None)(dropout_layer)
return output_layer
def create_advanced_model(input, out_dims):
with default_options(activation=relu):
model = Sequential([
For(range(2), lambda i: [ # lambda with one parameter
Convolution((3,3), [32,64][i], pad=True), # depth depends on i
Convolution((5,5), [32,64][i], pad=True),
Convolution((9,9), [32,64][i], pad=True),
MaxPooling((3,3), strides=(2,2))
]),
For(range(2), lambda : [ # lambda without parameter
Dense(512),
Dropout(0.5)
]),
Dense(out_dims, activation=None)
])
output_layer=model(input)
return output_layer
def create_model(base_model_file, input_features, params):
num_classes = params['num_classes']
dropout_rate = params['dropout_rate']
freeze_weights = params['freeze_weights']
# Load the pretrained classification net and find nodes
base_model = load_model(base_model_file)
log = logging.getLogger("neuralnets1.utils.create_model")
log.info('Loaded base model - %s with layers:' % base_model_file)
node_outputs = get_node_outputs(base_model)
[log.info('%s , %s' % (layer.name, layer.shape)) for layer in node_outputs]
graph.plot(base_model,
filename="base_model.pdf") # Write graph visualization
feature_node = find_by_name(base_model, 'features')
beforePooling_node = find_by_name(base_model, "z.x.x.r")
# Clone model until right before the pooling layer, ie. until including z.x.x.r
modelCloned = combine([beforePooling_node.owner]).clone(
CloneMethod.freeze if freeze_weights else CloneMethod.clone,
{feature_node: placeholder(name='features')})
# Center the input around zero and set model input.
# Do this early, to avoid CNTK bug with wrongly estimated layer shapes
feat_norm = input_features - constant(114)
model = modelCloned(feat_norm)
# Add pool layer
avgPool = GlobalAveragePooling(name="poolingLayer")(
model) # assign name to the layer and add to the model
# Add drop out layer
if dropout_rate > 0:
avgPoolDrop = Dropout(dropout_rate)(
avgPool
) # add drop out layer with specified drop out rate and add it to the model
else:
avgPoolDrop = avgPool
# Add new dense layer for class prediction
finalModel = Dense(num_classes, activation=None, name="Dense")(avgPoolDrop)
return finalModel
def inception_v3_cifar_model(input, labelDim, bnTimeConst):
# 32 x 32 x 3
conv1 = conv_bn_relu_layer(input, 32, (3, 3), (1, 1), True, bnTimeConst)
# 32 x 32 x 32
conv2 = conv_bn_relu_layer(conv1, 32, (3, 3), (1, 1), True, bnTimeConst)
# 32 x 32 x 32
conv3 = conv_bn_relu_layer(conv2, 64, (3, 3), (1, 1), True, bnTimeConst)
# 32 x 32 x 64
conv4 = conv_bn_relu_layer(conv3, 80, (1, 1), (1, 1), True, bnTimeConst)
# 32 x 32 x 80
conv5 = conv_bn_relu_layer(conv4, 128, (3, 3), (1, 1), True, bnTimeConst)
# 32 x 32 x 128
pool1 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=True)(conv5)
#
# Inception Blocks
# 16 x 16 x 128
mixed1 = inception_block_1(pool1, 32, [32, 48], [48, 64, 64], 32,
bnTimeConst)
# 16 x 16 x 160
mixed2 = inception_block_1(mixed1, 32, [32, 48], [48, 64, 64], 64,
bnTimeConst)
# 16 x 16 x 160
mixed3 = inception_block_1(mixed2, 32, [32, 48], [48, 64, 64], 64,
bnTimeConst)
# 16 x 16 x 160
#mixed4 = inception_block_2(mixed3, 32, [48, 64, 64], bnTimeConst)
mixed4 = inception_block_pass_through(mixed3, 0, 64, 80, 32, 48, 0,
bnTimeConst)
# 8 x 8 x 256
mixed5 = inception_block_3(mixed4, 192, [48, 64, 64],
[128, 128, 128, 128, 192], 192, bnTimeConst)
# 8 x 8 x
mixed6 = inception_block_3(mixed5, 192, [160, 160, 192],
[160, 160, 160, 160, 192], 192, bnTimeConst)
# 8 x 8 x 768
mixed7 = inception_block_3(mixed6, 192, [160, 160, 192],
[160, 160, 160, 160, 192], 192, bnTimeConst)
# 8 x 8 x 768
mixed8 = inception_block_3(mixed7, 192, [192, 192, 192],
[192, 192, 192, 192, 192], 192, bnTimeConst)
# 8 x 8 x 768
mixed9 = inception_block_3(mixed8, 192, [192, 192, 192],
[192, 192, 192, 192, 192], 192, bnTimeConst)
# 8 x 8 x 1280
mixed10 = inception_block_5(mixed9, 320, [384, 384, 384],
[448, 384, 384, 384], 192, bnTimeConst)
# 8 x 8 x 2048
mixed11 = inception_block_5(mixed10, 320, [384, 384, 384],
[448, 384, 384, 384], 192, bnTimeConst)
# 8 x 8 x 2048
#
# Prediction
#
pool3 = AveragePooling(filter_shape=(8, 8))(mixed11)
# 1 x 1 x 2048
drop = Dropout(dropout_rate=0.2)(pool3)
# 1 x 1 x 2048
z = Dense(labelDim, init=he_normal())(drop)
#
# Auxiliary
#
# 8 x 8 x 768
auxPool = AveragePooling(filter_shape=(3, 3), strides=(1, 1),
pad=True)(mixed8)
# 5 x 5 x 768
auxConv1 = conv_bn_relu_layer(auxPool, 128, (1, 1), (1, 1), True,
bnTimeConst)
# 5 x 5 x 128
auxConv2 = conv_bn_relu_layer(auxConv1, 256, (3, 3), (1, 1), True,
bnTimeConst)
# 1 x 1 x 768
aux = Dense(labelDim, init=he_normal())(auxConv2)
return {'z': z, 'aux': aux}
input_folder_path = "src/ml/data/autcar_training"
output_folder_path = "src/ml/data/autcar_training_balanced"
image_width = 224
image_height = 168
trainer = Trainer(deeplearning_framework="cntk", image_height=image_height, image_width=image_width)
trainer.create_balanced_dataset(input_folder_path, output_folder_path=output_folder_path)
model = Sequential([
Convolution2D(filter_shape=(5,5), num_filters=32, strides=(1,1), pad=True, name="first_conv"),
BatchNormalization(map_rank=1),
Activation(relu),
MaxPooling(filter_shape=(3,3), strides=(2,2), name="first_max"),
Convolution2D(filter_shape=(3,3), num_filters=48, strides=(1,1), pad=True, name="second_conv"),
BatchNormalization(map_rank=1),
Activation(relu),
MaxPooling(filter_shape=(3,3), strides=(2,2), name="second_max"),
Convolution2D(filter_shape=(3,3), num_filters=64, strides=(1,1), pad=True, name="third_conv"),
BatchNormalization(map_rank=1),
Activation(relu),
MaxPooling(filter_shape=(3,3), strides=(2,2), name="third_max"),
Convolution2D(filter_shape=(5,5), num_filters=32, strides=(1,1), pad=True, name="fourth_conv"),
BatchNormalization(map_rank=1),
Activation(relu),
Dense(100, activation=relu),
Dropout(0.1),
Dense(12, activation=softmax)
])
trainer.train(output_folder_path, model, epochs=4, output_model_path="driver_cntk.onnx")
trainer.test("driver_cntk.onnx", output_folder_path+"/test_map.txt")
def create_vgg19():
# Input variables denoting the features and label data
feature_var = input((num_channels, image_height, image_width))
label_var = input((num_classes))
# apply model to input
# remove mean value
input = minus(feature_var,
constant([[[104]], [[117]], [[124]]]),
name='mean_removed_input')
with default_options(activation=None, pad=True, bias=True):
z = Sequential([
# we separate Convolution and ReLU to name the output for feature extraction (usually before ReLU)
For(
range(2), lambda i: [
Convolution2D((3, 3), 64, name='conv1_{}'.format(i)),
Activation(activation=relu, name='relu1_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool1'),
For(
range(2), lambda i: [
Convolution2D((3, 3), 128, name='conv2_{}'.format(i)),
Activation(activation=relu, name='relu2_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool2'),
For(
range(4), lambda i: [
Convolution2D((3, 3), 256, name='conv3_{}'.format(i)),
Activation(activation=relu, name='relu3_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool3'),
For(
range(4), lambda i: [
Convolution2D((3, 3), 512, name='conv4_{}'.format(i)),
Activation(activation=relu, name='relu4_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool4'),
For(
range(4), lambda i: [
Convolution2D((3, 3), 512, name='conv5_{}'.format(i)),
Activation(activation=relu, name='relu5_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool5'),
Dense(4096, name='fc6'),
Activation(activation=relu, name='relu6'),
Dropout(0.5, name='drop6'),
Dense(4096, name='fc7'),
Activation(activation=relu, name='relu7'),
Dropout(0.5, name='drop7'),
Dense(num_classes, name='fc8')
])(input)
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
pe5 = classification_error(z, label_var, topN=5)
log_number_of_parameters(z)
print()
return {
'feature': feature_var,
'label': label_var,
'ce': ce,
'pe': pe,
'pe5': pe5,
'output': z
}
def VGG16(num_classes: int):
""" for image classification """
layer1 = Conv2DMaxPool(2,
conv_filter_shape=(3, 3),
pool_filter_shape=(2, 2),
conv_num_filters=64,
activation=C.relu,
conv_pad=True,
pool_strides=(2, 2),
name_prefix='layer1')
layer2 = Conv2DMaxPool(2,
conv_filter_shape=(3, 3),
pool_filter_shape=(2, 2),
conv_num_filters=128,
activation=C.relu,
conv_pad=True,
pool_strides=(2, 2),
name_prefix='layer2')
layer3 = Conv2DMaxPool(2,
conv_filter_shape=(3, 3),
pool_filter_shape=(2, 2),
conv_num_filters=256,
activation=C.relu,
conv_pad=True,
pool_strides=(2, 2),
name_prefix='layer3')
layer4 = Conv2DMaxPool(3,
conv_filter_shape=(3, 3),
pool_filter_shape=(2, 2),
conv_num_filters=512,
activation=C.relu,
conv_pad=True,
pool_strides=(2, 2),
name_prefix='layer4')
layer5 = Conv2DMaxPool(3,
conv_filter_shape=(3, 3),
pool_filter_shape=(2, 2),
conv_num_filters=512,
activation=C.relu,
conv_pad=True,
pool_strides=(2, 2),
name_prefix='layer5')
dense1 = Dense(4096, activation=C.relu, name='layer6')
dropout1 = Dropout(0.5)
dense2 = Dense(4096, activation=C.relu, name='layer7')
dropout2 = Dropout(0.5)
dense3 = Dense(num_classes, activation=C.relu, name='layer8')
def model(x):
x = layer1(x)
x = layer2(x)
x = layer3(x)
x = layer4(x)
x = layer5(x)
x = dropout1(dense1(x))
x = dropout2(dense2(x))
x = dense3(x)
return x
return model
def create_model_ext(input, ext_values, out_dims):
# in VGG style
#https://www.cs.toronto.edu/~frossard/post/vgg16/
convolutional_layer_1_1 = Convolution((3, 3),
16,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(input)
convolutional_layer_1_2 = Convolution(
(5, 5),
32,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(convolutional_layer_1_1)
pooling_layer_1 = MaxPooling((2, 2),
strides=(2, 2))(convolutional_layer_1_2)
convolutional_layer_2_1 = Convolution((3, 3),
32,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(pooling_layer_1)
convolutional_layer_2_2 = Convolution(
(7, 7),
64,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(convolutional_layer_2_1)
pooling_layer_2 = MaxPooling((2, 2),
strides=(1, 1))(convolutional_layer_2_2)
convolutional_layer_3_1 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(pooling_layer_2)
convolutional_layer_3_2 = Convolution(
(7, 7),
96,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(convolutional_layer_3_1)
pooling_layer_3 = MaxPooling((2, 2),
strides=(1, 1))(convolutional_layer_3_2)
convolutional_layer_4_1 = Convolution((3, 3),
96,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(pooling_layer_3)
pooling_layer_4 = MaxPooling((2, 2),
strides=(1, 1))(convolutional_layer_4_1)
##
fully_connected_layer_1 = Dense(512,
init=glorot_uniform())(pooling_layer_4)
dropout_layer_1 = Dropout(0.5)(fully_connected_layer_1)
fully_connected_with_extra_values = splice(dropout_layer_1,
ext_values,
axis=0)
fully_connected_layer_2 = Dense(
256, init=glorot_uniform())(fully_connected_with_extra_values)
fully_connected_layer_3 = Dense(
128, init=glorot_uniform())(fully_connected_layer_2)
dropout_layer_2 = Dropout(0.5)(fully_connected_layer_3)
output_layer = Dense(out_dims, init=glorot_uniform(),
activation=None)(dropout_layer_2)
return output_layer
def inception_v3_model(input, labelDim, dropRate, bnTimeConst):
# 299 x 299 x 3
conv1 = conv_bn_relu_layer(input, 32, (3, 3), (2, 2), False, bnTimeConst)
# 149 x 149 x 32
conv2 = conv_bn_relu_layer(conv1, 32, (3, 3), (1, 1), False, bnTimeConst)
# 147 x 147 x 32
conv3 = conv_bn_relu_layer(conv2, 64, (3, 3), (1, 1), True, bnTimeConst)
# 147 x 147 x 64
pool1 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=False)(conv3)
# 73 x 73 x 64
conv4 = conv_bn_relu_layer(pool1, 80, (1, 1), (1, 1), False, bnTimeConst)
# 73 x 73 x 80
conv5 = conv_bn_relu_layer(conv4, 192, (3, 3), (1, 1), False, bnTimeConst)
# 71 x 71 x 192
pool2 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=False)(conv5)
# 35 x 35 x 192
#
# Inception Blocks
#
mixed1 = inception_block_1(pool2, 64, [48, 64], [64, 96, 96], 32,
bnTimeConst)
# 35 x 35 x 256
mixed2 = inception_block_1(mixed1, 64, [48, 64], [64, 96, 96], 64,
bnTimeConst)
# 35 x 35 x 288
mixed3 = inception_block_1(mixed2, 64, [48, 64], [64, 96, 96], 64,
bnTimeConst)
# 35 x 35 x 288
mixed4 = inception_block_2(mixed3, 384, [64, 96, 96], bnTimeConst)
# 17 x 17 x 768
mixed5 = inception_block_3(mixed4, 192, [128, 128, 192],
[128, 128, 128, 128, 192], 192, bnTimeConst)
# 17 x 17 x 768
mixed6 = inception_block_3(mixed5, 192, [160, 160, 192],
[160, 160, 160, 160, 192], 192, bnTimeConst)
# 17 x 17 x 768
mixed7 = inception_block_3(mixed6, 192, [160, 160, 192],
[160, 160, 160, 160, 192], 192, bnTimeConst)
# 17 x 17 x 768
mixed8 = inception_block_3(mixed7, 192, [192, 192, 192],
[192, 192, 192, 192, 192], 192, bnTimeConst)
# 17 x 17 x 768
mixed9 = inception_block_4(mixed8, [192, 320], [192, 192, 192, 192],
bnTimeConst)
# 8 x 8 x 1280
mixed10 = inception_block_5(mixed9, 320, [384, 384, 384],
[448, 384, 384, 384], 192, bnTimeConst)
# 8 x 8 x 2048
mixed11 = inception_block_5(mixed10, 320, [384, 384, 384],
[448, 384, 384, 384], 192, bnTimeConst)
# 8 x 8 x 2048
#
# Prediction
#
pool3 = AveragePooling(filter_shape=(8, 8), pad=False)(mixed11)
# 1 x 1 x 2048
drop = Dropout(dropout_rate=dropRate)(pool3)
# 1 x 1 x 2048
z = Dense(labelDim, init=he_normal())(drop)
#
# Auxiliary
#
# 17 x 17 x 768
auxPool = AveragePooling(filter_shape=(5, 5), strides=(3, 3),
pad=False)(mixed8)
# 5 x 5 x 768
auxConv1 = conv_bn_relu_layer(auxPool, 128, (1, 1), (1, 1), True,
bnTimeConst)
# 5 x 5 x 128
auxConv2 = conv_bn_relu_layer(auxConv1, 768, (5, 5), (1, 1), False,
bnTimeConst)
# 1 x 1 x 768
aux = Dense(labelDim, init=he_normal())(auxConv2)
return {'z': z, 'aux': aux}