def test_layers_conv_pool_unpool_deconv():
pass
inC, inH, inW = 1, 4, 4
y = input((inC, inH, inW))
cMap = 1
zero_pad = True
conv_init = 1
filter_shape = (2, 2)
pooling_strides = (2, 2)
dat = np.arange(0, 16, dtype=np.float32).reshape(1, 1, 4, 4)
conv = Convolution(filter_shape,
cMap,
pad=zero_pad,
init=conv_init,
activation=None)(y)
pool = MaxPooling(filter_shape, pooling_strides)(conv)
unpool = MaxUnpooling(filter_shape, pooling_strides)(pool, conv)
z = ConvolutionTranspose(filter_shape, cMap, init=conv_init,
pad=zero_pad)(unpool)
assert z.shape == y.shape
res = z(dat)
expected_res = np.asarray([[30, 64, 34], [76, 160, 84], [46, 96, 50]],
np.float32)
np.testing.assert_array_almost_equal(
res[0][0][1:, 1:],
expected_res,
decimal=6,
err_msg="Wrong values in conv/pooling/unpooling/conv_transposed")
def test_max_pooling_layer(self):
# Test a model with a single CNTK MaxPooling layer against the equivalent ELL predictor
# This verifies that the import functions reshape and reorder values appropriately and
# that the equivalent ELL layer produces comparable output
# Create a MaxPooling CNTK layer
poolingLayer = MaxPooling((2, 2), strides=2)
# Input order for CNTK is channels, rows, columns
x = input((3, 12, 12))
cntkModel = poolingLayer(x)
# Create the equivalent ELL predictor
layerParameters = ELL.LayerParameters(
ELL.LayerShape(
12, 12, 3), # Input order for ELL is rows, columns, channels
ELL.NoPadding(),
ELL.LayerShape(6, 6, 3),
ELL.NoPadding())
poolingParameters = ELL.PoolingParameters(2, 2)
layer = ELL.FloatPoolingLayer(layerParameters, poolingParameters,
ELL.PoolingType.max)
predictor = ELL.FloatNeuralNetworkPredictor([layer])
# Get the results for both
inputValues = np.arange(432, dtype=np.float32).reshape(3, 12, 12)
cntkResults = cntkModel(inputValues)
orderedCntkResults = cntk_to_ell.get_float_vector_from_cntk_array(
cntkResults
) # Note that cntk inserts an extra dimension of 1 in the front
orderedInputValues = cntk_to_ell.get_float_vector_from_cntk_array(
inputValues)
ellResults = predictor.Predict(orderedInputValues)
# Compare them
np.testing.assert_array_equal(
orderedCntkResults, ellResults,
'results for MaxPooling layer do not match!')
return
def test_depth_first_search_blocks(depth, prefix_count):
from cntk.layers import Sequential, Convolution, MaxPooling, Dense
from cntk.default_options import default_options
with default_options(activation=C.relu):
image_to_vec = Sequential([
Convolution((5, 5), 32, pad=True),
MaxPooling((3, 3), strides=(2, 2)),
Dense(10, activation=None)
])
in1 = C.input_variable(shape=(3, 256, 256), name='image')
img = image_to_vec(in1)
found = C.logging.graph.depth_first_search(img,
lambda x: True,
depth=depth)
found_str = [str(v) for v in found]
assert len(found) == sum(prefix_count.values())
for prefix, count in prefix_count.items():
assert sum(f.startswith(prefix) for f in found_str) == count
def inception_block_pass_through(input, num1x1, num3x3r, num3x3, num3x3dblr,
num3x3dbl, numPool, bnTimeConst):
# 1x1 -> 3x3 branch
branch3x3_reduce = conv_bn_relu_layer(input, num3x3r, (1, 1), (1, 1), True,
bnTimeConst)
branch3x3 = conv_bn_relu_layer(branch3x3_reduce, num3x3, (3, 3), (2, 2),
True, bnTimeConst)
# 1x1 -> 3x3 -> 3x3 branch
branch3x3dbl_reduce = conv_bn_relu_layer(input, num3x3dblr, (1, 1), (1, 1),
True, bnTimeConst)
branch3x3dbl_conv = conv_bn_relu_layer(branch3x3dbl_reduce, num3x3dbl,
(3, 3), (1, 1), True, bnTimeConst)
branch3x3dbl = conv_bn_relu_layer(branch3x3dbl_conv, num3x3dbl, (3, 3),
(2, 2), True, bnTimeConst)
# Max Pooling
branchPool = MaxPooling((3, 3), strides=(2, 2), pad=True)(input)
out = splice(branch3x3, branch3x3dbl, branchPool, axis=0)
return out
def inception_block_4(input, num3x3, num7x7_3x3, bnTimeConst):
# 3x3 Convolution
branch3x3_1 = conv_bn_relu_layer(input, num3x3[0], (1, 1), (1, 1), True,
bnTimeConst)
branch3x3 = conv_bn_relu_layer(branch3x3_1, num3x3[1], (3, 3), (2, 2),
False, bnTimeConst)
# 7x7 3x3 Convolution
branch7x7_3x3_1 = conv_bn_relu_layer(input, num7x7_3x3[0], (1, 1), (1, 1),
True, bnTimeConst)
branch7x7_3x3_2 = conv_bn_relu_layer(branch7x7_3x3_1, num7x7_3x3[1],
(1, 7), (1, 1), True, bnTimeConst)
branch7x7_3x3_3 = conv_bn_relu_layer(branch7x7_3x3_2, num7x7_3x3[2],
(7, 1), (1, 1), True, bnTimeConst)
branch7x7_3x3 = conv_bn_relu_layer(branch7x7_3x3_3, num7x7_3x3[3], (3, 3),
(2, 2), False, bnTimeConst)
# Max Pooling
branchPool = MaxPooling((3, 3), strides=(2, 2), pad=False)(input)
out = splice(branch3x3, branch7x7_3x3, branchPool, axis=0)
return out
def create_advanced_model(input, out_dims):
with default_options(activation=relu):
model = Sequential([
For(
range(2),
lambda i: [ # lambda with one parameter
Convolution(
(3, 3), [16, 32][i], pad=True), # depth depends on i
#Convolution((5,5), [16,16][i], pad=True),
Convolution((9, 9), [16, 32][i], pad=True),
MaxPooling((3, 3), strides=(2, 2))
]),
For(
range(2),
lambda: [ # lambda without parameter
Dense(128),
Dropout(0.5)
]),
Dense(out_dims, activation=None)
])
output_layer = model(input)
return output_layer
def create_model(input):
#print (input.shape)
conv1 = Convolution((3,3), 64, init=glorot_uniform(), activation=relu, pad=True)(input)
conv1 = Convolution((3,3), 64, init=glorot_uniform(), activation=relu, pad=True)(conv1)
pool1 = MaxPooling((2,2), strides=(2,2))(conv1)
conv2 = Convolution((3,3), 128, init=glorot_uniform(), activation=relu, pad=True)(pool1)
conv2 = Convolution((3,3), 128, init=glorot_uniform(), activation=relu, pad=True)(conv2)
pool2 = MaxPooling((2,2), strides=(2,2))(conv2)
conv3 = Convolution((3,3), 256, init=glorot_uniform(), activation=relu, pad=True)(pool2)
conv3 = Convolution((3,3), 256, init=glorot_uniform(), activation=relu, pad=True)(conv3)
pool3 = MaxPooling((2,2), strides=(2,2))(conv3)
conv4 = Convolution((3,3), 512, init=glorot_uniform(), activation=relu, pad=True)(pool3)
conv4 = Convolution((3,3), 512, init=glorot_uniform(), activation=relu, pad=True)(conv4)
pool4 = MaxPooling((2,2), strides=(2,2))(conv4)
conv5 = Convolution((3,3), 1024, init=glorot_uniform(), activation=relu, pad=True)(pool4)
conv5 = Convolution((3,3), 1024, init=glorot_uniform(), activation=relu, pad=True)(conv5)
print("conv1"+str(conv1.shape))
print("pool1"+str(pool1.shape))
print("conv2"+str(conv2.shape))
print("pool2"+str(pool2.shape))
print("conv3"+str(conv3.shape))
print("pool3"+str(pool3.shape))
print("conv4"+str(conv4.shape))
print("pool4"+str(pool4.shape))
print("conv5"+str(conv5.shape))
#up5 =UpSampling2D(conv5)
#print("upsamplingC5"+str(up5.shape))
#up convolution
deconv6 = ConvolutionTranspose2D((2,2),512,strides=2,pad=False,activation=C.relu)(conv5)
print("deconv6" + str(deconv6.shape))
upconv6 = C.splice(deconv6, conv4, axis=0)
print("upconv6" + str(upconv6.shape))
conv6 = Convolution((3,3), 512, init=glorot_uniform(), activation=relu, pad=True)(upconv6)
conv6 = Convolution((3,3), 512, init=glorot_uniform(), activation=relu, pad=True)(conv6)
print("conv6"+str(conv6.shape))
#up convolution
deconv7 = ConvolutionTranspose2D((2,2),256,strides=2,pad=False,activation=C.relu)(conv6)
print("deconv7" + str(deconv7.shape))
upconv7 = C.splice(deconv7, conv3, axis=0)
print("upconv7" + str(upconv7.shape))
conv7 = Convolution((3,3), 256, init=glorot_uniform(), activation=relu, pad=True)(upconv7)
conv7 = Convolution((3,3), 256, init=glorot_uniform(), activation=relu, pad=True)(conv7)
print("conv7"+str(conv7.shape))
deconv8 = ConvolutionTranspose2D((2,2),128,strides=2,pad=False,activation=C.relu)(conv7)
print("(deconv8)"+str(deconv8.shape))
upconv8 = C.splice(deconv8, conv2, axis=0)
print("upconv8"+str(upconv8.shape))
#up8 = C.splice(UpSampling2D(conv7), conv2, axis=0)
conv8 = Convolution((3,3), 128, init=glorot_uniform(), activation=relu, pad=True)(upconv8)
conv8 = Convolution((3,3), 128, init=glorot_uniform(), activation=relu, pad=True)(conv8)
print("conv8"+str(conv8.shape))
deconv9 = ConvolutionTranspose2D((2,2),64,strides=2,pad=False,activation=C.relu)(conv8)
print("deconv9"+str(deconv9.shape))
upconv9 = C.splice(deconv9, conv1, axis=0)
print("upconv9"+str(upconv9.shape))
#up9 = C.splice(UpSampling2D(conv8), conv1, axis=0)
conv9 = Convolution((3,3), 64, init=glorot_uniform(), activation=relu, pad=True)(upconv9)
conv9 = Convolution((3,3), 64, init=glorot_uniform(), activation=relu, pad=True)(conv9)
print("conv9" + str(conv9.shape))
conv10 = Convolution((1,1), 1, init=glorot_uniform(), activation=sigmoid, pad=True)(conv9)
return conv10
def create_model(input, num_classes):
print(input.shape)
conv1 = Convolution((3, 3),
32,
init=glorot_uniform(),
activation=relu,
pad=True,
name="bir")(input)
conv1 = Convolution((3, 3),
32,
init=glorot_uniform(),
activation=relu,
pad=True,
name="iki")(conv1)
print(conv1.shape)
pool1 = MaxPooling((2, 2), strides=(2, 2))(conv1)
conv2 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(pool1)
conv2 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv2)
print(conv2.shape)
pool2 = MaxPooling((2, 2), strides=(2, 2))(conv2)
conv3 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(pool2)
conv3 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(conv3)
print(conv3.shape)
pool3 = MaxPooling((2, 2), strides=(2, 2))(conv3)
conv4 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(pool3)
conv4 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(conv4)
print(conv4.shape)
pool4 = MaxPooling((2, 2), strides=(2, 2))(conv4)
conv5 = Convolution((3, 3),
512,
init=glorot_uniform(),
activation=relu,
pad=True)(pool4)
conv5 = Convolution((3, 3),
512,
init=glorot_uniform(),
activation=relu,
pad=True)(conv5)
print(conv5.shape)
up6 = C.splice(UpSampling2D(conv5), conv4, axis=0)
conv6 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(up6)
conv6 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(conv6)
print(conv6.shape)
up7 = C.splice(UpSampling2D(conv6), conv3, axis=0)
conv7 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(up7)
conv7 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(conv7)
print(conv7.shape)
up8 = C.splice(UpSampling2D(conv7), conv2, axis=0)
conv8 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(up8)
conv8 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv8)
print(conv8.shape)
up9 = C.splice(UpSampling2D(conv8), conv1, axis=0)
conv9 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(up9)
conv9 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True,
name="beforelast")(conv9)
print(conv9.shape)
conv10 = Convolution((1, 1),
num_classes,
init=glorot_uniform(),
activation=sigmoid,
pad=True,
name="last")(conv9)
print(conv10.shape)
return conv10
def clone_cntk_layer(self, feature):
"""Returns a clone of the CNTK layer for per-layer forward prop validation"""
pad, filterShape, stride = self.get_cntk_parameters()
return MaxPooling(filterShape, strides=(stride, stride),
pad=pad)(feature)
def create_model_ext(input, ext_values, out_dims):
# in VGG style
#https://www.cs.toronto.edu/~frossard/post/vgg16/
convolutional_layer_1_1 = Convolution((3, 3),
16,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(input)
convolutional_layer_1_2 = Convolution(
(5, 5),
32,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(convolutional_layer_1_1)
pooling_layer_1 = MaxPooling((2, 2),
strides=(2, 2))(convolutional_layer_1_2)
convolutional_layer_2_1 = Convolution((3, 3),
32,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(pooling_layer_1)
convolutional_layer_2_2 = Convolution(
(7, 7),
64,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(convolutional_layer_2_1)
pooling_layer_2 = MaxPooling((2, 2),
strides=(1, 1))(convolutional_layer_2_2)
convolutional_layer_3_1 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(pooling_layer_2)
convolutional_layer_3_2 = Convolution(
(7, 7),
96,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(convolutional_layer_3_1)
pooling_layer_3 = MaxPooling((2, 2),
strides=(1, 1))(convolutional_layer_3_2)
convolutional_layer_4_1 = Convolution((3, 3),
96,
init=glorot_uniform(),
activation=relu,
pad=True,
strides=(1, 1))(pooling_layer_3)
pooling_layer_4 = MaxPooling((2, 2),
strides=(1, 1))(convolutional_layer_4_1)
##
fully_connected_layer_1 = Dense(512,
init=glorot_uniform())(pooling_layer_4)
dropout_layer_1 = Dropout(0.5)(fully_connected_layer_1)
fully_connected_with_extra_values = splice(dropout_layer_1,
ext_values,
axis=0)
fully_connected_layer_2 = Dense(
256, init=glorot_uniform())(fully_connected_with_extra_values)
fully_connected_layer_3 = Dense(
128, init=glorot_uniform())(fully_connected_layer_2)
dropout_layer_2 = Dropout(0.5)(fully_connected_layer_3)
output_layer = Dense(out_dims, init=glorot_uniform(),
activation=None)(dropout_layer_2)
return output_layer
def create_model(input):
# print (input.shape)
conv1 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(input)
conv1 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv1)
pool1 = MaxPooling((2, 2), strides=(2, 2))(conv1)
conv2 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(pool1)
conv2 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(conv2)
pool2 = MaxPooling((2, 2), strides=(2, 2))(conv2)
conv3 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(pool2)
conv3 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(conv3)
up8 = C.splice(UpSampling2D(conv3), conv2, axis=0)
conv8 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(up8)
conv8 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv8)
up9 = C.splice(UpSampling2D(conv8), conv1, axis=0)
conv9 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(up9)
conv9 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv9)
conv10 = Convolution((1, 1),
1,
init=glorot_uniform(),
activation=sigmoid,
pad=True)(conv9)
print("conv1" + str(conv1.shape))
print("pool1" + str(pool1.shape))
print("conv2" + str(conv2.shape))
print("pool2" + str(pool2.shape))
print("conv3" + str(conv3.shape))
print("conv8" + str(conv8.shape))
print("up9" + str(up9.shape))
print("conv9" + str(conv9.shape))
print("conv10" + str(conv10.shape))
return conv10
from cntk.layers import Dense, Sequential, Activation, Convolution2D, MaxPooling, Dropout, BatchNormalization
from cntk import softmax, relu
input_folder_path = "src/ml/data/autcar_training"
output_folder_path = "src/ml/data/autcar_training_balanced"
image_width = 224
image_height = 168
trainer = Trainer(deeplearning_framework="cntk", image_height=image_height, image_width=image_width)
trainer.create_balanced_dataset(input_folder_path, output_folder_path=output_folder_path)
model = Sequential([
Convolution2D(filter_shape=(5,5), num_filters=32, strides=(1,1), pad=True, name="first_conv"),
BatchNormalization(map_rank=1),
Activation(relu),
MaxPooling(filter_shape=(3,3), strides=(2,2), name="first_max"),
Convolution2D(filter_shape=(3,3), num_filters=48, strides=(1,1), pad=True, name="second_conv"),
BatchNormalization(map_rank=1),
Activation(relu),
MaxPooling(filter_shape=(3,3), strides=(2,2), name="second_max"),
Convolution2D(filter_shape=(3,3), num_filters=64, strides=(1,1), pad=True, name="third_conv"),
BatchNormalization(map_rank=1),
Activation(relu),
MaxPooling(filter_shape=(3,3), strides=(2,2), name="third_max"),
Convolution2D(filter_shape=(5,5), num_filters=32, strides=(1,1), pad=True, name="fourth_conv"),
BatchNormalization(map_rank=1),
Activation(relu),
Dense(100, activation=relu),
Dropout(0.1),
Dense(12, activation=softmax)
])
def inception_v3_cifar_model(input, labelDim, bnTimeConst):
# 32 x 32 x 3
conv1 = conv_bn_relu_layer(input, 32, (3,3), (1,1), True, bnTimeConst)
# 32 x 32 x 32
conv2 = conv_bn_relu_layer(conv1, 32, (3,3), (1,1), True, bnTimeConst)
# 32 x 32 x 32
conv3 = conv_bn_relu_layer(conv2, 64, (3,3), (1,1), True, bnTimeConst)
# 32 x 32 x 64
conv4 = conv_bn_relu_layer(conv3, 80, (1,1), (1,1), True, bnTimeConst)
# 32 x 32 x 80
conv5 = conv_bn_relu_layer(conv4, 128, (3,3), (1,1), True, bnTimeConst)
# 32 x 32 x 128
pool1 = MaxPooling(filter_shape=(3,3), strides=(2,2), pad=True)(conv5)
#
# Inception Blocks
# 16 x 16 x 128
mixed1 = inception_block_1(pool1, 32, [32, 48], [48, 64, 64], 32, bnTimeConst)
# 16 x 16 x 176
mixed2 = inception_block_1(mixed1, 32, [32, 48], [48, 64, 64], 32, bnTimeConst)
# 16 x 16 x 176
mixed3 = inception_block_1(mixed2, 32, [32, 48], [48, 64, 64], 32, bnTimeConst)
# 16 x 16 x 176
mixed4 = inception_block_pass_through(mixed3, 0, 32, 48, 32, 48, 0, bnTimeConst)
# 8 x 8 x 256
mixed5 = inception_block_3(mixed4, 64, [48, 64, 64], [48, 48, 48, 48, 64], 64, bnTimeConst)
# 8 x 8 x 256
mixed6 = inception_block_3(mixed5, 64, [48, 64, 64], [48, 48, 48, 48, 64], 64, bnTimeConst)
# 8 x 8 x 256
mixed7 = inception_block_3(mixed6, 64, [48, 64, 64], [48, 48, 48, 48, 64], 64, bnTimeConst)
# 8 x 8 x 256
mixed8 = inception_block_3(mixed7, 80, [48, 64, 64], [48, 48, 48, 48, 64], 80, bnTimeConst)
# 8 x 8 x 288
mixed9 = inception_block_3(mixed8, 128, [64, 128, 128], [64, 64, 64, 64, 128], 128, bnTimeConst)
# 8 x 8 x 512
mixed10 = inception_block_5(mixed9, 128, [64, 128, 128], [64, 64, 64, 128], 128, bnTimeConst)
# 8 x 8 x 512
mixed11 = inception_block_5(mixed10, 128, [64, 128, 128], [64, 64, 64, 128], 128, bnTimeConst)
# 8 x 8 x 512
#
# Prediction
#
pool2 = AveragePooling(filter_shape=(8,8))(mixed11)
# 1 x 1 x 512
z = Dense(labelDim, init=he_normal())(pool2)
#
# Auxiliary
#
# 8 x 8 x 288
auxPool = AveragePooling(filter_shape=(3,3), strides=(1,1), pad=True)(mixed8)
# 8 x 8 x 288
auxConv1 = conv_bn_relu_layer(auxPool, 320, (1,1), (1,1), True, bnTimeConst)
# 8 x 8 x 320
auxConv2 = conv_bn_relu_layer(auxConv1, 512, (3,3), (1,1), True, bnTimeConst)
# 8 x 8 x 512
aux = Dense(labelDim, init=he_normal())(auxConv2)
return {
'z': z,
'aux': aux
}
def create_model(input):
#print (input.shape)
conv1 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(input)
conv1 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv1)
pool1 = MaxPooling((2, 2), strides=(2, 2))(conv1)
conv2 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(pool1)
conv2 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(conv2)
pool2 = MaxPooling((2, 2), strides=(2, 2))(conv2)
conv3 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(pool2)
conv3 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(conv3)
pool3 = MaxPooling((2, 2), strides=(2, 2))(conv3)
conv4 = Convolution((3, 3),
512,
init=glorot_uniform(),
activation=relu,
pad=True)(pool3)
conv4 = Convolution((3, 3),
512,
init=glorot_uniform(),
activation=relu,
pad=True)(conv4)
pool4 = MaxPooling((2, 2), strides=(2, 2))(conv4)
conv5 = Convolution((3, 3),
1024,
init=glorot_uniform(),
activation=relu,
pad=True)(pool4)
conv5 = Convolution((3, 3),
1024,
init=glorot_uniform(),
activation=relu,
pad=True)(conv5)
print("conv1" + str(conv1.shape))
print("pool1" + str(pool1.shape))
print("conv2" + str(conv2.shape))
print("pool2" + str(pool2.shape))
print("conv3" + str(conv3.shape))
print("pool3" + str(pool3.shape))
print("conv4" + str(conv4.shape))
print("pool4" + str(pool4.shape))
print("conv5" + str(conv5.shape))
up5 = UpSampling2D(conv5)
print("upsamplingC5" + str(up5.shape))
#print("upsamplingC5"+str(C
# .reshape(up5,conv4.shape[]).shape))
##some how we need crop this conv4
#a = cntk.io.transforms.crop(crop_type='center', crop_size=(2, 2))(conv4)
#print("crop" + str(a))
##up7 = C.splice(UpSampling2D(conv4), conv3, axis=0)
#print("up6"+str(up6))
conv6 = Convolution((3, 3),
512,
init=glorot_uniform(),
activation=relu,
pad=True)(up5)
conv6 = Convolution((3, 3),
512,
init=glorot_uniform(),
activation=relu,
pad=True)(conv6)
print("conv6" + str(conv6.shape))
print("up6" + str((UpSampling2D(conv6)).shape))
up7 = C.splice(UpSampling2D(conv6), conv3, axis=0)
conv7 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(up7)
conv7 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(conv7)
print("conv7" + str(conv7.shape))
print("up8" + str((UpSampling2D(conv7)).shape))
up8 = C.splice(UpSampling2D(conv7), conv2, axis=0)
conv8 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(up8)
conv8 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(conv8)
up9 = C.splice(UpSampling2D(conv8), conv1, axis=0)
conv9 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(up9)
conv9 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv9)
conv10 = Convolution((1, 1),
1,
init=glorot_uniform(),
activation=sigmoid,
pad=True)(conv9)
return conv10
def inception_v3_cifar_model(input, labelDim, bnTimeConst):
# 32 x 32 x 3
conv1 = conv_bn_relu_layer(input, 32, (3, 3), (1, 1), True, bnTimeConst)
# 32 x 32 x 32
conv2 = conv_bn_relu_layer(conv1, 32, (3, 3), (1, 1), True, bnTimeConst)
# 32 x 32 x 32
conv3 = conv_bn_relu_layer(conv2, 64, (3, 3), (1, 1), True, bnTimeConst)
# 32 x 32 x 64
conv4 = conv_bn_relu_layer(conv3, 80, (1, 1), (1, 1), True, bnTimeConst)
# 32 x 32 x 80
conv5 = conv_bn_relu_layer(conv4, 128, (3, 3), (1, 1), True, bnTimeConst)
# 32 x 32 x 128
pool1 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=True)(conv5)
#
# Inception Blocks
# 16 x 16 x 128
mixed1 = inception_block_1(pool1, 32, [32, 48], [48, 64, 64], 32,
bnTimeConst)
# 16 x 16 x 160
mixed2 = inception_block_1(mixed1, 32, [32, 48], [48, 64, 64], 64,
bnTimeConst)
# 16 x 16 x 160
mixed3 = inception_block_1(mixed2, 32, [32, 48], [48, 64, 64], 64,
bnTimeConst)
# 16 x 16 x 160
#mixed4 = inception_block_2(mixed3, 32, [48, 64, 64], bnTimeConst)
mixed4 = inception_block_pass_through(mixed3, 0, 64, 80, 32, 48, 0,
bnTimeConst)
# 8 x 8 x 256
mixed5 = inception_block_3(mixed4, 192, [48, 64, 64],
[128, 128, 128, 128, 192], 192, bnTimeConst)
# 8 x 8 x
mixed6 = inception_block_3(mixed5, 192, [160, 160, 192],
[160, 160, 160, 160, 192], 192, bnTimeConst)
# 8 x 8 x 768
mixed7 = inception_block_3(mixed6, 192, [160, 160, 192],
[160, 160, 160, 160, 192], 192, bnTimeConst)
# 8 x 8 x 768
mixed8 = inception_block_3(mixed7, 192, [192, 192, 192],
[192, 192, 192, 192, 192], 192, bnTimeConst)
# 8 x 8 x 768
mixed9 = inception_block_3(mixed8, 192, [192, 192, 192],
[192, 192, 192, 192, 192], 192, bnTimeConst)
# 8 x 8 x 1280
mixed10 = inception_block_5(mixed9, 320, [384, 384, 384],
[448, 384, 384, 384], 192, bnTimeConst)
# 8 x 8 x 2048
mixed11 = inception_block_5(mixed10, 320, [384, 384, 384],
[448, 384, 384, 384], 192, bnTimeConst)
# 8 x 8 x 2048
#
# Prediction
#
pool3 = AveragePooling(filter_shape=(8, 8))(mixed11)
# 1 x 1 x 2048
drop = Dropout(dropout_rate=0.2)(pool3)
# 1 x 1 x 2048
z = Dense(labelDim, init=he_normal())(drop)
#
# Auxiliary
#
# 8 x 8 x 768
auxPool = AveragePooling(filter_shape=(3, 3), strides=(1, 1),
pad=True)(mixed8)
# 5 x 5 x 768
auxConv1 = conv_bn_relu_layer(auxPool, 128, (1, 1), (1, 1), True,
bnTimeConst)
# 5 x 5 x 128
auxConv2 = conv_bn_relu_layer(auxConv1, 256, (3, 3), (1, 1), True,
bnTimeConst)
# 1 x 1 x 768
aux = Dense(labelDim, init=he_normal())(auxConv2)
return {'z': z, 'aux': aux}
def create_vgg19():
# Input variables denoting the features and label data
feature_var = input((num_channels, image_height, image_width))
label_var = input((num_classes))
# apply model to input
# remove mean value
input = minus(feature_var,
constant([[[104]], [[117]], [[124]]]),
name='mean_removed_input')
with default_options(activation=None, pad=True, bias=True):
z = Sequential([
# we separate Convolution and ReLU to name the output for feature extraction (usually before ReLU)
For(
range(2), lambda i: [
Convolution2D((3, 3), 64, name='conv1_{}'.format(i)),
Activation(activation=relu, name='relu1_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool1'),
For(
range(2), lambda i: [
Convolution2D((3, 3), 128, name='conv2_{}'.format(i)),
Activation(activation=relu, name='relu2_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool2'),
For(
range(4), lambda i: [
Convolution2D((3, 3), 256, name='conv3_{}'.format(i)),
Activation(activation=relu, name='relu3_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool3'),
For(
range(4), lambda i: [
Convolution2D((3, 3), 512, name='conv4_{}'.format(i)),
Activation(activation=relu, name='relu4_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool4'),
For(
range(4), lambda i: [
Convolution2D((3, 3), 512, name='conv5_{}'.format(i)),
Activation(activation=relu, name='relu5_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool5'),
Dense(4096, name='fc6'),
Activation(activation=relu, name='relu6'),
Dropout(0.5, name='drop6'),
Dense(4096, name='fc7'),
Activation(activation=relu, name='relu7'),
Dropout(0.5, name='drop7'),
Dense(num_classes, name='fc8')
])(input)
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
pe5 = classification_error(z, label_var, topN=5)
log_number_of_parameters(z)
print()
return {
'feature': feature_var,
'label': label_var,
'ce': ce,
'pe': pe,
'pe5': pe5,
'output': z
}
def create_model(input):
# print (input.shape)
conv1 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(input)
conv1 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv1)
pool1 = MaxPooling((2, 2), strides=(2, 2))(conv1)
conv2 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(pool1)
conv2 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(conv2)
pool2 = MaxPooling((2, 2), strides=(2, 2))(conv2)
conv3 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(pool2)
conv3 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(conv3)
pool3 = MaxPooling((2, 2), strides=(2, 2))(conv3)
conv4 = Convolution((3, 3),
512,
init=glorot_uniform(),
activation=relu,
pad=True)(pool3)
conv4 = Convolution((3, 3),
512,
init=glorot_uniform(),
activation=relu,
pad=True)(conv4)
#pool4 = MaxPooling((2, 2), strides=(2, 2))(conv4)
#conv5 = Convolution((3, 3), 1024, init=glorot_uniform(), activation=relu, pad=True)(pool4)
#conv5 = Convolution((3, 3), 1024, init=glorot_uniform(), activation=relu, pad=True)(conv5)
print("conv1" + str(conv1.shape))
print("pool1" + str(pool1.shape))
print("conv2" + str(conv2.shape))
print("pool2" + str(pool2.shape))
print("conv3" + str(conv3.shape))
print("pool3" + str(pool3.shape))
print("conv4" + str(conv4.shape))
print("up6" + str((UpSampling2D(conv4)).shape))
up7 = C.splice(UpSampling2D(conv4), conv3, axis=0)
conv7 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(up7)
conv7 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(conv7)
print("conv7" + str(conv7.shape))
print("up8" + str((UpSampling2D(conv7)).shape))
up8 = C.splice(UpSampling2D(conv7), conv2, axis=0)
conv8 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(up8)
conv8 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv8)
up9 = C.splice(UpSampling2D(conv8), conv1, axis=0)
conv9 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(up9)
conv9 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv9)
conv10 = Convolution((1, 1),
1,
init=glorot_uniform(),
activation=sigmoid,
pad=True)(conv9)
return conv10
def test_max_pooling_layer(self):
"""Test a model with a single CNTK MaxPooling layer against the
equivalent ELL predictor. This verifies that the import functions
reshape and reorder values appropriately and that the equivalent ELL
layer produces comparable output
"""
x = input((2, 15, 15))
count = 0
inputValues = np.random.uniform(low=-5, high=5,
size=(2, 15, 15)).astype(np.float32)
for pool_size, stride_size in product(range(2, 4), range(2, 3)):
count += 1
_logger.info("test pooling size ({0},{0}) and stride {1}".format(
pool_size, stride_size))
# Create a MaxPooling CNTK layer
poolingLayer = MaxPooling((pool_size, pool_size),
pad=True,
strides=stride_size)
# Input order for CNTK is channels, rows, columns
cntkModel = poolingLayer(x)
# Get the results for both
cntkResults = cntkModel(inputValues)[0]
outputShape = cntkResults.shape
padding = int((pool_size - 1) / 2)
rows = int(inputValues.shape[1] + 2 * padding)
columns = int(inputValues.shape[2] + 2 * padding)
channels = int(inputValues.shape[0])
# Create the equivalent ELL predictor
layerParameters = ell.neural.LayerParameters(
# Input order for ELL is rows, columns, channels
ell.math.TensorShape(rows, columns, channels),
ell.neural.MinPadding(padding),
ell.math.TensorShape(outputShape[1], outputShape[2],
outputShape[0]),
ell.neural.NoPadding())
poolingParameters = ell.neural.PoolingParameters(
pool_size, stride_size)
layer = ell.neural.FloatPoolingLayer(layerParameters,
poolingParameters,
ell.neural.PoolingType.max)
predictor = ell.neural.FloatNeuralNetworkPredictor([layer])
# Note that cntk inserts an extra dimension of 1 in the front
orderedCntkResults = cntk_converters.\
get_float_vector_from_cntk_array(cntkResults)
orderedInputValues = cntk_converters.\
get_float_vector_from_cntk_array(inputValues)
ellResults = predictor.Predict(orderedInputValues)
# Compare them
np.testing.assert_array_almost_equal(
orderedCntkResults, ellResults, 5,
('results for MaxPooling layer do not match! (poolsize = '
'{}, stride = {}').format(pool_size, stride_size))
# now run same over ELL compiled model
self.verify_compiled(
predictor, orderedInputValues, orderedCntkResults,
'max_pooling{}_{}'.format(pool_size,
stride_size), 'test_' + str(count))
def create_model(input, num_classes):
conv1 = Convolution((3, 3),
32,
init=glorot_uniform(),
activation=relu,
pad=True)(input)
conv1 = Convolution((3, 3),
32,
init=glorot_uniform(),
activation=relu,
pad=True)(conv1)
pool1 = MaxPooling((2, 2), strides=(2, 2))(conv1)
conv2 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(pool1)
conv2 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv2)
pool2 = MaxPooling((2, 2), strides=(2, 2))(conv2)
conv3 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(pool2)
conv3 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(conv3)
pool3 = MaxPooling((2, 2), strides=(2, 2))(conv3)
conv4 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(pool3)
conv4 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(conv4)
pool4 = MaxPooling((2, 2), strides=(2, 2))(conv4)
conv5 = Convolution((3, 3),
512,
init=glorot_uniform(),
activation=relu,
pad=True)(pool4)
conv5 = Convolution((3, 3),
512,
init=glorot_uniform(),
activation=relu,
pad=True)(conv5)
up6 = C.splice(UpSampling2D(conv5), conv4, axis=0)
conv6 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(up6)
conv6 = Convolution((3, 3),
256,
init=glorot_uniform(),
activation=relu,
pad=True)(conv6)
up7 = C.splice(UpSampling2D(conv6), conv3, axis=0)
conv7 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(up7)
conv7 = Convolution((3, 3),
128,
init=glorot_uniform(),
activation=relu,
pad=True)(conv7)
up8 = C.splice(UpSampling2D(conv7), conv2, axis=0)
conv8 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(up8)
conv8 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv8)
up9 = C.splice(UpSampling2D(conv8), conv1, axis=0)
conv9 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(up9)
conv9 = Convolution((3, 3),
64,
init=glorot_uniform(),
activation=relu,
pad=True)(conv9)
conv10 = Convolution((1, 1),
num_classes,
init=glorot_uniform(),
activation=sigmoid,
pad=True)(conv9)
return conv10
def create_alexnet():
# Input variables denoting the features and label data
feature_var = C.input_variable((num_channels, image_height, image_width))
label_var = C.input_variable((num_classes))
# apply model to input
# remove mean value
mean_removed_features = minus(feature_var,
constant(114),
name='mean_removed_input')
with default_options(activation=None, pad=True, bias=True):
z = Sequential([
# we separate Convolution and ReLU to name the output for feature extraction (usually before ReLU)
Convolution2D((11, 11),
96,
init=normal(0.01),
pad=False,
strides=(4, 4),
name='conv1'),
Activation(activation=relu, name='relu1'),
LocalResponseNormalization(1.0, 2, 0.0001, 0.75, name='norm1'),
MaxPooling((3, 3), (2, 2), name='pool1'),
Convolution2D((5, 5),
192,
init=normal(0.01),
init_bias=0.1,
name='conv2'),
Activation(activation=relu, name='relu2'),
LocalResponseNormalization(1.0, 2, 0.0001, 0.75, name='norm2'),
MaxPooling((3, 3), (2, 2), name='pool2'),
Convolution2D((3, 3), 384, init=normal(0.01), name='conv3'),
Activation(activation=relu, name='relu3'),
Convolution2D((3, 3),
384,
init=normal(0.01),
init_bias=0.1,
name='conv4'),
Activation(activation=relu, name='relu4'),
Convolution2D((3, 3),
256,
init=normal(0.01),
init_bias=0.1,
name='conv5'),
Activation(activation=relu, name='relu5'),
MaxPooling((3, 3), (2, 2), name='pool5'),
Dense(4096, init=normal(0.005), init_bias=0.1, name='fc6'),
Activation(activation=relu, name='relu6'),
Dropout(0.5, name='drop6'),
Dense(4096, init=normal(0.005), init_bias=0.1, name='fc7'),
Activation(activation=relu, name='relu7'),
Dropout(0.5, name='drop7'),
Dense(num_classes, init=normal(0.01), name='fc8')
])(mean_removed_features)
# loss and metric
ce = cross_entropy_with_softmax(z, label_var)
pe = classification_error(z, label_var)
pe5 = classification_error(z, label_var, topN=5)
log_number_of_parameters(z)
print()
return {
'feature': feature_var,
'label': label_var,
'ce': ce,
'pe': pe,
'pe5': pe5,
'output': z
}
def inception_v3_model(input, labelDim, dropRate, bnTimeConst):
# 299 x 299 x 3
conv1 = conv_bn_relu_layer(input, 32, (3, 3), (2, 2), False, bnTimeConst)
# 149 x 149 x 32
conv2 = conv_bn_relu_layer(conv1, 32, (3, 3), (1, 1), False, bnTimeConst)
# 147 x 147 x 32
conv3 = conv_bn_relu_layer(conv2, 64, (3, 3), (1, 1), True, bnTimeConst)
# 147 x 147 x 64
pool1 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=False)(conv3)
# 73 x 73 x 64
conv4 = conv_bn_relu_layer(pool1, 80, (1, 1), (1, 1), False, bnTimeConst)
# 73 x 73 x 80
conv5 = conv_bn_relu_layer(conv4, 192, (3, 3), (1, 1), False, bnTimeConst)
# 71 x 71 x 192
pool2 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=False)(conv5)
# 35 x 35 x 192
#
# Inception Blocks
#
mixed1 = inception_block_1(pool2, 64, [48, 64], [64, 96, 96], 32,
bnTimeConst)
# 35 x 35 x 256
mixed2 = inception_block_1(mixed1, 64, [48, 64], [64, 96, 96], 64,
bnTimeConst)
# 35 x 35 x 288
mixed3 = inception_block_1(mixed2, 64, [48, 64], [64, 96, 96], 64,
bnTimeConst)
# 35 x 35 x 288
mixed4 = inception_block_2(mixed3, 384, [64, 96, 96], bnTimeConst)
# 17 x 17 x 768
mixed5 = inception_block_3(mixed4, 192, [128, 128, 192],
[128, 128, 128, 128, 192], 192, bnTimeConst)
# 17 x 17 x 768
mixed6 = inception_block_3(mixed5, 192, [160, 160, 192],
[160, 160, 160, 160, 192], 192, bnTimeConst)
# 17 x 17 x 768
mixed7 = inception_block_3(mixed6, 192, [160, 160, 192],
[160, 160, 160, 160, 192], 192, bnTimeConst)
# 17 x 17 x 768
mixed8 = inception_block_3(mixed7, 192, [192, 192, 192],
[192, 192, 192, 192, 192], 192, bnTimeConst)
# 17 x 17 x 768
mixed9 = inception_block_4(mixed8, [192, 320], [192, 192, 192, 192],
bnTimeConst)
# 8 x 8 x 1280
mixed10 = inception_block_5(mixed9, 320, [384, 384, 384],
[448, 384, 384, 384], 192, bnTimeConst)
# 8 x 8 x 2048
mixed11 = inception_block_5(mixed10, 320, [384, 384, 384],
[448, 384, 384, 384], 192, bnTimeConst)
# 8 x 8 x 2048
#
# Prediction
#
pool3 = AveragePooling(filter_shape=(8, 8), pad=False)(mixed11)
# 1 x 1 x 2048
drop = Dropout(dropout_rate=dropRate)(pool3)
# 1 x 1 x 2048
z = Dense(labelDim, init=he_normal())(drop)
#
# Auxiliary
#
# 17 x 17 x 768
auxPool = AveragePooling(filter_shape=(5, 5), strides=(3, 3),
pad=False)(mixed8)
# 5 x 5 x 768
auxConv1 = conv_bn_relu_layer(auxPool, 128, (1, 1), (1, 1), True,
bnTimeConst)
# 5 x 5 x 128
auxConv2 = conv_bn_relu_layer(auxConv1, 768, (5, 5), (1, 1), False,
bnTimeConst)
# 1 x 1 x 768
aux = Dense(labelDim, init=he_normal())(auxConv2)
return {'z': z, 'aux': aux}