def cgan_generator(z, y):
with C.layers.default_options(init=C.normal(scale=0.02), bias=False, map_rank=1, use_cntk_engine=True):
h = C.splice(z, y, axis=0)
h = C.relu(BatchNormalization()(Dense(1024)(h)))
h = C.relu(BatchNormalization()(Dense((128, 7, 7))(h)))
h = C.relu(BatchNormalization()(ConvolutionTranspose2D(
(5, 5), 128, strides=(2, 2), pad=True, output_shape=(14, 14))(h)))
h = ConvolutionTranspose2D((5, 5), 1, activation=C.sigmoid, strides=(2, 2), pad=True, output_shape=(28, 28))(h)
return C.reshape(h, input_dim)
def cgan_discriminator(x, y):
with C.layers.default_options(init=C.normal(scale=0.02), map_rank=1, use_cntk_engine=True):
hx = C.reshape(x, (1, 28, 28))
hy = C.ones_like(hx) * C.reshape(y, (label_dim, 1, 1))
h = C.splice(hx, hy, axis=0)
h = C.leaky_relu((Convolution2D((5, 5), 1, strides=(2, 2))(h)), alpha=0.2)
h = C.leaky_relu(BatchNormalization()(Convolution2D((5, 5), 64, strides=(2, 2))(h)), alpha=0.2)
h = C.leaky_relu(BatchNormalization()(Dense(1024)(h)), alpha=0.2)
h = Dense(1, activation=C.sigmoid)(h)
return h
def _create_convolution_model_with_skip_level_links():
with C.layers.default_options(init=C.glorot_uniform(), activation=C.relu):
h = feature_var
# The first two layers has bias=False to test, the conversion
# work with and without bias in the Convolution.
a = C.layers.Convolution2D(filter_shape=(5,5),
num_filters=64,
strides=(2,2),
pad=True, bias=False, name='first_convo')(h)
a = BatchNormalization(map_rank=1,
normalization_time_constant=4096,
use_cntk_engine=True, init_scale=1,
disable_regularization=True)(a)
b = C.layers.Convolution2D(filter_shape=(5,5),
num_filters=64,
strides=(2,2),
pad=True, bias=False, name='second_convo')(h)
b = BatchNormalization(map_rank=1,
normalization_time_constant=4096,
use_cntk_engine=True, init_scale=1,
disable_regularization=True)(b)
h = a + b
h = C.layers.Convolution2D(filter_shape=(5,5),
num_filters=64,
strides=(1,1),
pad=True, name='thrid_convo')(h)
h = BatchNormalization(map_rank=1,
normalization_time_constant=4096,
use_cntk_engine=True, init_scale=1,
disable_regularization=True)(h)
h = C.layers.Convolution2D(filter_shape=(5,5),
num_filters=64,
strides=(1,1),
pad=True, name='fourth_convo')(h)
h = BatchNormalization(map_rank=1,
normalization_time_constant=4096,
use_cntk_engine=True, init_scale=1,
disable_regularization=True)(h)
r = C.layers.Dense(num_classes, activation=None, name='classify')(h)
return r
def pix2pix_discriminator(y, x):
with C.layers.default_options(init=C.normal(0.02), pad=True, bias=False, map_rank=1, use_cntk_engine=True):
x = C.leaky_relu(Convolution2D((3, 3), 32, strides=2, bias=True)(x), alpha=0.2)
y = C.leaky_relu(Convolution2D((3, 3), 32, strides=2, bias=True)(y), alpha=0.2)
h = C.splice(x, y, axis=0)
h = C.leaky_relu(BatchNormalization()(Convolution2D((3, 3), 128, strides=2)(h)), alpha=0.2)
h = C.leaky_relu(BatchNormalization()(Convolution2D((3, 3), 256, strides=2)(h)), alpha=0.2)
h = C.leaky_relu(BatchNormalization()(Convolution2D((3, 3), 512, strides=2)(h)), alpha=0.2)
h = Convolution2D((1, 1), 1, activation=None, bias=True)(h)
return h
def create_model(feature_dimensions, classes):
with default_options(activation=relu, init=glorot_uniform()):
model = Sequential([
For(
range(3), lambda i: [
Convolution((5, 5), [32, 32, 64][i], pad=True),
BatchNormalization(map_rank=1),
MaxPooling((3, 3), strides=(2, 2))
]),
Dense(64),
BatchNormalization(map_rank=1),
Dense(len(classes), activation=None)
])
return model(feature_dimensions)
def convolution_bn(input, filter_size, num_filters, strides=(1,1), init=he_normal(), activation=relu):
if activation is None:
activation = lambda x: x
r = Convolution(filter_size, num_filters, strides=strides, init=init, activation=None, pad=True, bias=False)(input)
r = BatchNormalization(map_rank=1)(r)
r = activation(r)
return r
def conv_bn(input, filter_size, num_filters, strides=(1, 1), init=he_normal()):
c = Convolution(filter_size,
num_filters,
activation=None,
init=init,
pad=True,
strides=strides,
bias=False)(input)
r = BatchNormalization(map_rank=1,
normalization_time_constant=4096,
use_cntk_engine=False)(c)
return r
def dcgan_discriminator(h):
with C.layers.default_options(init=C.normal(0.02),
pad=True,
bias=False,
map_rank=1,
use_cntk_engine=True):
h = C.leaky_relu(Convolution2D((3, 3), 32, strides=2, bias=True)(h),
alpha=0.2)
h = C.leaky_relu(BatchNormalization()(Convolution2D((3, 3),
64,
strides=2)(h)),
alpha=0.2)
h = C.leaky_relu(BatchNormalization()(Convolution2D((3, 3),
128,
strides=2)(h)),
alpha=0.2)
h = C.leaky_relu(BatchNormalization()(Convolution2D((3, 3),
256,
strides=2)(h)),
alpha=0.2)
h = C.leaky_relu(BatchNormalization()(Convolution2D((3, 3),
512,
strides=2)(h)),
alpha=0.2)
h = C.leaky_relu(BatchNormalization()(Convolution2D((3, 3),
1024,
strides=2)(h)),
alpha=0.2)
h = Convolution2D((4, 4),
1,
activation=C.sigmoid,
pad=False,
bias=True,
strides=1)(h)
return h
def conv_bn_relu_layer(input,
num_filters,
filter_size,
strides=(1, 1),
pad=True,
bnTimeConst=4096,
init=he_normal()):
conv = Convolution(filter_size,
num_filters,
activation=None,
init=init,
pad=pad,
strides=strides,
bias=False)(input)
bn = BatchNormalization(map_rank=1,
normalization_time_constant=bnTimeConst,
use_cntk_engine=False)(conv)
return relu(bn)
def conv_bn(input,
filter_size,
num_filters,
strides=(1, 1),
init=C.he_normal(),
bn_init_scale=1):
c = Convolution2D(filter_size,
num_filters,
activation=None,
init=init,
pad=True,
strides=strides,
bias=False)(input)
r = BatchNormalization(map_rank=1,
normalization_time_constant=4096,
use_cntk_engine=False,
init_scale=bn_init_scale,
disable_regularization=True)(c)
return r
def conv_bn(layer_input,
filter_size,
num_filters,
strides,
init=he_normal(),
name=''):
"""
Returns a convolutional layer followed by a batch normalization layer
"""
r = Convolution(filter_size,
num_filters,
activation=None,
init=init,
pad=True,
strides=strides,
bias=True,
name=name)(layer_input)
r = BatchNormalization(map_rank=1,
normalization_time_constant=4096,
name='{}_bn'.format(name))(r)
return r
def create_network():
input_var = cntk.sequence.input_variable((num_channels, frame_height, frame_width), name='input_var')
target_var = cntk.input_variable((num_classes,), is_sparse=True, name='target_var')
with cntk.layers.default_options(enable_self_stabilization=True):
model = Sequential([
resnet_model(cntk.placeholder()), Label('resnet'),
Dense(hidden_dim, name='cnn_fc'),
cntk.layers.Stabilizer(),
bidirectional_recurrence(LSTM(hidden_dim // 2), LSTM(hidden_dim // 2)),
cntk.sequence.last,
BatchNormalization(),
Dense(num_classes)
])(input_var)
return {
'input': input_var,
'target': target_var,
'model': model,
'loss': cntk.cross_entropy_with_softmax(model, target_var),
'metric': cntk.classification_error(model, target_var)
}
def dcgan_generator(h):
with C.layers.default_options(init=C.normal(0.02),
pad=True,
bias=False,
map_rank=1,
use_cntk_engine=True):
h = C.reshape(h, (-1, 1, 1))
h = ConvolutionTranspose2D((4, 4),
1024,
pad=False,
strides=1,
output_shape=(4, 4))(h)
h = BatchNormalization()(h)
h = C.relu(h)
h = ConvolutionTranspose2D(
(5, 5),
512,
strides=2,
output_shape=(img_height // 32, img_width // 32))(h)
h = BatchNormalization()(h)
h = C.relu(h)
h = ConvolutionTranspose2D(
(5, 5),
256,
strides=2,
output_shape=(img_height // 16, img_width // 16))(h)
h = BatchNormalization()(h)
h = C.relu(h)
h = ConvolutionTranspose2D(
(5, 5),
128,
strides=2,
output_shape=(img_height // 8, img_width // 8))(h)
h = BatchNormalization()(h)
h = C.relu(h)
h = ConvolutionTranspose2D(
(5, 5),
64,
strides=2,
output_shape=(img_height // 4, img_width // 4))(h)
h = BatchNormalization()(h)
h = C.relu(h)
h = ConvolutionTranspose2D(
(5, 5),
32,
strides=2,
output_shape=(img_height // 2, img_width // 2))(h)
h = BatchNormalization()(h)
h = C.relu(h)
h = ConvolutionTranspose2D((5, 5),
3,
strides=2,
bias=True,
output_shape=(img_height, img_width))(h)
h = C.tanh(h)
return h
def test_batch_normalization_layer(self):
# Test a model with a single CNTK BatchNormalization layer against the equivalent ELL predictor
# This verifies that the import functions reshape and reorder values appropriately and
# that the equivalent ELL layer produces comparable output
# Create a BatchNormalization CNTK layer
batchNorm = BatchNormalization(map_rank=1)
# Input order for CNTK is channels, rows, columns
x = input((16, 10, 10))
cntkModel = batchNorm(x)
# Create a test set of scales and biases to use for both CNTK and ELL layers
scaleValues = np.linspace(0.1, 0.5, num=16, dtype=np.float_)
batchNorm.parameters[0].value = scaleValues
scaleVector = ELL.FloatVector(scaleValues)
biasValues = np.linspace(1, 2, num=16, dtype=np.float_)
batchNorm.parameters[1].value = biasValues
biasVector = ELL.FloatVector(biasValues)
# CNTK's BatchNormalization does not support overriding the running mean and variance,
# so we use zeros, which are the default values
meanVector = ELL.FloatVector(16)
varianceVector = ELL.FloatVector(16)
# Create the equivalent ELL predictor
layers = []
layerParameters = ELL.LayerParameters(
ELL.LayerShape(
10, 10, 16), # Input order for ELL is rows, columns, channels
ELL.NoPadding(),
ELL.LayerShape(10, 10, 16),
ELL.NoPadding())
# CNTK BatchNorm = ELL's BatchNorm + Scaling + Bias
# 1e-5 is the default epsilon for CNTK's BatchNormalization Layer
epsilon = 1e-5
layers.append(
ELL.FloatBatchNormalizationLayer(layerParameters, meanVector,
varianceVector, epsilon,
ELL.EpsilonSummand_variance))
layers.append(ELL.FloatScalingLayer(layerParameters, scaleVector))
layers.append(ELL.FloatBiasLayer(layerParameters, biasVector))
predictor = ELL.FloatNeuralNetworkPredictor(layers)
# Compare the results
inputValues = np.linspace(-5, 5, num=16 * 10 * 10,
dtype=np.float32).reshape(16, 10, 10)
cntkResults = cntkModel(inputValues)
orderedCntkResults = cntk_to_ell.get_float_vector_from_cntk_array(
cntkResults
) # Note that cntk inserts an extra dimension of 1 in the front
orderedInputValues = cntk_to_ell.get_float_vector_from_cntk_array(
inputValues)
ellResults = predictor.Predict(orderedInputValues)
# Compare them (precision is 1 less decimal place from epsilon)
np.testing.assert_array_almost_equal(
orderedCntkResults, ellResults, 4,
'results for BatchNormalization layer do not match!')
return
from autcar import Trainer
from cntk.layers import Dense, Sequential, Activation, Convolution2D, MaxPooling, Dropout, BatchNormalization
from cntk import softmax, relu
input_folder_path = "src/ml/data/autcar_training"
output_folder_path = "src/ml/data/autcar_training_balanced"
image_width = 224
image_height = 168
trainer = Trainer(deeplearning_framework="cntk", image_height=image_height, image_width=image_width)
trainer.create_balanced_dataset(input_folder_path, output_folder_path=output_folder_path)
model = Sequential([
Convolution2D(filter_shape=(5,5), num_filters=32, strides=(1,1), pad=True, name="first_conv"),
BatchNormalization(map_rank=1),
Activation(relu),
MaxPooling(filter_shape=(3,3), strides=(2,2), name="first_max"),
Convolution2D(filter_shape=(3,3), num_filters=48, strides=(1,1), pad=True, name="second_conv"),
BatchNormalization(map_rank=1),
Activation(relu),
MaxPooling(filter_shape=(3,3), strides=(2,2), name="second_max"),
Convolution2D(filter_shape=(3,3), num_filters=64, strides=(1,1), pad=True, name="third_conv"),
BatchNormalization(map_rank=1),
Activation(relu),
MaxPooling(filter_shape=(3,3), strides=(2,2), name="third_max"),
Convolution2D(filter_shape=(5,5), num_filters=32, strides=(1,1), pad=True, name="fourth_conv"),
BatchNormalization(map_rank=1),
Activation(relu),
Dense(100, activation=relu),
Dropout(0.1),
Dense(12, activation=softmax)
def pix2pix_generator(h):
with C.layers.default_options(init=C.normal(0.02), pad=True, bias=False, map_rank=1, use_cntk_engine=True):
h_enc1 = C.leaky_relu(Convolution2D((4, 4), 64, strides=2, bias=True)(h), alpha=0.2)
h_enc2 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 128, strides=2)(h_enc1)), alpha=0.2)
h_enc3 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 256, strides=2)(h_enc2)), alpha=0.2)
h_enc4 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=2)(h_enc3)), alpha=0.2)
h_enc5 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=2)(h_enc4)), alpha=0.2)
h_enc6 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=2)(h_enc5)), alpha=0.2)
h_enc7 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=1)(h_enc6)), alpha=0.2)
h_enc8 = C.leaky_relu(BatchNormalization()(Convolution2D((4, 4), 512, strides=1)(h_enc7)), alpha=0.2)
h_dec8 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 512, strides=1, pad=True, output_shape=(img_height // 64, img_width // 64))(h_enc8)))
h_dec8 = C.splice(h_dec8, h_enc8, axis=0)
h_dec8 = C.relu(h_dec8)
h_dec7 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 512, strides=1, pad=True, output_shape=(img_height // 64, img_width // 64))(h_dec8)))
h_dec7 = C.splice(h_dec7, h_enc7, axis=0)
h_dec7 = C.relu(h_dec7)
h_dec6 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 512, strides=1, pad=True, output_shape=(img_height // 64, img_width // 64))(h_dec7)))
h_dec6 = C.splice(h_dec6, h_enc6, axis=0)
h_dec6 = C.relu(h_dec6)
h_dec5 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 512, strides=2, pad=True, output_shape=(img_height // 32, img_width // 32))(h_dec6)))
h_dec5 = C.splice(h_dec5, h_enc5, axis=0)
h_dec5 = C.relu(h_dec5)
h_dec4 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 512, strides=2, pad=True, output_shape=(img_height // 16, img_width // 16))(h_dec5)))
h_dec4 = C.splice(h_dec4, h_enc4, axis=0)
h_dec4 = C.relu(h_dec4)
h_dec3 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 256, strides=2, pad=True, output_shape=(img_height // 8, img_width // 8))(h_dec4)))
h_dec3 = C.splice(h_dec3, h_enc3, axis=0)
h_dec3 = C.relu(h_dec3)
h_dec2 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 128, strides=2, pad=True, output_shape=(img_height // 4, img_width // 4))(h_dec3)))
h_dec2 = C.splice(h_dec2, h_enc2, axis=0)
h_dec2 = C.relu(h_dec2)
h_dec1 = Dropout(0.5)(BatchNormalization()(ConvolutionTranspose2D(
(4, 4), 64, strides=2, pad=True, output_shape=(img_height // 2, img_width // 2))(h_dec2)))
h_dec1 = C.splice(h_dec1, h_enc1, axis=0)
h_dec1 = C.relu(h_dec1)
h = ConvolutionTranspose2D((4, 4), 3, activation=C.tanh, strides=2, pad=True, bias=True,
output_shape=(img_height, img_width))(h_dec1)
return h