def create_model(base_model_file,
input_features,
num_classes,
dropout_rate=0.5,
freeze_weights=False):
# Load the pretrained classification net and find nodes
base_model = load_model(base_model_file)
feature_node = find_by_name(base_model, 'features')
beforePooling_node = find_by_name(base_model, "z.x.x.r")
#graph.plot(base_model, filename="base_model.pdf") # Write graph visualization
# Clone model until right before the pooling layer, ie. until including z.x.x.r
modelCloned = combine([beforePooling_node.owner]).clone(
CloneMethod.freeze if freeze_weights else CloneMethod.clone,
{feature_node: placeholder(name='features')})
# Center the input around zero and set model input.
# Do this early, to avoid CNTK bug with wrongly estimated layer shapes
feat_norm = input_features - constant(114)
model = modelCloned(feat_norm)
# Pool over all spatial dimensions and add dropout layer
avgPool = GlobalAveragePooling(name="poolingLayer")(model)
if dropout_rate > 0:
avgPoolDrop = Dropout(dropout_rate)(avgPool)
else:
avgPoolDrop = avgPool
# Add new dense layer for class prediction
finalModel = Dense(num_classes, activation=None,
name="prediction")(avgPoolDrop)
return finalModel
def ffnet(optimizer):
inputs = 2
outputs = 2
layers = 2
hidden_dimension = 50
# input variables denoting the features and label data
features = C.input_variable((inputs), np.float32)
label = C.input_variable((outputs), np.float32)
# Instantiate the feedforward classification model
my_model = Sequential([
Dense(hidden_dimension,
activation=C.sigmoid,
init=C.glorot_uniform(seed=98052)),
Dense(outputs, init=C.glorot_uniform(seed=98052))
])
z = my_model(features)
ce = C.cross_entropy_with_softmax(z, label)
pe = C.classification_error(z, label)
# Instantiate the trainer object to drive the model training
lr_per_minibatch = learning_rate_schedule(0.125, UnitType.minibatch)
progress_printer = ProgressPrinter(0)
trainer = C.Trainer(z, (ce, pe),
[optimizer(z.parameters, lr_per_minibatch)],
progress_printer)
# Get minibatches of training data and perform model training
minibatch_size = 25
num_minibatches_to_train = 63
for i in range(num_minibatches_to_train):
train_features, labels = generate_random_data(minibatch_size, inputs,
outputs)
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
trainer.train_minibatch({features: train_features, label: labels})
test_features, test_labels = generate_random_data(minibatch_size, inputs,
outputs)
avg_error = trainer.test_minibatch({
features: test_features,
label: test_labels
})
print(' error rate on an unseen minibatch: {}'.format(avg_error))
return z.parameters
def create_model(X):
# defines few stacked GRUs
l1 = Recurrence(step_function=GRU(shape=20))(X)
l2 = Recurrence(step_function=GRU(shape=20))(l1)
l3 = Dense(shape=1)(l2)
return l3
def Blocked_Dense(dim, activation=None):
dense = Dense(dim, activation=activation)
@C.layers.BlockFunction('blocked_dense', 'blocked_dense')
def func(x):
return dense(x)
return func
def create_shallow_model(input, out_dims):
convolutional_layer_1_1 = Convolution((7,7), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(input)
convolutional_layer_1_2 = Convolution((25,25), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(convolutional_layer_1_1)
pooling_layer_1 = MaxPooling((25,25), strides=(5,5))(convolutional_layer_1_2 )
convolutional_layer_2_1 = Convolution((3,3), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(pooling_layer_1)
pooling_layer_2 = MaxPooling((2,2), strides=(2,2))(convolutional_layer_2_1)
fully_connected_layer_1 = Dense(512, init=glorot_uniform())(pooling_layer_2)
fully_connected_layer_2 = Dense(128, init=glorot_uniform())(fully_connected_layer_1)
dropout_layer = Dropout(0.5)(fully_connected_layer_2)
output_layer = Dense(out_dims, init=glorot_uniform(), activation=None)(dropout_layer)
return output_layer
def _setup_test_model(self):
inputs = input_variable(shape=(1, ), dtype=np.float32)
outputs = input_variable(shape=(1, ), dtype=np.float32)
q = Dense(1, activation=None)(inputs)
loss = squared_error(q, outputs)
return {'inputs': inputs, 'outputs': outputs, 'f': q, 'loss': loss}
def create_vgg16(feature_var, num_classes, dropout=0.9):
with default_options(activation=None, pad=True, bias=True):
z = Sequential([
# we separate Convolution and ReLU to name the output for feature
# extraction (usually before ReLU)
For(
range(2), lambda i: [
Convolution2D((3, 3), 64, name='conv1_{}'.format(i)),
Activation(activation=relu, name='relu1_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool1'),
For(
range(2), lambda i: [
Convolution2D((3, 3), 128, name='conv2_{}'.format(i)),
Activation(activation=relu, name='relu2_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool2'),
For(
range(3), lambda i: [
Convolution2D((3, 3), 256, name='conv3_{}'.format(i)),
Activation(activation=relu, name='relu3_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool3'),
For(
range(3), lambda i: [
Convolution2D((3, 3), 512, name='conv4_{}'.format(i)),
Activation(activation=relu, name='relu4_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool4'),
For(
range(3), lambda i: [
Convolution2D((3, 3), 512, name='conv5_{}'.format(i)),
Activation(activation=relu, name='relu5_{}'.format(i)),
]),
MaxPooling((2, 2), (2, 2), name='pool5'),
Dense(4096, name='fc6'),
Activation(activation=relu, name='relu6'),
Dropout(dropout, name='drop6'),
Dense(4096, name='fc7'),
Activation(activation=relu, name='relu7'),
Dropout(dropout, name='drop7'),
Dense(num_classes, name='fc8')
])(feature_var)
return z
def _setup_test_model(self, *args, **kwargs):
inputs = placeholder(shape=(1, ))
outputs = input_variable(shape=(1, ), dtype=np.float32)
q = Dense(1, activation=None)(inputs)
loss = cross_entropy_with_softmax(q, outputs)
return {'inputs': inputs, 'outputs': outputs, 'f': q, 'loss': loss}
def create_basic_model(input, out_dims):
# convolutional_layer_1 = Convolution((3,3), 64, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(input)
# pooling_layer_1 = MaxPooling((2,2), strides=(1,1))(convolutional_layer_1 )
convolutional_layer_2 = Convolution((5,5), 64, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(input)
pooling_layer_2 = MaxPooling((2,2), strides=(1,1))(convolutional_layer_2)
convolutional_layer_3 = Convolution((9,9), 32, init=glorot_uniform(), activation=relu, pad=True, strides=(1,1))(pooling_layer_2)
pooling_layer_3 = MaxPooling((3,3), strides=(2,2))(convolutional_layer_3)
##
fully_connected_layer = Dense(256, init=glorot_uniform())(pooling_layer_3)
dropout_layer = Dropout(0.5)(fully_connected_layer)
output_layer = Dense(out_dims, init=glorot_uniform(), activation=None)(dropout_layer)
return output_layer
def LSTM_sequence_classifier_net(input, num_output_classes, embedding_dim,
LSTM_dim, cell_dim):
lstm_classifier = Sequential([
Embedding(embedding_dim),
Recurrence(LSTM(LSTM_dim, cell_dim)), sequence.last,
Dense(num_output_classes)
])
return lstm_classifier(input)
def test_htk_deserializers():
mbsize = 640
epoch_size = 1000 * mbsize
lr = [0.001]
feature_dim = 33
num_classes = 132
context = 2
os.chdir(data_path)
features_file = "glob_0000.scp"
labels_file = "glob_0000.mlf"
label_mapping_file = "state.list"
fd = HTKFeatureDeserializer(
StreamDefs(amazing_features=StreamDef(
shape=feature_dim, context=(context, context), scp=features_file)))
ld = HTKMLFDeserializer(
label_mapping_file,
StreamDefs(
awesome_labels=StreamDef(shape=num_classes, mlf=labels_file)))
reader = MinibatchSource([fd, ld])
features = C.input_variable(((2 * context + 1) * feature_dim))
labels = C.input_variable((num_classes))
model = Sequential(
[For(range(3), lambda: Recurrence(LSTM(256))),
Dense(num_classes)])
z = model(features)
ce = C.cross_entropy_with_softmax(z, labels)
errs = C.classification_error(z, labels)
learner = C.adam_sgd(z.parameters,
lr=C.learning_rate_schedule(lr, C.UnitType.sample,
epoch_size),
momentum=C.momentum_as_time_constant_schedule(1000),
low_memory=True,
gradient_clipping_threshold_per_sample=15,
gradient_clipping_with_truncation=True)
trainer = C.Trainer(z, (ce, errs), learner)
input_map = {
features: reader.streams.amazing_features,
labels: reader.streams.awesome_labels
}
pp = C.ProgressPrinter(freq=0)
# just run and verify it doesn't crash
for i in range(3):
mb_data = reader.next_minibatch(mbsize, input_map=input_map)
trainer.train_minibatch(mb_data)
pp.update_with_trainer(trainer, with_metric=True)
assert True
os.chdir(abs_path)
def ffnet(learner, trainer=None):
inputs = 5
outputs = 3
layers = 2
hidden_dimension = 3
if trainer is None:
# input variables denoting the features and label data
features = C.input_variable((inputs), np.float32)
label = C.input_variable((outputs), np.float32)
# Instantiate the feedforward classification model
my_model = Sequential ([
Dense(hidden_dimension, activation=C.sigmoid, init=C.glorot_uniform(seed=98052)),
Dense(outputs, init=C.glorot_uniform(seed=98052))])
z = my_model(features)
ce = C.cross_entropy_with_softmax(z, label)
pe = C.classification_error(z, label)
# Instantiate the trainer object to drive the model training
progress_printer = ProgressPrinter(0)
trainer = C.Trainer(z, (ce, pe), [learner(z.parameters)], [progress_printer])
else:
features = trainer.loss_function.arguments[0]
label = trainer.loss_function.arguments[1]
# Get minibatches of training data and perform model training
minibatch_size = 25
num_minibatches_to_train = 100
aggregate_loss = 0.0
for i in range(num_minibatches_to_train):
train_features, labels = generate_random_data(minibatch_size, inputs, outputs)
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
trainer.train_minibatch({features : train_features, label : labels})
sample_count = trainer.previous_minibatch_sample_count
aggregate_loss += trainer.previous_minibatch_loss_average * sample_count
last_avg_error = aggregate_loss / trainer.total_number_of_samples_seen
test_features, test_labels = generate_random_data(minibatch_size, inputs, outputs)
avg_error = trainer.test_minibatch({features : test_features, label : test_labels})
print(' error rate on an unseen minibatch: {}'.format(avg_error))
return last_avg_error, avg_error, trainer
def test_layers_dense(device_id):
y = input(2)
dat = np.array([[-1., 1.]], dtype=np.float32)
####################################################
# Test 1: no activation
####################################################
p = Dense(2, activation=None, name='foo')(y)
res = p(y).eval({y: dat})
npout = np.matrix(dat[0]) * p.foo.W.value + p.foo.b.value
np.testing.assert_array_equal(res, npout, err_msg='Error in dense layer')
####################################################
# Test 2: with activation
####################################################
p = Dense(2, activation=sigmoid, name='foo')(y)
res = p(y).eval({y: dat})
def _sigmoid(x):
return 1./(1 + np.exp(-x))
npout = _sigmoid(np.matrix(dat[0]) * p.foo.W.value + p.foo.b.value)
np.testing.assert_array_almost_equal(res, npout, decimal=7, err_msg='Error in dense layer with sigmoid')
####################################################
# Test 3: 2-dense layer
####################################################
p = Dense(3, activation=None, name='foo')(y)
q = Dense(3, activation=None, name='bar')(p)
res = q(y).eval({y: dat})
npout1 = np.matrix(dat[0]) * p.foo.W.value + p.foo.b.value
npout = npout1 * q.bar.W.value + q.bar.b.value
np.testing.assert_array_almost_equal(res, npout, decimal=7, err_msg='Error in 2-dense layer')
####################################################
# Test 4: Failing configuration
####################################################
with pytest.raises(ValueError):
Dense(2, input_rank=1, map_rank=1) # input_rank and map_rank can be specified at the same time
def create_vgg9_model(input, num_classes):
with default_options(activation=relu):
model = Sequential([
LayerStack(
3, lambda i: [
Convolution((3, 3), [64, 96, 128][i],
init=glorot_uniform(),
pad=True),
Convolution((3, 3), [64, 96, 128][i],
init=glorot_uniform(),
pad=True),
MaxPooling((3, 3), strides=(2, 2))
]),
LayerStack(2, lambda: [Dense(1024, init=glorot_uniform())]),
Dense(num_classes, init=glorot_uniform(), activation=None)
])
return model(input)
def ffnet():
input_dim = 2
num_output_classes = 2
num_hidden_layers = 2
hidden_layers_dim = 50
# Input variables denoting the features and label data
feature = input_variable((input_dim), np.float32)
label = input_variable((num_output_classes), np.float32)
netout = Sequential([
For(range(num_hidden_layers),
lambda i: Dense(hidden_layers_dim, activation=sigmoid)),
Dense(num_output_classes)
])(feature)
ce = cross_entropy_with_softmax(netout, label)
pe = classification_error(netout, label)
lr_per_minibatch = learning_parameter_schedule(0.5)
# Instantiate the trainer object to drive the model training
learner = sgd(netout.parameters, lr=lr_per_minibatch)
progress_printer = ProgressPrinter(128)
trainer = Trainer(netout, (ce, pe), learner, progress_printer)
# Get minibatches of training data and perform model training
minibatch_size = 25
for i in range(1024):
features, labels = generate_random_data(minibatch_size, input_dim,
num_output_classes)
# Specify the mapping of input variables in the model to actual
# minibatch data to be trained with
trainer.train_minibatch({feature: features, label: labels})
trainer.summarize_training_progress()
test_features, test_labels = generate_random_data(minibatch_size,
input_dim,
num_output_classes)
avg_error = trainer.test_minibatch({
feature: test_features,
label: test_labels
})
return avg_error
def create_advanced_model(input, out_dims):
with default_options(activation=relu):
model = Sequential([
For(range(2), lambda i: [ # lambda with one parameter
Convolution((3,3), [32,64][i], pad=True), # depth depends on i
Convolution((5,5), [32,64][i], pad=True),
Convolution((9,9), [32,64][i], pad=True),
MaxPooling((3,3), strides=(2,2))
]),
For(range(2), lambda : [ # lambda without parameter
Dense(512),
Dropout(0.5)
]),
Dense(out_dims, activation=None)
])
output_layer=model(input)
return output_layer
def create_model(output_dim):
return Sequential([
For(
range(num_layers), lambda: Sequential([
Stabilizer(),
Recurrence(LSTM(hidden_dim), go_backwards=False)
])),
Dense(output_dim)
])
def ffnet():
inputs = 2
outputs = 2
layers = 2
hidden_dimension = 50
# input variables denoting the features and label data
features = C.input_variable((inputs), np.float32)
label = C.input_variable((outputs), np.float32)
# Instantiate the feedforward classification model
my_model = Sequential ([
Dense(hidden_dimension, activation=C.sigmoid),
Dense(outputs)])
z = my_model(features)
ce = C.cross_entropy_with_softmax(z, label)
pe = C.classification_error(z, label)
# Instantiate the trainer object to drive the model training
lr_per_minibatch = C.learning_parameter_schedule(0.125)
progress_printer = ProgressPrinter(0)
trainer = C.Trainer(z, (ce, pe), [sgd(z.parameters, lr=lr_per_minibatch)], [progress_printer])
# Get minibatches of training data and perform model training
minibatch_size = 25
num_minibatches_to_train = 1024
aggregate_loss = 0.0
for i in range(num_minibatches_to_train):
train_features, labels = generate_random_data(minibatch_size, inputs, outputs)
# Specify the mapping of input variables in the model to actual minibatch data to be trained with
trainer.train_minibatch({features : train_features, label : labels})
sample_count = trainer.previous_minibatch_sample_count
aggregate_loss += trainer.previous_minibatch_loss_average * sample_count
last_avg_error = aggregate_loss / trainer.total_number_of_samples_seen
test_features, test_labels = generate_random_data(minibatch_size, inputs, outputs)
avg_error = trainer.test_minibatch({features : test_features, label : test_labels})
print(' error rate on an unseen minibatch: {}'.format(avg_error))
return last_avg_error, avg_error
def create_symbol():
# Weight initialiser from uniform distribution
# Activation (unless states) is None
with cntk.layers.default_options(init=cntk.glorot_uniform(),
activation=cntk.relu):
x = Convolution2D(filter_shape=(3, 3), num_filters=50,
pad=True)(features)
x = Convolution2D(filter_shape=(3, 3), num_filters=50, pad=True)(x)
x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x)
x = Dropout(0.25)(x)
x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x)
x = Convolution2D(filter_shape=(3, 3), num_filters=100, pad=True)(x)
x = MaxPooling((2, 2), strides=(2, 2), pad=False)(x)
x = Dropout(0.25)(x)
x = Dense(512)(x)
x = Dropout(0.5)(x)
x = Dense(N_CLASSES, activation=None)(x)
return x
def test_dense_layer(self):
"""Test a model with a single CNTK Dense layer against the equivalent
ELL predictor. This verifies that the import functions reshape and
reorder values appropriately and that the equivalent ELL layer
produces comparable output
"""
# Create a Dense CNTK layer with no bias or activation
denseLayer = Dense(5, bias=False)
x = input((2, 3, 4)) # Input order for CNTK is channels, rows, columns
cntkModel = denseLayer(x)
# Create a test set of weights to use for both CNTK and ELL layers
# CNTK has these in channels, rows, columns, [output shape] order
weightValues = np.arange(120, dtype=np.float_).reshape(2, 3, 4, 5)
# Set the weights
denseLayer.parameters[0].value = weightValues
# create an ELL Tensor from the cntk weights, which re-orders the
# weights and produces an appropriately dimensioned tensor
weightTensor = cntk_converters.\
get_tensor_from_cntk_dense_weight_parameter(
denseLayer.parameters[0])
# Create the equivalent ELL predictor
layerParameters = ell.neural.LayerParameters(
# Input order for ELL is rows, columns, channels
ell.math.TensorShape(3, 4, 2),
ell.neural.NoPadding(),
ell.math.TensorShape(1, 1, 5),
ell.neural.NoPadding(),
ell.nodes.PortType.smallReal)
layer = ell.neural.FullyConnectedLayer(layerParameters, weightTensor)
predictor = ell.neural.NeuralNetworkPredictor([layer])
# Get the results for both
inputValues = np.arange(24, dtype=np.float32).reshape(2, 3, 4)
orderedInputValues = cntk_converters.get_vector_from_cntk_array(
inputValues)
cntkResults = cntkModel(inputValues)
orderedCntkResults = cntk_converters.get_vector_from_cntk_array(
cntkResults)
ellResults = predictor.Predict(orderedInputValues)
# Compare the results
np.testing.assert_array_equal(
orderedCntkResults, ellResults,
'results for Dense layer do not match!')
# now run same over ELL compiled model
self.verify_compiled(predictor, orderedInputValues, orderedCntkResults,
"dense", "test")
def create_model():
'''
Creates the model to train
:return: Returns the last output of a sequential model using LSTMs
'''
return Sequential([
For(range(NUMBER_LAYERS),
lambda: Sequential([Recurrence(LSTM(HIDDEN_LAYER_DIMENSIONS))])),
sequence.last,
Dense(NUM_OUTPUT_CLASSES)
])
def test_failing_for_constructor():
with pytest.raises((ValueError, TypeError)):
network = For(range(3), Dense(5))
class MyFunction:
def __call__(self, x):
return Dense(x)
with pytest.raises((ValueError, TypeError)):
network = For(range(3), MyFunction())
with pytest.raises((ValueError, TypeError)):
network = For(range(3), MyFunction()(5))
def test_for_constructor_layer(layers_count, dense_units):
x = input(4)
network = For(range(layers_count), lambda i: Dense(dense_units))
expected_num_of_parameters = 2 * layers_count
assert len(network.parameters) == expected_num_of_parameters
res = network(x)
expected_output_shape = (dense_units, )
assert res.shape == expected_output_shape
def bn_inception_model(input, labelDim, bnTimeConst):
# 224 x 224 x 3
conv1 = conv_bn_relu_layer(input, 64, (7, 7), (2, 2), True, bnTimeConst)
# 112 x 112 x 64
pool1 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=True)(conv1)
# 56 x 56 x 64
conv2a = conv_bn_relu_layer(pool1, 64, (1, 1), (1, 1), True, bnTimeConst)
# 56 x 56 x 64
conv2b = conv_bn_relu_layer(conv2a, 192, (3, 3), (1, 1), True, bnTimeConst)
# 56 x 56 x 192
pool2 = MaxPooling(filter_shape=(3, 3), strides=(2, 2), pad=True)(conv2b)
# Inception Blocks
# 28 x 28 x 192
inception3a = inception_block_with_avgpool(pool2, 64, 64, 64, 64, 96, 32,
bnTimeConst)
# 28 x 28 x 256
inception3b = inception_block_with_avgpool(inception3a, 64, 64, 96, 64, 96,
64, bnTimeConst)
# 28 x 28 x 320
inception3c = inception_block_pass_through(inception3b, 0, 128, 160, 64,
96, 0, bnTimeConst)
# 14 x 14 x 576
inception4a = inception_block_with_avgpool(inception3c, 224, 64, 96, 96,
128, 128, bnTimeConst)
# 14 x 14 x 576
inception4b = inception_block_with_avgpool(inception4a, 192, 96, 128, 96,
128, 128, bnTimeConst)
# 14 x 14 x 576
inception4c = inception_block_with_avgpool(inception4b, 160, 128, 160, 128,
160, 128, bnTimeConst)
# 14 x 14 x 576
inception4d = inception_block_with_avgpool(inception4c, 96, 128, 192, 160,
192, 128, bnTimeConst)
# 14 x 14 x 576
inception4e = inception_block_pass_through(inception4d, 0, 128, 192, 192,
256, 0, bnTimeConst)
# 7 x 7 x 1024
inception5a = inception_block_with_avgpool(inception4e, 352, 192, 320, 160,
224, 128, bnTimeConst)
# 7 x 7 x 1024
inception5b = inception_block_with_maxpool(inception5a, 352, 192, 320, 192,
224, 128, bnTimeConst)
# Global Average
# 7 x 7 x 1024
pool3 = AveragePooling(filter_shape=(7, 7))(inception5b)
# 1 x 1 x 1024
z = Dense(labelDim, init=he_normal())(pool3)
return z
def modify_model(features, n_classes):
loaded_model = load_model(model_file)
feature_node = find_by_name(loaded_model, 'features')
last_node = find_by_name(loaded_model, 'h2_d')
all_layers = combine([last_node.owner
]).clone(CloneMethod.freeze,
{feature_node: placeholder()})
feat_norm = features - Constant(114)
fc_out = all_layers(feat_norm)
z = Dense(num_classes)(fc_out)
return (z)
def create_network():
input_var = cntk.sequence.input_variable((num_channels, frame_height, frame_width), name='input_var')
target_var = cntk.input_variable((num_classes,), is_sparse=True, name='target_var')
with cntk.layers.default_options(enable_self_stabilization=True):
model = Sequential([
resnet_model(cntk.placeholder()), Label('resnet'),
Dense(hidden_dim, name='cnn_fc'),
cntk.layers.Stabilizer(),
bidirectional_recurrence(LSTM(hidden_dim // 2), LSTM(hidden_dim // 2)),
cntk.sequence.last,
BatchNormalization(),
Dense(num_classes)
])(input_var)
return {
'input': input_var,
'target': target_var,
'model': model,
'loss': cntk.cross_entropy_with_softmax(model, target_var),
'metric': cntk.classification_error(model, target_var)
}
def bn_inceptionv2_cifar_model(input, labelDim, bnTimeConst):
# 32 x 32 x 3
conv1a = conv_bn_relu_layer(input, 32, (3,3), (1,1), True, bnTimeConst)
# 32 x 32 x 32
conv1b = conv_bn_relu_layer(conv1a, 32, (3,3), (1,1), True, bnTimeConst)
# 32 x 32 x 32
conv1c = conv_bn_relu_layer(conv1b, 32, (3,3), (1,1), True, bnTimeConst)
# 32 x 32 x 64
conv2a = conv_bn_relu_layer(conv1c, 64, (3,3), (1,1), True, bnTimeConst)
# 32 x 32 x 64
conv2b = conv_bn_relu_layer(conv2a, 64, (3,3), (1,1), True, bnTimeConst)
# 32 x 32 x 64
conv2c = conv_bn_relu_layer(conv2b, 64, (3,3), (1,1), True, bnTimeConst)
# Inception Blocks
# 32 x 32 x 64
inception3a = inception_block1(conv2b, 32, 32, 32, 32, 48, 16, bnTimeConst)
# 32 x 32 x 128
inception3b = inception_block1(inception3a, 0, 64, 80, 32, 48, 0, bnTimeConst)
# 32 x 32 x 128
inception3c = inception_block1(inception3b, 0, 64, 80, 32, 48, 0, bnTimeConst)
# 32 x 32 x 128
pool1 = AveragePooling(filter_shape=(3,3), strides = (2,2), pad = True)(inception3c)
# 16 x 16 x 256
inception4a = inception_block2(inception3c, 96, 48, 64, 48, 64, 64, bnTimeConst)
# 16 x 16 x 256
inception4b = inception_block2(inception4a, 96, 48, 64, 48, 64, 64, bnTimeConst)
# 16 x 16 x 256
inception4c = inception_block2(inception4b, 96, 48, 64, 48, 64, 64, bnTimeConst)
# 16 x 16 x 288
inception4d = inception_block2(inception4c, 48, 64, 96, 80, 96, 64, bnTimeConst)
# 16 x 16 x 288
inception4e = inception_block2(inception4d, 0, 128, 192, 192, 256, 0, bnTimeConst)
# 16 x 16 x 288
pool2 = AveragePooling(filter_shape=(3,3), strides = (2,2), pad = True)(inception4e)
# Inception Blocks
# 8 x 8 x 512
inception5a = inception_block1(pool2, 176, 96, 160, 96, 112, 64, bnTimeConst)
# 8 x 8 x 512
inception5b = inception_block1(inception5a, 176, 96, 160, 96, 112, 64, bnTimeConst)
# Global Average
# 8 x 8 x 512
pool3 = AveragePooling(filter_shape=(8,8))(inception5a)
# 1 x 1 x 512
z = Dense(labelDim, init=he_normal())(pool3)
return z
def test_sequential_clique_with_layers(input_elements, expected):
x = C.input_variable(input_elements)
np_data = np.arange(input_elements, dtype=np.float32)
unit_dense = Dense(input_elements, activation=None, init=1)
seq_clique = SequentialClique([unit_dense, unit_dense, unit_dense])(x)
assert seq_clique.shape == x.shape
res = seq_clique.eval(np_data)
assert res[0].shape == (input_elements, )
assert np.unique(res[0])[0] == expected
def create_convolutional_neural_network(self,
input_vars,
out_dims,
dropout_prob=0.0):
convolutional_layer_1 = Convolution((5, 5),
32,
strides=1,
activation=cntk.ops.relu,
pad=True)(input_vars)
pooling_layer_1 = MaxPooling((2, 2), strides=(2, 2),
pad=True)(convolutional_layer_1)
convolutional_layer_2 = Convolution((5, 5),
64,
strides=1,
activation=cntk.ops.relu,
pad=True)(pooling_layer_1)
pooling_layer_2 = MaxPooling((2, 2), strides=(2, 2),
pad=True)(convolutional_layer_2)
fully_connected_layer = Dense(
1024, activation=cntk.ops.relu)(pooling_layer_2)
dropout_layer = Dropout(dropout_prob)(fully_connected_layer)
output_layer = Dense(out_dims, activation=None)(dropout_layer)
return output_layer
def inceptionv1_cifar_model2(input, labelDim, bnTimeConst):
# 32 x 32 x 3
conv1 = conv_bn_relu_layer(input, 32, (3, 3), (1, 1), True, bnTimeConst)
# 32 x 32 x 32
conv2 = conv_bn_relu_layer(conv1, 32, (3, 3), (1, 1), True, bnTimeConst)
# Inception Blocks
# 32 x 32 x 64
inception3a = inception_block_with_maxpool(conv2, 32, 32, 32, 32, 48, 16,
bnTimeConst)
# 32 x 32 x 128
inception3b = inception_block_with_maxpool(inception3a, 32, 32, 32, 32, 48,
16, bnTimeConst)
maxpool1 = MaxPooling((3, 3), strides=(2, 2), pad=True)(inception3b)
# 16 x 16 x 128
inception4a = inception_block_with_maxpool(maxpool1, 96, 48, 64, 48, 64,
64, bnTimeConst)
# 16 x 16 x 288
inception4b = inception_block_with_maxpool(inception4a, 96, 48, 64, 48, 64,
64, bnTimeConst)
# 16 x 16 x 288
inception4c = inception_block_with_maxpool(inception4b, 96, 48, 64, 48, 64,
64, bnTimeConst)
# 16 x 16 x 288
inception4d = inception_block_with_maxpool(inception4c, 96, 48, 64, 48, 64,
64, bnTimeConst)
# 16 x 16 x 288
inception4e = inception_block_with_maxpool(inception4d, 96, 48, 64, 48, 64,
64, bnTimeConst)
maxpool2 = MaxPooling((3, 3), strides=(2, 2), pad=True)(inception4e)
# 8 x 8 x 288
inception5a = inception_block_with_maxpool(inception4e, 176, 96, 160, 96,
112, 64, bnTimeConst)
# 8 x 8 x 512
inception5b = inception_block_with_maxpool(inception5a, 176, 96, 160, 96,
112, 64, bnTimeConst)
# Global Average
# 8 x 8 x 512
pool1 = AveragePooling(filter_shape=(8, 8))(inception5b)
# 1 x 1 x 512
z = Dense(labelDim, init=he_normal())(pool1)
return z
def test_large_model_serialization_float(tmpdir):
import os;
from cntk.layers import Recurrence, LSTM, Dense
type_size = np.dtype(np.float32).itemsize
two_gb = 2**31
size = (2097152 + 4, 256, 512, 4096)
assert size[0] * size[1] * type_size > two_gb
device = C.device.cpu()
i = C.sequence.input(size[0])
w = C.Parameter((size[0], size[1]), init=C.uniform(3.0, seed=12345),
device=device)
e = C.times(i, w)
h_fwd = Recurrence(LSTM(size[2]))(e)
h_bwd = Recurrence(LSTM(size[2]), go_backwards=True)(e)
h_last_fwd = C.sequence.last(h_fwd)
h_first_bwd = C.sequence.first(h_bwd)
t = C.splice(h_last_fwd, h_first_bwd)
z1 = Dense(size[2], activation=C.relu)(t)
z = Dense(2, activation=None)(z1)
filename = str(tmpdir / 'test_large_model_serialization_float.out')
z.save(filename)
assert os.path.getsize(filename) > two_gb
y = C.Function.load(filename, device=device)
assert (len(z.parameters) == len(y.parameters))
for param_pair in zip(z.parameters, y.parameters):
assert param_pair[0].shape == param_pair[1].shape
assert np.allclose(param_pair[0].value, param_pair[1].value)
