def learning_word_embeddings_with_the_embedding_layer_cntk():
x_train, y_train, x_test, y_test = load_from_files()
max_features = 10000
maxlen = 20
embedding_dim = 8
x = cntk.input_variable(shape=(maxlen, ), dtype=np.float32)
y = cntk.input_variable(shape=(1, ), dtype=np.float32)
model = cntk.one_hot(x, num_classes=max_features, sparse_output=True)
model = cntk.layers.Embedding(embedding_dim)(model)
model = cntk.layers.Dense(1, activation=cntk.sigmoid)(model)
loss_function = cntk.binary_cross_entropy(model.output, y)
round_predictions = cntk.round(model.output)
equal_elements = cntk.equal(round_predictions, y)
accuracy_function = cntk.reduce_mean(equal_elements, axis=0)
max_epochs = 30
batch_size = 32
learner = cntk.adam(model.parameters,
cntk.learning_parameter_schedule_per_sample(0.0001),
cntk.learning_parameter_schedule_per_sample(0.99))
progress_printer = cntk.logging.ProgressPrinter(tag='Training',
num_epochs=max_epochs)
trainer = cntk.Trainer(model, (loss_function, accuracy_function),
[learner], progress_printer)
evaluator = cntk.Evaluator(accuracy_function)
cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer,
evaluator)
def implementing_1d_convnet_cntk():
max_features = 10000 # number of words to consider as features
max_len = 500 # cut texts after this number of words (among top max_features most common words)
x_train, y_train, x_test, y_test = load_data(max_features, max_len)
model = build_model_cntk(max_features, max_len)
x = cntk.input_variable(shape=(max_len, ), dtype=np.float32)
y = cntk.input_variable(shape=(1, ), dtype=np.float32)
model.replace_placeholders({model.placeholders[0]: x})
loss_function = cntk.binary_cross_entropy(model.output, y)
round_predictions = cntk.round(model.output)
equal_elements = cntk.equal(round_predictions, y)
accuracy_function = cntk.reduce_mean(equal_elements, axis=0)
max_epochs = 10
batch_size = 32
learner = cntk.adam(model.parameters,
cntk.learning_parameter_schedule_per_sample(0.0001),
cntk.learning_parameter_schedule_per_sample(0.99))
progress_printer = cntk.logging.ProgressPrinter(tag='Training',
num_epochs=max_epochs)
trainer = cntk.Trainer(model, (loss_function, accuracy_function),
[learner], progress_printer)
evaluator = cntk.Evaluator(accuracy_function)
cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer,
evaluator)
def train_mse_cntk(x, y, model, train_gen, val_gen, epochs, val_steps):
loss_function = cntk.squared_error(model, y)
accuracy_function = loss_function
learner = cntk.adam(model.parameters,
cntk.learning_parameter_schedule_per_sample(0.001),
cntk.learning_parameter_schedule_per_sample(0.9))
trainer = cntk.Trainer(model, (loss_function, accuracy_function),
[learner])
evaluator = cntk.Evaluator(accuracy_function)
history = fit_generator(x,
y,
model=model,
trainer=trainer,
evaluator=evaluator,
train_gen=train_gen,
steps_per_epoch=500,
epochs=epochs,
val_gen=val_gen,
validation_steps=val_steps)
plot_results(history)
def use_glove_word_embeddings_cntk(preload_weights=False):
tokenizer, x_train, y_train, x_val, y_val = from_raw_text_to_word_embeddings(
)
x = cntk.input_variable(shape=(Constants.maxlen, ), dtype=np.float32)
y = cntk.input_variable(shape=(1, ), dtype=np.float32)
model = cntk.one_hot(x,
num_classes=Constants.max_words,
sparse_output=True)
if preload_weights is True:
embedding_matrix = compute_embedding_matrix(tokenizer)
assert (Constants.embedding_dim
== embedding_matrix.shape[0]) or (Constants.embedding_dim
== embedding_matrix.shape[1])
model = cntk.layers.Embedding(weights=embedding_matrix)(model)
else:
model = cntk.layers.Embedding(Constants.embedding_dim)(model)
model = cntk.layers.Dense(32, activation=cntk.relu)(model)
model = cntk.layers.Dense(1, activation=cntk.sigmoid)(model)
loss_function = cntk.binary_cross_entropy(model.output, y)
round_predictions = cntk.round(model.output)
equal_elements = cntk.equal(round_predictions, y)
accuracy_function = cntk.reduce_mean(equal_elements, axis=0)
max_epochs = 10
batch_size = 32
learner = cntk.adam(model.parameters,
cntk.learning_parameter_schedule_per_sample(0.0001),
cntk.learning_parameter_schedule_per_sample(0.99))
progress_printer = cntk.logging.ProgressPrinter(tag='Training',
num_epochs=max_epochs)
trainer = cntk.Trainer(model, (loss_function, accuracy_function),
[learner], progress_printer)
evaluator = cntk.Evaluator(accuracy_function)
cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer,
evaluator)
progress_writer = cntk.logging.ProgressPrinter(
50) # helper for logging progress; log every 50 minibatches
trainer = cntk.Trainer(None, criterion, [learner], [progress_writer])
# Train!
for i in range(0, len(X_train), minibatch_size): # loop over minibatches
x = X_train[i:i + minibatch_size] # get one minibatch worth of data
y = Y_train[i:i + minibatch_size]
trainer.train_minibatch({
data: x,
label_one_hot: y
}) # update model from one minibatch
trainer.summarize_training_progress()
# Test error rate on the test set.
evaluator = cntk.Evaluator(metric, [progress_writer])
for i in range(0, len(X_test), minibatch_size): # loop over minibatches
x = X_test[i:i + minibatch_size] # get one minibatch worth of data
y = Y_test[i:i + minibatch_size]
evaluator.test_minibatch({data: x, label_one_hot: y}) # test one minibatch
evaluator.summarize_test_progress()
# Inspect predictions on one minibatch, for illustration.
# For evaluation, we map the output of the network between 0-1 and convert them into probabilities
# for the two classes. We use a softmax function to get the probabilities of each of the class.
get_probability = cntk.softmax(model)
X_check, Y_check = generate_synthetic_data(25) # a small batch of 25 examples
result = get_probability.eval(X_check)
print("Label :", [label.todense().argmax() for label in Y_check])
def run_experiment_cntk():
if os.path.isfile('x_train_imdb.bin'):
print('Loading from .bin files')
x_train, y_train, x_test, y_test = load_from_files(x_shape=(25000,
500),
y_shape=(25000, ))
else:
print('Loading data...')
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(
num_words=Constants.max_words)
print(len(x_train), 'train sequences')
print(len(x_test), 'test sequences')
print('Pad sequences (samples x time)')
x_train = keras.preprocessing.sequence.pad_sequences(
x_train, maxlen=Constants.maxlen)
x_test = keras.preprocessing.sequence.pad_sequences(
x_test, maxlen=Constants.maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
print('Saving to .bin files')
save_to_files(x_train, y_train, x_test, y_test)
x = cntk.sequence.input_variable(shape=(), dtype=np.float32)
y = cntk.input_variable(shape=(), dtype=np.float32)
x_placeholder = cntk.placeholder(shape=(),
dynamic_axes=[
cntk.Axis.default_batch_axis(),
cntk.Axis.default_dynamic_axis()
])
model = cntk.one_hot(x_placeholder,
num_classes=Constants.max_words,
sparse_output=True)
model = cntk.layers.Embedding(Constants.embedding_dim)(model)
model = cntk.layers.Recurrence(cntk.layers.LSTM(32))(model)
model = cntk.sequence.last(model)
model = cntk.layers.Dense(1, activation=cntk.sigmoid)(model)
model.save('ch6-2.cntk.model')
model = None
model = cntk.load_model('ch6-2.cntk.model')
model.replace_placeholders({model.placeholders[0]: x})
loss_function = cntk.binary_cross_entropy(model.output, y)
round_predictions = cntk.round(model.output)
equal_elements = cntk.equal(round_predictions, y)
accuracy_function = cntk.reduce_mean(equal_elements,
axis=cntk.Axis.all_static_axes())
max_epochs = 10
batch_size = 128
learner = cntk.adam(model.parameters,
cntk.learning_parameter_schedule_per_sample(0.01),
cntk.learning_parameter_schedule_per_sample(0.9))
progress_printer = cntk.logging.ProgressPrinter(tag='Training',
num_epochs=max_epochs)
trainer = cntk.Trainer(model, (loss_function, accuracy_function),
[learner], progress_printer)
evaluator = cntk.Evaluator(accuracy_function)
cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer,
evaluator)