def learning_word_embeddings_with_the_embedding_layer_cntk(): x_train, y_train, x_test, y_test = load_from_files() max_features = 10000 maxlen = 20 embedding_dim = 8 x = cntk.input_variable(shape=(maxlen, ), dtype=np.float32) y = cntk.input_variable(shape=(1, ), dtype=np.float32) model = cntk.one_hot(x, num_classes=max_features, sparse_output=True) model = cntk.layers.Embedding(embedding_dim)(model) model = cntk.layers.Dense(1, activation=cntk.sigmoid)(model) loss_function = cntk.binary_cross_entropy(model.output, y) round_predictions = cntk.round(model.output) equal_elements = cntk.equal(round_predictions, y) accuracy_function = cntk.reduce_mean(equal_elements, axis=0) max_epochs = 30 batch_size = 32 learner = cntk.adam(model.parameters, cntk.learning_parameter_schedule_per_sample(0.0001), cntk.learning_parameter_schedule_per_sample(0.99)) progress_printer = cntk.logging.ProgressPrinter(tag='Training', num_epochs=max_epochs) trainer = cntk.Trainer(model, (loss_function, accuracy_function), [learner], progress_printer) evaluator = cntk.Evaluator(accuracy_function) cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer, evaluator)
def implementing_1d_convnet_cntk(): max_features = 10000 # number of words to consider as features max_len = 500 # cut texts after this number of words (among top max_features most common words) x_train, y_train, x_test, y_test = load_data(max_features, max_len) model = build_model_cntk(max_features, max_len) x = cntk.input_variable(shape=(max_len, ), dtype=np.float32) y = cntk.input_variable(shape=(1, ), dtype=np.float32) model.replace_placeholders({model.placeholders[0]: x}) loss_function = cntk.binary_cross_entropy(model.output, y) round_predictions = cntk.round(model.output) equal_elements = cntk.equal(round_predictions, y) accuracy_function = cntk.reduce_mean(equal_elements, axis=0) max_epochs = 10 batch_size = 32 learner = cntk.adam(model.parameters, cntk.learning_parameter_schedule_per_sample(0.0001), cntk.learning_parameter_schedule_per_sample(0.99)) progress_printer = cntk.logging.ProgressPrinter(tag='Training', num_epochs=max_epochs) trainer = cntk.Trainer(model, (loss_function, accuracy_function), [learner], progress_printer) evaluator = cntk.Evaluator(accuracy_function) cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer, evaluator)
def train_mse_cntk(x, y, model, train_gen, val_gen, epochs, val_steps): loss_function = cntk.squared_error(model, y) accuracy_function = loss_function learner = cntk.adam(model.parameters, cntk.learning_parameter_schedule_per_sample(0.001), cntk.learning_parameter_schedule_per_sample(0.9)) trainer = cntk.Trainer(model, (loss_function, accuracy_function), [learner]) evaluator = cntk.Evaluator(accuracy_function) history = fit_generator(x, y, model=model, trainer=trainer, evaluator=evaluator, train_gen=train_gen, steps_per_epoch=500, epochs=epochs, val_gen=val_gen, validation_steps=val_steps) plot_results(history)
def use_glove_word_embeddings_cntk(preload_weights=False): tokenizer, x_train, y_train, x_val, y_val = from_raw_text_to_word_embeddings( ) x = cntk.input_variable(shape=(Constants.maxlen, ), dtype=np.float32) y = cntk.input_variable(shape=(1, ), dtype=np.float32) model = cntk.one_hot(x, num_classes=Constants.max_words, sparse_output=True) if preload_weights is True: embedding_matrix = compute_embedding_matrix(tokenizer) assert (Constants.embedding_dim == embedding_matrix.shape[0]) or (Constants.embedding_dim == embedding_matrix.shape[1]) model = cntk.layers.Embedding(weights=embedding_matrix)(model) else: model = cntk.layers.Embedding(Constants.embedding_dim)(model) model = cntk.layers.Dense(32, activation=cntk.relu)(model) model = cntk.layers.Dense(1, activation=cntk.sigmoid)(model) loss_function = cntk.binary_cross_entropy(model.output, y) round_predictions = cntk.round(model.output) equal_elements = cntk.equal(round_predictions, y) accuracy_function = cntk.reduce_mean(equal_elements, axis=0) max_epochs = 10 batch_size = 32 learner = cntk.adam(model.parameters, cntk.learning_parameter_schedule_per_sample(0.0001), cntk.learning_parameter_schedule_per_sample(0.99)) progress_printer = cntk.logging.ProgressPrinter(tag='Training', num_epochs=max_epochs) trainer = cntk.Trainer(model, (loss_function, accuracy_function), [learner], progress_printer) evaluator = cntk.Evaluator(accuracy_function) cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer, evaluator)
progress_writer = cntk.logging.ProgressPrinter( 50) # helper for logging progress; log every 50 minibatches trainer = cntk.Trainer(None, criterion, [learner], [progress_writer]) # Train! for i in range(0, len(X_train), minibatch_size): # loop over minibatches x = X_train[i:i + minibatch_size] # get one minibatch worth of data y = Y_train[i:i + minibatch_size] trainer.train_minibatch({ data: x, label_one_hot: y }) # update model from one minibatch trainer.summarize_training_progress() # Test error rate on the test set. evaluator = cntk.Evaluator(metric, [progress_writer]) for i in range(0, len(X_test), minibatch_size): # loop over minibatches x = X_test[i:i + minibatch_size] # get one minibatch worth of data y = Y_test[i:i + minibatch_size] evaluator.test_minibatch({data: x, label_one_hot: y}) # test one minibatch evaluator.summarize_test_progress() # Inspect predictions on one minibatch, for illustration. # For evaluation, we map the output of the network between 0-1 and convert them into probabilities # for the two classes. We use a softmax function to get the probabilities of each of the class. get_probability = cntk.softmax(model) X_check, Y_check = generate_synthetic_data(25) # a small batch of 25 examples result = get_probability.eval(X_check) print("Label :", [label.todense().argmax() for label in Y_check])
def run_experiment_cntk(): if os.path.isfile('x_train_imdb.bin'): print('Loading from .bin files') x_train, y_train, x_test, y_test = load_from_files(x_shape=(25000, 500), y_shape=(25000, )) else: print('Loading data...') (x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data( num_words=Constants.max_words) print(len(x_train), 'train sequences') print(len(x_test), 'test sequences') print('Pad sequences (samples x time)') x_train = keras.preprocessing.sequence.pad_sequences( x_train, maxlen=Constants.maxlen) x_test = keras.preprocessing.sequence.pad_sequences( x_test, maxlen=Constants.maxlen) print('x_train shape:', x_train.shape) print('x_test shape:', x_test.shape) print('Saving to .bin files') save_to_files(x_train, y_train, x_test, y_test) x = cntk.sequence.input_variable(shape=(), dtype=np.float32) y = cntk.input_variable(shape=(), dtype=np.float32) x_placeholder = cntk.placeholder(shape=(), dynamic_axes=[ cntk.Axis.default_batch_axis(), cntk.Axis.default_dynamic_axis() ]) model = cntk.one_hot(x_placeholder, num_classes=Constants.max_words, sparse_output=True) model = cntk.layers.Embedding(Constants.embedding_dim)(model) model = cntk.layers.Recurrence(cntk.layers.LSTM(32))(model) model = cntk.sequence.last(model) model = cntk.layers.Dense(1, activation=cntk.sigmoid)(model) model.save('ch6-2.cntk.model') model = None model = cntk.load_model('ch6-2.cntk.model') model.replace_placeholders({model.placeholders[0]: x}) loss_function = cntk.binary_cross_entropy(model.output, y) round_predictions = cntk.round(model.output) equal_elements = cntk.equal(round_predictions, y) accuracy_function = cntk.reduce_mean(equal_elements, axis=cntk.Axis.all_static_axes()) max_epochs = 10 batch_size = 128 learner = cntk.adam(model.parameters, cntk.learning_parameter_schedule_per_sample(0.01), cntk.learning_parameter_schedule_per_sample(0.9)) progress_printer = cntk.logging.ProgressPrinter(tag='Training', num_epochs=max_epochs) trainer = cntk.Trainer(model, (loss_function, accuracy_function), [learner], progress_printer) evaluator = cntk.Evaluator(accuracy_function) cntk_train(x, y, x_train, y_train, max_epochs, batch_size, trainer, evaluator)