def train_classifier_model(options): # Load data logger.info("Loading data...") [word_index, x, _, _, _] = \ data_processor.get_text_sequences( options['text_file_path'], options['vocab_size'], global_config.classifier_vocab_save_path) x = np.asarray(x) [y, _] = data_processor.get_labels(options['label_file_path'], False) shuffle_indices = np.random.permutation(np.arange(len(y))) x_shuffled = x[shuffle_indices] y_shuffled = y[shuffle_indices] # Split train/test set dev_sample_index = -1 * int(0.01 * float(len(y))) x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[ dev_sample_index:] y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[ dev_sample_index:] del x, y, x_shuffled, y_shuffled logger.info("Vocabulary Size: {:d}".format(global_config.vocab_size)) logger.info("Train/Dev split: {:d}/{:d}".format(len(y_train), len(y_dev))) # Training sess = tf_session_helper.get_tensorflow_session() with sess.as_default(): cnn = TextCNN(sequence_length=x_train.shape[1], num_classes=y_train.shape[1], vocab_size=options['vocab_size'], embedding_size=128, filter_sizes=list(map(int, [3, 4, 5])), num_filters=128, l2_reg_lambda=0.0) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Output directory for models and summaries out_dir = global_config.classifier_save_directory logger.info("Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries # train_summary_op = tf.summary.merge([loss_summary, acc_summary, grad_summaries_merged]) train_summary_op = tf.summary.merge([loss_summary, acc_summary]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) dev_summary_dir = os.path.join(out_dir, "summaries", "dev") dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=1) # Write vocabulary with open(global_config.classifier_vocab_save_path, 'w') as json_file: json.dump(word_index, json_file) logger.info("Saved vocabulary") # Initialize all variables sess.run(tf.global_variables_initializer()) def train_step(x_batch, y_batch): """ A single training step """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 0.5 } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) logger.info("step {}: loss {:g}, acc {:g}".format( step, loss, accuracy)) train_summary_writer.add_summary(summaries, step) def dev_step(x_batch, y_batch, writer=None): """ Evaluates model on a dev set """ feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: 1.0 } step, summaries, loss, accuracy = sess.run( [global_step, dev_summary_op, cnn.loss, cnn.accuracy], feed_dict) time_str = datetime.datetime.now().isoformat() logger.info("{}: step {}, loss {:g}, acc {:g}".format( time_str, step, loss, accuracy)) if writer: writer.add_summary(summaries, step) # Generate batches batches = data_processor.batch_iter(list(zip(x_train, y_train)), mconf.batch_size, options['training_epochs']) # Training loop. For each batch... for batch in batches: x_batch, y_batch = zip(*batch) train_step(x_batch, y_batch) current_step = tf.train.global_step(sess, global_step) if current_step % 100 == 0: logger.info("\nEvaluation:") dev_step(x_dev, y_dev, writer=dev_summary_writer) logger.info("") if current_step % 100 == 0: path = saver.save(sess, checkpoint_prefix, global_step=current_step) logger.info("Saved model checkpoint to {}\n".format(path))
def main(argv): options = Options() parser = argparse.ArgumentParser() parser.add_argument("--logging-level", type=str, default="INFO") run_mode = parser.add_mutually_exclusive_group(required=True) run_mode.add_argument("--train-model", action="store_true", default=False) run_mode.add_argument("--transform-text", action="store_true", default=False) run_mode.add_argument("--generate-novel-text", action="store_true", default=False) parser.parse_known_args(args=argv, namespace=options) if options.train_model: parser.add_argument("--vocab-size", type=int, default=1000) parser.add_argument("--training-epochs", type=int, default=10) parser.add_argument("--text-file-path", type=str, required=True) parser.add_argument("--label-file-path", type=str, required=True) parser.add_argument("--validation-text-file-path", type=str, required=True) parser.add_argument("--validation-label-file-path", type=str, required=True) parser.add_argument("--training-embeddings-file-path", type=str) parser.add_argument("--validation-embeddings-file-path", type=str, required=True) parser.add_argument("--dump-embeddings", action="store_true", default=False) parser.add_argument("--classifier-saved-model-path", type=str, required=True) if options.transform_text: parser.add_argument("--saved-model-path", type=str, required=True) parser.add_argument("--evaluation-text-file-path", type=str, required=True) parser.add_argument("--evaluation-label-file-path", type=str, required=True) if options.generate_novel_text: parser.add_argument("--saved-model-path", type=str, required=True) parser.add_argument("--num-sentences-to-generate", type=int, default=1000, required=True) parser.add_argument("--label-index", type=int, default=1000, required=False) parser.parse_known_args(args=argv, namespace=options) global logger logger = log_initializer.setup_custom_logger(global_config.logger_name, options.logging_level) if not (options.train_model or options.transform_text or options.generate_novel_text): logger.info("Nothing to do. Exiting ...") sys.exit(0) global_config.training_epochs = options.training_epochs logger.info("experiment_timestamp: {}".format( global_config.experiment_timestamp)) # Train and save model if options.train_model: os.makedirs(global_config.save_directory) with open(global_config.model_config_file_path, 'w') as model_config_file: json.dump(obj=mconf.__dict__, fp=model_config_file, indent=4) logger.info("Saved model config to {}".format( global_config.model_config_file_path)) # Retrieve all data logger.info("Reading data ...") [ word_index, padded_sequences, text_sequence_lengths, one_hot_labels, num_labels, text_tokenizer, inverse_word_index ] = get_data(options) data_size = padded_sequences.shape[0] encoder_embedding_matrix, decoder_embedding_matrix = \ get_word_embeddings(options.training_embeddings_file_path, word_index) # Build model logger.info("Building model architecture ...") network = adversarial_autoencoder.AdversarialAutoencoder() network.build_model(word_index, encoder_embedding_matrix, decoder_embedding_matrix, num_labels) logger.info("Training model ...") sess = tf_session_helper.get_tensorflow_session() [_, validation_actual_word_lists, validation_sequences, validation_sequence_lengths] = \ data_processor.get_test_sequences( options.validation_text_file_path, text_tokenizer, word_index, inverse_word_index) [_, validation_labels] = \ data_processor.get_test_labels(options.validation_label_file_path, global_config.save_directory) network.train(sess, data_size, padded_sequences, text_sequence_lengths, one_hot_labels, num_labels, word_index, encoder_embedding_matrix, decoder_embedding_matrix, validation_sequences, validation_sequence_lengths, validation_labels, inverse_word_index, validation_actual_word_lists, options) sess.close() logger.info("Training complete!") elif options.transform_text: # Enforce a particular style embedding and regenerate text logger.info("Transforming text style ...") with open( os.path.join(options.saved_model_path, global_config.model_config_file), 'r') as json_file: model_config_dict = json.load(json_file) mconf.init_from_dict(model_config_dict) logger.info("Restored model config from saved JSON") with open( os.path.join(options.saved_model_path, global_config.vocab_save_file), 'r') as json_file: word_index = json.load(json_file) with open( os.path.join(options.saved_model_path, global_config.index_to_label_dict_file), 'r') as json_file: index_to_label_map = json.load(json_file) with open( os.path.join(options.saved_model_path, global_config.average_label_embeddings_file), 'rb') as pickle_file: average_label_embeddings = pickle.load(pickle_file) global_config.vocab_size = len(word_index) num_labels = len(index_to_label_map) text_tokenizer = tf.keras.preprocessing.text.Tokenizer( num_words=global_config.vocab_size, filters=global_config.tokenizer_filters) text_tokenizer.word_index = word_index inverse_word_index = {v: k for k, v in word_index.items()} [actual_sequences, _, padded_sequences, text_sequence_lengths] = \ data_processor.get_test_sequences( options.evaluation_text_file_path, text_tokenizer, word_index, inverse_word_index) [label_sequences, _] = \ data_processor.get_test_labels(options.evaluation_label_file_path, options.saved_model_path) logger.info("Building model architecture ...") network = adversarial_autoencoder.AdversarialAutoencoder() encoder_embedding_matrix, decoder_embedding_matrix = get_word_embeddings( None, word_index) network.build_model(word_index, encoder_embedding_matrix, decoder_embedding_matrix, num_labels) sess = tf_session_helper.get_tensorflow_session() total_nll = 0 for i in range(num_labels): logger.info("Style chosen: {}".format(i)) filtered_actual_sequences = list() filtered_padded_sequences = list() filtered_text_sequence_lengths = list() for k in range(len(actual_sequences)): if label_sequences[k] != i: filtered_actual_sequences.append(actual_sequences[k]) filtered_padded_sequences.append(padded_sequences[k]) filtered_text_sequence_lengths.append( text_sequence_lengths[k]) style_embedding = np.asarray(average_label_embeddings[i]) [generated_sequences, final_sequence_lengths, _, _, _, cross_entropy_scores] = \ network.transform_sentences( sess, filtered_padded_sequences, filtered_text_sequence_lengths, style_embedding, num_labels, os.path.join(options.saved_model_path, global_config.model_save_file)) nll = -np.mean(a=cross_entropy_scores, axis=0) total_nll += nll logger.info("NLL: {}".format(nll)) actual_word_lists = \ [data_processor.generate_words_from_indices(x, inverse_word_index) for x in filtered_actual_sequences] execute_post_inference_operations( actual_word_lists, generated_sequences, final_sequence_lengths, inverse_word_index, global_config.experiment_timestamp, i) logger.info("Generation complete for label {}".format(i)) logger.info("Mean NLL: {}".format(total_nll / num_labels)) logger.info("Predicting labels from latent spaces ...") _, _, overall_label_predictions, style_label_predictions, adversarial_label_predictions, _ = \ network.transform_sentences( sess, padded_sequences, text_sequence_lengths, average_label_embeddings[0], num_labels, os.path.join(options.saved_model_path, global_config.model_save_file)) # write label predictions to file output_file_path = "output/{}-inference/overall_labels_prediction.txt".format( global_config.experiment_timestamp) os.makedirs(os.path.dirname(output_file_path), exist_ok=True) with open(output_file_path, 'w') as output_file: for one_hot_label in overall_label_predictions: output_file.write("{}\n".format( one_hot_label.tolist().index(1))) output_file_path = "output/{}-inference/style_labels_prediction.txt".format( global_config.experiment_timestamp) os.makedirs(os.path.dirname(output_file_path), exist_ok=True) with open(output_file_path, 'w') as output_file: for one_hot_label in style_label_predictions: output_file.write("{}\n".format( one_hot_label.tolist().index(1))) output_file_path = "output/{}-inference/adversarial_labels_prediction.txt".format( global_config.experiment_timestamp) os.makedirs(os.path.dirname(output_file_path), exist_ok=True) with open(output_file_path, 'w') as output_file: for one_hot_label in adversarial_label_predictions: output_file.write("{}\n".format( one_hot_label.tolist().index(1))) logger.info("Inference run complete") sess.close() elif options.generate_novel_text: logger.info("Generating novel text") with open( os.path.join(options.saved_model_path, global_config.model_config_file), 'r') as json_file: model_config_dict = json.load(json_file) mconf.init_from_dict(model_config_dict) logger.info("Restored model config from saved JSON") with open( os.path.join(options.saved_model_path, global_config.vocab_save_file), 'r') as json_file: word_index = json.load(json_file) with open( os.path.join(options.saved_model_path, global_config.index_to_label_dict_file), 'r') as json_file: index_to_label_map = json.load(json_file) with open( os.path.join(options.saved_model_path, global_config.average_label_embeddings_file), 'rb') as pickle_file: average_label_embeddings = pickle.load(pickle_file) global_config.vocab_size = len(word_index) inverse_word_index = {v: k for k, v in word_index.items()} num_labels = len(index_to_label_map) text_tokenizer = tf.keras.preprocessing.text.Tokenizer( num_words=global_config.vocab_size, filters=global_config.tokenizer_filters) text_tokenizer.word_index = word_index data_processor.populate_word_blacklist(word_index) logger.info("Building model architecture ...") network = adversarial_autoencoder.AdversarialAutoencoder() encoder_embedding_matrix, decoder_embedding_matrix = get_word_embeddings( None, word_index) network.build_model(word_index, encoder_embedding_matrix, decoder_embedding_matrix, num_labels) sess = tf_session_helper.get_tensorflow_session() for label_index in index_to_label_map: if options.label_index and label_index != options.label_index: continue style_embedding = np.asarray( average_label_embeddings[int(label_index)]) generated_sequences, final_sequence_lengths = \ network.generate_novel_sentences( sess, style_embedding, options.num_sentences_to_generate, num_labels, os.path.join(options.saved_model_path, global_config.model_save_file)) # first trims the generates sentences down to the length the decoder returns # then trim any <eos> token trimmed_generated_sequences = \ [[index for index in sequence if index != global_config.predefined_word_index[global_config.eos_token]] for sequence in [x[:(y - 1)] for (x, y) in zip(generated_sequences, final_sequence_lengths)]] generated_word_lists = \ [data_processor.generate_words_from_indices(x, inverse_word_index) for x in trimmed_generated_sequences] generated_sentences = [" ".join(x) for x in generated_word_lists] output_file_path = "output/{}-generation/generated_sentences_{}.txt".format( global_config.experiment_timestamp, label_index) os.makedirs(os.path.dirname(output_file_path), exist_ok=True) with open(output_file_path, 'w') as output_file: for sentence in generated_sentences: output_file.write(sentence + "\n") logger.info("Generated {} sentences of label {} at path {}".format( options.num_sentences_to_generate, index_to_label_map[label_index], output_file_path)) sess.close() logger.info("Generation run complete")
def get_style_transfer_score(classifier_saved_model_path, text_file_path, label): with open( os.path.join(classifier_saved_model_path, global_config.vocab_save_file), 'r') as json_file: word_index = json.load(json_file) vocab_size = len(word_index) text_tokenizer = tf.keras.preprocessing.text.Tokenizer( num_words=global_config.vocab_size, filters=global_config.tokenizer_filters) text_tokenizer.word_index = word_index with open(text_file_path) as text_file: actual_sequences = text_tokenizer.texts_to_sequences(text_file) trimmed_sequences = [[ x if x < vocab_size else word_index[global_config.unk_token] for x in sequence ] for sequence in actual_sequences] text_sequences = tf.keras.preprocessing.sequence.pad_sequences( trimmed_sequences, maxlen=global_config.max_sequence_length, padding='post', truncating='post', value=word_index[global_config.eos_token]) x_test = np.asarray(text_sequences) y_test = np.asarray([int(label)] * len(text_sequences)) checkpoint_file = tf.train.latest_checkpoint( os.path.join(classifier_saved_model_path, "checkpoints")) graph = tf.Graph() with graph.as_default(): sess = tf_session_helper.get_tensorflow_session() with sess.as_default(): # Load the saved meta graph and restore variables saver = tf.train.import_meta_graph( "{}.meta".format(checkpoint_file)) saver.restore(sess, checkpoint_file) # Get the placeholders from the graph by name input_x = graph.get_operation_by_name("input_x").outputs[0] # input_y = graph.get_operation_by_name("input_y").outputs[0] dropout_keep_prob = graph.get_operation_by_name( "dropout_keep_prob").outputs[0] # Tensors we want to evaluate predictions = graph.get_operation_by_name( "output/predictions").outputs[0] # Generate batches for one epoch batches = data_processor.batch_iter(list(x_test), mconf.batch_size, 1, shuffle=False) # Collect the predictions here all_predictions = [] for x_test_batch in batches: batch_predictions = sess.run(predictions, { input_x: x_test_batch, dropout_keep_prob: 1.0 }) all_predictions = np.concatenate( [all_predictions, batch_predictions]) sess.close() # Print accuracy if y_test is defined if y_test is not None: correct_predictions = float(sum(all_predictions == y_test)) accuracy = correct_predictions / float(len(y_test)) # f1_score = metrics.f1_score(y_true=y_test, y_pred=all_predictions) confusion_matrix = metrics.confusion_matrix(y_true=y_test, y_pred=all_predictions) return [accuracy, confusion_matrix] logger.info("Nothing to evaluate") return [0.0, None]