def main(unused_argv): if not FLAGS.data_dir: raise ValueError("--data_dir is required.") encoder = encoder_manager.EncoderManager() # Maybe load unidirectional encoder. if FLAGS.uni_checkpoint_path: print("Loading unidirectional model...") uni_config = configuration.model_config() encoder.load_model( uni_config, FLAGS.uni_vocab_file, FLAGS.uni_embeddings_file, FLAGS.uni_checkpoint_path) # Maybe load bidirectional encoder. if FLAGS.bi_checkpoint_path: print("Loading bidirectional model...") bi_config = configuration.model_config(bidirectional_encoder=True) encoder.load_model( bi_config, FLAGS.bi_vocab_file, FLAGS.bi_embeddings_file, FLAGS.bi_checkpoint_path) if FLAGS.eval_task in ["MR", "CR", "SUBJ", "MPQA"]: eval_classification.eval_nested_kfold( encoder, FLAGS.eval_task, FLAGS.data_dir, use_nb=False) elif FLAGS.eval_task == "SICK": eval_sick.evaluate(encoder, evaltest=True, loc=FLAGS.data_dir) elif FLAGS.eval_task == "MSRP": eval_msrp.evaluate(encoder, evalcv=True, evaltest=True, use_feats=True, loc=FLAGS.data_dir) elif FLAGS.eval_task == "TREC": eval_trec.evaluate(encoder, evalcv=True, evaltest=True, loc=FLAGS.data_dir) else: raise ValueError("Unrecognized eval_task: %s" % FLAGS.eval_task) encoder.close()
def main(unused_argv): if not FLAGS.data_dir: raise ValueError("--data_dir is required.") encoder = encoder_manager.EncoderManager() # Maybe load unidirectional encoder. if FLAGS.uni_checkpoint_path: print("Loading unidirectional model...") uni_config = configuration.model_config() encoder.load_model(uni_config, FLAGS.uni_vocab_file, FLAGS.uni_embeddings_file, FLAGS.uni_checkpoint_path) # Maybe load bidirectional encoder. if FLAGS.bi_checkpoint_path: print("Loading bidirectional model...") bi_config = configuration.model_config(bidirectional_encoder=True) encoder.load_model(bi_config, FLAGS.bi_vocab_file, FLAGS.bi_embeddings_file, FLAGS.bi_checkpoint_path) if FLAGS.eval_task in ["MR", "CR", "SUBJ", "MPQA"]: eval_classification.eval_nested_kfold( encoder, FLAGS.eval_task, FLAGS.data_dir, use_nb=False) elif FLAGS.eval_task == "SICK": eval_sick.evaluate(encoder, evaltest=True, loc=FLAGS.data_dir) elif FLAGS.eval_task == "MSRP": eval_msrp.evaluate( encoder, evalcv=True, evaltest=True, use_feats=True, loc=FLAGS.data_dir) elif FLAGS.eval_task == "TREC": eval_trec.evaluate(encoder, evalcv=True, evaltest=True, loc=FLAGS.data_dir) else: raise ValueError("Unrecognized eval_task: %s" % FLAGS.eval_task) encoder.close()
def main(unused_argv): if not FLAGS.data_dir: raise ValueError("--data_dir is required.") if not FLAGS.output_dir: raise ValueError("--output_dir is required.") encoder = encoder_manager.EncoderManager() # Maybe load unidirectional encoder. if FLAGS.uni_checkpoint_path: print("Loading unidirectional model...") uni_config = configuration.model_config() encoder.load_model(uni_config, FLAGS.uni_vocab_file, FLAGS.uni_embeddings_file, FLAGS.uni_checkpoint_path) # Maybe load bidirectional encoder. if FLAGS.bi_checkpoint_path: print("Loading bidirectional model...") bi_config = configuration.model_config(bidirectional_encoder=True) encoder.load_model(bi_config, FLAGS.bi_vocab_file, FLAGS.bi_embeddings_file, FLAGS.bi_checkpoint_path) evaluate(encoder, FLAGS.output_dir, evaltest=True, loc=FLAGS.data_dir) encoder.close()
def extract_by_skip_thought(sent_list: List[str]): """ To make it compatible with the toolkit, we need the input to be a list of sentences :param sent_list: :return: """ skip_thought_dir = os.path.join('/home/junpeiz/Project/Twitter/data', 'skipThoughts', 'pretrained', 'skip_thoughts_uni_2017_02_02') # Set paths to the model. VOCAB_FILE = os.path.join(skip_thought_dir, "vocab.txt") EMBEDDING_MATRIX_FILE = os.path.join(skip_thought_dir, "embeddings.npy") CHECKPOINT_PATH = os.path.join(skip_thought_dir, "model.ckpt-501424") # The following directory should contain files rt-polarity.neg and # rt-polarity.pos. # MR_DATA_DIR = "/dir/containing/mr/data" # Set up the encoder. Here we are using a single unidirectional model. # To use a bidirectional model as well, call load_model() again with # configuration.model_config(bidirectional_encoder=True) and paths to the # bidirectional model's files. The encoder will use the concatenation of # all loaded models. encoder = encoder_manager.EncoderManager() encoder.load_model(configuration.model_config(), vocabulary_file=VOCAB_FILE, embedding_matrix_file=EMBEDDING_MATRIX_FILE, checkpoint_path=CHECKPOINT_PATH) encoding_list = encoder.encode(sent_list) return encoding_list
def get_encoder(): # Download and extract the bidirectional model. (shell script) # cd models/ # wget "http://download.tensorflow.org/models/skip_thoughts_bi_2017_02_16.tar.gz" # tar -xvf skip_thoughts_bi_2017_02_16.tar.gz # rm skip_thoughts_bi_2017_02_16.tar.gz # cd .. # # Set paths to the model. pretrained_path = 'models/skip_thoughts_bi_2017_02_16/' VOCAB_FILE = os.path.join(pretrained_path, 'vocab.txt') EMBEDDING_MATRIX_FILE = os.path.join(pretrained_path, 'embeddings.npy') CHECKPOINT_PATH = os.path.join(pretrained_path, 'model.ckpt-500008') # Set up the encoder. Here we are using a single unidirectional model. # To use a bidirectional model as well, call load_model() again with # configuration.model_config(bidirectional_encoder=True) and paths to the # bidirectional model's files. The encoder will use the concatenation of # all loaded models. encoder = encoder_manager.EncoderManager() encoder.load_model(configuration.model_config(bidirectional_encoder=True), vocabulary_file=VOCAB_FILE, embedding_matrix_file=EMBEDDING_MATRIX_FILE, checkpoint_path=CHECKPOINT_PATH) return encoder
def __init__(self, use_char=False): super(SkipThought, self).__init__() self.use_char2vec = use_char cur_path = os.path.abspath(os.path.dirname(__file__)) # Set paths to the model. VOCAB_FILE = os.path.join( cur_path, "../../models/skip_thoughts_uni_2017_02_02/vocab.txt") EMBEDDING_MATRIX_FILE = os.path.join( cur_path, "../../models/skip_thoughts_uni_2017_02_02/embeddings.txt") CHECKPOINT_PATH = os.path.join( cur_path, "../../models/skip_thoughts_uni_2017_02_02/model.ckpt-501424") self.encoder = encoder_manager.EncoderManager() self.encoder.load_model(configuration.model_config(), vocabulary_file=VOCAB_FILE, embedding_matrix_file=EMBEDDING_MATRIX_FILE, checkpoint_path=CHECKPOINT_PATH) if self.use_char2vec: PROJ_MODEL_PATH = os.path.join( cur_path, "../../models/char_word2vec/skip-thought_linear_projection.m") self.char_w2v = CharWord2vec() with open(PROJ_MODEL_PATH) as f: self.proj = pickle.load(f)
def load_model(vocab_file, embedding_matrix_file, checkpoint_path, bidirectional_encoder): encoder = encoder_manager.EncoderManager() encoder.load_model(configuration.model_config( bidirectional_encoder=bidirectional_encoder), vocabulary_file=vocab_file, embedding_matrix_file=embedding_matrix_file, checkpoint_path=checkpoint_path) return encoder
def setup_encoder(): VOCAB_FILE = '/data/ryli/kcli/skip-thoughts/pretrained/skip_thoughts_uni_2017_02_02/vocab.txt' EMBEDDING_MATRIX_FILE = '/data/ryli/kcli/skip-thoughts/pretrained/skip_thoughts_uni_2017_02_02/embeddings.npy' CHECKPOINT_PATH = '/data/ryli/kcli/skip-thoughts/pretrained/skip_thoughts_uni_2017_02_02/model.ckpt-501424' encoder = encoder_manager.EncoderManager() encoder.load_model(configuration.model_config(), vocabulary_file=VOCAB_FILE, embedding_matrix_file=EMBEDDING_MATRIX_FILE, checkpoint_path=CHECKPOINT_PATH) return encoder
def main(): parser = argparse.ArgumentParser( description="encoding sentences example for skip_thoughts.", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('vocab_file', help="specify the vocab_file") parser.add_argument('embedding_matrix_file', help='specify the embedding_matrix_file') parser.add_argument('checkpoint_path', help="specify the checkpoint_path") parser.add_argument('mr_data_dir', help="specify the mr_data_dir") parser.add_argument('--model_name', default="skip_thoughts") parser.add_argument('--bidirect', choices=["True", "False"], default="False") args = parser.parse_args() if args.bidirect == "True": args.bidirect = True else: args.bidirect = False encoder = encoder_manager.EncoderManager(args.model_name) encoder.load_model( configuration.model_config(bidirectional_encoder=args.bidirect), vocabulary_file=args.vocab_file, embedding_matrix_file=args.embedding_matrix_file, checkpoint_path=args.checkpoint_path) data = [] with open(os.path.join(args.mr_data_dir, 'rt-polarity.neg'), 'rb') as f: data.extend([line.decode('latin-1').strip() for line in f]) with open(os.path.join(args.mr_data_dir, 'rt-polarity.pos'), 'rb') as f: data.extend([line.decode('latin-1').strip() for line in f]) encodings = encoder.encode(data) def get_nn(ind, num=10): encoding = encodings[ind] scores = sd.cdist([encoding], encodings, 'cosine')[0] sorted_ids = np.argsort(scores) print("Senetence:") print("", data[ind]) print("\nNearest neighbors:") for i in range(1, num + 1): print(" %d. %s (%.3f)" % (i, data[sorted_ids[i]], scores[sorted_ids[i]])) get_nn(0)
def __init__(self, withSVM=False): [lib, con, neu] = cPickle.load(open(os.getcwd() + '/sampleData.pkl', 'rb')) self.bias_dict = {} for tree in lib: sentence = tree.get_words() self.bias_dict[sentence] = 1 for tree in con: sentence = tree.get_words() self.bias_dict[sentence] = -1 for tree in neu: sentence = tree.get_words() self.bias_dict[sentence] = 0 self.encoder = encoder_manager.EncoderManager() self.data_encodings = [] self.data = self.bias_dict.keys() self.blacklist = [] #f = open('skipthoughts.pkl', 'rb') # right now, we're using a unidirectional skip model; # we can try the bidirectional model later dir_path = os.path.dirname(os.path.realpath(__file__)) VOCAB_FILE = dir_path + "/../data/vocab.txt" EMBEDDING_MATRIX_FILE = dir_path + "/../data/embeddings.npy" CHECKPOINT_PATH = dir_path + "/../data/model.ckpt-501424" self.encoder.load_model(configuration.model_config(), vocabulary_file=VOCAB_FILE, embedding_matrix_file=EMBEDDING_MATRIX_FILE, checkpoint_path=CHECKPOINT_PATH) self.sentiment = SentimentIntensityAnalyzer() self.clf = None self.withSVM = withSVM if withSVM: print('using the SVM!') f = open('./svm.pkl', 'rb') self.clf = cPickle.load(f)
def main(unused_argv): if not FLAGS.input_file_pattern: raise ValueError("--input_file_pattern is required.") if not FLAGS.checkpoint_dir: raise ValueError("--checkpoint_dir is required.") if not FLAGS.eval_dir: raise ValueError("--eval_dir is required.") # Create the evaluation directory if it doesn't exist. eval_dir = FLAGS.eval_dir if not tf.gfile.IsDirectory(eval_dir): tf.logging.info("Creating eval directory: %s", eval_dir) tf.gfile.MakeDirs(eval_dir) g = tf.Graph() with g.as_default(): # Build the model for evaluation. model_config = configuration.model_config( input_file_pattern=FLAGS.input_file_pattern, input_queue_capacity=FLAGS.num_eval_examples, shuffle_input_data=False) model = skip_thoughts_model.SkipThoughtsModel(model_config, mode="eval") model.build() losses = tf.concat(model.target_cross_entropy_losses, 0) weights = tf.concat(model.target_cross_entropy_loss_weights, 0) # Create the Saver to restore model Variables. saver = tf.train.Saver() # Create the summary operation and the summary writer. summary_op = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(eval_dir) g.finalize() # Run a new evaluation run every eval_interval_secs. while True: start = time.time() tf.logging.info( "Starting evaluation at " + time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime())) run_once(model, losses, weights, saver, summary_writer, summary_op) time_to_next_eval = start + FLAGS.eval_interval_secs - time.time() if time_to_next_eval > 0: time.sleep(time_to_next_eval)
def __init__(self, modelPath, checkpointPath): """Initialize skip though model. Arguments: modelPath {str} -- the path to model checkpointPath {str} -- the filename of mode.ckpt-xxxx """ self.modelPath = modelPath self.checkpointPath = os.path.join(modelPath, "..", checkpointPath) self.vocabFile = os.path.join(modelPath, "vocab.txt") self.embeddingMatrixFile = os.path.join(modelPath, "embeddings.npy") self.encoder = encoder_manager.EncoderManager() self.encoder.load_model(configuration.model_config(), vocabulary_file=self.vocabFile, embedding_matrix_file=self.embeddingMatrixFile, checkpoint_path=self.checkpointPath)
def restore_skipthought(model_dir, model_name, skipthought_embedding, skipthought_vocab): """ :rtype: encoder_manager.EncoderManager() :return: """ check_point_path = os.path.join(model_dir, model_name) skip_thought_embedding_matrix = os.path.join(model_dir, skipthought_embedding) skip_thought_vocab = os.path.join(model_dir, skipthought_vocab) encoder = encoder_manager.EncoderManager() encoder.load_model(configuration.model_config(), vocabulary_file=skip_thought_vocab, embedding_matrix_file=skip_thought_embedding_matrix, checkpoint_path=check_point_path) return encoder
def main(unused_argv): if not FLAGS.input_file_pattern: raise ValueError("--input_file_pattern is required.") if not FLAGS.checkpoint_dir: raise ValueError("--checkpoint_dir is required.") if not FLAGS.eval_dir: raise ValueError("--eval_dir is required.") # Create the evaluation directory if it doesn't exist. eval_dir = FLAGS.eval_dir if not tf.gfile.IsDirectory(eval_dir): tf.logging.info("Creating eval directory: %s", eval_dir) tf.gfile.MakeDirs(eval_dir) g = tf.Graph() with g.as_default(): # Build the model for evaluation. model_config = configuration.model_config( input_file_pattern=FLAGS.input_file_pattern, input_queue_capacity=FLAGS.num_eval_examples, shuffle_input_data=False) model = skip_thoughts_model.SkipThoughtsModel(model_config, mode="eval") model.build() losses = tf.concat(model.target_cross_entropy_losses, 0) weights = tf.concat(model.target_cross_entropy_loss_weights, 0) # Create the Saver to restore model Variables. saver = tf.train.Saver() # Create the summary operation and the summary writer. summary_op = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(eval_dir) g.finalize() # Run a new evaluation run every eval_interval_secs. while True: start = time.time() tf.logging.info("Starting evaluation at " + time.strftime( "%Y-%m-%d-%H:%M:%S", time.localtime())) run_once(model, losses, weights, saver, summary_writer, summary_op) time_to_next_eval = start + FLAGS.eval_interval_secs - time.time() if time_to_next_eval > 0: time.sleep(time_to_next_eval)
def __init__(self, withSVM=False): [lib, con, neu] = cPickle.load(open('sampleData.pkl', 'rb')) self.bias_dict = {} for tree in lib: sentence = tree.get_words() self.bias_dict[sentence] = 1 for tree in con: sentence = tree.get_words() self.bias_dict[sentence] = -1 for tree in neu: sentence = tree.get_words() self.bias_dict[sentence] = 0 self.encoder = encoder_manager.EncoderManager() self.data_encodings = [] self.data = self.bias_dict.keys() self.blacklist = [] #f = open('skipthoughts.pkl', 'rb') # right now, we're using a unidirectional skip model; # we can try the bidirectional model later VOCAB_FILE = "/Users/az/Desktop/projects/modemo/backend/modules/tf/skip_thoughts/pretrained/skip_thoughts_uni_2017_02_02/vocab.txt" EMBEDDING_MATRIX_FILE = "/Users/az/Desktop/projects/modemo/backend/modules/tf/skip_thoughts/pretrained/skip_thoughts_uni_2017_02_02/embeddings.npy" CHECKPOINT_PATH = "/Users/az/Desktop/projects/modemo/backend/modules/tf/skip_thoughts/pretrained/skip_thoughts_uni_2017_02_02/model.ckpt-501424" self.encoder.load_model(configuration.model_config(), vocabulary_file=VOCAB_FILE, embedding_matrix_file=EMBEDDING_MATRIX_FILE, checkpoint_path=CHECKPOINT_PATH) self.sentiment = SentimentIntensityAnalyzer() self.clf = None if withSVM: print('using the SVM!') f = open('./svm.pkl', 'rb') self.clf = cPickle.load(f)
def main(unused_argv): if not FLAGS.input_file_pattern: raise ValueError("--input_file_pattern is required.") if not FLAGS.train_dir: raise ValueError("--train_dir is required.") model_config = configuration.model_config( input_file_pattern=FLAGS.input_file_pattern) training_config = configuration.training_config() tf.logging.info("Building training graph.") g = tf.Graph() with g.as_default(): model = skip_thoughts_model.SkipThoughtsModel(model_config, mode="train") model.build() learning_rate = _setup_learning_rate(training_config, model.global_step) optimizer = tf.train.AdamOptimizer(learning_rate) train_tensor = tf.contrib.slim.learning.create_train_op( total_loss=model.total_loss, optimizer=optimizer, global_step=model.global_step, clip_gradient_norm=training_config.clip_gradient_norm) saver = tf.train.Saver() tf.contrib.slim.learning.train( train_op=train_tensor, logdir=FLAGS.train_dir, graph=g, global_step=model.global_step, number_of_steps=training_config.number_of_steps, save_summaries_secs=training_config.save_summaries_secs, saver=saver, save_interval_secs=training_config.save_model_secs)
def main(unused_argv): if not FLAGS.input_file_pattern: raise ValueError("--input_file_pattern is required.") if not FLAGS.train_dir: raise ValueError("--train_dir is required.") model_config = configuration.model_config( input_file_pattern=FLAGS.input_file_pattern) training_config = configuration.training_config() tf.logging.info("Building training graph.") g = tf.Graph() with g.as_default(): model = skip_thoughts_model.SkipThoughtsModel(model_config, mode="train") model.build() learning_rate = _setup_learning_rate(training_config, model.global_step) optimizer = tf.train.AdamOptimizer(learning_rate) train_tensor = tf.contrib.slim.learning.create_train_op( total_loss=model.total_loss, optimizer=optimizer, global_step=model.global_step, clip_gradient_norm=training_config.clip_gradient_norm) saver = tf.train.Saver() tf.contrib.slim.learning.train( train_op=train_tensor, logdir=FLAGS.train_dir, graph=g, global_step=model.global_step, number_of_steps=training_config.number_of_steps, save_summaries_secs=training_config.save_summaries_secs, saver=saver, save_interval_secs=training_config.save_model_secs)
def main(unused_argv): if not FLAGS.train_dir: raise ValueError("--train_dir is required.") #read_vocab(FLAGS.vocab) model_config = configuration.model_config() training_config = configuration.training_config() ################ define discriminator model ################ disc_model = Discriminator(sequence_length=MAXLEN, num_classes=1, vocab_size=model_config.vocab_size, embedding_size=model_config.word_embedding_dim, filter_sizes=[1, 2, 3, 4, 5, 7, 10], num_filters=[100, 100, 100, 100, 100, 100, 100]) ################# define training model ################# model = skip_thoughts_model.SkipThoughtsModel(model_config, mode="train") model.build() learning_rate = _setup_learning_rate(training_config, model.global_step) optimizer = tf.train.AdamOptimizer(learning_rate) variables_to_train = [v for v in tf.trainable_variables()] variables_to_restore = [ v for v in tf.all_variables() if ('discriminator' not in v.name) ] print(len(variables_to_train)) train_tensor = tf.contrib.slim.learning.create_train_op( total_loss=model.total_loss, optimizer=optimizer, clip_gradient_norm=training_config.clip_gradient_norm, variables_to_train=variables_to_train) ######################define target lstm #################### #target_lstm = skip_thoughts_model.TargetLSTM(config=model_config) #synthesized = True target_lstm = None synthesized = False ################ define testing model ################ #model_config_test = configuration.model_config() #model_test = skip_thoughts_model.SkipThoughtsModel(model_config_test, mode="eval") #model_test.build(is_testing=True) ################ define savers ################ reloader = tf.train.Saver(var_list=variables_to_restore) reloader_all = tf.train.Saver() saver = tf.train.Saver(max_to_keep=1000) gpu_config = tf.ConfigProto(gpu_options=tf.GPUOptions( per_process_gpu_memory_fraction=1.0, allow_growth=True), allow_soft_placement=True, log_device_placement=False) init_op = tf.global_variables_initializer() sess = tf.Session(config=gpu_config) run_metadata = tf.RunMetadata() sess.run(init_op, options=tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE, output_partition_graphs=True), run_metadata=run_metadata) with open("/tmp/meta.txt", 'w') as f: f.write(str(run_metadata)) if FLAGS.reload_model: reloader.restore(sess, FLAGS.reload_model) if FLAGS.reload_model_all: reloader_all.restore(sess, FLAGS.reload_model_all) ################ load training data ############## train_data_loader = DataLoader(128) train_data_loader.load(FLAGS.train_corpus_en, FLAGS.train_corpus_fr) total_loss_sup_list = [] total_loss_rl_list = [] bleu_list = [] fake_list, real_list, neglikely_list = [], [], [] outf = open(os.path.join(FLAGS.train_dir, 'log.txt'), 'a') logf = open(os.path.join(FLAGS.train_dir, 'debug_log.txt'), 'w') ############### run training and testing ############# for i in xrange(1000000): model_prefix = "" if i < FLAGS.pretrain_G_steps: model_prefix = "preG_" np_global_step, total_loss_sup, total_loss_rl, avg_bleu, avg_fake, avg_real, avg_neglikely = my_train_step( sess, train_tensor, model, train_data_loader, logf, train_sup=True, train_rl=False, disc_model=disc_model, adjustD=False, adjustG=True, given_num=MAXLEN) elif i < FLAGS.pretrain_G_steps + FLAGS.pretrain_D_steps: model_prefix = "preD_" np_global_step, total_loss_sup, total_loss_rl, avg_bleu, avg_fake, avg_real, avg_neglikely = my_train_step( sess, train_tensor, model, train_data_loader, logf, train_sup=False, train_rl=True, disc_model=disc_model, adjustD=True, adjustG=False, given_num=0) elif FLAGS.mixer_period and FLAGS.mixer_step and FLAGS.mixer_period > 0: gn = default_given_num - ( i - FLAGS.pretrain_G_steps - FLAGS.pretrain_D_steps ) // FLAGS.mixer_period * FLAGS.mixer_step if gn < 0: gn = 0 model_prefix = "mixGN" + str(gn) + "_" if i % 10 == 0: adjustD = FLAGS.adjustD else: adjustD = False if i % 200 == 0: print("gn=", gn) np_global_step, total_loss_sup, total_loss_rl, avg_bleu, avg_fake, avg_real, avg_neglikely = my_train_step( \ sess, train_tensor, model, train_data_loader, logf, train_sup=False, train_rl=True, \ disc_model=disc_model, adjustD=adjustD, adjustG=FLAGS.adjustG, given_num=gn) else: model_prefix = "" np_global_step, total_loss_sup, total_loss_rl, avg_bleu, avg_fake, avg_real, avg_neglikely = my_train_step( sess, train_tensor, model, train_data_loader, logf, train_sup=False, train_rl=True, disc_model=disc_model, adjustD=FLAGS.adjustD, adjustG=FLAGS.adjustG) total_loss_sup_list.append(total_loss_sup) total_loss_rl_list.append(total_loss_rl) fake_list.append(avg_fake) real_list.append(avg_real) bleu_list.append(avg_bleu) neglikely_list.append(avg_neglikely) if np_global_step % 2000 == 0: saver.save( sess, os.path.join(FLAGS.train_dir, model_prefix + "model-" + str(np_global_step))) if np_global_step % 20 == 0: # my_test_step(sess, model_test, FLAGS.test_result+'-'+str(np_global_step)) print(np_global_step, np.mean(total_loss_sup_list), np.mean(total_loss_rl_list)) print(np.mean(bleu_list), np.mean(fake_list), np.mean(real_list)) print(np.mean(neglikely_list)) outf.write( str(np_global_step) + " " + str(np.mean(total_loss_sup_list)) + " " + str(np.mean(total_loss_rl_list)) + " " + str(np.mean(bleu_list)) + " " + str(np.mean(fake_list)) + " " + str(np.mean(real_list)) + " " + str(np.mean(neglikely_list)) + "\n") total_loss_sup_list, total_loss_rl_list, bleu_list, fake_list, real_list, neglikely_list = [],[],[],[],[],[]
# traditional measurement like levenstein distance, dynamic time wrapping, jaro, etc. print(_generate_log("Average Embedding", ae_sims, sim_names)) print(_generate_log("InferSent", inf_sims, sim_names)) print(_generate_log("SkipThought", st_sims, sim_names)) if __name__ == '__main__': # Load in InferSent infersent = torch.load(MODEL_PATH) # rely on "models.py" as well infersent.set_glove_path(GLOVE_PATH) # Load in SkipThought config_gpu = tf.ConfigProto() config_gpu.gpu_options.allow_growth = True with tf.Graph().as_default(), tf.Session(config=config_gpu) as session: skipthought = encoder_manager.EncoderManager() skipthought.load_model( configuration.model_config(bidirectional_encoder=True), vocabulary_file=VOCAB_FILE, embedding_matrix_file=EMBEDDING_MATRIX_FILE, checkpoint_path=CHECKPOINT_PATH) # Load in average embedding avg_emb = AverageEmbedder(word_emb_dim=300) avg_emb.set_glove_path(GLOVE_PATH) IPython.embed()
import pandas as pd from skip_thoughts import configuration from skip_thoughts import encoder_manager from sklearn.feature_extraction.text import TfidfVectorizer VOCAB_FILE = ".\\skip_thoughts_bi_2017_02_16\\vocab.txt" EMBEDDING_MATRIX_FILE = ".\\skip_thoughts_bi_2017_02_16\\embeddings.npy" CHECKPOINT_PATH = ".\\skip_thoughts_bi_2017_02_16\\model.ckpt-500008" encoder = encoder_manager.EncoderManager() encoder.load_model(configuration.model_config(bidirectional_encoder=True),vocabulary_file=VOCAB_FILE,embedding_matrix_file=EMBEDDING_MATRIX_FILE,checkpoint_path=CHECKPOINT_PATH) def neural_features(dataset_loc): english_dataset = pd.read_csv(dataset_loc) headline = english_dataset['headline'] body = english_dataset['content'] labels = [int(x) for x in english_dataset['label']] labels_done = [] flag = True body_encodings = np.zeros((len(body),2400)) j = 0 for i in range(len(body)): flag=True try: current_body_encoding = encoder.encode(body[i:i+1]) except: flag=False
def _int64_feature(value): return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) def _float_feature(value): return tf.train.Feature(float_list=tf.train.FloatList(value=[value])) VOCAB_FILE = "skip_thoughts_model/pretrained/skip_thoughts_uni_2017_02_02/vocab.txt" EMBEDDING_MATRIX_FILE = "skip_thoughts_model/pretrained/skip_thoughts_uni_2017_02_02/embeddings.npy" CHECKPOINT_PATH = "skip_thoughts_model/pretrained/skip_thoughts_uni_2017_02_02/model.ckpt-501424" print ("loading model ...") encoder = encoder_manager.EncoderManager() encoder.load_model(configuration.model_config(), vocabulary_file=VOCAB_FILE, embedding_matrix_file=EMBEDDING_MATRIX_FILE, checkpoint_path=CHECKPOINT_PATH) data_name, name_to_annotations, _ = pickle.load(open("data/train_info.pkl", "rb")) tfrecords_filename = 'train_image.tfrecords' writer = tf.python_io.TFRecordWriter(tfrecords_filename) img_dir = 'data/train2014/'
skipthought_checkpoint_path = 'checkpoints/skipthought/model.ckpt-501424' # Loading Captions for COCO #cap_path = 'coco/annotations/captions_train2014.json' #cat_path = 'coco/annotations/instances_train2014.json' #print('Loading Captions') #coco_cap = COCO(cap_path) #captions = coco_cap.loadAnns(coco_cap.getAnnIds()) #words = [caption['caption'] for caption in captions] # CUB-2011 Captions generated by the model - Show & Tell [Vinyals et. al 2016] cap_path = 'bird_caps_big.npy' encoder = encoder_manager.EncoderManager() encoder.load_model(configuration.model_config(), vocabulary_file=skipthought_vocab_path, embedding_matrix_file=skipthought_embedding_matrix_path, checkpoint_path=skipthought_checkpoint_path) compressor = PCA(n_components=128) print('Loading Captions') bird_caps = np.load(cap_path) print('loaded captions -', len(bird_caps)) words = [bird_cap[0] for bird_cap in bird_caps] ##words = [key for key in dictionary.keys()] encodings = encoder.encode(words, verbose=True) print('Performing PCA...') compressor.fit(np.array(encodings)) encodings = compressor.transform(encodings) mapping = []
def setUp(self): super(SkipThoughtsModelTest, self).setUp() self._model_config = configuration.model_config()
def setUp(self): super(SkipThoughtsModelTest, self).setUp() self._model_config = configuration.model_config()
def main(unused_argv): if not FLAGS.input_file_pattern: raise ValueError("--input_file_pattern is required.") if not FLAGS.run_dir: raise ValueError("--run_dir is required.") if not FLAGS.decoder: raise ValueError("--decoder is required.") if not FLAGS.train_dir: train_dir = os.path.join(FLAGS.run_dir, 'run_{t}'.format(t=time.time())) tf.logging.info( "No specified --train_dir. Creating {d}.".format(d=train_dir)) os.makedirs(train_dir) write_config(train_dir=train_dir, flags=FLAGS) else: tf.logging.info("Specified --train_dir {d}; Not autocreating.".format( d=FLAGS.train_dir)) train_dir = FLAGS.train_dir decoder_config = experiments.get_decoder_config(flags=FLAGS) model_config = configuration.model_config( input_file_pattern=FLAGS.input_file_pattern, vocab_size=FLAGS.vocab_size, batch_size=FLAGS.batch_size, word_embedding_dim=FLAGS.word_dim, pretrained_word_emb_file=FLAGS.pretrained_word_emb_file, word_emb_trainable=FLAGS.word_emb_trainable, encoder_dim=FLAGS.encoder_dim, skipgram_encoder=FLAGS.skipgram_encoder, sequence_decoder_pre=decoder_config.sequence_decoder_pre, sequence_decoder_cur=decoder_config.sequence_decoder_cur, sequence_decoder_post=decoder_config.sequence_decoder_post, skipgram_decoder_pre=decoder_config.skipgram_decoder_pre, skipgram_decoder_cur=decoder_config.skipgram_decoder_cur, skipgram_decoder_post=decoder_config.skipgram_decoder_post, share_weights_logits=FLAGS.share_weights_logits, normalise_decoder_losses=FLAGS.normalise_decoder_losses, skipgram_prefactor=FLAGS.skipgram_prefactor, sequence_prefactor=FLAGS.sequence_prefactor) training_config = configuration.training_config( number_of_steps=FLAGS.number_of_steps) tf.logging.info("Building training graph.") g = tf.Graph() with g.as_default(): tf.set_random_seed(1234) model = skip_thoughts_model.SkipThoughtsModel(model_config, mode="train") model.build() learning_rate = _setup_learning_rate(training_config, model.global_step) optimizer = tf.train.AdamOptimizer(learning_rate) train_tensor = tf.contrib.slim.learning.create_train_op( total_loss=model.total_loss, optimizer=optimizer, global_step=model.global_step, clip_gradient_norm=training_config.clip_gradient_norm, summarize_gradients=True, check_numerics=True) saver = tf.train.Saver() gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=FLAGS.gpu_fraction) tf.contrib.slim.learning.train( train_op=train_tensor, logdir=train_dir, graph=g, global_step=model.global_step, number_of_steps=training_config.number_of_steps, session_config=tf.ConfigProto(gpu_options=gpu_options), save_summaries_secs=training_config.save_summaries_secs, saver=saver, save_interval_secs=training_config.save_model_secs)
def main(unused_argv): if not FLAGS.input_file_pattern: raise ValueError("--input_file_pattern is required.") if not FLAGS.train_dir: raise ValueError("--train_dir is required.") model_config = configuration.model_config( input_file_pattern=FLAGS.input_file_pattern, bidirectional_encoder=True) training_config = configuration.training_config() tf.logging.info("Building training graph.") g = tf.Graph() with g.as_default(): model = skip_thoughts_model.SkipThoughtsModel(model_config, mode="train") model.build() encoder_variables = [v for v in tf.global_variables() if v.name.startswith("encoder") and "Adam" not in v.name] embedding_variables = [v for v in tf.global_variables() if v.name.startswith("word_embedding") and "Adam" not in v.name] print([v.name for v in (encoder_variables+embedding_variables)]) learning_rate = _setup_learning_rate(training_config, model.global_step) optimizer = tf.train.AdamOptimizer(learning_rate) encoder_mult = 0.1 embedding_mult = 0.01 multiply = dict([(v, encoder_mult) for v in encoder_variables] + [(v, embedding_mult) for v in embedding_variables]) train_tensor = tf.contrib.slim.learning.create_train_op( total_loss=model.total_loss, optimizer=optimizer, gradient_multipliers=multiply, global_step=model.global_step, clip_gradient_norm=training_config.clip_gradient_norm) saver = tf.train.Saver() model_path = tf.train.latest_checkpoint(FLAGS.train_dir) pretrain_saver = tf.train.Saver(encoder_variables+embedding_variables) print(model_path) if model_path: def restore_fn(sess): tf.logging.info( "Restoring SA&T variables from checkpoint file") saver.restore(sess, model_path) else: def restore_fn(sess): tf.logging.info( "Restoring SA&T variables from pretrained model") #saver.restore(sess, "/home/ubuntu/code/A_skip_thoughts_2/skip_thoughts/model/backup/run1/model.ckpt-2111") pretrain_saver.restore(sess, "/home/ubuntu/code/pretrained/bi/model.ckpt-500008") tf.contrib.slim.learning.train( train_op=train_tensor, logdir=FLAGS.train_dir, graph=g, global_step=model.global_step, number_of_steps=training_config.number_of_steps, save_summaries_secs=training_config.save_summaries_secs, saver=saver, save_interval_secs=training_config.save_model_secs, init_fn = restore_fn)
FLAGS_PICKLE_PATH = os.path.join(FLAGS.model_dir, "flags.pkl") # Load the configuration used to make the model with open(FLAGS_PICKLE_PATH, 'r') as f: model_flags = cPickle.load(f) decoder_config = experiments.get_decoder_config(flags=model_flags) model_config = configuration.model_config( input_file_pattern=model_flags.input_file_pattern, vocab_size=model_flags.vocab_size, batch_size=model_flags.batch_size, word_embedding_dim=model_flags.word_dim, encoder_dim=model_flags.encoder_dim, skipgram_encoder=model_flags.skipgram_encoder, sequence_decoder_pre=decoder_config.sequence_decoder_pre, sequence_decoder_cur=decoder_config.sequence_decoder_cur, sequence_decoder_post=decoder_config.sequence_decoder_post, skipgram_decoder_pre=decoder_config.skipgram_decoder_pre, skipgram_decoder_cur=decoder_config.skipgram_decoder_cur, skipgram_decoder_post=decoder_config.skipgram_decoder_post, share_weights_logits=model_flags.share_weights_logits, normalise_decoder_losses=model_flags.normalise_decoder_losses, skipgram_prefactor=model_flags.skipgram_prefactor, sequence_prefactor=model_flags.sequence_prefactor) # Set up the encoder. Here we are using a single unidirectional model. # To use a bidirectional model as well, call load_model() again with # configuration.model_config(bidirectional_encoder=True) and paths to the # bidirectional model's files. The encoder will use the concatenation of # all loaded models. encoder = encoder_manager.EncoderManager()
def main(unused_argv): if not FLAGS.input_file_pattern: raise ValueError("--input_file_pattern is required.") if not FLAGS.train_dir: raise ValueError("--train_dir is required.") model_config = configuration.model_config( input_file_pattern=FLAGS.input_file_pattern) training_config = configuration.training_config() tf.logging.info("Building training graph.") g = tf.Graph() with g.as_default(): grads_tower = [] for dev_ind in range(4): with tf.device('/gpu:%d' % dev_ind): model = skip_thoughts_model.SkipThoughtsModel(model_config, mode="train") model.build() learning_rate = _setup_learning_rate(training_config, model.global_step) optimizer = tf.train.AdamOptimizer(learning_rate) total_loss = model.total_loss # Update ops use GraphKeys.UPDATE_OPS collection if update_ops is None. update_ops = set(ops.get_collection(ops.GraphKeys.UPDATE_OPS)) # Make sure update_ops are computed before total_loss. if update_ops: with ops.control_dependencies(update_ops): barrier = control_flow_ops.no_op(name='update_barrier') total_loss = control_flow_ops.with_dependencies([barrier], total_loss) variables_to_train = tf_variables.trainable_variables() assert variables_to_train gate_gradients = tf_optimizer.Optimizer.GATE_OP # Create the gradients. Note that apply_gradients adds the gradient # computation to the current graph. grads = optimizer.compute_gradients( total_loss, variables_to_train, gate_gradients=gate_gradients, aggregation_method=None, colocate_gradients_with_ops=False) grads = tf.contrib.slim.learning.clip_gradient_norms( grads, training_config.clip_gradient_norm) grads_tower.append(grads) avg_grads = average_gradients.average_gradients(grads_tower) # Create gradient updates. grad_updates = optimizer.apply_gradients(avg_grads, global_step=model.global_step) with ops.name_scope('train_op'): # Make sure total_loss is valid. total_loss = array_ops.check_numerics(total_loss, 'LossTensor is inf or nan') # Ensure the train_tensor computes grad_updates. train_op = control_flow_ops.with_dependencies([grad_updates], total_loss) # Add the operation used for training to the 'train_op' collection train_ops = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) if train_op not in train_ops: train_ops.append(train_op) saver = tf.train.Saver() tf.contrib.slim.learning.train( train_op=train_op, logdir=FLAGS.train_dir, graph=g, global_step=model.global_step, number_of_steps=training_config.number_of_steps, save_summaries_secs=training_config.save_summaries_secs, saver=saver, save_interval_secs=training_config.save_model_secs)