def main(): dataloader = IMDB() model = VRAE(dataloader.word2idx, dataloader.idx2word) sess = tf.Session() sess.run(tf.global_variables_initializer()) for epoch in range(args.num_epoch): dataloader.update_word_dropout() print("\nWord Dropout") dataloader.shuffle() print("Data Shuffled", end='\n\n') for i, (enc_inp, dec_inp, dec_out) in enumerate(dataloader.next_batch()): log = model.train_session(sess, enc_inp, dec_inp, dec_out) if i % args.display_loss_step == 0: print("Step %d | [%d/%d] | [%d/%d]" % (log['step'], epoch+1, args.num_epoch, i, len(dataloader.enc_inp)//args.batch_size), end='') print(" | nll_loss:%.1f | kl_w:%.3f | kl_loss:%.2f \n" % (log['nll_loss'], log['kl_w'], log['kl_loss'])) model.reconstruct(sess, enc_inp[-1], dec_inp[-1]) #model.generate(sess) model.customized_reconstruct(sess, 'i love this film and i think it is one of the best films') model.customized_reconstruct(sess, 'this movie is a waste of time and there is no point to watch it') save_path = model.saver.save(sess, model.model_path) print("Model saved in file: %s" % save_path)
def main(): dataloader = IMDB() model = VRAE(dataloader.word2idx) sess = tf.Session() sess.run(tf.global_variables_initializer()) for epoch in range(args.num_epoch): dataloader.update_word_dropout() dataloader.shuffle() print("\nData Shuffled", end='\n\n') for i, (seq, seq_dropped, seq_len) in enumerate(dataloader.next_batch()): log = model.train_session(sess, seq, seq_dropped, seq_len) if i % args.display_loss_step == 0: print("Step %d | [%d/%d] | [%d/%d]" % (log['step'], epoch + 1, args.num_epoch, i, len(dataloader._X) // args.batch_size), end='') """ if args.mutinfo_loss: print("\n\tnll_loss:%.1f | kl_w:%.3f | kl_loss:%.2f | temper:%.2f | mutinfo_loss:%.2f \n" % ( log['nll_loss'], log['kl_w'], log['kl_loss'], log['temperature'], log['mutinfo_loss'])) else: """ print(" | nll_loss:%.1f | kl_w:%.3f | kl_loss:%.2f \n" % (log['nll_loss'], log['kl_w'], log['kl_loss'])) model.reconstruct(sess, seq[-1], seq_dropped[-1]) model.generate(sess) save_path = model.saver.save(sess, model.model_path) print("Model saved in file: %s" % save_path)
def main(): dataloader = IMDB() model = VRAE(dataloader.word2idx) sess = tf.Session() sess.run(tf.global_variables_initializer()) print("Loading trained model ...") model.saver.restore(sess, model.model_path) # lowercase, no punctuation, please model.customized_reconstruct(sess, 'i love this firm it is one of the best') model.customized_reconstruct( sess, 'i want to see this movie it seems interesting')
def main(): dataloader = IMDB() model = VRAE(dataloader.word2idx) sess = tf.Session() sess.run(tf.global_variables_initializer()) for epoch in range(args.num_epoch): dataloader.update_word_dropout() dataloader.shuffle() print("\nData Shuffled", end='\n\n') for i, (seq, seq_dropped, seq_len) in enumerate(dataloader.next_batch()): log = model.train_session(sess, seq, seq_dropped, seq_len) if i % args.display_loss_step == 0: bar = '[%d/%d] | [%d/%d] | nll_loss: %.1f | kl_w: %.3f | kl_loss: %.1f' vars = (epoch+1, args.num_epoch, i+1, len(dataloader._X)//args.batch_size, log['nll_loss'], log['kl_w'], log['kl_loss']) print(bar % vars) if i % args.display_text_step == 0: model.reconstruct(sess, seq[-1], seq_dropped[-1]) model.generate(sess)
def main(): os.environ["CUDA_VISIBLE_DEVICES"] = "1" args.max_len = 100 args.batch_size = 64 args.max_dec_len = 100 args.display_info_step = 1000 args.isPointer = False args.vocab_limit = 35000 train_data_len = 209940 args.diff_input = True print(args) dataloader = IWSLT(batch_size=args.batch_size, vocab_limit=args.vocab_limit, max_input_len=args.max_len, max_output_len=args.max_dec_len) params = { 'vocab_size_encoder': len(dataloader.idx2token), 'vocab_size': len(dataloader.word2idx), 'word2idx': dataloader.word2idx, 'idx2word': dataloader.idx2word, 'idx2token': dataloader.idx2token} print('Vocab Size:', params['vocab_size']) model = VRAE(params) saver = tf.train.Saver() exp_path = "./saved/iwslt_seq2seq/" model_name = "seq2seq.ckpt" config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth=True sess = tf.Session(config=config) sess.run(tf.global_variables_initializer()) EPOCH_STEPS = (train_data_len-1)//args.batch_size+1 summary_writer = tf.summary.FileWriter(exp_path, sess.graph) restore_path = tf.train.latest_checkpoint(exp_path) if restore_path: saver.restore(sess, restore_path) last_train_step = int(restore_path.split("-")[-1]) % EPOCH_STEPS print("Model restore from file: %s, last train step: %d" % (restore_path, last_train_step)) # summary_writer = tf.summary.FileWriter(exp_path) # saver.restore(sess, exp_path+model_name) for epoch in range(args.num_epoch): # dataloader.update_word_dropout() # print("\nWord Dropout") # dataloader.shuffle() # print("Data Shuffled", end='\n\n') batcher = dataloader.load_data() step = -1 while True: try: # enc_inp, dec_inp_full, dec_out = next(dataloader.data_loader) (x_enc_inp, x_dec_inp_full, x_dec_out, y_enc_inp, y_dec_inp_full, y_dec_out), x_enc_inp_oovs, data_oovs, _ = next(batcher) # enc_inp, dec_inp_full, dec_out = dataloader.next_batch() enc_inp, dec_inp_full, dec_out = x_enc_inp, y_dec_inp_full, y_dec_out dec_inp = dataloader.update_word_dropout(dec_inp_full) step += 1 except StopIteration: print("there are no more examples") break # print(step, "enc_inp.shape:", enc_inp.shape) # print(step, "dec_inp_full.shape:", dec_inp_full.shape) # print(step, "dec_out.shape:", dec_out.shape) log = model.train_session(sess, enc_inp, dec_inp, dec_out) # get the summaries and iteration number so we can write summaries to tensorboard summaries, train_step = log['summaries'], log['step'] summary_writer.add_summary(summaries, train_step) # write the summaries if train_step % 100 == 0: # flush the summary writer every so often summary_writer.flush() if step % args.display_loss_step == 0: print("Step %d | [%d/%d] | [%d/%d]" % (log['step'], epoch+1, args.num_epoch, step, train_data_len//args.batch_size), end='') print(" | loss:%.3f" % (log['loss'])) if step % args.display_info_step == 0: model.reconstruct(sess, enc_inp[-1], dec_out[-1]) save_path = saver.save(sess, exp_path+model_name, global_step=train_step) print("Model saved in file: %s" % save_path) model.reconstruct(sess, enc_inp[-1], dec_out[-1]) save_path = saver.save(sess, exp_path+model_name, global_step=train_step) print("Model saved in file: %s" % save_path)
def main(): ## CUDA os.environ["CUDA_VISIBLE_DEVICES"] = "1" ## Parameters dataloader = REDDIT(batch_size=64, vocab_limit=35000, max_input_len=150, max_output_len=150) params = { 'vocab_size': len(dataloader.word2idx), 'word2idx': dataloader.word2idx, 'idx2word': dataloader.idx2word, } print('Vocab Size:', params['vocab_size']) args.max_len = 150 args.batch_size = 64 args.max_dec_len = 151 args.display_info_step = 10000 print(args) ## ModelInit model = VRAE(params) exp_path = "./saved/seq2seq/" model_name = "seq2seq.ckpt" ## Session saver = tf.train.Saver() config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) restore_path = tf.train.latest_checkpoint(exp_path) # restore_path = "./saved/seq2seq/seq2seq.ckpt-70002" saver.restore(sess, restore_path) print("Model restore from file: %s" % (restore_path)) # Parpear Dir ref_file = exp_path + "test.input.txt" trans_file = exp_path + "test.output.txt" result_file = exp_path + "test." + restore_path.split( "-")[-1] + ".result.txt" test_file = "./corpus/reddit/test.txt" # Test Dir dataloader.trans_in_ref(finpath=test_file, foutpath=ref_file) with open(trans_file, "w") as f: f.write("") print("[PAEPEAR DATASET]") # Test DataSet test_len = 20000 batcher = dataloader._load_data(fpath=test_file) for _ in tqdm(range((test_len - 1) // args.batch_size + 1)): try: enc_inp, dec_inp_full, dec_out = next(batcher) # dec_inp = dataloader.update_word_dropout(dec_inp_full) except StopIteration: print("there are no more examples") break model.evaluation(sess, enc_inp, trans_file) # Evaluation eval_log = {} for metric in ['bleu', 'rouge', 'accuracy', 'word_accuracy']: score = evaluation_utils.evaluate(ref_file, trans_file, metric) eval_log[metric] = score if metric == "bleu": print( " bleu-1, bleu-2, bleu-3, bleu-4: %.5f, %.5f, %.5f, %.5f" % score) elif metric == "rouge": print(" rouge-1, rouge-2, rouge-l: %.5f, %.5f, %.5f" % score) else: print(" %s: %.5f" % (metric, score)) from measures import selfbleu selfbleuobj = selfbleu.SelfBleu(trans_file, 1) print(" selfbleu-1", selfbleuobj.get_score()) eval_log['selfbleu-1'] = selfbleuobj.get_score() selfbleuobj = selfbleu.SelfBleu(trans_file, 2) print(" selfbleu-2", selfbleuobj.get_score()) eval_log['selfbleu-2'] = selfbleuobj.get_score() # Record Log dataloader.record_result(eval_log, finpath=test_file, frespaht=trans_file, foutpath=result_file)
def main(): dataloader = IMDB() model = VRAE(dataloader.word2idx, dataloader.idx2word) sess = tf.Session() sess.run(tf.global_variables_initializer()) print("Loading trained model ...") model.saver.restore(sess, model.model_path) # lowercase, no punctuation, please model.customized_reconstruct(sess, 'i love this firm and it is beyond my expectation') model.customized_reconstruct(sess, 'i want to watch this movie again because it is so interesting') model.customized_reconstruct(sess, 'the time taken to develop the characters is quite long') model.customized_reconstruct(sess, 'is there any point to make a bad movie like this') model.customized_reconstruct(sess, 'sorry but there is no point to watch this movie again') model.customized_reconstruct(sess, 'to be honest this movie is not worth my time and money')
def main(): dataloader = IMDB() params = { 'vocab_size': len(dataloader.word2idx), 'word2idx': dataloader.word2idx, 'idx2word': dataloader.idx2word, } print('Vocab Size:', params['vocab_size']) model = VRAE(params) saver = tf.train.Saver() config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True sess = tf.Session(config=config) sess.run(tf.global_variables_initializer()) for epoch in range(args.num_epoch): dataloader.update_word_dropout() print("\nWord Dropout") dataloader.shuffle() print("Data Shuffled", end='\n\n') for i, (enc_inp, dec_inp, dec_out) in enumerate(dataloader.next_batch()): log = model.train_session(sess, enc_inp, dec_inp, dec_out) if i % args.display_loss_step == 0: print("Step %d | [%d/%d] | [%d/%d]" % (log['step'], epoch + 1, args.num_epoch, i, len(dataloader.enc_inp) // args.batch_size), end='') print(" | nll_loss:%.1f | kl_w:%.3f | kl_loss:%.2f \n" % (log['nll_loss'], log['kl_w'], log['kl_loss'])) model.generate(sess) model.reconstruct(sess, enc_inp[-1], dec_inp[-1]) model.customized_reconstruct( sess, 'i love this film and i think it is one of the best films') model.customized_reconstruct( sess, 'this movie is a waste of time and there is no point to watch it') save_path = saver.save(sess, './saved/vrae.ckpt') print("Model saved in file: %s" % save_path)
def main(_): if not FLAGS.data_path: raise ValueError("Must set --data_path to PTB data directory") config = get_config() eval_config = get_config() batch_size = config.batch_size text = codecs.open(FLAGS.data_path, "r", encoding="utf-8").read() seq_list = re.split(u"[,。\n]+", text) cts_seq_len = 5 word_to_id, id_to_word, num_chars = char_index_mapping(text) ind_seq_list = [[word_to_id[char] for char in seq] for seq in seq_list] data = get_batch_from_seq_list(ind_seq_list, config.batch_size) config.vocab_size = num_chars eval_config.vocab_size = num_chars eval_config.batch_size = 1 data_split = (int(len(data) * 0.9) // (batch_size)) * batch_size train_data = data[:data_split] test_data = data[data_split:] with tf.Graph().as_default(): initializer = tf.random_uniform_initializer(-config.init_scale, config.init_scale) with tf.name_scope("Train"): with tf.variable_scope("Model", reuse=None, initializer=initializer): m = VRAE(is_training=True, config=config, seq_len=cts_seq_len) tf.scalar_summary("Training Loss", m.cost) with tf.name_scope("Test"): with tf.variable_scope("Model", reuse=True, initializer=initializer): mtest = VRAE(is_training=True, config=eval_config, seq_len=cts_seq_len) tf.scalar_summary("Training Loss", mtest.cost) sv = tf.train.Supervisor(logdir=FLAGS.save_path) with sv.managed_session() as session: for i in range(config.max_max_epoch): KL_rate = 0 if (i > 3): KL_rate = (i - 3) * 0.1 m.assign_KL_rate(session, KL_rate) train_costs, train_KL_term, train_reconstruction_cost = run_epoch( session, m, train_data) print("Epoch: %d Train costs: %.3f" % (i + 1, train_costs)) print("Epoch: %d Train KL divergence: %.3f" % (i + 1, train_KL_term)) print("Epoch: %d Train reconstruction costs: %.3f" % (i + 1, train_reconstruction_cost)) test_costs, test_KL_term, test_reconstruction_cost = run_epoch( session, mtest, test_data) print("Epoch: %d test costs: %.3f" % (i + 1, test_costs)) print("Epoch: %d test KL divergence: %.3f" % (i + 1, test_KL_term)) print("Epoch: %d test reconstruction costs: %.3f" % (i + 1, test_reconstruction_cost)) sampled_cov = np.identity(config.latent_dim) sampled_mean = np.zeros(config.latent_dim) start_pt = np.random.multivariate_normal(sampled_mean, sampled_cov) end_pt = np.random.multivariate_normal(sampled_mean, sampled_cov) sampling_outputs = linear_interpolate(session, mtest, start_pt, end_pt, 21, id_to_word) print("sampling outputs: ", sampling_outputs) reconstruct_inputs = seq_list[-32:] reconstruct_outputs = [] for recon_input in reconstruct_inputs: reconstruct_outputs.append( reconstruct(session, mtest, reconstruct_input, word_to_id, id_to_word)) for recon_input, recon_output in zip(reconstruct_inputs, reconstruct_outputs): print("reconstruct inputs: ", recon_input) print("reconstruct outputs: ", recon_output) if FLAGS.save_path: print("Saving model to %s." % FLAGS.save_path) sv.saver.save(session, FLAGS.save_path, global_step=sv.global_step)