def main():
dataloader = IMDB()
model = VRAE(dataloader.word2idx, dataloader.idx2word)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for epoch in range(args.num_epoch):
dataloader.update_word_dropout()
print("\nWord Dropout")
dataloader.shuffle()
print("Data Shuffled", end='\n\n')
for i, (enc_inp, dec_inp, dec_out) in enumerate(dataloader.next_batch()):
log = model.train_session(sess, enc_inp, dec_inp, dec_out)
if i % args.display_loss_step == 0:
print("Step %d | [%d/%d] | [%d/%d]" % (log['step'], epoch+1, args.num_epoch, i, len(dataloader.enc_inp)//args.batch_size), end='')
print(" | nll_loss:%.1f | kl_w:%.3f | kl_loss:%.2f \n" % (log['nll_loss'], log['kl_w'], log['kl_loss']))
model.reconstruct(sess, enc_inp[-1], dec_inp[-1])
#model.generate(sess)
model.customized_reconstruct(sess, 'i love this film and i think it is one of the best films')
model.customized_reconstruct(sess, 'this movie is a waste of time and there is no point to watch it')
save_path = model.saver.save(sess, model.model_path)
print("Model saved in file: %s" % save_path)
def main():
dataloader = IMDB()
model = VRAE(dataloader.word2idx)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for epoch in range(args.num_epoch):
dataloader.update_word_dropout()
dataloader.shuffle()
print("\nData Shuffled", end='\n\n')
for i, (seq, seq_dropped,
seq_len) in enumerate(dataloader.next_batch()):
log = model.train_session(sess, seq, seq_dropped, seq_len)
if i % args.display_loss_step == 0:
print("Step %d | [%d/%d] | [%d/%d]" %
(log['step'], epoch + 1, args.num_epoch, i,
len(dataloader._X) // args.batch_size),
end='')
"""
if args.mutinfo_loss:
print("\n\tnll_loss:%.1f | kl_w:%.3f | kl_loss:%.2f | temper:%.2f | mutinfo_loss:%.2f \n" % (
log['nll_loss'], log['kl_w'], log['kl_loss'], log['temperature'], log['mutinfo_loss']))
else:
"""
print(" | nll_loss:%.1f | kl_w:%.3f | kl_loss:%.2f \n" %
(log['nll_loss'], log['kl_w'], log['kl_loss']))
model.reconstruct(sess, seq[-1], seq_dropped[-1])
model.generate(sess)
save_path = model.saver.save(sess, model.model_path)
print("Model saved in file: %s" % save_path)
def main():
dataloader = IMDB()
model = VRAE(dataloader.word2idx)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
print("Loading trained model ...")
model.saver.restore(sess, model.model_path)
# lowercase, no punctuation, please
model.customized_reconstruct(sess,
'i love this firm it is one of the best')
model.customized_reconstruct(
sess, 'i want to see this movie it seems interesting')
def main():
dataloader = IMDB()
model = VRAE(dataloader.word2idx)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
for epoch in range(args.num_epoch):
dataloader.update_word_dropout()
dataloader.shuffle()
print("\nData Shuffled", end='\n\n')
for i, (seq, seq_dropped, seq_len) in enumerate(dataloader.next_batch()):
log = model.train_session(sess, seq, seq_dropped, seq_len)
if i % args.display_loss_step == 0:
bar = '[%d/%d] | [%d/%d] | nll_loss: %.1f | kl_w: %.3f | kl_loss: %.1f'
vars = (epoch+1, args.num_epoch, i+1, len(dataloader._X)//args.batch_size, log['nll_loss'],
log['kl_w'], log['kl_loss'])
print(bar % vars)
if i % args.display_text_step == 0:
model.reconstruct(sess, seq[-1], seq_dropped[-1])
model.generate(sess)
def main():
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
args.max_len = 100
args.batch_size = 64
args.max_dec_len = 100
args.display_info_step = 1000
args.isPointer = False
args.vocab_limit = 35000
train_data_len = 209940
args.diff_input = True
print(args)
dataloader = IWSLT(batch_size=args.batch_size, vocab_limit=args.vocab_limit, max_input_len=args.max_len, max_output_len=args.max_dec_len)
params = {
'vocab_size_encoder': len(dataloader.idx2token),
'vocab_size': len(dataloader.word2idx),
'word2idx': dataloader.word2idx,
'idx2word': dataloader.idx2word,
'idx2token': dataloader.idx2token}
print('Vocab Size:', params['vocab_size'])
model = VRAE(params)
saver = tf.train.Saver()
exp_path = "./saved/iwslt_seq2seq/"
model_name = "seq2seq.ckpt"
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
EPOCH_STEPS = (train_data_len-1)//args.batch_size+1
summary_writer = tf.summary.FileWriter(exp_path, sess.graph)
restore_path = tf.train.latest_checkpoint(exp_path)
if restore_path:
saver.restore(sess, restore_path)
last_train_step = int(restore_path.split("-")[-1]) % EPOCH_STEPS
print("Model restore from file: %s, last train step: %d" % (restore_path, last_train_step))
# summary_writer = tf.summary.FileWriter(exp_path)
# saver.restore(sess, exp_path+model_name)
for epoch in range(args.num_epoch):
# dataloader.update_word_dropout()
# print("\nWord Dropout")
# dataloader.shuffle()
# print("Data Shuffled", end='\n\n')
batcher = dataloader.load_data()
step = -1
while True:
try:
# enc_inp, dec_inp_full, dec_out = next(dataloader.data_loader)
(x_enc_inp, x_dec_inp_full, x_dec_out, y_enc_inp, y_dec_inp_full, y_dec_out), x_enc_inp_oovs, data_oovs, _ = next(batcher)
# enc_inp, dec_inp_full, dec_out = dataloader.next_batch()
enc_inp, dec_inp_full, dec_out = x_enc_inp, y_dec_inp_full, y_dec_out
dec_inp = dataloader.update_word_dropout(dec_inp_full)
step += 1
except StopIteration:
print("there are no more examples")
break
# print(step, "enc_inp.shape:", enc_inp.shape)
# print(step, "dec_inp_full.shape:", dec_inp_full.shape)
# print(step, "dec_out.shape:", dec_out.shape)
log = model.train_session(sess, enc_inp, dec_inp, dec_out)
# get the summaries and iteration number so we can write summaries to tensorboard
summaries, train_step = log['summaries'], log['step']
summary_writer.add_summary(summaries, train_step) # write the summaries
if train_step % 100 == 0: # flush the summary writer every so often
summary_writer.flush()
if step % args.display_loss_step == 0:
print("Step %d | [%d/%d] | [%d/%d]" % (log['step'], epoch+1, args.num_epoch, step, train_data_len//args.batch_size), end='')
print(" | loss:%.3f" % (log['loss']))
if step % args.display_info_step == 0:
model.reconstruct(sess, enc_inp[-1], dec_out[-1])
save_path = saver.save(sess, exp_path+model_name, global_step=train_step)
print("Model saved in file: %s" % save_path)
model.reconstruct(sess, enc_inp[-1], dec_out[-1])
save_path = saver.save(sess, exp_path+model_name, global_step=train_step)
print("Model saved in file: %s" % save_path)
def main():
## CUDA
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
## Parameters
dataloader = REDDIT(batch_size=64,
vocab_limit=35000,
max_input_len=150,
max_output_len=150)
params = {
'vocab_size': len(dataloader.word2idx),
'word2idx': dataloader.word2idx,
'idx2word': dataloader.idx2word,
}
print('Vocab Size:', params['vocab_size'])
args.max_len = 150
args.batch_size = 64
args.max_dec_len = 151
args.display_info_step = 10000
print(args)
## ModelInit
model = VRAE(params)
exp_path = "./saved/seq2seq/"
model_name = "seq2seq.ckpt"
## Session
saver = tf.train.Saver()
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
restore_path = tf.train.latest_checkpoint(exp_path)
# restore_path = "./saved/seq2seq/seq2seq.ckpt-70002"
saver.restore(sess, restore_path)
print("Model restore from file: %s" % (restore_path))
# Parpear Dir
ref_file = exp_path + "test.input.txt"
trans_file = exp_path + "test.output.txt"
result_file = exp_path + "test." + restore_path.split(
"-")[-1] + ".result.txt"
test_file = "./corpus/reddit/test.txt"
# Test Dir
dataloader.trans_in_ref(finpath=test_file, foutpath=ref_file)
with open(trans_file, "w") as f:
f.write("")
print("[PAEPEAR DATASET]")
# Test DataSet
test_len = 20000
batcher = dataloader._load_data(fpath=test_file)
for _ in tqdm(range((test_len - 1) // args.batch_size + 1)):
try:
enc_inp, dec_inp_full, dec_out = next(batcher)
# dec_inp = dataloader.update_word_dropout(dec_inp_full)
except StopIteration:
print("there are no more examples")
break
model.evaluation(sess, enc_inp, trans_file)
# Evaluation
eval_log = {}
for metric in ['bleu', 'rouge', 'accuracy', 'word_accuracy']:
score = evaluation_utils.evaluate(ref_file, trans_file, metric)
eval_log[metric] = score
if metric == "bleu":
print(
" bleu-1, bleu-2, bleu-3, bleu-4: %.5f, %.5f, %.5f, %.5f" %
score)
elif metric == "rouge":
print(" rouge-1, rouge-2, rouge-l: %.5f, %.5f, %.5f" % score)
else:
print(" %s: %.5f" % (metric, score))
from measures import selfbleu
selfbleuobj = selfbleu.SelfBleu(trans_file, 1)
print(" selfbleu-1", selfbleuobj.get_score())
eval_log['selfbleu-1'] = selfbleuobj.get_score()
selfbleuobj = selfbleu.SelfBleu(trans_file, 2)
print(" selfbleu-2", selfbleuobj.get_score())
eval_log['selfbleu-2'] = selfbleuobj.get_score()
# Record Log
dataloader.record_result(eval_log,
finpath=test_file,
frespaht=trans_file,
foutpath=result_file)
def main():
dataloader = IMDB()
model = VRAE(dataloader.word2idx, dataloader.idx2word)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
print("Loading trained model ...")
model.saver.restore(sess, model.model_path)
# lowercase, no punctuation, please
model.customized_reconstruct(sess, 'i love this firm and it is beyond my expectation')
model.customized_reconstruct(sess, 'i want to watch this movie again because it is so interesting')
model.customized_reconstruct(sess, 'the time taken to develop the characters is quite long')
model.customized_reconstruct(sess, 'is there any point to make a bad movie like this')
model.customized_reconstruct(sess, 'sorry but there is no point to watch this movie again')
model.customized_reconstruct(sess, 'to be honest this movie is not worth my time and money')
def main():
dataloader = IMDB()
params = {
'vocab_size': len(dataloader.word2idx),
'word2idx': dataloader.word2idx,
'idx2word': dataloader.idx2word,
}
print('Vocab Size:', params['vocab_size'])
model = VRAE(params)
saver = tf.train.Saver()
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
sess.run(tf.global_variables_initializer())
for epoch in range(args.num_epoch):
dataloader.update_word_dropout()
print("\nWord Dropout")
dataloader.shuffle()
print("Data Shuffled", end='\n\n')
for i, (enc_inp, dec_inp,
dec_out) in enumerate(dataloader.next_batch()):
log = model.train_session(sess, enc_inp, dec_inp, dec_out)
if i % args.display_loss_step == 0:
print("Step %d | [%d/%d] | [%d/%d]" %
(log['step'], epoch + 1, args.num_epoch, i,
len(dataloader.enc_inp) // args.batch_size),
end='')
print(" | nll_loss:%.1f | kl_w:%.3f | kl_loss:%.2f \n" %
(log['nll_loss'], log['kl_w'], log['kl_loss']))
model.generate(sess)
model.reconstruct(sess, enc_inp[-1], dec_inp[-1])
model.customized_reconstruct(
sess, 'i love this film and i think it is one of the best films')
model.customized_reconstruct(
sess,
'this movie is a waste of time and there is no point to watch it')
save_path = saver.save(sess, './saved/vrae.ckpt')
print("Model saved in file: %s" % save_path)
def main(_):
if not FLAGS.data_path:
raise ValueError("Must set --data_path to PTB data directory")
config = get_config()
eval_config = get_config()
batch_size = config.batch_size
text = codecs.open(FLAGS.data_path, "r", encoding="utf-8").read()
seq_list = re.split(u"[，。\n]+", text)
cts_seq_len = 5
word_to_id, id_to_word, num_chars = char_index_mapping(text)
ind_seq_list = [[word_to_id[char] for char in seq] for seq in seq_list]
data = get_batch_from_seq_list(ind_seq_list, config.batch_size)
config.vocab_size = num_chars
eval_config.vocab_size = num_chars
eval_config.batch_size = 1
data_split = (int(len(data) * 0.9) // (batch_size)) * batch_size
train_data = data[:data_split]
test_data = data[data_split:]
with tf.Graph().as_default():
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.name_scope("Train"):
with tf.variable_scope("Model",
reuse=None,
initializer=initializer):
m = VRAE(is_training=True, config=config, seq_len=cts_seq_len)
tf.scalar_summary("Training Loss", m.cost)
with tf.name_scope("Test"):
with tf.variable_scope("Model",
reuse=True,
initializer=initializer):
mtest = VRAE(is_training=True,
config=eval_config,
seq_len=cts_seq_len)
tf.scalar_summary("Training Loss", mtest.cost)
sv = tf.train.Supervisor(logdir=FLAGS.save_path)
with sv.managed_session() as session:
for i in range(config.max_max_epoch):
KL_rate = 0
if (i > 3):
KL_rate = (i - 3) * 0.1
m.assign_KL_rate(session, KL_rate)
train_costs, train_KL_term, train_reconstruction_cost = run_epoch(
session, m, train_data)
print("Epoch: %d Train costs: %.3f" % (i + 1, train_costs))
print("Epoch: %d Train KL divergence: %.3f" %
(i + 1, train_KL_term))
print("Epoch: %d Train reconstruction costs: %.3f" %
(i + 1, train_reconstruction_cost))
test_costs, test_KL_term, test_reconstruction_cost = run_epoch(
session, mtest, test_data)
print("Epoch: %d test costs: %.3f" % (i + 1, test_costs))
print("Epoch: %d test KL divergence: %.3f" %
(i + 1, test_KL_term))
print("Epoch: %d test reconstruction costs: %.3f" %
(i + 1, test_reconstruction_cost))
sampled_cov = np.identity(config.latent_dim)
sampled_mean = np.zeros(config.latent_dim)
start_pt = np.random.multivariate_normal(sampled_mean, sampled_cov)
end_pt = np.random.multivariate_normal(sampled_mean, sampled_cov)
sampling_outputs = linear_interpolate(session, mtest, start_pt,
end_pt, 21, id_to_word)
print("sampling outputs: ", sampling_outputs)
reconstruct_inputs = seq_list[-32:]
reconstruct_outputs = []
for recon_input in reconstruct_inputs:
reconstruct_outputs.append(
reconstruct(session, mtest, reconstruct_input, word_to_id,
id_to_word))
for recon_input, recon_output in zip(reconstruct_inputs,
reconstruct_outputs):
print("reconstruct inputs: ", recon_input)
print("reconstruct outputs: ", recon_output)
if FLAGS.save_path:
print("Saving model to %s." % FLAGS.save_path)
sv.saver.save(session,
FLAGS.save_path,
global_step=sv.global_step)
