def main():
vocab_sizeen, vocab_sizech, vocaben, rev_vocabch = getfanyiInfo()
if not os.path.exists(checkpoint_dir):
os.mkdir(checkpoint_dir)
print("checkpoint_dir is {0}".format(checkpoint_dir))
with tf.Session() as sess:
model = createModel(sess, True, vocab_sizeen, vocab_sizech)
print(_buckets)
model.batch_size = 1
conversation_history = []
while True:
prompt = "请输入: "
sentence = input(prompt)
conversation_history.append(sentence.strip())
conversation_history = conversation_history[-convo_hist_limit:]
token_ids = list(
reversed(
datautil.sentence_to_ids(" ".join(conversation_history),
vocaben,
normalize_digits=True,
Isch=False)))
#token_ids = list(reversed(vocab.tokens2Indices(" ".join(conversation_history))))
print(token_ids)
#token_ids = list(reversed(vocab.tokens2Indices(sentence)))
bucket_id = min([
b for b in xrange(len(_buckets))
if _buckets[b][0] > len(token_ids)
])
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
{bucket_id: [(token_ids, [])]}, bucket_id)
_, _, output_logits = model.step(sess, encoder_inputs,
decoder_inputs, target_weights,
bucket_id, True)
#TODO implement beam search
outputs = [
int(np.argmax(logit, axis=1)) for logit in output_logits
]
print("outputs", outputs, datautil.EOS_ID)
if datautil.EOS_ID in outputs:
outputs = outputs[:outputs.index(datautil.EOS_ID)]
#print(vocab.indices2Tokens(outputs))
#print("结果",datautil.ids2texts(outputs,rev_vocabch))
convo_output = " ".join(
datautil.ids2texts(outputs, rev_vocabch))
conversation_history.append(convo_output)
print(convo_output)
else:
print("can not translation！")
def main():
vocab_sizeen, vocab_sizech, rev_vocaben, rev_vocabch, source_train_file_path, target_train_file_path = getdialogInfo(
)
if not os.path.exists(checkpoint_dir):
os.mkdir(checkpoint_dir)
print("checkpoint_dir is {0}".format(checkpoint_dir))
with tf.Session() as sess:
model = createModel(sess, False, vocab_sizeen, vocab_sizech)
print("Using bucket sizes:")
print(_buckets)
source_test_file_path = source_train_file_path
target_test_file_path = target_train_file_path
print(source_train_file_path)
print(target_train_file_path)
train_set = readData(source_train_file_path, target_train_file_path,
max_train_data_size)
test_set = readData(source_test_file_path, target_test_file_path,
max_train_data_size)
train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_buckets))]
print("bucket sizes = {0}".format(train_bucket_sizes))
train_total_size = float(sum(train_bucket_sizes))
# 桶秤是一个从0增加到1的数字列表，我们将用它来选择一个桶。[标度[i]，标度[i+1]的长度与第i个训练桶的大小成比例，稍后使用。
train_buckets_scale = [
sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in xrange(len(train_bucket_sizes))
]
step_time, loss = 0.0, 0.0
current_step = 0
previous_losses = []
while True:
random_number_01 = np.random.random_sample()
bucket_id = min([
i for i in xrange(len(train_buckets_scale))
if train_buckets_scale[i] > random_number_01
])
start_time = time.time()
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
train_set, bucket_id)
_, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,
target_weights, bucket_id, False)
step_time += (time.time() - start_time) / steps_per_checkpoint
loss += step_loss / steps_per_checkpoint
current_step += 1
if current_step % steps_per_checkpoint == 0:
perplexity = math.exp(loss) if loss < 300 else float('inf')
print(
"global step %d learning rate %.4f step-time %.2f perplexity "
"%.2f" %
(model.global_step.eval(), model.learning_rate.eval(),
step_time, perplexity))
if len(previous_losses) > 2 and loss > max(
previous_losses[-3:]):
sess.run(model.learning_rate_decay_op)
previous_losses.append(loss)
checkpoint_path = os.path.join(checkpoint_dir,
"seq2seqtest.ckpt")
print(checkpoint_path)
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
step_time, loss = 0.0, 0.0
for bucket_id in xrange(len(_buckets)):
if len(test_set[bucket_id]) == 0:
print(" eval: empty bucket %d" % (bucket_id))
continue
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
test_set, bucket_id)
_, eval_loss, output_logits = model.step(
sess, encoder_inputs, decoder_inputs, target_weights,
bucket_id, True)
eval_ppx = math.exp(
eval_loss) if eval_loss < 300 else float('inf')
print(" eval: bucket %d perplexity %.2f" %
(bucket_id, eval_ppx))
inputstr = datautil.ids2texts(
reversed([en[0] for en in encoder_inputs]),
rev_vocaben)
print("输入", inputstr)
print(
"输出",
datautil.ids2texts([en[0] for en in decoder_inputs],
rev_vocabch))
outputs = [
np.argmax(logit, axis=1)[0] for logit in output_logits
]
if datautil.EOS_ID in outputs:
outputs = outputs[:outputs.index(datautil.EOS_ID)]
print("结果", datautil.ids2texts(outputs, rev_vocabch))
sys.stdout.flush()
def main():
vocab_sizeen, vocab_sizech, rev_vocaben, rev_vocabch, source_train_file_path, target_train_file_path = getfanyi(
)
if not os.path.exists(checkpoint_dir):
os.mkdir(checkpoint_dir)
with tf.Session() as sess:
model = createModel(sess, False, vocab_sizeen, vocab_sizech)
source_test_file_path = source_train_file_path
target_test_file_path = target_train_file_path
train_set = readData(source_train_file_path, target_train_file_path,
max_train_data_size)
test_set = readData(source_test_file_path, target_test_file_path,
max_train_data_size)
train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_buckets))]
train_total_size = float(sum(train_bucket_sizes))
train_buckets_scale = [
sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in xrange(len(train_bucket_sizes))
]
step_time, loss = 0.0, 0.0
current_step = 0
previous_losses = []
while True:
random_number_01 = np.random.random_sample()
bucket_id = min([
i for i in xrange(len(train_buckets_scale))
if train_buckets_scale[i] > random_number_01
])
start_time = time.time()
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
train_set, bucket_id)
_, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,
target_weights, bucket_id, False)
step_time += (time.time() - start_time) / steps_per_checkpoint
loss += step_loss / steps_per_checkpoint
current_step += 1
if current_step % steps_per_checkpoint == 0:
perplexity = math.exp(loss) if loss < 300 else float('inf')
if len(previous_losses) > 2 and loss > max(
previous_losses[-3:]):
sess.run(model.learning_rate_decay_op)
previous_losses.append(loss)
checkpoint_path = os.path.join(checkpoint_dir, 'seq2seqtest.ckpt')
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
step_time = 0.0, 0.0
if len(test_set[bucket_id]) == 0:
continue
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
test_set, bucket_id)
_, eval_loss, output_logits = model.step(sess, encoder_inputs,
decoder_inputs,
target_weights, bucket_id,
True)
eval_ppx = math.exp(eval_loss) if eval_loss < 300 else float('inf')
inputstr = datautil.ids2texts(
reversed([en[0] for en in encoder_inputs]), rev_vocaben)
outputs = [np.argmax(logit, axis=1)[0] for logit in output_logits]
if datautil.EOS_ID in outputs:
outputs = outputs[:outputs.index(datautil.EOS_ID)]
sys.stdout.flush()
def main():
vocab_sizeen,vocab_sizech,rev_vocaben,rev_vocabch,source_train_file_path,target_train_file_path = getdialogInfo()
if not os.path.exists(checkpoint_dir):
os.mkdir(checkpoint_dir)
print ("checkpoint_dir is {0}".format(checkpoint_dir))
with tf.Session() as sess:
model = createModel(sess,False,vocab_sizeen,vocab_sizech)
print ("Using bucket sizes:")
print (_buckets)
source_test_file_path = source_train_file_path
target_test_file_path = target_train_file_path
print (source_train_file_path)
print (target_train_file_path)
train_set = readData(source_train_file_path, target_train_file_path,max_train_data_size)
test_set = readData(source_test_file_path, target_test_file_path,max_train_data_size)
train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_buckets))]
print( "bucket sizes = {0}".format(train_bucket_sizes))
train_total_size = float(sum(train_bucket_sizes))
# A bucket scale is a list of increasing numbers from 0 to 1 that we'll use
# to select a bucket. Length of [scale[i], scale[i+1]] is proportional to
# the size if i-th training bucket, as used later.
train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size for i in xrange(len(train_bucket_sizes))]
step_time, loss = 0.0, 0.0
current_step = 0
previous_losses = []
while True:
# Choose a bucket according to data distribution. We pick a random number
# in [0, 1] and use the corresponding interval in train_buckets_scale.
random_number_01 = np.random.random_sample()
bucket_id = min([i for i in xrange(len(train_buckets_scale)) if train_buckets_scale[i] > random_number_01])
# 开始训练.
start_time = time.time()
encoder_inputs, decoder_inputs, target_weights = model.get_batch(train_set, bucket_id)
_, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,target_weights, bucket_id, False)
step_time += (time.time() - start_time) / steps_per_checkpoint
loss += step_loss / steps_per_checkpoint
current_step += 1
# 保存检查点，测试数据
if current_step % steps_per_checkpoint == 0:
# Print statistics for the previous epoch.
perplexity = math.exp(loss) if loss < 300 else float('inf')
print ("global step %d learning rate %.4f step-time %.2f perplexity "
"%.2f" % (model.global_step.eval(), model.learning_rate.eval(),step_time, perplexity))
# Decrease learning rate if no improvement was seen over last 3 times.
if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):
sess.run(model.learning_rate_decay_op)
previous_losses.append(loss)
# Save checkpoint and zero timer and loss.
checkpoint_path = os.path.join(checkpoint_dir, "seq2seqtest.ckpt")
print(checkpoint_path)
model.saver.save(sess, checkpoint_path, global_step=model.global_step)
step_time, loss = 0.0, 0.0
# Run evals on development set and print their perplexity.
for bucket_id in xrange(len(_buckets)):
if len(test_set[bucket_id]) == 0:
print(" eval: empty bucket %d" % (bucket_id))
continue
encoder_inputs, decoder_inputs, target_weights = model.get_batch(test_set, bucket_id)
_, eval_loss,output_logits = model.step(sess, encoder_inputs, decoder_inputs,target_weights, bucket_id, True)
eval_ppx = math.exp(eval_loss) if eval_loss < 300 else float('inf')
print(" eval: bucket %d perplexity %.2f" % (bucket_id, eval_ppx))
inputstr = datautil.ids2texts(reversed([en[0] for en in encoder_inputs]) ,rev_vocaben)
print("输入",inputstr)
print("输出",datautil.ids2texts([en[0] for en in decoder_inputs] ,rev_vocabch))
outputs = [np.argmax(logit, axis=1)[0] for logit in output_logits]
#outputs = [int(np.argmax(logit, axis=1)) for logit in output_logits]
#print("outputs",outputs,datautil.EOS_ID)
if datautil.EOS_ID in outputs:
outputs = outputs[:outputs.index(datautil.EOS_ID)]
print("结果",datautil.ids2texts(outputs,rev_vocabch))
sys.stdout.flush()
