def test_from_input():
reader = Reader()
model = DynamicLSTM(None, is_training=False, reuse=False)
model_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="model")
model_saver = tf.train.Saver(model_variables)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
ckpt_path = tf.train.latest_checkpoint(config.model_dir)
if ckpt_path:
model_saver.restore(sess,
tf.train.latest_checkpoint(config.model_dir))
print("Read model parameters from %s" %
tf.train.latest_checkpoint(config.model_dir))
else:
print("model doesn't exists")
user_input = raw_input("input: ")
while user_input:
inputs, inputs_len = reader.get_batch_from_input(user_input)
feed_dict = {model.x: inputs, model.x_len: inputs_len}
x, inputs, last_state, last_output, output_prob = sess.run(
[
model.x, model.inputs, model.last_state, model.last_output,
model.output_prob
],
feed_dict=feed_dict)
pdb.set_trace()
user_input = raw_input("input: ")
def test_top_3(filepath, loop_idx=-1, if_long=False):
reader = Reader()
model = DynamicLSTM(None, is_training=False, reuse=False)
model_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="model")
model_saver = tf.train.Saver(model_variables)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
sess.run(init_op)
ckpt_path = tf.train.latest_checkpoint(config.model_dir)
if loop_idx > 0:
ckpt_path = ckpt_path.split('-')[0] + '-' + str(loop_idx)
if ckpt_path:
model_saver.restore(sess, ckpt_path)
print("Read model parameters from %s" % ckpt_path)
else:
print("model doesn't exists")
f = open(filepath)
total_count = 0.0
top5_hit = 0.0
top3_hit = 0.0
top1_hit = 0.0
for line in f:
line = line.strip()
line_items = line.split()
line_len = len(line_items)
if line_len == 1:
continue
beg_idx = 1
if if_long:
beg_idx = line_len - 1
for i in range(beg_idx, line_len):
total_count += 1
context = " ".join(line_items[:i])
word = line_items[i]
inputs, inputs_len, outputs = reader.get_batch_from_input(
context)
feed_dict = {model.x: inputs, model.x_len: inputs_len}
prob = sess.run([model.output_prob], feed_dict=feed_dict)
top5_ind = prob[-1][-1].argsort()[-5:][::-1]
words = [reader.words[i] for i in top5_ind]
if words[0] == word:
top1_hit += 1
if words[:3].count(word) > 0:
top3_hit += 1
if words[:5].count(word) > 0:
top5_hit += 1
print("Input is : " + context)
print("Expected word is : " + word)
print("Word_predict is : " + "#".join(words))
print(
filepath +
"'s predict acc is >>> top5_acc is : %.2f%%, top3_acc is : %.2f%%, top1_acc is : %.2f%%, top5_hit_count is : %f, top3_hit_count is : %f, top1_hit_count is : %f, total count is : %f."
% (top5_hit * 100 / total_count, top3_hit * 100 / total_count,
top1_hit * 100 / total_count, top5_hit, top3_hit, top1_hit,
total_count))
def test_from_input():
reader = Reader()
model = DynamicLSTM(None, is_training=False, reuse=False)
model_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="model")
output_graph = "./model/dynamic_lstm_1layer_embed96_hid256_share_model_newdic.pb"
model_saver = tf.train.Saver(model_variables)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess:
output_node_names = 'model/predict/Softmax'
#ckpt_path = tf.train.latest_checkpoint(config.model_dir)
#ckpt_path = tf.train.get_checkpoint_state(config.model_dir)
checkpoint = tf.train.get_checkpoint_state(config.model_dir)
input_checkpoint = checkpoint.model_checkpoint_path
saver_meta = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices = True)
graph = tf.get_default_graph()
tf.train.write_graph(graph, FILEDIR, 'dynamic_lstm_1layer_embed96_hid256_share_model_newdic_meta.pb',as_text=True)
input_graph_def = graph.as_graph_def()
print ('ckpt:', input_checkpoint)
init_op = tf.group(tf.global_variables_initializer(),tf.local_variables_initializer())
sess.run(init_op)
saver_meta.restore(sess, input_checkpoint)
output_graph_def = graph_util.convert_variables_to_constants(sess,input_graph_def, output_node_names.split(","))
tf.train.write_graph(output_graph_def,FILEDIR,'fuck2_accuracy.pb',as_text=True)
with tf.gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." % len(output_graph_def.node))
if ckpt_path:
model_saver.restore(sess, tf.train.latest_checkpoint(config.model_dir))
print("Read model parameters from %s" % tf.train.latest_checkpoint(config.model_dir))
else:
print("model doesn't exists")
freeze_graph(sess, 'model')
#model_path = os.path.join(config.model_dir, config.model_name)
#model_saver.save(sess, model_path, global_step=0)
user_input = raw_input("input: ")
while user_input:
inputs, inputs_len, outputs = reader.get_batch_from_input(user_input)
feed_dict={ model.x: inputs, model.x_len: inputs_len}
prob = sess.run([model.output_prob], feed_dict=feed_dict)
top3_ind = prob[-1][-1].argsort()[-3:][::-1]
sentence = []
user_input_words = user_input.split()
#for w in range(len(user_input_words)):
#import pdb
#pdb.set_trace()
#sentence.append(user_input_words[w])
print("input: " + " ".join(user_input))
#print("input: " + " ".join(sentence))
print("top answers are:")
words = [reader.words[i] for i in top3_ind]
print(" ".join(words))
user_input = raw_input("input: ")
def test_from_input():
reader = Reader()
header_map = words_2_map(reader)
if len(header_map) == 0:
return
model = DynamicLSTM(None, is_training=False, reuse=False)
model_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="model")
#output_graph = "./model/dynamic_lstm_1layer_embed96_hid256_share_model_newdic.pb"
model_saver = tf.train.Saver(model_variables)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
#ckpt_path = tf.train.latest_checkpoint('dy_share_lstm_batch13')
print(config.model_dir)
ckpt_path = tf.train.latest_checkpoint(config.model_dir)
print(ckpt_path)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init_op)
if ckpt_path:
model_saver.restore(sess, ckpt_path)
print("Read model parameters from %s" % ckpt_path)
else:
print("model doesn't exists")
user_context = raw_input("Context: ").strip()
input_header = raw_input("User Input : ").strip()
while user_context:
inputs, inputs_len, outputs = reader.get_batch_from_input(
user_context)
feed_dict = {model.x: inputs, model.x_len: inputs_len}
prob = sess.run([model.output_prob], feed_dict=feed_dict)
#prob, last_state, embed, outputsflat = sess.run([model.output_prob, model.last_state, model.inputs, model.outputs_flat], feed_dict=feed_dict)
last_prob = prob[-1][-1]
#pdb.set_trace()
if not input_header or not header_map.has_key(input_header):
top3_ind = last_prob.argsort()[-3:][::-1]
else:
top3_ind = get_header_prob(header_map, last_prob, input_header,
3)
sentence = []
user_input_words = user_context.split()
print("input: " + " ".join(user_context))
print("top answers are:")
words = [reader.words[i] for i in top3_ind]
print(" ".join(words))
user_context = raw_input("input: ").strip()
input_header = raw_input("User Input : ").strip()
def test_top_3(filepath):
reader = Reader()
queue = Queue("test", config.batch_size)
model = DynamicLSTM(queue, is_training=False, reuse=False)
model_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="model")
model_saver = tf.train.Saver(model_variables)
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess:
sess.run(init_op)
ckpt_path = tf.train.latest_checkpoint(config.model_dir)
if ckpt_path:
model_saver.restore(sess, tf.train.latest_checkpoint(config.model_dir))
print("Read model parameters from %s" % tf.train.latest_checkpoint(config.model_dir))
else:
print("model doesn't exists")
data_gen = reader.get_custom_batch_from_file(filepath, config.batch_size)
correct = 0
total = 0
line_num = 0
for inputs, inputs_len, outputs in data_gen:
line_num += 1
sess.run(queue.enqueue_op, feed_dict={ queue.inputs: inputs,
queue.inputs_len: inputs_len,
queue.outputs: outputs })
x, cands, last_state, last_output, probs = sess.run([model.x, model.y, model.last_state, model.last_output, model.output_prob])
for i in range(config.batch_size):
#pdb.set_trace()
for word_ind in range(3):
print("%s "%reader.words[x[i][word_ind]], end=' ')
max_val = probs[i][cands[i][0]]
#if max_val > probs[i][cands[i][1]] and max_val > probs[i][cands[i][2]] and max_val > probs[i][cands[i][3]] and max_val > probs[i][cands[i][4]] and max_val > probs[i][cands[i][5]] and max_val > probs[i][cands[i][6]] and max_val > probs[i][cands[i][7]] and max_val > probs[i][cands[i][8]] and max_val > probs[i][cands[i][9]]:
#if max_val > probs[i][cands[i][1]] and max_val > probs[i][cands[i][2]] and max_val > probs[i][cands[i][3]] and max_val > probs[i][cands[i][4]]:
if max_val > probs[i][cands[i][1]] and max_val > probs[i][cands[i][2]]:
correct += 1
print("correct ", end='')
else:
print("incorrect ", end='')
for word_ind in range(10):
print("%s(%f) "%(reader.words[cands[i][word_ind]], probs[i][cands[i][word_ind]]), end=' ')
print("\n")
total += config.batch_size
#print("total: %d correct: %d correct rate: %.4f" %(total, correct, float(correct)/float(total)))
print("total: %d correct: %d correct rate: %.4f" %(total, correct, float(correct)/float(total)))
def test_top_3(filepath):
reader = Reader()
queue = Queue("test", config.batch_size)
model = DynamicLSTM(queue, is_training=False, reuse=False)
model_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="model")
model_saver = tf.train.Saver(model_variables)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
sess.run(init_op)
ckpt_path = tf.train.latest_checkpoint(config.model_dir)
if ckpt_path:
model_saver.restore(sess,
tf.train.latest_checkpoint(config.model_dir))
print("Read model parameters from %s" %
tf.train.latest_checkpoint(config.model_dir))
else:
print("model doesn't exists")
data_gen = reader.get_custom_batch_from_file(filepath,
config.batch_size)
correct = 0
total = 0
line_num = 0
for inputs, inputs_len, outputs in data_gen:
line_num += 1
sess.run(queue.enqueue_op,
feed_dict={
queue.inputs: inputs,
queue.inputs_len: inputs_len,
queue.outputs: outputs
})
probs = sess.run(model.output_prob)
pdb.set_trace()
for i in range(config.batch_size):
max_val = probs[i][outputs[i][0]]
if max_val >= probs[i][outputs[i][1]] and max_val >= probs[i][
outputs[i][2]]:
correct += 1
total += config.batch_size
print("total: %d correct: %d correct rate: %.4f" %
(total, correct, float(correct) / float(total)))
def test_from_file(filepath):
reader = Reader()
model = DynamicLSTM(None, is_training=False, reuse=False)
model_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="model")
model_saver = tf.train.Saver(model_variables)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess:
ckpt_path = tf.train.latest_checkpoint(config.model_dir)
if ckpt_path:
model_saver.restore(sess, tf.train.latest_checkpoint(config.model_dir))
print("Read model parameters from %s" % tf.train.latest_checkpoint(config.model_dir))
else:
print("model doesn't exists")
model_path = os.path.join(config.model_dir, config.model_name)
model_saver.save(sess, model_path, global_step=0)
data_gen = reader.get_custom_line_from_file(filepath)
for inputs, inputs_len in data_gen:
feed_dict={ model.x: inputs, model.x_len: inputs_len}
prob = sess.run([model.output_prob], feed_dict=feed_dict)
def test_from_input():
reader = Reader()
model = DynamicLSTM(None, is_training=False, reuse=False)
model_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="model")
#pdb.set_trace()
#output_graph = "./model/dynamic_lstm_1layer_embed96_hid256_share_model_newdic.pb"
model_saver = tf.train.Saver(model_variables)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
ckpt_path = tf.train.latest_checkpoint(config.model_dir)
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init_op)
if ckpt_path:
model_saver.restore(sess,
tf.train.latest_checkpoint(config.model_dir))
print("Read model parameters from %s" %
tf.train.latest_checkpoint(config.model_dir))
else:
print("model doesn't exists")
#fetch_model(sess,model_variables)
user_input = raw_input("input: ")
while user_input:
inputs, inputs_len, outputs = reader.get_batch_from_input(
user_input)
feed_dict = {model.x: inputs, model.x_len: inputs_len}
prob = sess.run([model.output_prob], feed_dict=feed_dict)
pdb.set_trace()
top3_ind = prob[-1][-1].argsort()[-10:][::-1]
sentence = []
user_input_words = user_input.split()
print("input: " + " ".join(user_input))
print("top answers are:")
words = [reader.words[i] for i in top3_ind]
print(" ".join(words))
user_input = raw_input("input: ")
def ajust_eval(filepath,loop_idx = -1):
#Init reader and load model
reader = Reader()
header_map = words_2_map(reader)
if len(header_map)==0:
return
model = DynamicLSTM(None, is_training=False, reuse=False)
model_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="model")
model_saver = tf.train.Saver(model_variables)
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess:
sess.run(init_op)
#Model load
ckpt_path = tf.train.latest_checkpoint(config.model_dir)
if loop_idx > 0:
ckpt_path = ckpt_path.split('-')[0] + '-' + str(loop_idx)
if ckpt_path:
model_saver.restore(sess, ckpt_path)
print("Read model parameters from %s" % ckpt_path)
else:
print("model doesn't exists")
#File handle
f = open(filepath)
click_total = {"total":0,1:0,3:0,5:0}
hit_total = {"total":0,1:0,3:0,5:0}
UPPERS=['NUM','TELEPHONE','DIGIT','SCORES','USER','YEARGAP']
#File loop, file must be format context \t(input\t)word
for line in f:
line = line.strip()
line_items = line.split('\t')
if len(line_items)<2:
sys.stderr.write("Error line format, items missing of line: "+line+'\n')
#line info handle
context = line_items[0].strip()
word = line_items[-1].strip()
input_str = word
if len(line_items)==3:
input_str = line_items[1].strip()
if input_str == "" or punctuation.count(word)>0 or word.isdigit() or UPPERS.count(word)>0:
continue
#Data model input output
inputs, inputs_len, outputs = reader.get_batch_from_input(context)
feed_dict={ model.x: inputs, model.x_len: inputs_len}
prob = sess.run([model.output_prob], feed_dict=feed_dict)
last_prob = prob[-1][-1]
#Exp starts
click_total["total"] += len(word)#Click rate's denominator
hit_total["total"] += 1
input_len = len(input_str)
tmp_click_count = {1:input_len,3:input_len,5:input_len}#list to record the click count for top1, top3, top5
tmp_hit_count = {1:0,3:0,5:0}
for i in range(1,(len(input_str)+1)):
input_header = input_str[:i]
top5_ind = get_header_prob(header_map,last_prob,input_header,5)
words = [reader.words[j] for j in top5_ind]
if len(words)==0:
words.append(input_header)
for idx in [1,3,5]:
if words[:idx].count(word)>0:
if tmp_click_count[idx]==input_len:
tmp_click_count[idx] = i
if i == 1:
tmp_hit_count[idx] = 1
print("Context is : "+context)
print("Input is : "+input_header)
print("Expected word is : " + word)
print("Word_predict is : "+"#".join(words))
if words[:1].count(word)>0:
break
for idx in [1,3,5]:
click_total[idx] += tmp_click_count[idx]
hit_total[idx] += tmp_hit_count[idx]
print(filepath + "'s type-rate is >>> top5_type-rate is : %.2f%%, top3_type-rate is : %.2f%%, top1_type-rate is : %.2f%%, top5_type_total is : %f, top3_type_total is : %f, top1_type_total is : %f, total type count is : %f."%(click_total[5]*100/click_total['total'], click_total[3]*100/click_total['total'],click_total[1]*100/click_total['total'],click_total[5], click_total[3],click_total[1],click_total['total']))
print(filepath + "'s uni-input-hit-rate is >>> top5_hit_rate is : %.2f%%, top3_hit_rate is : %.2f%%, top1_hit_rate is : %.2f%%, top5_hit_total is %f,top3_hit_total is %f,top1_hit_total is %f,total count is : %f."%(hit_total[5]*100/hit_total['total'], hit_total[3]*100/hit_total['total'],hit_total[1]*100/hit_total['total'],hit_total[5], hit_total[3],hit_total[1],hit_total['total']))
def main(_):
ps_hosts = FLAGS.ps_hosts.split(",")
worker_hosts = FLAGS.worker_hosts.split(",")
worker_num = len(worker_hosts)
# Create a cluster from the parameter server and worker hosts.
cluster = tf.train.ClusterSpec({ "ps": ps_hosts, "worker" : worker_hosts })
# Start a server for a specific task
server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index)
#print("I'm worker %d and my server target is, "%FLAGS.task_index, server.target)
if FLAGS.job_name == "ps":
server.join()
elif FLAGS.job_name == "worker":
is_chief = FLAGS.task_index == 0
if is_chief:
# Create reader for reading raw data
reader = Reader()
# Count step for valid epoch
train_dir = os.path.join(config.data_path, config.dataset, "train")
valid_dir = os.path.join(config.data_path, config.dataset, "valid")
total_valid_step = util.count_step(valid_dir, config.batch_size)
# Build graph
with tf.variable_scope("input"):
train_queue = Queue("train", config.batch_size)
valid_queue = Queue("clean", config.batch_size)
with tf.variable_scope("helper"):
# Define training variables and ops
do_valid = tf.get_variable("do_valid", shape=[], dtype=tf.bool, initializer=tf.constant_initializer(False), trainable=False)
train_done = tf.get_variable("train_done", shape=[], dtype=tf.bool, initializer=tf.constant_initializer(False), trainable=False)
valid_done = tf.get_variable("valid_done", shape=[], dtype=tf.bool, initializer=tf.constant_initializer(False), trainable=False)
temp_val = tf.placeholder(tf.float32, shape=[])
global_loss = tf.get_variable("global_loss", shape=[], dtype=tf.float32, initializer=tf.zeros_initializer(), trainable=False)
global_loss_update = tf.assign_add(global_loss, temp_val, use_locking=True)
global_acc_1 = tf.get_variable("global_acc_1", shape=[], dtype=tf.float32, initializer=tf.zeros_initializer(), trainable=False)
global_acc_k = tf.get_variable("global_acc_k", shape=[], dtype=tf.float32, initializer=tf.zeros_initializer(), trainable=False)
global_acc_1_update = tf.assign_add(global_acc_1, temp_val, use_locking=True)
global_acc_k_update = tf.assign_add(global_acc_k, temp_val, use_locking=True)
global_train_step = tf.get_variable(name='global_train_step', dtype=tf.int32, shape=[], initializer=tf.constant_initializer(0), trainable=False)
increment_train_step = tf.assign_add(global_train_step, 1, use_locking=True)
global_valid_step = tf.get_variable(name='global_valid_step', dtype=tf.int32, shape=[], initializer=tf.constant_initializer(0), trainable=False)
increment_valid_step = tf.assign_add(global_valid_step, 1, use_locking=True)
with tf.device(tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % FLAGS.task_index, cluster=cluster)):
print("Creating %d layers of %d units." % (config.num_layer, config.hidden_size))
train_model = DynamicLSTM(train_queue, is_training=True, reuse=False)
valid_model = DynamicLSTM(valid_queue, is_training=False, reuse=True)
model_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="model")
# Add global_train_step to restore if necessary
model_vars.append(global_train_step)
model_saver = tf.train.Saver(model_vars, max_to_keep=0)
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
with tf.Session(server.target, config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess:
# Create a FileWriter to write summaries
#summary_writer = tf.summary.FileWriter(config.log_dir, sess.graph)
sess.run(init_op)
print("Variables initialized ...")
if FLAGS.restore_model:
ckpt_path = tf.train.latest_checkpoint(config.model_dir)
if ckpt_path:
model_saver.restore(sess, tf.train.latest_checkpoint(config.model_dir))
print("Read model parameters from %s" % tf.train.latest_checkpoint(config.model_dir))
else:
print("model doesn't exists")
if is_chief:
if FLAGS.fetch_model:
model_vals = sess.run(model_vars)
model_vals_name = ['model_vals/' + i.name.replace("/", "-") for i in model_vars]
for i in range(len(model_vals)):
with open(model_vals_name[i], 'wb') as f:
model_vals[i].tofile(f)
print("finish fetching model")
# Create coordinator to control threads
coord = tf.train.Coordinator()
# Create start threads function
start_threads = start_threads_func(reader, sess, coord)
train_threads = start_threads(train_dir, train_queue)
valid_threads = start_threads(valid_dir, valid_queue)
valid_ppl_history = []
while not train_done.eval():
# Training phase
while not do_valid.eval():
start_time = time.time()
word_num, train_loss, _ = sess.run([train_model.word_num, train_model.loss, train_model.train_op])
train_speed = word_num // (time.time()-start_time)
train_ppl = np.exp(train_loss)
train_step = sess.run(increment_train_step)
if train_step % config.step_per_log == 0:
print("TaskID: {} Train step: {} Train ppl: {:.2f} Train speed: {}".format(FLAGS.task_index, train_step, train_ppl, train_speed))
if train_step % config.step_per_validation == 0:
sess.run(do_valid.assign(True))
sess.run(valid_done.assign(False))
# Validation phase
if is_chief:
print("start validation")
while global_valid_step.eval() < total_valid_step:
loss, acc_1, acc_k = sess.run([valid_model.loss, valid_model.top_1_accuracy, valid_model.top_k_accuracy])
sess.run(increment_valid_step)
sess.run(global_loss_update, feed_dict={temp_val: loss})
sess.run(global_acc_1_update, feed_dict={temp_val: acc_1})
sess.run(global_acc_k_update, feed_dict={temp_val: acc_k})
if is_chief:
true_valid_step = global_valid_step.eval()
valid_loss = sess.run(global_loss)/true_valid_step
valid_ppl = np.exp(valid_loss)
valid_acc_1 = sess.run(global_acc_1)/true_valid_step
valid_acc_k = sess.run(global_acc_k)/true_valid_step
print("Valid step: {} Valid acc_1: {:.4f}, Valid acc_k: {:.4f} Valid ppl: {:.2f}".format(true_valid_step, valid_acc_1, valid_acc_k, valid_ppl))
# If converged, finish training
if(len(valid_ppl_history) >= 3 and valid_ppl > max(valid_ppl_history[-3:])):
#if len(valid_ppl_history) >= 2 and valid_ppl > valid_ppl_history[-1]:
print("Training is converged")
sess.run(train_done.assign(True))
else:
print("Saving model...")
# Save graph and model parameters
model_path = os.path.join(config.model_dir, config.model_name)
model_saver.save(sess, model_path, global_step=global_train_step.eval())
valid_ppl_history.append(valid_ppl)
sess.run(do_valid.assign(False))
sess.run(valid_done.assign(True))
else:
while not valid_done.eval():
pass
sess.run(tf.variables_initializer([global_valid_step, global_loss, global_acc_1, global_acc_k]))
if is_chief:
# Stop data threads
coord.request_stop()
# Clean queue before finish
while train_queue.size.eval() > 0:
sess.run(train_queue.dequeue_op)
while valid_queue.size.eval() > 0:
sess.run(valid_queue.dequeue_op)
coord.join(train_threads)
coord.join(valid_threads)