def run_eval(dataset, hps, logdir, mode, num_eval_steps):
with tf.variable_scope("model"):
hps.num_sampled = 0 # Always using full softmax at evaluation. run out of memory
hps.keep_prob = 1.0
model = LM(hps, "eval", "/cpu:0")
if hps.average_params:
print("Averaging parameters for evaluation.")
saver = tf.train.Saver(model.avg_dict)
else:
saver = tf.train.Saver()
# Use only 4 threads for the evaluation.
config = tf.ConfigProto(allow_soft_placement=True,
intra_op_parallelism_threads=20,
inter_op_parallelism_threads=1)
sess = tf.Session(config=config)
sw = tf.summary.FileWriter(logdir + "/" + mode, sess.graph)
ckpt_loader = CheckpointLoader(saver, model.global_step, logdir + "/train")
with sess.as_default():
ckpt_loader.load_checkpoint() # FOR ONLY ONE CHECKPOINT
global_step = ckpt_loader.last_global_step
data_iterator = dataset.iterate_once(hps.batch_size * hps.num_gpus,
hps.num_steps)
sess.run(tf.local_variables_initializer())
print("global_step:", global_step)
loss_nom = 0.0
loss_den = 0.0
for i, (x, y, w) in enumerate(data_iterator):
if i >= num_eval_steps:
break
loss = sess.run(model.loss, {model.x: x, model.y: y, model.w: w})
loss_nom += loss
loss_den += w.mean()
loss = loss_nom / loss_den
sys.stdout.write("%d: %.3f (%.3f) ... " % (i, loss, np.exp(loss)))
sys.stdout.flush()
sys.stdout.write("\n")
log_perplexity = loss_nom / loss_den
print("Results at %d: log_perplexity = %.3f perplexity = %.3f" %
(global_step, log_perplexity, np.exp(log_perplexity)))
summary = tf.Summary()
summary.value.add(tag='eval/log_perplexity',
simple_value=log_perplexity)
summary.value.add(tag='eval/perplexity',
simple_value=np.exp(log_perplexity))
sw.add_summary(summary, global_step)
sw.flush()
def sentence_ppl(prefix_words, dataset, hps, logdir, mode):
inputs, targets = process_sentence(prefix_words, dataset, hps)
with tf.variable_scope("model"):
hps.num_sampled = 0 # Always using full softmax at evaluation.
hps.keep_prob = 1.0
# model = LM(hps, "eval", "/cpu:0")
model = LM(hps, "eval", "/gpu:0")
if hps.average_params:
print("Averaging parameters for evaluation.")
saver = tf.train.Saver(model.avg_dict)
else:
saver = tf.train.Saver()
config = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(config=config)
sw = tf.summary.FileWriter(logdir + "/" + mode, sess.graph)
ckpt_loader = CheckpointLoader(saver, model.global_step, logdir + "/train")
with sess.as_default():
while ckpt_loader.load_checkpoint():
tf.local_variables_initializer().run()
ppl = sess.run(model.loss, {model.x: inputs, model.y: targets})
print(np.exp(ppl))
return np.exp(ppl)
def run_eval(dataset, hps, logdir, mode, num_eval_steps):
with tf.variable_scope("model"):
hps.num_sampled = 0 # Always using full softmax at evaluation.
hps.keep_prob = 1.0
#model = LM(hps, "eval", "/cpu:0")
model = LM(hps, "eval", "/gpu:0")
if hps.average_params:
print("Averaging parameters for evaluation.")
saver = tf.train.Saver(model.avg_dict)
else:
saver = tf.train.Saver()
# Use only 4 threads for the evaluation.
#config = tf.ConfigProto(allow_soft_placement=True,
# intra_op_parallelism_threads=20,
# inter_op_parallelism_threads=1)
config = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(config=config)
sw = tf.summary.FileWriter(logdir + "/" + mode, sess.graph)
ckpt_loader = CheckpointLoader(saver, model.global_step, logdir + "/train")
with sess.as_default():
while ckpt_loader.load_checkpoint():
global_step = ckpt_loader.last_global_step
data_iterator = dataset.iterate_once(hps.batch_size * hps.num_gpus, hps.num_steps)
#tf.initialize_local_variables().run()
tf.local_variables_initializer().run()
loss_nom = 0.0
loss_den = 0.0
#for i, (x, y, w) in enumerate(data_iterator):
for i, (x, y) in enumerate(data_iterator):
if i >= num_eval_steps and mode!="eval_full":
break
#loss = sess.run(model.loss, {model.x: x, model.y: y, model.w: w})
start_time = time.time()
loss = sess.run(model.loss, {model.x: x, model.y: y})
dt = time.time() - start_time
wps = hps.batch_size * hps.num_steps / dt
loss_nom += loss
loss_den += 1 # ???
#loss_den += w.mean()
loss = loss_nom / loss_den
sys.stdout.write("%d: %.3f (%.3f), wps: %.3f ... \n" % (i, loss, np.exp(loss), wps))
sys.stdout.flush()
sys.stdout.write("\n")
log_perplexity = loss_nom / loss_den
print("Results at %d: log_perplexity = %.3f perplexity = %.3f" % (
global_step, log_perplexity, np.exp(log_perplexity)))
summary = tf.Summary()
summary.value.add(tag='eval/log_perplexity', simple_value=log_perplexity)
summary.value.add(tag='eval/perplexity', simple_value=np.exp(log_perplexity))
sw.add_summary(summary, global_step)
sw.flush()
if mode == "eval_full":
break #we don't need to wait for other checkpoints in this mode
def run_eval(dataset, hps, logdir, mode, num_eval_steps):
with tf.variable_scope('model'):
hps.num_sampled = 0 # Always using full softmax at evaluation.
hps.keep_prob = 1.0
model = LM(hps, 'eval', '/cpu:0')
if hps.average_params:
print('Averaging parameters for evaluation.')
saver = tf.train.Saver(model.avg_dict)
else:
saver = tf.train.Saver()
# Use only 4 threads for the evaluation.
config = tf.ConfigProto(allow_soft_placement=True,
intra_op_parallelism_threads=4,
inter_op_parallelism_threads=1)
sess = tf.Session(config=config)
sw = tf.summary.FileWriter(logdir + '/' + mode, sess.graph)
ckpt_loader = CheckpointLoader(saver, model.global_step, logdir + '/train')
with sess.as_default():
while ckpt_loader.load_checkpoint():
global_step = ckpt_loader.last_global_step
data_iterator = dataset.iterate_once(
hps.batch_size * hps.num_gpus, hps.num_steps)
tf.initialize_local_variables().run()
for v in tf.get_collection('initial_state'):
sess.run(v.initializer,
feed_dict={model.batch_size: hps.batch_size})
loss_nom = 0.0
loss_den = 0.0
for i, (x, y, w) in enumerate(data_iterator):
if i >= num_eval_steps:
break
loss = sess.run(model.loss, {
model.x: x, model.y: y, model.w: w,
model.batch_size: hps.batch_size})
loss_nom += loss
loss_den += w.mean()
loss = loss_nom / loss_den
sys.stdout.write('%d: %.3f (%.3f) ... ' % (
i, loss, np.exp(loss)))
sys.stdout.flush()
sys.stdout.write('\n')
log_perplexity = loss_nom / loss_den
print('Results at %d: log_perplexity = %.3f perplexity = %.3f' % (
global_step, log_perplexity, np.exp(log_perplexity)))
summary = tf.Summary()
summary.value.add(
tag='eval/log_perplexity', simple_value=log_perplexity)
summary.value.add(
tag='eval/perplexity', simple_value=np.exp(log_perplexity))
sw.add_summary(summary, global_step)
sw.flush()
def predict_next(dataset, hps, logdir, mode, num_eval_steps, vocab):
with tf.variable_scope("model"):
hps.num_sampled = 0 # Always using full softmax at evaluation. run out of memory
hps.keep_prob = 1.0
model = LM(hps, "predict_next", "/cpu:0")
if hps.average_params:
print("Averaging parameters for evaluation.")
saver = tf.train.Saver(model.avg_dict)
else:
saver = tf.train.Saver()
# Use only 4 threads for the evaluation.
config = tf.ConfigProto(allow_soft_placement=True,
intra_op_parallelism_threads=20,
inter_op_parallelism_threads=1)
sess = tf.Session(config=config)
sw = tf.summary.FileWriter(logdir + "/" + mode, sess.graph)
ckpt_loader = CheckpointLoader(saver, model.global_step, logdir + "/train")
with sess.as_default():
ckpt_loader.load_checkpoint() # FOR ONLY ONE CHECKPOINT
global_step = ckpt_loader.last_global_step
data_iterator = dataset.iterate_once(hps.batch_size * hps.num_gpus,
hps.num_steps)
sess.run(tf.local_variables_initializer())
print("global_step:", global_step)
loss_nom = 0.0
loss_den = 0.0
cur_time = time.time()
savedKey = 0
totalKey = 0
'''
text = open("data/news.en.heldout-00001-of-00050","r")
for kk,line in enumerate(text):
totalKey += len(line.strip())
if kk==0:
print len(line)
print "totalKey:",totalKey
'''
predicted_words = []
for i, (x, y, w) in enumerate(data_iterator):
#if i >= num_eval_steps:
# break
'''
print "i",i
print "x",x
for j in x[:]:
print j
for jj in j:
print vocab.get_token(jj)
'''
#print "x:",[vocab.get_token(ix) for ix in x[0]]
#print "y:",[vocab.get_token(ix) for ix in y[0]]
inputs = [vocab.get_token(ix) for ix in x[0]]
labels = [vocab.get_token(ix) for ix in y[0]]
loss, logits, indexes = sess.run(
[model.loss, model.logits, model.index], {
model.x: x,
model.y: y,
model.w: w
})
#print logits.shape,indexes
#print indexes[0]
tmpKS = 0
tmpAllKey = 0
for step in range(hps.num_steps):
words = []
totalKey += len(inputs[step])
tmpAllKey += len(inputs[step])
if step > 0:
totalKey += 1 # for space between two keys
tmpAllKey += 1
for j in range(hps.arg_max):
word = vocab.get_token(indexes[0][step][j])
words += [word]
if word == labels[step]:
predicted_words += [word]
tmpKS += len(labels[step])
savedKey += len(labels[step])
#print "predict: ", words
# print "x:",x
print("i:%6d, savedKey:%d , totalKey:%d, ksr : %.3f " %
(i, tmpKS, tmpAllKey, tmpKS * 1.0 / tmpAllKey))
print("savedKey:%d , totalKey:%d, ksr : %.3f " %
(savedKey, totalKey, savedKey * 1.0 / totalKey))
print("predicted_words:")
print(predicted_words)
now = time.time()
print "time:", now - cur_time
def run_eval(dataset, hps, logdir, mode, num_eval_steps):
print_debug('run_eval logdir=%s ' % (logdir))
with tf.variable_scope("model"):
hps.num_sampled = 0 # Always using full softmax at evaluation.
hps.keep_prob = 1.0
#model = LM(hps, "eval", "/cpu:0")
model = LM(hps, "eval", "/gpu:0")
if hps.average_params:
print("Averaging parameters for evaluation.")
saver = tf.train.Saver(model.avg_dict)
else:
saver = tf.train.Saver()
# Use only 4 threads for the evaluation.
#config = tf.ConfigProto(allow_soft_placement=True,
# intra_op_parallelism_threads=20,
# inter_op_parallelism_threads=1)
config = tf.ConfigProto(allow_soft_placement=True)
sess = tf.Session(config=config)
sw = tf.summary.FileWriter(logdir + "/" + mode, sess.graph)
print_debug('run_eval tf.summary.FileWriter=%s ' % (logdir + "/" + mode))
ckpt_loader = CheckpointLoader(saver, model.global_step, logdir + "/train")
print_debug('run_eval ckpt_loader=%s ' % (ckpt_loader.logdir))
with sess.as_default():
print_debug('run_eval sess.as_default iteration')
while ckpt_loader.load_checkpoint():
print_debug('eval load_checkpoint chunk Loader done!')
global_step = ckpt_loader.last_global_step
if mode == "eval_full":
data_iterator = dataset.iterate_forever(
hps.batch_size * hps.num_gpus, hps.num_steps)
else:
data_iterator = dataset.iterate_once(
hps.batch_size * hps.num_gpus, hps.num_steps)
print_debug('eval run local variables initalizer')
#tf.initialize_local_variables().run()
tf.local_variables_initializer().run()
loss_nom = 0.0
loss_den = 0.0
print_debug('eval run for loop of enumerated data iterator mode=' +
mode + ' eval_steps=' + str(num_eval_steps))
#for i, (x, y, w) in enumerate(data_iterator):
for i, (x, y) in enumerate(data_iterator):
if i >= num_eval_steps and mode != "eval_full":
break
#loss = sess.run(model.loss, {model.x: x, model.y: y, model.w: w})
loss = sess.run(model.loss, {model.x: x, model.y: y})
loss_nom += loss
loss_den += 1 # ???
#loss_den += w.mean()
loss = loss_nom / loss_den
#sys.stdout.write("%d: %.3f (%.3f) ... " % (i, loss, np.exp(loss)))
#sys.stdout.flush()
#sys.stdout.write("\n")
log_perplexity = loss_nom / loss_den
print("Results at %d: log_perplexity = %.3f perplexity = %.3f" %
(global_step, log_perplexity, np.exp(log_perplexity)))
summary = tf.Summary()
summary.value.add(tag='eval/log_perplexity',
simple_value=log_perplexity)
summary.value.add(tag='eval/perplexity',
simple_value=np.exp(log_perplexity))
sw.add_summary(summary, global_step)
sw.flush()
#if mode == "eval_full":
# break #we don't need to wait for other checkpoints in this mode
break #we always break
print_debug('run_eval END OF WHILE loader loop')
print_debug('run_eval END OF WHILE session loop')
sess.close()
tf.reset_default_graph()