def _Decode(self, article_text):
"""Restore a checkpoint and decode it.
Args:
saver: Tensorflow checkpoint saver.
sess: Tensorflow session.
Returns:
If success, returns true, otherwise, false.
"""
bs = beam_search.BeamSearch(
self._model, self._hps.batch_size,
self._vocab.WordToId(data.SENTENCE_START),
self._vocab.WordToId(data.SENTENCE_END),
self._hps.dec_timesteps)
###################
#article_text = "How do I know the difference, between class and object"
article = "<d><p><s>"+article_text+"</s></p></d>"
article_sentences = [sent.strip() for sent in data.ToSentences(article, include_token=False)]
pad_id = self._vocab.WordToId(data.PAD_TOKEN)
enc_inputs = []
for i in xrange(min(100,len(article_sentences))):
enc_inputs += data.GetWordIds(article_sentences[i], self._vocab)
enc_input_len = len(enc_inputs)
while len(enc_inputs) < self._hps.enc_timesteps:
enc_inputs.append(pad_id)
###################
w, h = 120, 4
article_batch_cp = [[0 for x in range(w)] for y in range(h)]
for i in range(0,4):
article_batch_cp[i] = enc_inputs#article_batch[i]
w, h = 1, 4
article_lens_cp = [[0 for x in range(w)] for y in range(h)]
#article_lens_cp = article_lens.copy()
for i in range(0,4):
article_lens_cp[i] = enc_input_len
best_beam = bs.BeamSearch(self._sess, article_batch_cp, article_lens_cp)
#print len(best_beam)
best_beam = best_beam[0]
decode_output = [int(t) for t in best_beam.tokens[1:]]
QUESTION = article_text
test = ' '.join(data.Ids2Words(decode_output, self._vocab))
end_p = test.find(data.SENTENCE_END, 0)
if end_p != -1:
test = test[:end_p]
#print "<Answer>"+test
ANSWER = test.replace('<UNK>','')
return QUESTION, ANSWER
def Decode(self, article):
"""Restore a checkpoint and decode it.
Args:
saver: Tensorflow checkpoint saver.
sess: Tensorflow session.
Returns:
If success, returns true, otherwise, false.
"""
(article_batch, article_lens) = self.get_article_inputs(article)
bs = beam_search.BeamSearch(self._model, 4,
self._vocab.WordToId(data.SENTENCE_START),
self._vocab.WordToId(data.SENTENCE_END),
self._hps.dec_timesteps)
article_batch_cp = article_batch.copy()
article_lens_cp = article_lens.copy()
best_beam = bs.BeamSearch(self.sess, article_batch_cp,
article_lens_cp)[0]
decode_output = [int(t) for t in best_beam.tokens[1:]]
decoded_text = ' '.join(data.Ids2Words(decode_output, self._vocab))
return decoded_text
def _Eval(model, data_batcher, vocab=None):
"""Runs model eval."""
model.build_graph()
saver = tf.train.Saver()
summary_writer = tf.summary.FileWriter(FLAGS.eval_dir)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.3)
sess = tf.Session(config=tf.ConfigProto( #gpu_options=gpu_options,
allow_soft_placement=True))
running_avg_loss = 0
step = 0
while True:
#time.sleep(FLAGS.eval_interval_secs)
try:
ckpt_state = tf.train.get_checkpoint_state(FLAGS.log_root)
except tf.errors.OutOfRangeError as e:
tf.logging.error('Cannot restore checkpoint: %s', e)
continue
if not (ckpt_state and ckpt_state.model_checkpoint_path):
tf.logging.info('No model to eval yet at %s', FLAGS.train_dir)
continue
tf.logging.info('Loading checkpoint %s',
ckpt_state.model_checkpoint_path)
saver.restore(sess, ckpt_state.model_checkpoint_path)
(article_batch, abstract_batch, targets, article_lens, abstract_lens,
loss_weights, _, _, cnts, bsize) = data_batcher.NextBatch()
(summaries, loss,
train_step) = model.run_eval_step(sess, article_batch, abstract_batch,
targets, article_lens,
abstract_lens, loss_weights)
tf.logging.info(
'article: %s',
' '.join(data.Ids2Words(article_batch[0][:].tolist(), vocab)))
tf.logging.info(
'abstract: %s',
' '.join(data.Ids2Words(abstract_batch[0][:].tolist(), vocab)))
summary_writer.add_summary(summaries, train_step)
running_avg_loss = _RunningAvgLoss(running_avg_loss, loss,
summary_writer, train_step)
step += 1
print('step: {} \t processing {} to {} \t avg_loss: {}'.format(
step, np.min(cnts), np.max(cnts), running_avg_loss))
if step % 100 == 0:
summary_writer.flush()
def _Eval(model, data_batcher, iteration, count, vocab=None):
"""Runs model eval."""
if count == 0:
model.build_graph()
saver = tf.train.Saver()
summary_writer = tf.summary.FileWriter(FLAGS.eval_dir)
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
running_avg_loss = 0
step = 0
while iteration > step:
time.sleep(FLAGS.eval_interval_secs)
try:
ckpt_state = tf.train.get_checkpoint_state(FLAGS.log_root)
except tf.errors.OutOfRangeError as e:
tf.logging.error('Cannot restore checkpoint: %s', e)
continue
if not (ckpt_state and ckpt_state.model_checkpoint_path):
tf.logging.info('No model to eval yet at %s', FLAGS.train_dir)
continue
tf.logging.info('Loading checkpoint %s',
ckpt_state.model_checkpoint_path)
saver.restore(sess, ckpt_state.model_checkpoint_path)
(article_batch, abstract_batch, targets, article_lens, abstract_lens,
loss_weights, _, _) = data_batcher.NextBatch()
(summaries, loss,
train_step) = model.run_eval_step(sess, article_batch, abstract_batch,
targets, article_lens,
abstract_lens, loss_weights)
tf.logging.info(
'article: %s',
' '.join(data.Ids2Words(article_batch[0][:].tolist(), vocab)))
tf.logging.info(
'abstract: %s',
' '.join(data.Ids2Words(abstract_batch[0][:].tolist(), vocab)))
summary_writer.add_summary(summaries, train_step)
running_avg_loss = _RunningAvgLoss(running_avg_loss, loss,
summary_writer, train_step)
if step % 10 == 0:
summary_writer.flush()
sys.stdout.write("eval_avg_loss: %f\n\n\n" % running_avg_loss)
return running_avg_loss
def _DecodeBatch(self, output_ids):
"""Convert id to words and writing results.
Args:
article: The original article string.
abstract: The human (correct) abstract string.
output_ids: The abstract word ids output by machine.
"""
decoded_output = ' '.join(data.Ids2Words(output_ids, self._vocab))
end_p = decoded_output.find(data.SENTENCE_END, 0)
if end_p != -1:
decoded_output = decoded_output[:end_p]
return decoded_output.strip()
def _DecodeBatch(self, article, abstract, output_ids):
"""Convert id to words and writing results.
Args:
article: The original article string.
abstract: The human (correct) abstract string.
output_ids: The abstract word ids output by machine.
"""
decoded_output = ' '.join(data.Ids2Words(output_ids, self._vocab))
end_p = decoded_output.find(data.SENTENCE_END, 0)
if end_p != -1:
decoded_output = decoded_output[:end_p]
tf.logging.info('article: %s', article)
tf.logging.info('abstract: %s', abstract)
tf.logging.info('decoded: %s', decoded_output)
self._decode_io.Write(abstract, decoded_output.strip())
def single_decode(self, one_sent_in):
"""Decoding loop for long running process."""
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
time.sleep(DECODE_LOOP_DELAY_SECS)
ckpt_state = tf.train.get_checkpoint_state(FLAGS.log_root)
if not (ckpt_state and ckpt_state.model_checkpoint_path):
tf.logging.info('No model to decode yet at %s', FLAGS.log_root)
return False
tf.logging.info('checkpoint path %s', ckpt_state.model_checkpoint_path)
ckpt_path = os.path.join(
FLAGS.log_root, os.path.basename(ckpt_state.model_checkpoint_path))
tf.logging.info('renamed checkpoint path %s', ckpt_path)
self._saver.restore(sess, ckpt_path)
for _ in range(FLAGS.decode_batches_per_ckpt):
(article_batch, _, _, article_lens, _, _, origin_articles,
origin_abstracts) = self._batch_reader.NextBatch()
for i in range(1):#range(self._hps.batch_size):
bs = beam_search.BeamSearch(
self._model, self._hps.batch_size,
self._vocab.WordToId(data.SENTENCE_START),
self._vocab.WordToId(data.SENTENCE_END),
self._hps.dec_timesteps)
one_sent = one_sent_in
one_sent = one_sent.strip().split()
article_lens_cp2 = [len(one_sent)]*2
one_sent = [self._vocab.WordToId(x) for x in one_sent]
if len(one_sent) < self._hps.enc_timesteps:
padid = self._vocab.WordToId(data.PAD_TOKEN)
one_sent = one_sent + [padid]*(self._hps.enc_timesteps-len(one_sent))
else:
one_sent = one_sent[:self._hps.enc_timesteps]
article_batch_cp2 = [one_sent]*2
best_beam = bs.BeamSearch(sess, article_batch_cp2, article_lens_cp2)[0]
decode_output = [int(t) for t in best_beam.tokens[1:]]
decoded_output = ' '.join(data.Ids2Words(decode_output, self._vocab))
end_p = decoded_output.find(data.SENTENCE_END, 0)
if end_p != -1:
decoded_output = decoded_output[:end_p]
decoded_output = decoded_output.strip()
print("decode output = {}".format(decoded_output))
print("##### decode over #####")
def _DecodeBatch(self, source, targets, dec_outputs):
"""Converts id to words and writes results.
Args:
source: The original source string.
targets: The human (correct) target string.
dec_outputs: The target word ids output by machine.
Returns:
List of metric scores for this batch.
"""
output = ['None'] * len(dec_outputs)
source_words = source.split()
for i in range(len(dec_outputs)):
if dec_outputs[i] < 0: # it's from copier
position = -1 - dec_outputs[i]
if position < len(source_words):
output[i] = source_words[position]
else:
output[i] = '<out_of_bound>'
elif dec_outputs[i] >= 0: # it's from generator or unk (if 0)
output[i] = data.Ids2Words([dec_outputs[i]], self._output_vocab)[0]
source = source.replace(data.SENTENCE_START + ' ', '').replace(
' ' + data.SENTENCE_END, '')
targets = [
x.replace(data.SENTENCE_START + ' ', '').replace(
' ' + data.SENTENCE_END, '') for x in targets
]
decoded = ' '.join(output)
end_p = decoded.find(data.SENTENCE_END, 0)
if end_p != -1:
decoded = decoded[:end_p].strip()
bleu_score = metrics.get_bleu(decoded, targets)
f1_score = metrics.get_f1(decoded, targets)
exact_score = metrics.get_exact(decoded, targets)
self._decode_io.Write(source, targets, decoded, bleu_score,
f1_score, exact_score)
return bleu_score, f1_score, exact_score
def start_generate_text(self, post_data):
quest = post_data['params']['question']
(article_batch, _, _, article_lens, _, _, origin_articles,
origin_abstracts) = self.get_batch_quest(quest)
bs = beam_search.BeamSearch(
myServer.decoder._model, myServer.decoder._hps.batch_size,
myServer.decoder._vocab.WordToId(data.SENTENCE_START),
myServer.decoder._vocab.WordToId(data.SENTENCE_END),
myServer.decoder._hps.dec_timesteps)
questions = []
article_batch_cp = article_batch.copy()
article_batch_cp[:] = article_batch[0]
article_lens_cp = article_lens.copy()
article_lens_cp[:] = article_lens[0]
best_beam = bs.BeamSearch(myServer.sess, article_batch_cp,
article_lens_cp)
print("quest:%s" % (origin_articles[0].replace(' ', '')))
for i in range(len(best_beam)):
result_beam = best_beam[i]
decode_output = [int(t) for t in result_beam.tokens[1:]]
decoded_output = ''.join(
data.Ids2Words(decode_output, myServer.decoder._vocab))
end_p = decoded_output.find(data.SENTENCE_END, 0)
if end_p != -1:
decoded_output = decoded_output[:end_p]
questions.append(decoded_output)
print("%doutput:%s" % (i, decoded_output))
posdata = {}
params = {}
params["success"] = "true"
params["user_id"] = post_data['params']['user_id']
params["questions"] = questions
posdata['id'] = post_data['id']
posdata['jsonrpc'] = '2.0'
posdata['result'] = params
self.wfile.write(
json.dumps(posdata).encode(encoding='utf_8', errors='strict'))
def _Train(model, eval_model, data_batcher, eval_batcher, vocab=None):
"""Runs model training."""
with tf.device('/cpu:0'):
model.build_graph()
saver = tf.train.Saver()
# Train dir is different from log_root to avoid summary directory
# conflict with Supervisor.
summary_writer = tf.summary.FileWriter(FLAGS.train_dir)
sv = tf.train.Supervisor(logdir=FLAGS.log_root,
is_chief=True,
saver=saver,
summary_op=None,
save_summaries_secs=60,
save_model_secs=FLAGS.checkpoint_secs,
global_step=model.global_step)
sess = sv.prepare_or_wait_for_session(config=tf.ConfigProto(
allow_soft_placement=True))
eval_model.build_graph()
eval_writer = tf.summary.FileWriter(FLAGS.eval_dir)
eval_sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
running_avg_loss = 0
eval_avg_loss = 0
step = 0
eval_step = 0
while not sv.should_stop() and step < FLAGS.max_run_steps:
(article_batch, abstract_batch, targets, article_lens, abstract_lens,
loss_weights, _, _) = data_batcher.NextBatch()
(_, summaries, loss, train_step) = model.run_train_step(
sess, article_batch, abstract_batch, targets, article_lens,
abstract_lens, loss_weights)
summary_writer.add_summary(summaries, train_step)
running_avg_loss = _RunningAvgLoss(
running_avg_loss, loss, summary_writer, train_step)
step += 1
if step % 1 == 0:
summary_writer.flush()
sys.stdout.write("running_avg_loss: %f\n" % running_avg_loss)
# Evaluation:
time.sleep(2)
for count in range(10):
try:
ckpt_state = tf.train.get_checkpoint_state(FLAGS.log_root)
except tf.errors.OutOfRangeError as e:
tf.logging.error('Cannot restore checkpoint: %s', e)
continue
if not (ckpt_state and ckpt_state.model_checkpoint_path):
tf.logging.info('No model to eval yet at %s', FLAGS.train_dir)
continue
tf.logging.info('Loading checkpoint %s', ckpt_state.model_checkpoint_path)
saver.restore(eval_sess, ckpt_state.model_checkpoint_path)
(ab, bb, tar, al, bl,
eval_loss_weights, _, _) = eval_batcher.NextBatch()
(eval_summaries, eval_loss, eval_step) = eval_model.run_eval_step(
eval_sess, ab, bb, tar, al,
bl, eval_loss_weights)
tf.logging.info(
'article: %s',
' '.join(data.Ids2Words(article_batch[0][:].tolist(), vocab)))
tf.logging.info(
'abstract: %s',
' '.join(data.Ids2Words(abstract_batch[0][:].tolist(), vocab)))
eval_writer.add_summary(eval_summaries, eval_step)
eval_avg_loss = _RunningAvgLoss(
eval_avg_loss, eval_loss, eval_writer, eval_step)
eval_step += 1
if eval_step % 1 == 0:
eval_writer.flush()
sys.stdout.write("eval_avg_loss: %f\n\n\n" % running_avg_loss)
# Calculate previous evaluation loss for early stopping
i = 0
eval_results = tf.contrib.estimator.read_eval_metrics(FLAGS.eval_dir)
for step, metrics in eval_results.items():
tmp += metrics['running_avg_loss']
i += 1
avg_eval_loss = tmp / i
if eval_avg_loss - avg_eval_loss < 1 and eval_avg_loss is not avg_eval_loss:
sys.stdout.write("Eval loss is not decreasing anymore!")
break
sv.Stop()
return running_avg_loss
