def testBleuWrapperWithUnicodeLineSeparator(self):
hyp_filename, ref_filename = self._generate_test_data(
"unicode-linesep", [u"a b a c", u"e f \u2028 d"],
[u"a b a z", u"y f g d k l m"])
bleu = bleu_hook.bleu_wrapper(ref_filename, hyp_filename)
actual_bleu = 0.2638
self.assertAllClose(bleu, actual_bleu, atol=1e-03)
def testBleuWrapperWithUnicodeLineSeparator(self):
hyp_filename, ref_filename = self._generate_test_data(
"unicode-linesep", [u"a b a c", u"e f \u2028 d"],
[u"a b a z", u"y f g d k l m"])
bleu = bleu_hook.bleu_wrapper(ref_filename, hyp_filename)
actual_bleu = 0.2638
self.assertAllClose(bleu, actual_bleu, atol=1e-03)
def testBleuWrapper(self):
hyp_filename, ref_filename = self._generate_test_data(
"standard", [u"a b a c", u"e f g d"],
[u"a b a z", u"y f g d k l m"])
bleu = bleu_hook.bleu_wrapper(ref_filename, hyp_filename)
actual_bleu = 0.3436
self.assertAllClose(bleu, actual_bleu, atol=1e-03)
def compute_bleu_summaries(hook_args):
"""Compute BLEU core summaries using the decoder output.
Args:
hook_args: DecodeHookArgs namedtuple
Returns:
A list of tf.Summary values if hook_args.hparams contains the
reference file and the translated file.
"""
decode_hparams = hook_args.decode_hparams
estimator = hook_args.estimator
current_step = estimator.get_variable_value(tf.GraphKeys.GLOBAL_STEP)
has_iters = hasattr(decode_hparams, "iterations_per_loop")
if current_step and has_iters and decode_hparams.iterations_per_loop:
iterations_per_loop = decode_hparams.iterations_per_loop
current_epoch = np.asscalar(current_step) // iterations_per_loop
else:
current_epoch = 0
if (decode_hparams.decode_reference is None
or decode_hparams.decode_to_file is None):
return None
values = []
bleu = 100 * bleu_hook.bleu_wrapper(decode_hparams.decode_reference,
decode_hparams.decode_to_file)
values.append(tf.Summary.Value(tag="BLEU", simple_value=bleu))
tf.logging.info("%s: BLEU = %6.2f" % (decode_hparams.decode_to_file, bleu))
if decode_hparams.mlperf_mode:
mlperf_log.transformer_print(key=mlperf_log.EVAL_TARGET,
value=decode_hparams.mlperf_threshold)
mlperf_log.transformer_print(key=mlperf_log.EVAL_ACCURACY,
value={
"epoch": max(current_epoch - 1, 0),
"value": bleu
})
mlperf_log.transformer_print(key=mlperf_log.EVAL_STOP)
if bleu >= decode_hparams.mlperf_threshold:
decode_hparams.set_hparam("mlperf_success", True)
return values
def compute_bleu_summaries(hook_args):
"""Compute BLEU core summaries using the decoder output.
Args:
hook_args: DecodeHookArgs namedtuple
Returns:
A list of tf.Summary values if hook_args.hparams contains the
reference file and the translated file.
"""
decode_hparams = hook_args.decode_hparams
if (decode_hparams.decode_reference is None or
decode_hparams.decode_to_file is None):
return None
values = []
bleu = 100 * bleu_hook.bleu_wrapper(
decode_hparams.decode_reference, decode_hparams.decode_to_file)
values.append(tf.Summary.Value(tag="BLEU", simple_value=bleu))
tf.logging.info("%s: BLEU = %6.2f" % (decode_hparams.decode_to_file, bleu))
return values
def compute_bleu_summaries(hook_args):
"""Compute BLEU core summaries using the decoder output.
Args:
hook_args: DecodeHookArgs namedtuple
Returns:
A list of tf.Summary values if hook_args.hparams contains the
reference file and the translated file.
"""
decode_hparams = hook_args.decode_hparams
if (decode_hparams.decode_reference is None
or decode_hparams.decode_to_file is None):
return None
values = []
bleu = 100 * bleu_hook.bleu_wrapper(decode_hparams.decode_reference,
decode_hparams.decode_to_file)
values.append(tf.Summary.Value(tag="BLEU", simple_value=bleu))
tf.logging.info("%s: BLEU = %6.2f" % (decode_hparams.decode_to_file, bleu))
if hook_args.hparams.mlperf_mode:
current_step = decode_hparams.mlperf_decode_step
mlperf_log.transformer_print(key=mlperf_log.EVAL_TARGET,
value=decode_hparams.mlperf_threshold)
mlperf_log.transformer_print(
key=mlperf_log.EVAL_ACCURACY,
value={
"epoch":
max(current_step // decode_hparams.iterations_per_loop - 1, 0),
"value":
bleu
})
mlperf_log.transformer_print(key=mlperf_log.EVAL_STOP)
if bleu >= decode_hparams.mlperf_threshold:
decode_hparams.set_hparam("mlperf_success", True)
return values
def compute_bleu_summaries(hook_args):
"""Compute BLEU core summaries using the decoder output.
Args:
hook_args: DecodeHookArgs namedtuple
Returns:
A list of tf.Summary values if hook_args.hparams contains the
reference file and the translated file.
"""
decode_hparams = hook_args.decode_hparams
if (decode_hparams.decode_reference is None or
decode_hparams.decode_to_file is None):
return None
values = []
bleu = 100 * bleu_hook.bleu_wrapper(
decode_hparams.decode_reference, decode_hparams.decode_to_file)
values.append(tf.Summary.Value(tag="BLEU", simple_value=bleu))
tf.logging.info("%s: BLEU = %6.2f" % (decode_hparams.decode_to_file, bleu))
if hook_args.hparams.mlperf_mode:
current_step = decode_hparams.mlperf_decode_step
mlperf_log.transformer_print(
key=mlperf_log.EVAL_TARGET, value=decode_hparams.mlperf_threshold)
mlperf_log.transformer_print(
key=mlperf_log.EVAL_ACCURACY,
value={
"epoch": max(current_step // decode_hparams.iterations_per_loop - 1,
0),
"value": bleu
})
mlperf_log.transformer_print(key=mlperf_log.EVAL_STOP)
if bleu >= decode_hparams.mlperf_threshold:
decode_hparams.set_hparam("mlperf_success", True)
return values
epoch_i % hparams['log_bleu_every'] == 0:
start = time.time()
so_file = open(hparams['pairs_path'] + 'dev_input.txt')
results_prediction = estimator.predict(
input_fn=lambda: i_input_fn(so_file, hparams))
pred_tmp_file = open(hparams['checkpoints_path'] + \
('dev_prediction_%i.txt' % epoch_i), 'w')
for j, r in enumerate(results_prediction):
if j % 1000 == 0:
print('Predicting...', j)
sent_vec = r['classes']
if 1 in sent_vec:
sent_vec = sent_vec[:list(sent_vec).index(1)]
sent = output_encoder.decode(sent_vec)
pred_tmp_file.write("%s\n" % sent)
pred_tmp_file.close()
print('Predictions took', time.time() - start, 'sec')
bleu = bleu_hook.bleu_wrapper(
ref_filename=hparams['pairs_path'] + 'dev_output.txt',
hyp_filename=hparams['checkpoints_path'] +
('dev_prediction_%i.txt' % epoch_i)) * 100
print('\033[32;1mBLEU = %f\033[0m' % bleu)
summary = tf.Summary(value=[
tf.Summary.Value(tag='BLEU_predictions', simple_value=bleu)
])
summary_writer.add_summary(summary, epoch_i)
summary_writer.flush()
print(eval_results)
print('Training finished')
def testBleuWrapper(self):
hyp_filename, ref_filename = self._generate_test_data(
"standard", [u"a b a c", u"e f g d"], [u"a b a z", u"y f g d k l m"])
bleu = bleu_hook.bleu_wrapper(ref_filename, hyp_filename)
actual_bleu = 0.3436
self.assertAllClose(bleu, actual_bleu, atol=1e-03)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
if FLAGS.translation:
if FLAGS.translations_dir:
raise ValueError(
"Cannot specify both --translation and --translations_dir.")
if FLAGS.bleu_variant in ("uncased", "both"):
bleu = 100 * bleu_hook.bleu_wrapper(FLAGS.reference, FLAGS.translation,
case_sensitive=False)
print("BLEU_uncased = %6.2f" % bleu)
if FLAGS.bleu_variant in ("cased", "both"):
bleu = 100 * bleu_hook.bleu_wrapper(FLAGS.reference, FLAGS.translation,
case_sensitive=True)
print("BLEU_cased = %6.2f" % bleu)
return
if not FLAGS.translations_dir:
raise ValueError(
"Either --translation or --translations_dir must be specified.")
transl_dir = os.path.expanduser(FLAGS.translations_dir)
if not os.path.exists(transl_dir):
exit_time = time.time() + FLAGS.wait_minutes * 60
tf.logging.info("Translation dir %s does not exist, waiting till %s.",
transl_dir, time.asctime(time.localtime(exit_time)))
while not os.path.exists(transl_dir):
time.sleep(10)
if time.time() > exit_time:
raise ValueError("Translation dir %s does not exist" % transl_dir)
last_step_file = os.path.join(FLAGS.event_dir, "last_evaluated_step.txt")
if FLAGS.min_steps == -1:
if tf.gfile.Exists(last_step_file):
with open(last_step_file) as ls_file:
FLAGS.min_steps = int(ls_file.read())
else:
FLAGS.min_steps = 0
if FLAGS.report_zero is None:
FLAGS.report_zero = FLAGS.min_steps == 0
writer = tf.summary.FileWriter(FLAGS.event_dir)
for transl_file in bleu_hook.stepfiles_iterator(
transl_dir, FLAGS.wait_minutes, FLAGS.min_steps, path_suffix=""):
# report_zero handling must be inside the for-loop,
# so we are sure the transl_dir is already created.
if FLAGS.report_zero:
all_files = (os.path.join(transl_dir, f) for f in os.listdir(transl_dir))
start_time = min(
os.path.getmtime(f) for f in all_files if os.path.isfile(f))
values = []
if FLAGS.bleu_variant in ("uncased", "both"):
values.append(tf.Summary.Value(
tag="BLEU_uncased" + FLAGS.tag_suffix, simple_value=0))
if FLAGS.bleu_variant in ("cased", "both"):
values.append(tf.Summary.Value(
tag="BLEU_cased" + FLAGS.tag_suffix, simple_value=0))
writer.add_event(tf.summary.Event(summary=tf.Summary(value=values),
wall_time=start_time, step=0))
FLAGS.report_zero = False
filename = transl_file.filename
tf.logging.info("Evaluating " + filename)
values = []
if FLAGS.bleu_variant in ("uncased", "both"):
bleu = 100 * bleu_hook.bleu_wrapper(FLAGS.reference, filename,
case_sensitive=False)
values.append(tf.Summary.Value(tag="BLEU_uncased" + FLAGS.tag_suffix,
simple_value=bleu))
tf.logging.info("%s: BLEU_uncased = %6.2f" % (filename, bleu))
if FLAGS.bleu_variant in ("cased", "both"):
bleu = 100 * bleu_hook.bleu_wrapper(FLAGS.reference, filename,
case_sensitive=True)
values.append(tf.Summary.Value(tag="BLEU_cased" + FLAGS.tag_suffix,
simple_value=bleu))
tf.logging.info("%s: BLEU_cased = %6.2f" % (transl_file.filename, bleu))
writer.add_event(tf.summary.Event(
summary=tf.Summary(value=values),
wall_time=transl_file.mtime, step=transl_file.steps))
writer.flush()
with open(last_step_file, "w") as ls_file:
ls_file.write(str(transl_file.steps) + "\n")
sent_vec = r['classes']
if 1 in sent_vec:
sent_vec = sent_vec[:list(sent_vec).index(1)]
if 0 in sent_vec:
sent_vec = sent_vec[:list(sent_vec).index(0)]
sent = output_encoder.decode(sent_vec)
predictions.append(sent)
file_path = exp_path + ('predictions/prediction_%s.txt' %
corpus_info['evals'][k]['name'])
print(file_path)
pred_tmp_file = open(file_path, 'w')
for s in predictions:
pred_tmp_file.write("%s\n" % s)
pred_tmp_file.close()
print("Comparing %s and %s" %
(corpus_info['eval_target_path'], file_path))
bleu = bleu_hook.bleu_wrapper(
ref_filename=corpus_info['eval_target_path'],
hyp_filename=file_path) * 100
print('\033[32;1mBLEU %s = %f\033[0m' %
(corpus_info['evals'][k]['name'], bleu))
summary = tf.Summary(value=[
tf.Summary.Value(tag=('BLEU_%s' % corpus_info['evals'][k]['name']),
simple_value=bleu)
])
summary_writer.add_summary(summary, i)
summary_writer.flush()
#break
print("END")
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
if FLAGS.translation:
if FLAGS.translations_dir:
raise ValueError(
"Cannot specify both --translation and --translations_dir.")
if FLAGS.bleu_variant in ("uncased", "both"):
bleu = 100 * bleu_hook.bleu_wrapper(
FLAGS.reference, FLAGS.translation, case_sensitive=False)
print("BLEU_uncased = %6.2f" % bleu)
if FLAGS.bleu_variant in ("cased", "both"):
bleu = 100 * bleu_hook.bleu_wrapper(
FLAGS.reference, FLAGS.translation, case_sensitive=True)
print("BLEU_cased = %6.2f" % bleu)
return
if not FLAGS.translations_dir:
raise ValueError(
"Either --translation or --translations_dir must be specified.")
transl_dir = os.path.expanduser(FLAGS.translations_dir)
if not os.path.exists(transl_dir):
exit_time = time.time() + FLAGS.wait_minutes * 60
tf.logging.info("Translation dir %s does not exist, waiting till %s.",
transl_dir, time.asctime(time.localtime(exit_time)))
while not os.path.exists(transl_dir):
time.sleep(10)
if time.time() > exit_time:
raise ValueError("Translation dir %s does not exist" %
transl_dir)
last_step_file = os.path.join(FLAGS.event_dir, "last_evaluated_step.txt")
if FLAGS.min_steps == -1:
if tf.gfile.Exists(last_step_file):
with open(last_step_file) as ls_file:
FLAGS.min_steps = int(ls_file.read())
else:
FLAGS.min_steps = 0
if FLAGS.report_zero is None:
FLAGS.report_zero = FLAGS.min_steps == 0
writer = tf.summary.FileWriter(FLAGS.event_dir)
for transl_file in bleu_hook.stepfiles_iterator(transl_dir,
FLAGS.wait_minutes,
FLAGS.min_steps,
path_suffix=""):
# report_zero handling must be inside the for-loop,
# so we are sure the transl_dir is already created.
if FLAGS.report_zero:
all_files = (os.path.join(transl_dir, f)
for f in os.listdir(transl_dir))
start_time = min(
os.path.getmtime(f) for f in all_files if os.path.isfile(f))
values = []
if FLAGS.bleu_variant in ("uncased", "both"):
values.append(
tf.Summary.Value(tag="BLEU_uncased" + FLAGS.tag_suffix,
simple_value=0))
if FLAGS.bleu_variant in ("cased", "both"):
values.append(
tf.Summary.Value(tag="BLEU_cased" + FLAGS.tag_suffix,
simple_value=0))
writer.add_event(
tf.summary.Event(summary=tf.Summary(value=values),
wall_time=start_time,
step=0))
FLAGS.report_zero = False
filename = transl_file.filename
tf.logging.info("Evaluating " + filename)
values = []
if FLAGS.bleu_variant in ("uncased", "both"):
bleu = 100 * bleu_hook.bleu_wrapper(
FLAGS.reference, filename, case_sensitive=False)
values.append(
tf.Summary.Value(tag="BLEU_uncased" + FLAGS.tag_suffix,
simple_value=bleu))
tf.logging.info("%s: BLEU_uncased = %6.2f" % (filename, bleu))
if FLAGS.bleu_variant in ("cased", "both"):
bleu = 100 * bleu_hook.bleu_wrapper(
FLAGS.reference, filename, case_sensitive=True)
values.append(
tf.Summary.Value(tag="BLEU_cased" + FLAGS.tag_suffix,
simple_value=bleu))
tf.logging.info("%s: BLEU_cased = %6.2f" %
(transl_file.filename, bleu))
writer.add_event(
tf.summary.Event(summary=tf.Summary(value=values),
wall_time=transl_file.mtime,
step=transl_file.steps))
writer.flush()
with open(last_step_file, "w") as ls_file:
ls_file.write(str(transl_file.steps) + "\n")
