def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
trainer_utils.log_registry()
trainer_utils.validate_flags()
assert FLAGS.schedule == "train_and_evaluate"
data_dir = os.path.expanduser(FLAGS.data_dir)
output_dir = os.path.expanduser(FLAGS.output_dir)
hparams = trainer_utils.create_hparams(FLAGS.hparams_set,
data_dir,
passed_hparams=FLAGS.hparams)
hparams = trainer_utils.add_problem_hparams(hparams, FLAGS.problems)
estimator, _ = trainer_utils.create_experiment_components(
data_dir=data_dir,
model_name=FLAGS.model,
hparams=hparams,
run_config=trainer_utils.create_run_config(output_dir))
decode_hp = decoding.decode_hparams(FLAGS.decode_hparams)
decode_hp.add_hparam("shards", FLAGS.decode_shards)
if FLAGS.decode_interactive:
decoding.decode_interactively(estimator, decode_hp)
elif FLAGS.decode_from_file:
decoding.decode_from_file(estimator, FLAGS.decode_from_file, decode_hp,
FLAGS.decode_to_file)
else:
decoding.decode_from_dataset(estimator, FLAGS.problems.split("-"),
decode_hp, FLAGS.decode_to_file)
def decode(estimator, hparams, decode_hp):
"""Decode from estimator. Interactive, from file, or from dataset."""
if FLAGS.decode_interactive:
if estimator.config.use_tpu:
raise ValueError("TPU can only decode from dataset.")
decoding.decode_interactively(estimator,
hparams,
decode_hp,
checkpoint_path=FLAGS.checkpoint_path)
elif FLAGS.decode_from_file:
decoding.decode_from_file(estimator,
FLAGS.decode_from_file,
hparams,
decode_hp,
FLAGS.decode_to_file,
checkpoint_path=FLAGS.checkpoint_path)
if FLAGS.checkpoint_path and FLAGS.keep_timestamp:
ckpt_time = os.path.getmtime(FLAGS.checkpoint_path + ".index")
os.utime(FLAGS.decode_to_file, (ckpt_time, ckpt_time))
else:
decoding.decode_from_dataset(
estimator,
FLAGS.problem,
hparams,
decode_hp,
decode_to_file=FLAGS.decode_to_file,
dataset_split="test" if FLAGS.eval_use_test_set else None)
def decode(estimator):
if FLAGS.decode_interactive:
decoding.decode_interactively(estimator)
elif FLAGS.decode_from_file is not None:
decoding.decode_from_file(estimator, FLAGS.decode_from_file)
elif FLAGS.decode_from_dataset:
decoding.decode_from_dataset(estimator)
def decode(self, dataset_split=None, decode_from_file=False):
"""Decodes from dataset or file."""
if decode_from_file:
decoding.decode_from_file(self._estimator,
self._decode_hparams.decode_from_file,
self._hparams, self._decode_hparams,
self._decode_hparams.decode_to_file)
else:
decoding.decode_from_dataset(self._estimator,
self._hparams.problem.name,
self._hparams,
self._decode_hparams,
dataset_split=dataset_split)
def decode(estimator, hparams, decode_hp):
if FLAGS.decode_interactive:
decoding.decode_interactively(estimator, hparams, decode_hp)
elif FLAGS.decode_from_file:
decoding.decode_from_file(estimator, FLAGS.decode_from_file, hparams,
decode_hp, FLAGS.decode_to_file)
else:
decoding.decode_from_dataset(
estimator,
FLAGS.problems.split("-"),
hparams,
decode_hp,
decode_to_file=FLAGS.decode_to_file,
dataset_split="test" if FLAGS.eval_use_test_set else None)
def decode(estimator, hparams, decode_hp):
if FLAGS.decode_interactive:
decoding.decode_interactively(estimator, hparams, decode_hp)
elif FLAGS.decode_from_file:
decoding.decode_from_file(estimator, FLAGS.decode_from_file, hparams,
decode_hp, FLAGS.decode_to_file)
else:
decoding.decode_from_dataset(
estimator,
FLAGS.problems.split("-"),
hparams,
decode_hp,
decode_to_file=FLAGS.decode_to_file,
dataset_split="test" if FLAGS.eval_use_test_set else None)
def decode(estimator, hparams, decode_hp):
"""Decode from estimator. Interactive, from file, or from dataset."""
if FLAGS.decode_interactive:
if estimator.config.use_tpu:
raise ValueError("TPU can only decode from dataset.")
decoding.decode_interactively(estimator,
hparams,
decode_hp,
checkpoint_path=FLAGS.checkpoint_path)
elif FLAGS.decode_from_file:
if estimator.config.use_tpu:
raise ValueError("TPU can only decode from dataset.")
decoding.decode_from_file(estimator,
FLAGS.decode_from_file,
hparams,
decode_hp,
FLAGS.decode_to_file,
checkpoint_path=FLAGS.checkpoint_path)
if FLAGS.checkpoint_path and FLAGS.keep_timestamp:
ckpt_time = os.path.getmtime(FLAGS.checkpoint_path + ".index")
os.utime(FLAGS.decode_to_file, (ckpt_time, ckpt_time))
else:
# Fathom
predictions = decoding.decode_from_dataset(
estimator,
FLAGS.problem,
hparams,
decode_hp,
decode_to_file=FLAGS.decode_to_file,
dataset_split=dataset_to_t2t_mode(FLAGS.dataset_split),
return_generator=FLAGS.fathom_output_predictions,
# save logs/summaries to a directory with the same name as decode_output_file
# in situations where we are calling decode without write permissions
# to the model directory
output_dir=os.path.splitext(FLAGS.decode_output_file)[0])
# Fathom
if FLAGS.fathom_output_predictions:
print('Assuming only one problem...')
assert '-' not in FLAGS.problems
# if we already have built problem instance in hparams, no need to create
# it second time (as it's downloading files from gcs)
if hasattr(hparams, 'problem'):
problem = hparams.problem
else:
problem = registry.problem(FLAGS.problems)
problem.output_predictions(predictions=predictions,
num_examples=FLAGS.num_examples)
def __init__(self):
tf.logging.set_verbosity(tf.logging.INFO)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
trainer_utils.log_registry()
#trainer_utils.validate_flags()
self.data_dir = os.path.expanduser(FLAGS.data_dir)
self.output_dir = os.path.expanduser(FLAGS.output_dir)
self.hparams = trainer_utils.create_hparams(
FLAGS.hparams_set, self.data_dir, passed_hparams=FLAGS.hparams)
trainer_utils.add_problem_hparams(self.hparams, FLAGS.problems)
self.estimator, _ = trainer_utils.create_experiment_components(
data_dir=self.data_dir,
model_name=FLAGS.model,
hparams=self.hparams,
run_config=trainer_utils.create_run_config(self.output_dir))
self.decode_hp = decoding.decode_hparams(FLAGS.decode_hparams)
self.decode_hp.add_hparam("shards", FLAGS.decode_shards)
self.decode_hp.add_hparam("shard_id", FLAGS.worker_id)
output_sentence = decoding.decode_from_file(
self.estimator,
FLAGS.decode_from_file,
self.decode_hp,
FLAGS.decode_to_file,
input_sentence=FLAGS.input_sentence)
def decode(estimator, hparams, decode_hp):
if FLAGS.decode_interactive:
decoding.decode_interactively(estimator, hparams, decode_hp)
elif FLAGS.decode_from_file:
for checkpoint_elem in get_checkpoint_path():
decoding.decode_from_file(estimator,
FLAGS.decode_from_file,
hparams,
decode_hp,
FLAGS.decode_to_file + checkpoint_elem,
checkpoint_path=FLAGS.checkpoint_path +
checkpoint_elem)
if FLAGS.keep_timestamp:
ckpt_time = os.path.getmtime(FLAGS.checkpoint_path +
checkpoint_elem + ".index")
os.utime(FLAGS.decode_to_file + checkpoint_elem,
(ckpt_time, ckpt_time))
'''
def decode(estimator, hparams, decode_hp):
if FLAGS.decode_interactive:
decoding.decode_interactively(estimator, hparams, decode_hp)
elif FLAGS.decode_from_file:
decoding.decode_from_file(estimator, FLAGS.decode_from_file, hparams,
decode_hp, FLAGS.decode_to_file,
checkpoint_path=FLAGS.checkpoint_path)
if FLAGS.checkpoint_path and FLAGS.keep_timestamp:
ckpt_time = os.path.getmtime(FLAGS.checkpoint_path + ".index")
os.utime(FLAGS.decode_to_file, (ckpt_time, ckpt_time))
else:
decoding.decode_from_dataset(
estimator,
FLAGS.problems.split("-"),
hparams,
decode_hp,
decode_to_file=FLAGS.decode_to_file,
dataset_split="test" if FLAGS.eval_use_test_set else None)
def decode(estimator, hparams, decode_hp):
if FLAGS.decode_interactive:
decoding.decode_interactively(estimator, hparams, decode_hp)
elif FLAGS.decode_from_file:
decoding.decode_from_file(estimator, FLAGS.decode_from_file, hparams,
decode_hp, FLAGS.decode_to_file,
checkpoint_path=FLAGS.checkpoint_path)
if FLAGS.checkpoint_path and FLAGS.keep_timestamp:
ckpt_time = os.path.getmtime(FLAGS.checkpoint_path + ".index")
os.utime(FLAGS.decode_to_file, (ckpt_time, ckpt_time))
else:
decoding.decode_from_dataset(
estimator,
FLAGS.problems.split("-"),
hparams,
decode_hp,
decode_to_file=FLAGS.decode_to_file,
dataset_split="test" if FLAGS.eval_use_test_set else None)
def decode(self,
dataset_split=None,
decode_from_file=False,
checkpoint_path=None):
"""Decodes from dataset or file."""
if decode_from_file:
decoding.decode_from_file(self._estimator,
self._decode_hparams.decode_from_file,
self._hparams,
self._decode_hparams,
self._decode_hparams.decode_to_file)
else:
decoding.decode_from_dataset(
self._estimator,
self._hparams.problem.name,
self._hparams,
self._decode_hparams,
dataset_split=dataset_split,
checkpoint_path=checkpoint_path)
def get_next(self, input_sentence):
input_sentence_modify = ""
for chr in input_sentence.strip():
input_sentence_modify += chr
input_sentence_modify += " "
input_sentence_modify = input_sentence_modify[:-1]
output_sentence_modify = decoding.decode_from_file(
self.estimator,
FLAGS.decode_from_file,
self.decode_hp,
FLAGS.decode_to_file,
input_sentence=input_sentence)
output_sentence = "".join(output_sentence_modify.split())
return output_sentence
def decode(estimator, hparams, decode_hp):
"""Decode from estimator. Interactive, from file, or from dataset."""
if FLAGS.decode_interactive:
if estimator.config.use_tpu:
raise ValueError("TPU can only decode from dataset.")
decoding.decode_interactively(estimator, hparams, decode_hp,
checkpoint_path=FLAGS.checkpoint_path)
elif FLAGS.decode_from_file:
if estimator.config.use_tpu:
raise ValueError("TPU can only decode from dataset.")
decoding.decode_from_file(estimator, FLAGS.decode_from_file, hparams,
decode_hp, FLAGS.decode_to_file,
checkpoint_path=FLAGS.checkpoint_path)
if FLAGS.checkpoint_path and FLAGS.keep_timestamp:
ckpt_time = os.path.getmtime(FLAGS.checkpoint_path + ".index")
os.utime(FLAGS.decode_to_file, (ckpt_time, ckpt_time))
else:
decoding.decode_from_dataset(
estimator,
FLAGS.problem,
hparams,
decode_hp,
decode_to_file=FLAGS.decode_to_file,
dataset_split="test" if FLAGS.eval_use_test_set else None)