def _prepare(self):
""" Prepares for evaluation.
Builds the model with reuse=True, mode=EVAL and preprocesses
data file(s).
"""
text_inputter = TextLineInputter(dataset=self._dataset,
data_field_name="eval_features_file",
batch_size=self._batch_size)
self._eval_feeding_data = text_inputter.make_feeding_data()
self._model_configs = update_infer_params( # update inference parameters
self._model_configs,
beam_size=self._beam_size,
maximum_labels_length=self._maximum_labels_length,
length_penalty=self._length_penalty)
estimator_spec = model_fn(model_configs=self._model_configs,
mode=ModeKeys.INFER,
dataset=self._dataset,
name=self._model_name,
reuse=True,
verbose=False)
self._predict_ops = estimator_spec.predictions
tmp_trans_dir = os.path.join(self._model_configs["model_dir"],
GlobalNames.TMP_TRANS_DIRNAME)
if not gfile.Exists(tmp_trans_dir):
gfile.MakeDirs(tmp_trans_dir)
self._tmp_trans_file_prefix = os.path.join(
tmp_trans_dir, GlobalNames.TMP_TRANS_FILENAME_PREFIX)
self._read_ckpt_bleulog()
self._eval_labels_file = self._dataset.eval_labels_file
self._check_bleu_script()
self._estop_patience = 0
self._best_bleu_score = 0.
def run(self):
""" Trains the model. """
# vocabulary
self._vocab_source = Vocab(
filename=self._model_configs["data"]["source_words_vocabulary"],
bpe_codes=self._model_configs["data"]["source_bpecodes"],
reverse_seq=False)
self._vocab_target = Vocab(
filename=self._model_configs["data"]["target_words_vocabulary"],
bpe_codes=self._model_configs["data"]["target_bpecodes"],
reverse_seq=self._model_configs["train"]["reverse_target"])
# build dataset
dataset = Dataset(
self._vocab_source,
self._vocab_target,
train_features_file=self._model_configs["data"]["train_features_file"],
train_labels_file=self._model_configs["data"]["train_labels_file"],
eval_features_file=self._model_configs["data"]["eval_features_file"],
eval_labels_file=self._model_configs["data"]["eval_labels_file"])
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
estimator_spec = model_fn(model_configs=self._model_configs,
mode=ModeKeys.TRAIN,
dataset=dataset,
name=self._model_configs["problem_name"])
train_op = estimator_spec.train_op
hooks = estimator_spec.training_hooks
# build training session
sess = tf.train.MonitoredSession(
session_creator=None,
hooks=hooks)
train_text_inputter = ParallelTextInputter(
dataset,
"train_features_file",
"train_labels_file",
self._model_configs["train"]["batch_size"],
self._model_configs["train"]["batch_tokens_size"],
self._model_configs["train"]["shuffle_every_epoch"])
train_data = train_text_inputter.make_feeding_data(
input_fields=estimator_spec.input_fields,
maximum_features_length=self._model_configs["train"]["maximum_features_length"],
maximum_labels_length=self._model_configs["train"]["maximum_labels_length"])
eidx = 0
while True:
if sess.should_stop():
break
tf.logging.info("STARTUP Epoch {}".format(eidx))
for data in train_data:
if sess.should_stop():
break
sess.run(train_op, feed_dict=data["feed_dict"])
eidx += 1
def _prepare(self):
""" Prepares for evaluation.
Builds the model with reuse=True, mode=EVAL and preprocesses
data file(s).
Furthermore, if the decay_type of optimizer is "loss_decay", creates
the controller variables/operations.
"""
features_file = self._dataset["features_file"]
labels_file = self._dataset["labels_file"]
vocab_source = self._dataset["vocab_source"]
vocab_target = self._dataset["vocab_target"]
text_inputter = ParallelTextInputter(
LineReader(data=features_file,
preprocessing_fn=lambda x: vocab_source.convert_to_idlist(x)),
LineReader(data=labels_file,
preprocessing_fn=lambda x: vocab_target.convert_to_idlist(x)),
vocab_source.pad_id,
vocab_target.pad_id,
batch_size=self._batch_size,
batch_tokens_size=None,
shuffle_every_epoch=None,
bucketing=True)
estimator_spec = model_fn(
model_configs=self._model_configs,
mode=ModeKeys.EVAL,
vocab_source=vocab_source,
vocab_target=vocab_target,
name=self._model_name,
reuse=True,
verbose=False)
self._eval_feeding_data = text_inputter.make_feeding_data(
input_fields=estimator_spec.input_fields, in_memory=True)
self._loss_op = estimator_spec.loss
# for learning decay decay
self._half_lr = False
self._start_decay_at = 0
if self._model_configs["optimizer_params"]["optimizer.lr_decay"]["decay_type"] == "loss_decay":
self._half_lr = True
lr_tensor_dict = get_dict_from_collection(Constants.LEARNING_RATE_VAR_NAME)
self._learning_rate = lr_tensor_dict[Constants.LEARNING_RATE_VAR_NAME]
self._max_patience = self._model_configs["optimizer_params"]["optimizer.lr_decay"]["patience"]
self._start_decay_at = self._model_configs["optimizer_params"]["optimizer.lr_decay"]["start_decay_at"]
assert self._start_decay_at >= self._start_at, (
"start_decay_at in optimizer.lr_decay should be no less than start_at in LossMetricSpec.")
self._half_lr_op = lr_tensor_dict[Constants.LR_AUTO_HALF_OP_NAME]
self._bad_count = 0
self._min_loss = 10000.
def _prepare(self):
""" Prepares for evaluation.
Builds the model with reuse=True, mode=EVAL and preprocesses
data file(s).
"""
features_file = self._dataset["features_file"]
labels_file = self._dataset["labels_file"]
vocab_source = self._dataset["vocab_source"]
vocab_target = self._dataset["vocab_target"]
self._model_configs = update_infer_params( # update inference parameters
self._model_configs,
beam_size=self._beam_size,
maximum_labels_length=self._maximum_labels_length,
length_penalty=self._length_penalty)
estimator_spec = model_fn(model_configs=self._model_configs,
mode=ModeKeys.INFER,
vocab_source=vocab_source,
vocab_target=vocab_target,
name=self._model_name, reuse=True,
verbose=False)
self._predict_ops = estimator_spec.predictions
text_inputter = TextLineInputter(
line_readers=LineReader(
data=features_file,
preprocessing_fn=lambda x: vocab_source.convert_to_idlist(x)),
padding_id=vocab_source.pad_id,
batch_size=self._batch_size)
self._infer_data = text_inputter.make_feeding_data(
input_fields=estimator_spec.input_fields)
tmp_trans_dir = os.path.join(self._model_configs["model_dir"], Constants.TMP_TRANS_DIRNAME)
if not gfile.Exists(tmp_trans_dir):
gfile.MakeDirs(tmp_trans_dir)
self._tmp_trans_file_prefix = os.path.join(tmp_trans_dir, Constants.TMP_TRANS_FILENAME_PREFIX)
self._read_ckpt_bleulog()
# load references
self._references = []
for rfile in access_multiple_files(labels_file):
with open_file(rfile) as fp:
if self._char_level:
self._references.append(to_chinese_char(fp.readlines()))
else:
self._references.append(fp.readlines())
self._references = list(map(list, zip(*self._references)))
with open_file(features_file) as fp:
self._sources = fp.readlines()
self._bad_count = 0
self._best_bleu_score = 0.
def _prepare(self):
""" Prepares for evaluation.
Builds the model with reuse=True, mode=EVAL and preprocesses
data file(s).
Furthermore, if the decay_type of optimizer is "loss_decay", creates
the controller variables/operations.
"""
text_inputter = ParallelTextInputter(
dataset=self._dataset,
features_field_name="eval_features_file",
labels_field_name="eval_labels_file",
batch_size=self._batch_size,
batch_tokens_size=None,
shuffle_every_epoch=None,
bucketing=True)
estimator_spec = model_fn(model_configs=self._model_configs,
mode=ModeKeys.EVAL,
dataset=self._dataset,
name=self._model_name,
reuse=True,
verbose=False)
self._eval_feeding_data = text_inputter.make_feeding_data(
input_fields=estimator_spec.input_fields, in_memory=True)
self._loss_op = estimator_spec.loss
# for learning decay decay
self._half_lr = False
self._start_decay_at = 0
if self._model_configs["optimizer_params"]["optimizer.lr_decay"][
"decay_type"] == "loss_decay":
self._half_lr = True
lr_tensor_dict = get_dict_from_collection(
Constants.LEARNING_RATE_VAR_NAME)
self._learning_rate = lr_tensor_dict[
Constants.LEARNING_RATE_VAR_NAME]
self._max_patience = self._model_configs["optimizer_params"][
"optimizer.lr_decay"]["patience"]
self._start_decay_at = self._model_configs["optimizer_params"][
"optimizer.lr_decay"]["start_decay_at"]
assert self._start_decay_at >= self._start_at, (
"start_decay_at in optimizer.lr_decay should be no less than start_at in LossMetricSpec."
)
div_factor = lr_tensor_dict[Constants.LR_ANNEAL_DIV_FACTOR_NAME]
self._half_lr_op = div_factor.assign(div_factor * 2.)
self._bad_count = 0
self._min_loss = 10000.
def _prepare(self):
""" Prepares for evaluation.
Builds the model with reuse=True, mode=EVAL and preprocesses
data file(s).
Furthermore, if the decay_type of optimizer is "loss_decay", creates
the controller variables/operations.
"""
text_inputter = ParallelTextInputter(
dataset=self._dataset,
features_field_name="eval_features_file",
labels_field_name="eval_labels_file",
batch_size=self._batch_size,
batch_tokens_size=None,
shuffle_every_epoch=None,
bucketing=True)
estimator_spec = model_fn(
model_configs=self._model_configs,
mode=ModeKeys.EVAL,
dataset=self._dataset,
name=self._model_name,
reuse=True,
verbose=False)
self._eval_feeding_data = text_inputter.make_feeding_data(
input_fields=estimator_spec.input_fields, in_memory=True)
self._loss_op = estimator_spec.loss
# for learning decay decay
self._half_lr = False
self._start_decay_at = 0
if self._model_configs["optimizer_params"]["optimizer.lr_decay"]["decay_type"] == "loss_decay":
self._half_lr = True
lr_tensor_dict = get_dict_from_collection(Constants.LEARNING_RATE_VAR_NAME)
self._learning_rate = lr_tensor_dict[Constants.LEARNING_RATE_VAR_NAME]
self._max_patience = self._model_configs["optimizer_params"]["optimizer.lr_decay"]["patience"]
self._start_decay_at = self._model_configs["optimizer_params"]["optimizer.lr_decay"]["start_decay_at"]
assert self._start_decay_at >= self._start_at, (
"start_decay_at in optimizer.lr_decay should be no less than start_at in LossMetricSpec.")
div_factor = lr_tensor_dict[Constants.LR_ANNEAL_DIV_FACTOR_NAME]
self._half_lr_op = div_factor.assign(div_factor * 2.)
self._bad_count = 0
self._min_loss = 10000.
def _prepare(self):
""" Prepares for evaluation.
Builds the model with reuse=True, mode=EVAL and preprocesses
data file(s).
"""
self._model_configs = update_infer_params( # update inference parameters
self._model_configs,
beam_size=self._beam_size,
maximum_labels_length=self._maximum_labels_length,
length_penalty=self._length_penalty)
estimator_spec = model_fn(model_configs=self._model_configs,
mode=ModeKeys.INFER, dataset=self._dataset,
name=self._model_name, reuse=True, verbose=False)
self._predict_ops = estimator_spec.predictions
text_inputter = TextLineInputter(
dataset=self._dataset,
data_field_name="eval_features_file",
batch_size=self._batch_size)
self._infer_data = text_inputter.make_feeding_data(
input_fields=estimator_spec.input_fields)
tmp_trans_dir = os.path.join(self._model_configs["model_dir"], Constants.TMP_TRANS_DIRNAME)
if not gfile.Exists(tmp_trans_dir):
gfile.MakeDirs(tmp_trans_dir)
self._tmp_trans_file_prefix = os.path.join(tmp_trans_dir, Constants.TMP_TRANS_FILENAME_PREFIX)
self._read_ckpt_bleulog()
# load references
self._references = []
for rfile in self._dataset.eval_labels_file:
with open_file(rfile) as fp:
if self._char_level:
self._references.append(to_chinese_char(fp.readlines()))
else:
self._references.append(fp.readlines())
self._references = list(map(list, zip(*self._references)))
with open_file(self._dataset.eval_features_file) as fp:
self._sources = fp.readlines()
self._bad_count = 0
self._best_bleu_score = 0.
def _prepare(self):
""" Prepares for evaluation.
Builds the model with reuse=True, mode=EVAL and preprocesses
data file(s).
"""
text_inputter = TextLineInputter(dataset=self._dataset,
data_field_name="eval_features_file",
batch_size=self._batch_size)
self._infer_data = text_inputter.make_feeding_data()
self._model_configs = update_infer_params( # update inference parameters
self._model_configs,
beam_size=self._beam_size,
maximum_labels_length=self._maximum_labels_length,
length_penalty=self._length_penalty)
estimator_spec = model_fn(model_configs=self._model_configs,
mode=ModeKeys.INFER,
dataset=self._dataset,
name=self._model_name,
reuse=True,
verbose=False)
self._predict_ops = estimator_spec.predictions
tmp_trans_dir = os.path.join(self._model_configs["model_dir"],
Constants.TMP_TRANS_DIRNAME)
if not gfile.Exists(tmp_trans_dir):
gfile.MakeDirs(tmp_trans_dir)
self._tmp_trans_file_prefix = os.path.join(
tmp_trans_dir, Constants.TMP_TRANS_FILENAME_PREFIX)
self._read_ckpt_bleulog()
# load references
self._references = []
for rfile in self._dataset.eval_labels_file:
with open_file(rfile) as fp:
self._references.append(fp.readlines())
self._references = list(map(list, zip(*self._references)))
with open_file(self._dataset.eval_features_file) as fp:
self._sources = fp.readlines()
self._bad_count = 0
self._best_bleu_score = 0.
def run(self):
"""Infers data files. """
# build datasets
self._vocab_source = Vocab(
filename=self._model_configs["eval"]["source_words_vocabulary"],
bpe_codes=self._model_configs["eval"]["source_bpecodes"],
reverse_seq=False)
self._vocab_target = Vocab(
filename=self._model_configs["eval"]["target_words_vocabulary"],
bpe_codes=self._model_configs["eval"]["target_bpecodes"],
reverse_seq=self._model_configs["train"]["reverse_target"])
# build dataset
dataset = Dataset(
self._vocab_source,
self._vocab_target,
eval_features_file=[p["features_file"] for p
in self._model_configs["eval_data"]],
eval_labels_file=[p["labels_file"] for p
in self._model_configs["eval_data"]])
# update evaluation model config
self._model_configs, metric_str = update_eval_metric(
self._model_configs, self._model_configs["eval"]["metric"])
tf.logging.info("Evaluating using {}".format(metric_str))
# build model
estimator_spec = model_fn(model_configs=self._model_configs,
mode=ModeKeys.EVAL,
dataset=dataset,
name=self._model_configs["problem_name"])
sess = self._build_default_session()
do_bucketing = (sum([p["output_attention"]
for p in self._model_configs["eval_data"]]) == 0)
text_inputter = ParallelTextInputter(
dataset=dataset,
features_field_name="eval_features_file",
labels_field_name="eval_labels_file",
batch_size=self._model_configs["eval"]["batch_size"],
bucketing=do_bucketing)
# reload
checkpoint_path = tf.train.latest_checkpoint(self._model_configs["model_dir"])
if checkpoint_path:
tf.logging.info("reloading models...")
saver = tf.train.Saver()
saver.restore(sess, checkpoint_path)
else:
raise OSError("File NOT Found. Fail to load checkpoint file from: {}"
.format(self._model_configs["model_dir"]))
tf.logging.info("Start evaluation.")
overall_start_time = time.time()
for eval_data, param in zip(text_inputter.make_feeding_data(
input_fields=estimator_spec.input_fields, in_memory=True),
self._model_configs["eval_data"]):
tf.logging.info("Evaluation Source File: {}.".format(param["features_file"]))
tf.logging.info("Evaluation Target File: {}.".format(param["labels_file"]))
start_time = time.time()
result = evaluate_with_attention(
sess=sess,
eval_op=estimator_spec.loss,
eval_data=eval_data,
vocab_source=self._vocab_source,
vocab_target=self._vocab_target,
attention_op=estimator_spec.predictions \
if param["output_attention"] else None,
output_filename_prefix=param["labels_file"].strip().split("/")[-1])
tf.logging.info("FINISHED {}. Elapsed Time: {}."
.format(param["features_file"], str(time.time() - start_time)))
tf.logging.info("Evaluation Score ({} on {}): {}"
.format(metric_str, param["features_file"], result))
tf.logging.info("Total Elapsed Time: %s" % str(time.time() - overall_start_time))
def run(self):
"""Infers data files. """
# build datasets
self._vocab_source = Vocab(
filename=self._model_configs["infer"]["source_words_vocabulary"],
bpe_codes=self._model_configs["infer"]["source_bpecodes"],
reverse_seq=False)
self._vocab_target = Vocab(
filename=self._model_configs["infer"]["target_words_vocabulary"],
bpe_codes=self._model_configs["infer"]["target_bpecodes"],
reverse_seq=self._model_configs["train"]["reverse_target"])
# build dataset
dataset = Dataset(
self._vocab_source,
self._vocab_target,
eval_features_file=[p["features_file"] for p
in self._model_configs["infer_data"]])
self._model_configs = update_infer_params(
self._model_configs,
beam_size=self._model_configs["infer"]["beam_size"],
maximum_labels_length=self._model_configs["infer"]["maximum_labels_length"],
length_penalty=self._model_configs["infer"]["length_penalty"])
# build model
estimator_spec = model_fn(model_configs=self._model_configs,
mode=ModeKeys.INFER,
dataset=dataset,
name=self._model_configs["problem_name"])
predict_op = estimator_spec.predictions
sess = self._build_default_session()
text_inputter = TextLineInputter(
dataset=dataset,
data_field_name="eval_features_file",
batch_size=self._model_configs["infer"]["batch_size"])
# reload
checkpoint_path = tf.train.latest_checkpoint(self._model_configs["model_dir"])
if checkpoint_path:
tf.logging.info("reloading models...")
saver = tf.train.Saver()
saver.restore(sess, checkpoint_path)
else:
raise OSError("File NOT Found. Fail to find checkpoint file from: {}"
.format(self._model_configs["model_dir"]))
tf.logging.info("Start inference.")
overall_start_time = time.time()
for infer_data, param in zip(text_inputter.make_feeding_data(
input_fields=estimator_spec.input_fields),
self._model_configs["infer_data"]):
tf.logging.info("Infer Source File: {}.".format(param["features_file"]))
start_time = time.time()
infer(sess=sess,
prediction_op=predict_op,
infer_data=infer_data,
output=param["output_file"],
vocab_source=self._vocab_source,
vocab_target=self._vocab_target,
delimiter=self._model_configs["infer"]["delimiter"],
output_attention=param["output_attention"],
tokenize_output=self._model_configs["infer"]["char_level"],
verbose=True)
tf.logging.info("FINISHED {}. Elapsed Time: {}."
.format(param["features_file"], str(time.time() - start_time)))
if param["labels_file"] is not None:
bleu_score = multi_bleu_score_from_file(
hypothesis_file=param["output_file"],
references_files=param["labels_file"],
char_level=self._model_configs["infer"]["char_level"])
tf.logging.info("BLEU score (%s): %.2f"
% (param["features_file"], bleu_score))
tf.logging.info("Total Elapsed Time: %s" % str(time.time() - overall_start_time))
def run(self):
""" Trains the model. """
# vocabulary
vocab_source = Vocab(
filename=self._model_configs["data"]["source_words_vocabulary"],
bpe_codes=self._model_configs["data"]["source_bpecodes"],
reverse_seq=self._model_configs["train"]["features_r2l"])
vocab_target = Vocab(
filename=self._model_configs["data"]["target_words_vocabulary"],
bpe_codes=self._model_configs["data"]["target_bpecodes"],
reverse_seq=self._model_configs["train"]["labels_r2l"])
eval_dataset = {
"vocab_source": vocab_source,
"vocab_target": vocab_target,
"features_file": self._model_configs["data"]["eval_features_file"],
"labels_file": self._model_configs["data"]["eval_labels_file"]
}
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
estimator_spec = model_fn(model_configs=self._model_configs,
mode=ModeKeys.TRAIN,
vocab_source=vocab_source,
vocab_target=vocab_target,
name=self._model_configs["problem_name"])
train_ops = estimator_spec.train_ops
hooks = estimator_spec.training_hooks
# build training session
sess = tf.train.MonitoredSession(
session_creator=tf.train.ChiefSessionCreator(
scaffold=tf.train.Scaffold(),
checkpoint_dir=None,
master="",
config=config),
hooks=tuple(hooks) + tuple(
build_eval_metrics(self._model_configs,
eval_dataset,
model_name=estimator_spec.name)))
train_text_inputter = ParallelTextInputter(
LineReader(
data=self._model_configs["data"]["train_features_file"],
maximum_length=self._model_configs["train"]
["maximum_features_length"],
preprocessing_fn=lambda x: vocab_source.convert_to_idlist(x)),
LineReader(
data=self._model_configs["data"]["train_labels_file"],
maximum_length=self._model_configs["train"]
["maximum_labels_length"],
preprocessing_fn=lambda x: vocab_target.convert_to_idlist(x)),
vocab_source.pad_id,
vocab_target.pad_id,
batch_size=self._model_configs["train"]["batch_size"],
batch_tokens_size=self._model_configs["train"]
["batch_tokens_size"],
shuffle_every_epoch=self._model_configs["train"]
["shuffle_every_epoch"],
fill_full_batch=True,
bucketing=True)
train_data = train_text_inputter.make_feeding_data(
input_fields=estimator_spec.input_fields)
eidx = [0, 0]
update_cycle = [self._model_configs["train"]["update_cycle"], 1]
def step_fn(step_context):
step_context.session.run(train_ops["zeros_op"])
try:
while update_cycle[0] != update_cycle[1]:
data = train_data.next()
step_context.session.run(train_ops["collect_op"],
feed_dict=data["feed_dict"])
update_cycle[1] += 1
data = train_data.next()
update_cycle[1] = 1
return step_context.run_with_hooks(train_ops["train_op"],
feed_dict=data["feed_dict"])
except StopIteration:
eidx[1] += 1
while not sess.should_stop():
if eidx[0] != eidx[1]:
tf.logging.info("STARTUP Epoch {}".format(eidx[1]))
eidx[0] = eidx[1]
sess.run_step_fn(step_fn)
def run(self):
"""Infers data files. """
# build datasets
self._vocab_source = Vocab(
filename=self._model_configs["eval"]["source_words_vocabulary"],
bpe_codes=self._model_configs["eval"]["source_bpecodes"],
reverse_seq=False)
self._vocab_target = Vocab(
filename=self._model_configs["eval"]["target_words_vocabulary"],
bpe_codes=self._model_configs["eval"]["target_bpecodes"],
reverse_seq=self._model_configs["train"]["reverse_target"])
# build dataset
dataset = Dataset(
self._vocab_source,
self._vocab_target,
eval_features_file=[p["features_file"] for p
in self._model_configs["eval_data"]],
eval_labels_file=[p["labels_file"] for p
in self._model_configs["eval_data"]])
# update evaluation model config
self._model_configs, metric_str = update_eval_metric(
self._model_configs, self._model_configs["eval"]["metric"])
tf.logging.info("Evaluating using {}".format(metric_str))
# build model
estimator_spec = model_fn(model_configs=self._model_configs,
mode=ModeKeys.EVAL,
dataset=dataset,
name=self._model_configs["problem_name"])
sess = self._build_default_session()
text_inputter = ParallelTextInputter(
dataset=dataset,
features_field_name="eval_features_file",
labels_field_name="eval_labels_file",
batch_size=self._model_configs["eval"]["batch_size"],
bucketing=(sum([p["output_attention"]
for p in self._model_configs["eval_data"]]) == 0))
# reload
checkpoint_path = tf.train.latest_checkpoint(self._model_configs["model_dir"])
if checkpoint_path:
tf.logging.info("reloading models...")
saver = tf.train.Saver()
saver.restore(sess, checkpoint_path)
else:
raise OSError("File NOT Found. Fail to load checkpoint file from: {}"
.format(self._model_configs["model_dir"]))
tf.logging.info("Start evaluation.")
overall_start_time = time.time()
for eval_data, param in zip(text_inputter.make_feeding_data(
input_fields=estimator_spec.input_fields, in_memory=True),
self._model_configs["eval_data"]):
tf.logging.info("Evaluation Source File: {}.".format(param["features_file"]))
tf.logging.info("Evaluation Target File: {}.".format(param["labels_file"]))
start_time = time.time()
result = evaluate_with_attention(
sess=sess,
loss_op=estimator_spec.loss,
eval_data=eval_data,
vocab_source=self._vocab_source,
vocab_target=self._vocab_target,
attention_op=estimator_spec.predictions \
if param["output_attention"] else None,
output_filename_prefix=param["labels_file"].strip().split("/")[-1])
tf.logging.info("FINISHED {}. Elapsed Time: {}."
.format(param["features_file"], str(time.time() - start_time)))
tf.logging.info("Evaluation Score ({} on {}): {}"
.format(metric_str, param["features_file"], result))
tf.logging.info("Total Elapsed Time: %s" % str(time.time() - overall_start_time))
def run(self):
"""Infers data files. """
# build datasets
self._vocab_source = Vocab(
filename=self._model_configs["infer"]["source_words_vocabulary"],
bpe_codes=self._model_configs["infer"]["source_bpecodes"],
reverse_seq=False)
self._vocab_target = Vocab(
filename=self._model_configs["infer"]["target_words_vocabulary"],
bpe_codes=self._model_configs["infer"]["target_bpecodes"],
reverse_seq=self._model_configs["train"]["reverse_target"])
# build dataset
dataset = Dataset(
self._vocab_source,
self._vocab_target,
eval_features_file=[p["features_file"] for p
in self._model_configs["infer_data"]])
self._model_configs = update_infer_params(
self._model_configs,
beam_size=self._model_configs["infer"]["beam_size"],
maximum_labels_length=self._model_configs["infer"]["maximum_labels_length"],
length_penalty=self._model_configs["infer"]["length_penalty"])
# build model
estimator_spec = model_fn(model_configs=self._model_configs,
mode=ModeKeys.INFER,
dataset=dataset,
name=self._model_configs["problem_name"])
predict_op = estimator_spec.predictions
sess = self._build_default_session()
text_inputter = TextLineInputter(
dataset=dataset,
data_field_name="eval_features_file",
batch_size=self._model_configs["infer"]["batch_size"])
# reload
checkpoint_path = tf.train.latest_checkpoint(self._model_configs["model_dir"])
if checkpoint_path:
tf.logging.info("reloading models...")
saver = tf.train.Saver()
saver.restore(sess, checkpoint_path)
else:
raise OSError("File NOT Found. Fail to find checkpoint file from: {}"
.format(self._model_configs["model_dir"]))
tf.logging.info("Start inference.")
overall_start_time = time.time()
for infer_data, param in zip(text_inputter.make_feeding_data(
input_fields=estimator_spec.input_fields),
self._model_configs["infer_data"]):
tf.logging.info("Infer Source File: {}.".format(param["features_file"]))
start_time = time.time()
infer(sess=sess,
prediction_op=predict_op,
infer_data=infer_data,
output=param["output_file"],
vocab_source=self._vocab_source,
vocab_target=self._vocab_target,
delimiter=self._model_configs["infer"]["delimiter"],
output_attention=param["output_attention"],
tokenize_output=self._model_configs["infer"]["char_level"],
verbose=True)
tf.logging.info("FINISHED {}. Elapsed Time: {}."
.format(param["features_file"], str(time.time() - start_time)))
if param["labels_file"] is not None:
bleu_score = multi_bleu_score_from_file(
hypothesis_file=param["output_file"],
references_files=param["labels_file"],
char_level=self._model_configs["infer"]["char_level"])
tf.logging.info("BLEU score (%s): %.2f"
% (param["features_file"], bleu_score))
tf.logging.info("Total Elapsed Time: %s" % str(time.time() - overall_start_time))
def run(self):
""" Trains the model. """
# vocabulary
self._vocab_source = Vocab(
filename=self._model_configs["data"]["source_words_vocabulary"],
bpe_codes=self._model_configs["data"]["source_bpecodes"],
reverse_seq=False)
self._vocab_target = Vocab(
filename=self._model_configs["data"]["target_words_vocabulary"],
bpe_codes=self._model_configs["data"]["target_bpecodes"],
reverse_seq=self._model_configs["train"]["reverse_target"])
# build dataset
dataset = Dataset(
self._vocab_source,
self._vocab_target,
train_features_file=self._model_configs["data"]["train_features_file"],
train_labels_file=self._model_configs["data"]["train_labels_file"],
eval_features_file=self._model_configs["data"]["eval_features_file"],
eval_labels_file=self._model_configs["data"]["eval_labels_file"])
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
estimator_spec = model_fn(model_configs=self._model_configs,
mode=ModeKeys.TRAIN,
dataset=dataset,
name=self._model_configs["problem_name"])
train_ops = estimator_spec.train_ops
hooks = estimator_spec.training_hooks
# build training session
sess = tf.train.MonitoredSession(
session_creator=tf.train.ChiefSessionCreator(
scaffold=tf.train.Scaffold(),
checkpoint_dir=None,
master="",
config=config),
hooks=hooks)
train_text_inputter = ParallelTextInputter(
dataset,
"train_features_file",
"train_labels_file",
self._model_configs["train"]["batch_size"],
self._model_configs["train"]["batch_tokens_size"],
self._model_configs["train"]["shuffle_every_epoch"],
fill_full_batch=True)
train_data = train_text_inputter.make_feeding_data(
input_fields=estimator_spec.input_fields,
maximum_features_length=self._model_configs["train"]["maximum_features_length"],
maximum_labels_length=self._model_configs["train"]["maximum_labels_length"])
eidx = [0, 0]
update_cycle = [self._model_configs["train"]["update_cycle"], 1]
def step_fn(step_context):
step_context.session.run(train_ops["zeros_op"])
try:
while update_cycle[0] != update_cycle[1]:
data = train_data.next()
step_context.session.run(
train_ops["collect_op"], feed_dict=data["feed_dict"])
update_cycle[1] += 1
data = train_data.next()
update_cycle[1] = 1
return step_context.run_with_hooks(
train_ops["train_op"], feed_dict=data["feed_dict"])
except StopIteration:
eidx[1] += 1
while not sess.should_stop():
if eidx[0] != eidx[1]:
tf.logging.info("STARTUP Epoch {}".format(eidx[1]))
eidx[0] = eidx[1]
sess.run_step_fn(step_fn)
