def create_or_load_hparams(load_dir, default_hparams, hparams_path,
save_hparams):
"""Create hparams or load hparams from out_dir."""
hparams = utils.load_hparams(load_dir)
if not hparams:
hparams = default_hparams
# Override hparams values with existing standard hparams config
hparams = utils.maybe_parse_standard_hparams(hparams, hparams_path)
hparams = process_input_path(hparams)
hparams = extend_hparams(hparams)
else:
hparams = ensure_compatible_hparams(hparams, default_hparams,
hparams_path)
hparams = process_input_path(hparams)
# Save HParams
if save_hparams:
utils.save_hparams(default_hparams.out_dir, hparams)
for metric in hparams.metrics:
utils.save_hparams(getattr(hparams, "best_" + metric + "_dir"),
hparams)
# Print HParams
utils.print_hparams(hparams)
return hparams
def create_or_load_hparams(out_dir, default_hparams, flags):
"""Create hparams or load hparams from out_dir."""
hparams = utils.load_hparams(out_dir, verbose=not flags.chat)
if not hparams:
# Parse the ones from the command line
hparams = default_hparams
hparams = utils.maybe_parse_standard_hparams(hparams,
flags.hparams_path,
verbose=not flags.chat)
hparams = extend_hparams(hparams)
else:
hparams = ensure_compatible_hparams(hparams, default_hparams, flags)
# Save HParams
utils.save_hparams(out_dir, hparams, verbose=not flags.chat)
for metric in hparams.metrics:
utils.save_hparams(getattr(hparams, "best_" + metric + "_dir"),
hparams,
verbose=not flags.chat)
# Print HParams
if not flags.chat:
utils.print_hparams(hparams)
return hparams
def create_or_load_hparams(out_dir, default_hparams, hparams_path):
"""Create hparams or load hparams from out_dir."""
hparams = utils.load_hparams(out_dir)
# print(hparams); assert False #debug
if not hparams:
hparams = default_hparams
hparams = utils.maybe_parse_standard_hparams(
hparams, hparams_path)
hparams = extend_hparams(hparams)
else:
hparams = ensure_compatible_hparams(hparams, default_hparams, hparams_path)
if FLAGS.inference_input_file:
hparams.src_vocab_file = os.path.join(out_dir, "../data/vocab.cor")
hparams.tgt_vocab_file = os.path.join(out_dir, "../data/vocab.man")
hparams.out_dir = out_dir
hparams.best_bleu_dir = os.path.join(out_dir, "best_bleu")
hparams.train_prefix = os.path.join(out_dir, "../data/train")
hparams.dev_prefix = os.path.join(out_dir, "../data/dev_test")
hparams.vocab_prefix = os.path.join(out_dir, "../data/vocab")
hparams.rc_vocab_file = os.path.join(out_dir, "../data/vocab.cor")
hparams.test_prefix = os.path.join(out_dir, "../data/test")
# Save HParams
utils.save_hparams(out_dir, hparams)
for metric in hparams.metrics:
utils.save_hparams(getattr(hparams, "best_" + metric + "_dir"), hparams)
# Print HParams
utils.print_hparams(hparams)
return hparams
def create_or_load_hparams(out_dir, default_hparams, save_hparams=True):
hparams = utils.load_hparams(out_dir)
if not hparams:
hparams = default_hparams
hparams = extend_hparams(hparams)
# Save HParams
if save_hparams:
utils.save_hparams(out_dir, hparams)
# Print HParams
utils.print_hparams(hparams)
return hparams
def run_prediction(input_file_path, output_file_path):
infile = 'input_file'
word_split(input_file_path, infile, jieba_split)
model_dir = 'jb_attention'
hparams = utils.load_hparams(model_dir)
hparams.inference_indices = [i for i in range(150)]
sample_src_dataset = inference.load_data(infile)
log_device_placement = hparams.log_device_placement
if not hparams.attention:
model_creator = nmt_model.Model
else:
if (hparams.encoder_type == 'gnmt'
or hparams.attention_architecture in ['gnmt', 'gnmt_v2']):
model_creator = gnmt_model.GNMTModel
elif hparams.attention_architecture == 'standard':
model_creator = attention_model.AttentionModel
else:
raise ValueError('Unknown attention architecture %s' %
(hparams.attention_architecture))
infer_model = model_helper.create_infer_model(model_creator,
hparams,
scope=None)
config_proto = utils.get_config_proto(
log_device_placement=log_device_placement,
num_intra_threads=hparams.num_intra_threads,
num_inter_threads=hparams.num_inter_threads)
infer_sess = tf.Session(target='',
config=config_proto,
graph=infer_model.graph)
with infer_model.graph.as_default():
loaded_infer_model, global_step = model_helper.create_or_load_model(
infer_model.model, model_dir, infer_sess, 'infer')
iterator_feed_dict = {
infer_model.src_placeholder: sample_src_dataset,
infer_model.batch_size_placeholder: 1,
}
infer_sess.run(infer_model.iterator.initalizer,
feed_dict=iterator_feed_dict)
while True:
try:
nmt_outputs, _ = infer_model.decode(infer_sess)
except tf.errors.OutOfRangeError:
break
def __init__(self, data_dir, id_speaker, num_translations_per_input=1,
scope=None):
self.data_dir = data_dir
self.hparams = utils.load_hparams(self.data_dir)
self.hparams.infer_batch_size = 1
self.ckpt = tf.train.latest_checkpoint(data_dir)
self.id = id_speaker
self.num_translations_per_input = num_translations_per_input
model_creator = gnmt_model.GNMTModel
self.infer_model = model_helper.create_infer_model(model_creator,
self.hparams,
scope)
def create_or_load_hparams(out_dir, default_hparams, hparams_path):
"""Create hparams or load hparams from out_dir."""
hparams = utils.load_hparams(out_dir)
if not hparams:
hparams = default_hparams
hparams = utils.maybe_parse_standard_hparams(hparams, hparams_path)
hparams = extend_hparams(hparams)
else:
hparams = ensure_compatible_hparams(hparams, default_hparams,
hparams_path)
# Save HParams
utils.save_hparams(out_dir, hparams)
for metric in hparams.metrics:
utils.save_hparams(getattr(hparams, "best_bleu_dir"), hparams)
# Print HParams
utils.print_hparams(hparams)
return hparams
def modelInit():
out_dir = 'tmp/nmt_attention_model'
print("loading parameters")
hparams = utils.load_hparams(out_dir)
ckpt = tf.train.latest_checkpoint(out_dir)
if not hparams.attention:
model_creator = nmt_model.Model
elif hparams.attention_architecture == "standard":
model_creator = attention_model.AttentionModel
elif hparams.attention_architecture in ["gnmt", "gnmt_v2"]:
model_creator = gnmt_model.GNMTModel
else:
raise ValueError("Unknown model architecture")
infer_model = model_helper.create_infer_model(model_creator,
hparams,
scope=None)
return infer_model, ckpt, hparams
def create_or_load_hparams(out_dir,
default_hparams,
hparams_path,
save_hparams=True):
hparams = utils.load_hparams(out_dir)
if not hparams:
hparams = default_hparams
hparams = utils.maybe_parse_standard_hparams(hparams, hparams_path)
hparams = extend_hparams(hparams)
else:
hparams = ensure_compatible_hparams(hparams, default_hparams,
hparams_path)
if save_hparams:
utils.save_hparams(out_dir, hparams)
for metric in hparams.metrics:
utils.save_hparams(getattr(hparams, "best_" + metric + "_dir"),
hparams)
utils.print_hparams(hparams)
return hparams
def create_or_load_hparams(out_dir, default_hparams):
"""
Create hparams or load hparams from out_dir.
"""
hparams = utils.load_hparams(out_dir)
if not hparams:
hparams = default_hparams
hparams.add_hparam("best_bleu", 0)
best_bleu_dir = os.path.join(out_dir, "best_bleu")
hparams.add_hparam("best_bleu_dir", best_bleu_dir)
os.makedirs(best_bleu_dir)
hparams.add_hparam("avg_best_bleu", 0)
best_bleu_dir = os.path.join(hparams.out_dir, "avg_best_bleu")
hparams.add_hparam("avg_best_bleu_dir",
os.path.join(hparams.out_dir, "avg_best_bleu"))
os.makedirs(best_bleu_dir)
# Set num_train_steps
train_src_file = "%s.%s" % (hparams.train_prefix, hparams.src)
train_tgt_file = "%s.%s" % (hparams.train_prefix, hparams.tgt)
with open(train_src_file, 'r', encoding='utf-8') as f:
train_src_steps = len(f.readlines())
with open(train_tgt_file, 'r', encoding='utf-8') as f:
train_tgt_steps = len(f.readlines())
hparams.add_hparam(
"num_train_steps",
min([train_src_steps, train_tgt_steps]) * hparams.epochs)
# Set encoder/decoder layers
hparams.add_hparam("num_encoder_layers", hparams.num_layers)
hparams.add_hparam("num_decoder_layers", hparams.num_layers)
# Set residual layers
num_encoder_residual_layers = 0
num_decoder_residual_layers = 0
if hparams.num_encoder_layers > 1:
num_encoder_residual_layers = hparams.num_encoder_layers - 1
if hparams.num_decoder_layers > 1:
num_decoder_residual_layers = hparams.num_decoder_layers - 1
# The first unidirectional layer (after the bi-directional layer) in
# the GNMT encoder can't have residual connection due to the input is
# the concatenation of fw_cell and bw_cell's outputs.
num_encoder_residual_layers = hparams.num_encoder_layers - 2
# Compatible for GNMT models
if hparams.num_encoder_layers == hparams.num_decoder_layers:
num_decoder_residual_layers = num_encoder_residual_layers
hparams.add_hparam("num_encoder_residual_layers",
num_encoder_residual_layers)
hparams.add_hparam("num_decoder_residual_layers",
num_decoder_residual_layers)
# Vocab
# Get vocab file names first
if hparams.vocab_prefix:
src_vocab_file = hparams.vocab_prefix + "." + hparams.src
tgt_vocab_file = hparams.vocab_prefix + "." + hparams.tgt
else:
raise ValueError("hparams.vocab_prefix must be provided.")
# Source vocab
src_vocab_size, src_vocab_file = vocab_utils.check_vocab(
src_vocab_file,
hparams.out_dir,
sos=hparams.sos,
eos=hparams.eos,
unk=vocab_utils.UNK)
# Target vocab
if hparams.share_vocab:
utils.log("Using source vocab for target")
tgt_vocab_file = src_vocab_file
tgt_vocab_size = src_vocab_size
else:
tgt_vocab_size, tgt_vocab_file = vocab_utils.check_vocab(
tgt_vocab_file,
hparams.out_dir,
sos=hparams.sos,
eos=hparams.eos,
unk=vocab_utils.UNK)
hparams.add_hparam("src_vocab_size", src_vocab_size)
hparams.add_hparam("tgt_vocab_size", tgt_vocab_size)
hparams.add_hparam("src_vocab_file", src_vocab_file)
hparams.add_hparam("tgt_vocab_file", tgt_vocab_file)
# Pretrained Embeddings:
hparams.add_hparam("src_embed_file", "")
hparams.add_hparam("tgt_embed_file", "")
if hparams.embed_prefix:
src_embed_file = hparams.embed_prefix + "." + hparams.src
tgt_embed_file = hparams.embed_prefix + "." + hparams.tgt
if os.path.exists(src_embed_file):
hparams.src_embed_file = src_embed_file
if os.path.exists(tgt_embed_file):
hparams.tgt_embed_file = tgt_embed_file
# Save HParams
utils.save_hparams(out_dir, hparams)
return hparams
