def load_model(model_dir, params=None):
"""Loads a model class from a given directory
"""
train_options = training_utils.TrainOptions.load(model_dir)
# Load vocabulary
source_vocab_info = vocab.get_vocab_info(train_options.source_vocab_path)
target_vocab_info = vocab.get_vocab_info(train_options.target_vocab_path)
# Find model class
model_class = getattr(models, train_options.model_class)
# Parse parameter and merge with defaults
hparams = model_class.default_params()
hparams.update(train_options.hparams)
if params is not None:
hparams.update(params)
training_utils.print_hparams(hparams)
# Create model instance
model = model_class(
source_vocab_info=source_vocab_info,
target_vocab_info=target_vocab_info,
params=hparams)
return model
def __init__(self, params, mode, name):
super(Seq2SeqModel, self).__init__(params, mode, name)
self._source_embedding = None
self._source_emb_scope = None
self._target_embedding = None
self._target_emb_scope = None
self.source_vocab_info = None
if "vocab_source" in self.params and self.params["vocab_source"]:
self.source_vocab_info = vocab.get_vocab_info(
self.params["vocab_source"])
if "pos_source" in self.params and self.params["pos_source"]:
self.source_pos_info = vocab.get_vocab_info(
self.params["pos_source"])
if "ner_source" in self.params and self.params["ner_source"]:
self.source_ner_info = vocab.get_vocab_info(
self.params["ner_source"])
self.target_vocab_info = None
if "vocab_target" in self.params and self.params["vocab_target"]:
self.target_vocab_info = vocab.get_vocab_info(
self.params["vocab_target"])
if "ner_target" in self.params and self.params["ner_target"]:
self.target_ner_info = vocab.get_vocab_info(
self.params["ner_target"])
def __init__(self, params, mode, name):
super(Seq2SeqModel, self).__init__(params, mode, name)
self.source_vocab_info = None
if "vocab_source" in self.params and self.params["vocab_source"]:
self.source_vocab_info = vocab.get_vocab_info(self.params["vocab_source"])
self.target_vocab_info = None
if "vocab_target" in self.params and self.params["vocab_target"]:
self.target_vocab_info = vocab.get_vocab_info(self.params["vocab_target"])
def __init__(self, params, mode, name):
super(Seq2SeqModel, self).__init__(params, mode, name)
self.source_vocab_info = None
if "vocab_source" in self.params and self.params["vocab_source"]:
self.source_vocab_info = vocab.get_vocab_info(self.params["vocab_source"])
self.target_vocab_info = None
if "vocab_target" in self.params and self.params["vocab_target"]:
self.target_vocab_info = vocab.get_vocab_info(self.params["vocab_target"])
def test_vocab_info(self): vocab_info = vocab.get_vocab_info(self.vocab_file.name) self.assertEqual(vocab_info.vocab_size, 3) self.assertEqual(vocab_info.path, self.vocab_file.name) self.assertEqual(vocab_info.special_vocab.UNK, 3) self.assertEqual(vocab_info.special_vocab.SEQUENCE_START, 4) self.assertEqual(vocab_info.special_vocab.SEQUENCE_END, 5) self.assertEqual(vocab_info.total_size, 6)
def test_vocab_info(self):
vocab_info = vocab.get_vocab_info(self.vocab_file.name)
self.assertEqual(vocab_info.vocab_size, 3)
self.assertEqual(vocab_info.path, self.vocab_file.name)
self.assertEqual(vocab_info.special_vocab.UNK, 3)
self.assertEqual(vocab_info.special_vocab.SEQUENCE_START, 4)
self.assertEqual(vocab_info.special_vocab.SEQUENCE_END, 5)
self.assertEqual(vocab_info.total_size, 6)
def __init__(self, params, mode, name="att_seq2seq"):
super(AttentionSeq2Seq, self).__init__(params, mode, name)
# add candidate answer part
self.source_candidate_vocab_info = None
if "vocab_source_candidate" in self.params and self.params[
"vocab_source_candidate"]:
self.source_candidate_vocab_info = vocab.get_vocab_info(
self.params["vocab_source_candidate"])
def __init__(self, params, mode, name="basic_biseq2seq"):
super(BasicBiSeq2Seq, self).__init__(params, mode, name)
# add candidate answer part
self.source_candidate_vocab_info = None
if "vocab_source_candidate" in self.params and self.params[
"vocab_source_candidate"]:
self.source_candidate_vocab_info = vocab.get_vocab_info(
self.params["vocab_source_candidate"])
self.encoder_class = locate(self.params["encoder.class"])
self.decoder_class = locate(self.params["decoder.class"])
def setUp(self):
super(EncoderDecoderTests, self).setUp()
tf.logging.set_verbosity(tf.logging.INFO)
self.batch_size = 2
self.input_depth = 4
self.sequence_length = 10
# Create vocabulary
self.vocab_list = [str(_) for _ in range(10)]
self.vocab_list += ["笑う", "泣く", "了解", "はい", "＾＿＾"]
self.vocab_size = len(self.vocab_list)
self.vocab_file = test_utils.create_temporary_vocab_file(self.vocab_list)
self.vocab_info = vocab.get_vocab_info(self.vocab_file.name)
def __init__(self, params, mode, name):
super(Seq2SeqModel, self).__init__(params, mode, name)
self.source_vocab_info = None
if "vocab_source" in self.params and self.params["vocab_source"]:
self.source_vocab_info = vocab.get_vocab_info(
self.params["vocab_source"])
self.target_vocab_info = None
if "vocab_target" in self.params and self.params["vocab_target"]:
self.target_vocab_info = vocab.get_vocab_info(
self.params["vocab_target"])
self.embedding_mat_source = None
self.embedding_mat_target = None
# added for pretrain
if "embedding.file" in self.params and self.params["embedding.file"]:
self.embedding_mat_source = read_embeddings(
self.params['embedding.file'], self.source_vocab_info.path,
self.params["embedding.dim"], "source")
self.embedding_mat_target = read_embeddings(
self.params['embedding.file'], self.target_vocab_info.path,
self.params["embedding.dim"], "target")
def setUp(self):
super(EncoderDecoderTests, self).setUp()
tf.logging.set_verbosity(tf.logging.INFO)
self.batch_size = 2
self.input_depth = 4
self.sequence_length = 10
# Create vocabulary
self.vocab_list = [str(_) for _ in range(10)]
self.vocab_list += ["笑う", "泣く", "了解", "はい", "＾＿＾"]
self.vocab_size = len(self.vocab_list)
self.vocab_file = test_utils.create_temporary_vocab_file(self.vocab_list)
self.vocab_info = vocab.get_vocab_info(self.vocab_file.name)
tf.contrib.framework.get_or_create_global_step()
def test_copy_gen_model(source_path=None, target_path=None, vocab_path=None):
tf.logging.set_verbosity(tf.logging.INFO)
batch_size = 2
input_depth = 4
sequence_length = 10
if vocab_path is None:
# Create vocabulary
vocab_list = [str(_) for _ in range(10)]
vocab_list += ["笑う", "泣く", "了解", "はい", "＾＿＾"]
vocab_size = len(vocab_list)
vocab_file = test_utils.create_temporary_vocab_file(vocab_list)
vocab_info = vocab.get_vocab_info(vocab_file.name)
vocab_path = vocab_file.name
tf.logging.info(vocab_file.name)
else:
vocab_info = vocab.get_vocab_info(vocab_path)
vocab_list = get_vocab_list(vocab_path)
extend_vocab = vocab_list + ["中国", "爱", "你"]
tf.contrib.framework.get_or_create_global_step()
source_len = sequence_length + 5
target_len = sequence_length + 10
source = " ".join(np.random.choice(extend_vocab, source_len))
target = " ".join(np.random.choice(extend_vocab, target_len))
is_tmp_file = False
if source_path is None and target_path is None:
is_tmp_file = True
sources_file, targets_file = test_utils.create_temp_parallel_data(
sources=[source], targets=[target])
source_path = sources_file.name
target_path = targets_file.name
# Build model graph
mode = tf.contrib.learn.ModeKeys.TRAIN
params_ = CopyGenSeq2Seq.default_params().copy()
params_.update({
"vocab_source": vocab_path,
"vocab_target": vocab_path,
})
model = CopyGenSeq2Seq(params=params_, mode=mode)
tf.logging.info(source_path)
tf.logging.info(target_path)
input_pipeline_ = input_pipeline.ParallelTextInputPipeline(params={
"source_files": [source_path],
"target_files": [target_path]
},
mode=mode)
input_fn = training_utils.create_input_fn(pipeline=input_pipeline_,
batch_size=batch_size)
features, labels = input_fn()
fetches = model(features, labels, None)
fetches = [_ for _ in fetches if _ is not None]
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
sess.run(tf.tables_initializer())
with tf.contrib.slim.queues.QueueRunners(sess):
fetches_ = sess.run(fetches)
if is_tmp_file:
sources_file.close()
targets_file.close()
return model, fetches_
def create_experiment(output_dir):
"""
Creates a new Experiment instance.
Args:
output_dir: Output directory for model checkpoints and summaries.
"""
config = run_config.RunConfig(
tf_random_seed=FLAGS.tf_random_seed,
save_checkpoints_secs=FLAGS.save_checkpoints_secs,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
keep_checkpoint_max=FLAGS.keep_checkpoint_max,
keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours)
# Load vocabulary info
source_vocab_info = vocab.get_vocab_info(FLAGS.vocab_source)
target_vocab_info = vocab.get_vocab_info(FLAGS.vocab_target)
# Find model class
model_class = getattr(models, FLAGS.model)
# Parse parameter and merge with defaults
hparams = model_class.default_params()
if FLAGS.hparams is not None and isinstance(FLAGS.hparams, str):
hparams = HParamsParser(hparams).parse(FLAGS.hparams)
elif isinstance(FLAGS.hparams, dict):
hparams.update(FLAGS.hparams)
# Print hparams
training_utils.print_hparams(hparams)
# One the main worker, save training options and vocabulary
if config.is_chief:
# Copy vocabulary to output directory
gfile.MakeDirs(output_dir)
source_vocab_path = os.path.join(output_dir, "vocab_source")
gfile.Copy(FLAGS.vocab_source, source_vocab_path, overwrite=True)
target_vocab_path = os.path.join(output_dir, "vocab_target")
gfile.Copy(FLAGS.vocab_target, target_vocab_path, overwrite=True)
# Save train options
train_options = training_utils.TrainOptions(
hparams=hparams,
model_class=FLAGS.model,
source_vocab_path=source_vocab_path,
target_vocab_path=target_vocab_path)
train_options.dump(output_dir)
# Create model
model = model_class(source_vocab_info=source_vocab_info,
target_vocab_info=target_vocab_info,
params=hparams)
bucket_boundaries = None
if FLAGS.buckets:
bucket_boundaries = list(map(int, FLAGS.buckets.split(",")))
# Create training input function
train_input_fn = training_utils.create_input_fn(
data_provider_fn=functools.partial(
data_utils.make_parallel_data_provider,
data_sources_source=FLAGS.train_source,
data_sources_target=FLAGS.train_target,
shuffle=True,
num_epochs=FLAGS.train_epochs,
delimiter=FLAGS.delimiter),
batch_size=FLAGS.batch_size,
bucket_boundaries=bucket_boundaries)
# Create eval input function
eval_input_fn = training_utils.create_input_fn(
data_provider_fn=functools.partial(
data_utils.make_parallel_data_provider,
data_sources_source=FLAGS.dev_source,
data_sources_target=FLAGS.dev_target,
shuffle=False,
num_epochs=1,
delimiter=FLAGS.delimiter),
batch_size=FLAGS.batch_size)
def model_fn(features, labels, params, mode):
"""Builds the model graph"""
return model(features, labels, params, mode)
estimator = tf.contrib.learn.estimator.Estimator(model_fn=model_fn,
model_dir=output_dir,
config=config)
train_hooks = training_utils.create_default_training_hooks(
estimator=estimator,
sample_frequency=FLAGS.sample_every_n_steps,
delimiter=FLAGS.delimiter)
eval_metrics = {
"log_perplexity": metrics.streaming_log_perplexity(),
"bleu": metrics.make_bleu_metric_spec(),
}
experiment = tf.contrib.learn.experiment.Experiment(
estimator=estimator,
train_input_fn=train_input_fn,
eval_input_fn=eval_input_fn,
min_eval_frequency=FLAGS.eval_every_n_steps,
train_steps=FLAGS.train_steps,
eval_steps=None,
eval_metrics=eval_metrics,
train_monitors=train_hooks)
return experiment
