def test_without_extra_args(self):
pipeline_def = yaml.load("""
class: ParallelTextInputPipeline
params:
source_files: ["file1"]
target_files: ["file2"]
num_epochs: 1
shuffle: True
""")
pipeline = input_pipeline.make_input_pipeline_from_def(
pipeline_def, tf.contrib.learn.ModeKeys.TRAIN)
self.assertIsInstance(pipeline, input_pipeline.ParallelTextInputPipeline)
#pylint: disable=W0212
self.assertEqual(pipeline.params["source_files"], ["file1"])
self.assertEqual(pipeline.params["target_files"], ["file2"])
self.assertEqual(pipeline.params["num_epochs"], 1)
self.assertEqual(pipeline.params["shuffle"], True)
def test_without_extra_args(self):
pipeline_def = yaml.load("""
class: ParallelTextInputPipeline
params:
source_files: ["file1"]
target_files: ["file2"]
num_epochs: 1
shuffle: True
""")
pipeline = input_pipeline.make_input_pipeline_from_def(
pipeline_def, tf.contrib.learn.ModeKeys.TRAIN)
self.assertIsInstance(pipeline, input_pipeline.ParallelTextInputPipeline)
#pylint: disable=W0212
self.assertEqual(pipeline.params["source_files"], ["file1"])
self.assertEqual(pipeline.params["target_files"], ["file2"])
self.assertEqual(pipeline.params["num_epochs"], 1)
self.assertEqual(pipeline.params["shuffle"], True)
def everySenPre(flags_input_pipeline, model, hooks, model_dir, sess):
flags_input_pipeline = _maybe_load_yaml(flags_input_pipeline)
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
flags_input_pipeline, mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False, num_epochs=1)
tf.reset_default_graph()
# Create the graph used for inference
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=flags_batch_size)
saver = tf.train.Saver()
checkpoint_path = flags_checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(model_dir)
def session_init_op(_scaffold, sess):
saver.restore(sess, checkpoint_path)
tf.logging.info("Restored model from %s", checkpoint_path)
scaffold = tf.train.Scaffold(init_fn=session_init_op)
session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
# logging.basicConfig(filename='new.log',level=logging.DEBUG)
sess = tf.train.MonitoredSession(session_creator=session_creator, hooks=hooks)
output,result = sess.run([])
# with tf.train.MonitoredSession(
# session_creator=session_creator,
# hooks=hooks) as sess:
# # sess.run(tf.global_variables_initializer())
# # Run until the inputs are exhausted
# while True:
# outputs, result = sess.run([])
return result, sess
def foo_test_our_dataset(self):
pipeline_def = yaml.load("""
class: ParallelTextInputPipeline
params:
source_files: "/data/input_tiny.txt"
target_files: "/data/output_tiny.txt"
num_epochs: 1
shuffle: True
""")
pdb.set_trace()
pipeline = input_pipeline.make_input_pipeline_from_def(
pipeline_def, tf.contrib.learn.ModeKeys.TRAIN)
self.assertIsInstance(pipeline, input_pipeline.ParallelTextInputPipeline)
data_provider = pipeline.make_data_provider()
features = pipeline.read_from_data_provider(data_provider)
with self.test_session() as sess:
foo = sess.run(features)
with tf.contrib.slim.queues.QueueRunners(sess):
res = sess.run(features)
def create_experiment(output_dir):
"""
Creates a new Experiment instance.
Args:
output_dir: Output directory for model checkpoints and summaries.
"""
config = run_config.RunConfig(
tf_random_seed=FLAGS.tf_random_seed,
save_checkpoints_secs=FLAGS.save_checkpoints_secs,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
keep_checkpoint_max=FLAGS.keep_checkpoint_max,
keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours,
gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config.tf_config.gpu_options.allow_growth = FLAGS.gpu_allow_growth
config.tf_config.log_device_placement = FLAGS.log_device_placement
train_options = training_utils.TrainOptions(
model_class=FLAGS.model, model_params=FLAGS.model_params)
# On the main worker, save training options
if config.is_chief:
gfile.MakeDirs(output_dir)
train_options.dump(output_dir)
bucket_boundaries = None
if FLAGS.buckets:
bucket_boundaries = list(map(int, FLAGS.buckets.split(",")))
# Training data input pipeline
train_input_pipeline = input_pipeline.make_input_pipeline_from_def(
def_dict=FLAGS.input_pipeline_train,
mode=tf.contrib.learn.ModeKeys.TRAIN)
# Create training input function
train_input_fn = training_utils.create_input_fn(
pipeline=train_input_pipeline,
batch_size=FLAGS.batch_size,
bucket_boundaries=bucket_boundaries,
mode=tf.contrib.learn.ModeKeys.TRAIN)
# Development data input pipeline
dev_input_pipeline = input_pipeline.make_input_pipeline_from_def(
def_dict=FLAGS.input_pipeline_dev,
mode=tf.contrib.learn.ModeKeys.EVAL,
shuffle=False,
num_epochs=1)
# Create eval input function
eval_input_fn = training_utils.create_input_fn(
pipeline=dev_input_pipeline,
batch_size=FLAGS.batch_size,
allow_smaller_final_batch=True,
mode=tf.contrib.learn.ModeKeys.EVAL)
def model_fn(features, labels, params, mode):
"""Builds the model graph"""
model = _create_from_dict(
{
"class": train_options.model_class,
"params": train_options.model_params
},
models,
mode=mode)
return model(features, labels, params)
estimator = tf.contrib.learn.Estimator(model_fn=model_fn,
model_dir=output_dir,
config=config,
params=FLAGS.model_params)
# Create hooks
train_hooks = []
for dict_ in FLAGS.hooks:
hook = _create_from_dict(dict_,
hooks,
model_dir=estimator.model_dir,
run_config=config)
train_hooks.append(hook)
# Create metrics
eval_metrics = {}
for dict_ in FLAGS.metrics:
metric = _create_from_dict(dict_, metric_specs)
eval_metrics[metric.name] = metric
experiment = PatchedExperiment(estimator=estimator,
train_input_fn=train_input_fn,
eval_input_fn=eval_input_fn,
min_eval_frequency=FLAGS.eval_every_n_steps,
train_steps=FLAGS.train_steps,
eval_steps=None,
eval_metrics=eval_metrics,
train_monitors=train_hooks)
return experiment
for tdict in fl_tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(str(tdict["class"])) or getattr(tasks, str(
tdict["class"]))
if (str(tdict["class"]) == "DecodeText"):
task = task_cls(
tdict["params"], callback_func=_save_prediction_to_dict)
elif (str(tdict["class"]) == "DumpAttention"):
task = task_cls(tdict["params"], callback_func=_handle_attention)
hooks.append(task)
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
fl_input_pipeline,
mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False,
num_epochs=1)
# Create the graph used for inference
predictions, _, _ = create_inference_graph(
model=model, input_pipeline=input_pipeline_infer, batch_size=batch_size)
graph = tf.get_default_graph()
# Function to run inference.
def run_inference():
# tf.reset_default_graph()
with graph.as_default():
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
if isinstance(FLAGS.input_pipeline, string_types):
FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
FLAGS.input_pipeline,
mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False,
num_epochs=1)
# Create output dir
if not os.path.exists(FLAGS.model_dir + '/pred'):
os.makedirs(FLAGS.model_dir + '/pred')
# Load saved training options
train_options = training_utils.TrainOptions.load(FLAGS.model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params,
_maybe_load_yaml(FLAGS.model_params))
model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
print("******", task_cls)
task = task_cls(tdict["params"],
callback_func=_save_prediction_to_dict)
hooks.append(task)
# Create the graph used for inference
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
saver = tf.train.Saver()
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
def session_init_op(_scaffold, sess):
saver.restore(sess, checkpoint_path)
tf.logging.info("Restored model from %s", checkpoint_path)
scaffold = tf.train.Scaffold(init_fn=session_init_op)
session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
with tf.train.MonitoredSession(session_creator=session_creator,
hooks=hooks) as sess:
# Run until the inputs are exhausted
while not sess.should_stop():
sess.run([])
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
if isinstance(FLAGS.input_pipeline, string_types):
FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
FLAGS.input_pipeline,
mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False,
num_epochs=1)
# Load saved training options
train_options = training_utils.TrainOptions.load(FLAGS.model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params,
_maybe_load_yaml(FLAGS.model_params))
model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
# Create the graph used for inference
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
saver = tf.train.Saver(tf.all_variables())
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
def session_init_op(_scaffold, sess):
saver.restore(sess, checkpoint_path)
tf.logging.info("Restored model from %s", checkpoint_path)
if FLAGS.save_pb_during_infer:
save_vars = {}
for v in tf.trainable_variables():
save_vars[v.value().name] = sess.run(v)
g2 = tf.Graph()
with g2.as_default():
consts = {}
for k in save_vars.keys():
consts[k] = tf.constant(save_vars[k])
tf.import_graph_def(
sess.graph_def,
input_map={name: consts[name]
for name in consts.keys()})
tf.train.write_graph(g2.as_graph_def(), FLAGS.save_pb_dir,
'rnn.pb', False)
tf.train.write_graph(g2.as_graph_def(), FLAGS.save_pb_dir,
'rnn.txt')
tf.logging.info("Save pb down! %s", FLAGS.save_pb_dir)
scaffold = tf.train.Scaffold(init_fn=session_init_op)
session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
with tf.train.MonitoredSession(session_creator=session_creator,
hooks=hooks) as sess:
# Run until the inputs are exhausted
while not sess.should_stop():
sess.run([])
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
if isinstance(FLAGS.input_pipeline, string_types):
FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
FLAGS.input_pipeline,
mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False,
num_epochs=1)
# Load saved training options
train_options = training_utils.TrainOptions.load(FLAGS.model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params,
_maybe_load_yaml(FLAGS.model_params))
model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
# Create the graph used for inference
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
# moving_average_variables = []
#
# for var in slim.get_model_variables():
# if 'resnet_v1_50' in var.name and 'fully_connected' not in var.name:
# moving_average_variables.append(var)
#
#
# tf_global_step = slim.get_or_create_global_step()
#
# variable_averages = tf.train.ExponentialMovingAverage(
# 0.99, tf_global_step)
#
# #get variable of the models and apply average to the concerned variables
# variables_to_restore = variable_averages.variables_to_restore(
# moving_average_variables)
#
#
# #TODO
# #current checkpoint have model/att_seq2seq/model/att_seq2seq/encode/.... for the exp moving
# #instead of model/att_seq2seq/
# #so need to rename these:
#
# def name_in_checkpoint(var):
# if 'ExponentialMovingAverage' in var:
# return var.replace('model/att_seq2seq/', 'model/att_seq2seq/model/att_seq2seq/')
# return var
#
#
# variables_to_restore = {name_in_checkpoint(k): v
# for k,v in variables_to_restore.items()}
#
#
# variables_to_restore[tf_global_step.op.name] = tf_global_step
#
# for k,v in variables_to_restore.items():
# print(k)
# print(v)
# print("#############")
# import sys
# sys.exit()
# saver = tf.train.Saver(var_list=variables_to_restore)
saver = tf.train.Saver()
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
def session_init_op(_scaffold, sess):
saver.restore(sess, checkpoint_path)
tf.logging.info("Restored model from %s", checkpoint_path)
scaffold = tf.train.Scaffold(init_fn=session_init_op)
session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
with tf.train.MonitoredSession(session_creator=session_creator,
hooks=hooks) as sess:
# Run until the inputs are exhausted
while not sess.should_stop():
sess.run([])
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
if isinstance(FLAGS.input_pipeline, string_types):
FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
FLAGS.input_pipeline,
mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False,
num_epochs=1)
# Load saved training options
train_options = training_utils.TrainOptions.load(FLAGS.model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params,
_maybe_load_yaml(FLAGS.model_params))
model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
# Create the graph used for inference
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
saver = tf.train.Saver()
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
#def session_init_op(_scaffold, sess):
# saver.restore(sess, checkpoint_path)
# tf.logging.info("Restored model from %s", checkpoint_path)
#scaffold = tf.train.Scaffold(init_fn=session_init_op)
#session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
#with tf.train.MonitoredSession(
# session_creator=session_creator,
# hooks=hooks) as sess:
# # Run until the inputs are exhausted
# while not sess.should_stop():
# sess.run([])
with tf.Session() as sess:
print("start to restore checkpoint:{} into session".format(
checkpoint_path))
saver.restore(sess, checkpoint_path)
saved_model_path = os.path.join(FLAGS.model_dir, FLAGS.export_dir)
if os.path.exists(saved_model_path):
print("remove old directory:{}".format(saved_model_path))
shutil.rmtree(saved_model_path)
print("start to export SavedModel")
builder = tf.saved_model.builder.SavedModelBuilder(saved_model_path)
builder.add_meta_graph_and_variables(sess, ["fin_biseq2seq"])
builder.save()
print("finish exporting SavedModel")
def create_experiment(output_dir):
"""
Creates a new Experiment instance.
Args:
output_dir: Output directory for model checkpoints and summaries.
"""
config = run_config.RunConfig(
tf_random_seed=FLAGS.tf_random_seed,
save_checkpoints_secs=FLAGS.save_checkpoints_secs,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
keep_checkpoint_max=FLAGS.keep_checkpoint_max,
keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours,
gpu_memory_fraction=FLAGS.gpu_memory_fraction)
config.tf_config.gpu_options.allow_growth = FLAGS.gpu_allow_growth
config.tf_config.log_device_placement = FLAGS.log_device_placement
train_options = training_utils.TrainOptions(
model_class=FLAGS.model,
model_params=FLAGS.model_params)
# On the main worker, save training options
if config.is_chief:
gfile.MakeDirs(output_dir)
train_options.dump(output_dir)
bucket_boundaries = None
if FLAGS.buckets:
bucket_boundaries = list(map(int, FLAGS.buckets.split(",")))
# Training data input pipeline
train_input_pipeline = input_pipeline.make_input_pipeline_from_def(
def_dict=FLAGS.input_pipeline_train,
mode=tf.contrib.learn.ModeKeys.TRAIN)
# Create training input function
train_input_fn = training_utils.create_input_fn(
pipeline=train_input_pipeline,
batch_size=FLAGS.batch_size,
bucket_boundaries=bucket_boundaries,
scope="train_input_fn")
# Development data input pipeline
dev_input_pipeline = input_pipeline.make_input_pipeline_from_def(
def_dict=FLAGS.input_pipeline_dev,
mode=tf.contrib.learn.ModeKeys.EVAL,
shuffle=False, num_epochs=1)
# Create eval input function
eval_input_fn = training_utils.create_input_fn(
pipeline=dev_input_pipeline,
batch_size=FLAGS.batch_size,
allow_smaller_final_batch=True,
scope="dev_input_fn")
def model_fn(features, labels, params, mode):
"""Builds the model graph"""
model = _create_from_dict({
"class": train_options.model_class,
"params": train_options.model_params
}, models, mode=mode)
return model(features, labels, params)
estimator = tf.contrib.learn.Estimator(
model_fn=model_fn,
model_dir=output_dir,
config=config,
params=FLAGS.model_params)
# Create hooks
train_hooks = []
for dict_ in FLAGS.hooks:
hook = _create_from_dict(
dict_, hooks,
model_dir=estimator.model_dir,
run_config=config)
train_hooks.append(hook)
# Create metrics
eval_metrics = {}
for dict_ in FLAGS.metrics:
metric = _create_from_dict(dict_, metric_specs)
eval_metrics[metric.name] = metric
experiment = PatchedExperiment(
estimator=estimator,
train_input_fn=train_input_fn,
eval_input_fn=eval_input_fn,
min_eval_frequency=FLAGS.eval_every_n_steps,
train_steps=FLAGS.train_steps,
eval_steps=None,
eval_metrics=eval_metrics,
train_monitors=train_hooks)
return experiment
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
if isinstance(FLAGS.input_pipeline, string_types):
FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
FLAGS.input_pipeline, mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False, num_epochs=1)
# Load saved training options
train_options = training_utils.TrainOptions.load(FLAGS.model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(
model_params, _maybe_load_yaml(FLAGS.model_params))
model = model_cls(
params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
# Create the graph used for inference
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
saver = tf.train.Saver()
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
def session_init_op(_scaffold, sess):
saver.restore(sess, checkpoint_path)
tf.logging.info("Restored model from %s", checkpoint_path)
scaffold = tf.train.Scaffold(init_fn=session_init_op)
session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
with tf.train.MonitoredSession(
session_creator=session_creator,
hooks=hooks) as sess:
# Run until the inputs are exhausted
while not sess.should_stop():
sess.run([])
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
if isinstance(FLAGS.input_pipeline, string_types):
FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
if isinstance(FLAGS.model_params, string_types):
FLAGS.model_params = _maybe_load_yaml(FLAGS.model_params)
if isinstance(FLAGS.models, string_types):
FLAGS.models = _maybe_load_yaml(FLAGS.models)
for mdict in FLAGS.models:
if 'params' not in mdict:
mdict['params'] = {}
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
FLAGS.input_pipeline,
mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False,
num_epochs=1)
# ---------- Load Models First to Load Model Paramerters ----------
model_variables = []
for mdict in FLAGS.models:
# Load saved training options
train_options = training_utils.TrainOptions.load(mdict['dir'])
# Get the model class
model_cls = locate(train_options.model_class) or getattr(
model_clsses, train_options.model_class)
# Load model params
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params, mdict['params'])
model_params = _deep_merge_dict(model_params, FLAGS.model_params)
# Create model
model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Create computation graph
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
# Get path to the checkpoint
checkpoint_path = mdict[
'checkpoint_path'] if 'checkpoint_path' in mdict else tf.train.latest_checkpoint(
mdict['dir'])
# Get Saver
saver = tf.train.Saver()
# Create session to load values
with tf.Session() as sess:
# Load model values from checkpoint
saver.restore(sess, checkpoint_path)
# List all variables
variables = {}
for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES):
variables[var.name] = var.eval()
model_variables.append(variables)
# Reset graph
tf.reset_default_graph()
# Create computation graph for ensemble
models = []
vocab_source = None
vocab_target = None
for n, (mdict, variables) in enumerate(zip(FLAGS.models, model_variables)):
# Load saved training options
train_options = training_utils.TrainOptions.load(mdict['dir'])
# Get the model class
model_cls = locate(train_options.model_class) or getattr(
model_clsses, train_options.model_class)
# Load model params
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params, mdict['params'])
model_params = _deep_merge_dict(model_params, FLAGS.model_params)
# Create model
model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
models.append(model)
# Predefine variables
with tf.variable_scope('model{}'.format(n)):
for name, value in variables.items():
varname = name.split(':')[0]
tf.get_variable(varname,
shape=value.shape,
initializer=tf.constant_initializer(value))
# Create computation graph
with tf.variable_scope('model{}'.format(n), reuse=True):
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
# Get vocab informatin
if 'vocab_source' in model_params:
vocab_source = vocab_source if vocab_source else model_params[
'vocab_source']
assert vocab_source == model_params[
'vocab_source'], 'Vocab Not Match'
if 'vocab_target' in model_params:
vocab_target = vocab_target if vocab_target else model_params[
'vocab_target']
assert vocab_target == model_params[
'vocab_target'], 'Vocab Not Match'
# Fill vocab info of model_params
if vocab_source:
FLAGS.model_params['vocab_source'] = vocab_source
if vocab_target:
FLAGS.model_params['vocab_target'] = vocab_target
# Create Ensemble Models
ensemble_model = EnsembleModel(models=models, params=FLAGS.model_params)
# Create Computation Graph
predictions, _, _ = create_inference_graph(ensemble_model,
input_pipeline_infer,
FLAGS.batch_size)
# DEBUG
#for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES):
# print(var.name)
#exit();
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
with tf.train.MonitoredSession(hooks=hooks) as sess:
# Run until the inputs are exhausted
while not sess.should_stop():
sess.run([])
def create_estimator_and_specs(output_dir):
sessionConfig = tf.ConfigProto(log_device_placement=True,
allow_soft_placement=True)
sessionConfig.gpu_options.allow_growth = FLAGS.gpu_allow_growth
sessionConfig.gpu_options.per_process_gpu_memory_fraction = FLAGS.gpu_memory_fraction
config = tf.estimator.RunConfig(
tf_random_seed=FLAGS.tf_random_seed,
save_checkpoints_secs=FLAGS.save_checkpoints_secs,
save_checkpoints_steps=FLAGS.save_checkpoints_steps,
session_config=sessionConfig,
keep_checkpoint_max=FLAGS.keep_checkpoint_max,
keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours)
train_options = training_utils.TrainOptions(
model_class=FLAGS.model, model_params=FLAGS.model_params)
# On the main worker, save training options
if config.is_chief:
gfile.MakeDirs(output_dir)
train_options.dump(output_dir)
bucket_boundaries = None
if FLAGS.buckets:
bucket_boundaries = list(map(int, FLAGS.buckets.split(",")))
# Training data input pipeline
train_input_pipeline = input_pipeline.make_input_pipeline_from_def(
def_dict=FLAGS.input_pipeline_train,
mode=tf.contrib.learn.ModeKeys.TRAIN)
# Create training input function
train_input_fn = training_utils.create_input_fn(
pipeline=train_input_pipeline,
batch_size=FLAGS.batch_size,
bucket_boundaries=bucket_boundaries,
scope="train_input_fn")
# Development data input pipeline
dev_input_pipeline = input_pipeline.make_input_pipeline_from_def(
def_dict=FLAGS.input_pipeline_dev,
mode=tf.contrib.learn.ModeKeys.EVAL,
shuffle=False,
num_epochs=1)
# Create eval input function
eval_input_fn = training_utils.create_input_fn(
pipeline=dev_input_pipeline,
batch_size=FLAGS.batch_size,
allow_smaller_final_batch=True,
scope="dev_input_fn")
def model_fn(features, labels, params, mode):
"""Builds the model graph"""
model = _create_from_dict(
{
"class": train_options.model_class,
"params": train_options.model_params
},
models,
mode=mode)
(predictions, loss, train_op) = model(features, labels, params)
# Create metrics
eval_metrics = {}
for dict_ in FLAGS.metrics:
metric = _create_from_dict(dict_, metric_specs)
eval_metrics[metric.name] = metric(features, labels, predictions)
return tf.estimator.EstimatorSpec(mode=mode,
predictions=predictions,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metrics)
estimator = tf.estimator.Estimator(model_fn=model_fn,
model_dir=output_dir,
config=config,
params=FLAGS.model_params)
# Create hooks
train_hooks = []
for dict_ in FLAGS.hooks:
hook = _create_from_dict(dict_,
hooks,
model_dir=estimator.model_dir,
run_config=config)
train_hooks.append(hook)
train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn,
max_steps=FLAGS.train_steps,
hooks=train_hooks)
eval_spec = tf.estimator.EvalSpec(input_fn=eval_input_fn)
return (estimator, train_spec, eval_spec)
