def main(_argv):
"""The entrypoint for the script"""
# Parse YAML FLAGS
FLAGS.hooks = _maybe_load_yaml(FLAGS.hooks)
FLAGS.metrics = _maybe_load_yaml(FLAGS.metrics)
FLAGS.model_params = _maybe_load_yaml(FLAGS.model_params)
FLAGS.input_pipeline_train = _maybe_load_yaml(FLAGS.input_pipeline_train)
FLAGS.input_pipeline_dev = _maybe_load_yaml(FLAGS.input_pipeline_dev)
# Load flags from config file
final_config = {}
if FLAGS.config_paths:
for config_path in FLAGS.config_paths.split(","):
config_path = config_path.strip()
if not config_path:
continue
config_path = os.path.abspath(config_path)
tf.logging.info("Loading config from %s", config_path)
with gfile.GFile(config_path.strip()) as config_file:
config_flags = yaml.load(config_file)
final_config = _deep_merge_dict(
final_config, config_flags
) ###merge the flags and values from all the files into a dict
tf.logging.info("Final Config:\n%s", yaml.dump(
final_config)) ###print the flags and values read from all the files
# Merge flags with config values
for flag_key, flag_value in final_config.items(
): ###map the flags and values to FLAGS in the code
if hasattr(FLAGS, flag_key) and isinstance(getattr(FLAGS, flag_key),
dict):
merged_value = _deep_merge_dict(
flag_value, getattr(FLAGS, flag_key)
) ###merge the values has been defined and the new values from the config files
setattr(FLAGS, flag_key, merged_value)
elif hasattr(FLAGS, flag_key):
setattr(FLAGS, flag_key, flag_value)
else:
tf.logging.warning("Ignoring config flag: %s", flag_key)
if FLAGS.save_checkpoints_secs is None \
and FLAGS.save_checkpoints_steps is None:
FLAGS.save_checkpoints_secs = 600
tf.logging.info("Setting save_checkpoints_secs to %d",
FLAGS.save_checkpoints_secs)
if not FLAGS.output_dir:
FLAGS.output_dir = tempfile.mkdtemp() ###creat temporary files
if not FLAGS.input_pipeline_train:
raise ValueError("You must specify input_pipeline_train")
if not FLAGS.input_pipeline_dev:
raise ValueError("You must specify input_pipeline_dev")
learn_runner.run(experiment_fn=create_experiment,
output_dir=FLAGS.output_dir,
schedule=FLAGS.schedule)
def main(_argv):
"""The entrypoint for the script"""
# Parse YAML FLAGS
FLAGS.hooks = _maybe_load_yaml(FLAGS.hooks)
FLAGS.metrics = _maybe_load_yaml(FLAGS.metrics)
FLAGS.model_params = _maybe_load_yaml(FLAGS.model_params)
FLAGS.input_pipeline_train = _maybe_load_yaml(FLAGS.input_pipeline_train)
FLAGS.input_pipeline_dev = _maybe_load_yaml(FLAGS.input_pipeline_dev)
# Load flags from config file
final_config = {}
if FLAGS.config_paths:
for config_path in FLAGS.config_paths.split(","):
config_path = config_path.strip()
if not config_path:
continue
config_path = os.path.abspath(config_path)
tf.logging.info("Loading config from %s", config_path)
with gfile.GFile(config_path.strip()) as config_file:
config_flags = yaml.load(config_file)
final_config = _deep_merge_dict(final_config, config_flags)
tf.logging.info("Final Config:\n%s", yaml.dump(final_config))
# Merge flags with config values
for flag_key, flag_value in final_config.items():
if hasattr(FLAGS, flag_key) and isinstance(getattr(FLAGS, flag_key),
dict):
merged_value = _deep_merge_dict(flag_value,
getattr(FLAGS, flag_key))
setattr(FLAGS, flag_key, merged_value)
elif hasattr(FLAGS, flag_key):
setattr(FLAGS, flag_key, flag_value)
else:
tf.logging.warning("Ignoring config flag: %s", flag_key)
if FLAGS.save_checkpoints_secs is None \
and FLAGS.save_checkpoints_steps is None:
FLAGS.save_checkpoints_secs = 600
tf.logging.info("Setting save_checkpoints_secs to %d",
FLAGS.save_checkpoints_secs)
if not FLAGS.output_dir:
FLAGS.output_dir = tempfile.mkdtemp()
if not FLAGS.input_pipeline_train:
raise ValueError("You must specify input_pipeline_train")
if not FLAGS.input_pipeline_dev:
raise ValueError("You must specify input_pipeline_dev")
(estimator, train_spec,
eval_spec) = create_estimator_and_specs(output_dir=FLAGS.output_dir)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
def main(_argv):
"""The entrypoint for the script"""
# Parse YAML FLAGS
FLAGS.hooks = _maybe_load_yaml(FLAGS.hooks)
FLAGS.metrics = _maybe_load_yaml(FLAGS.metrics)
FLAGS.model_params = _maybe_load_yaml(FLAGS.model_params)
FLAGS.input_pipeline_train = _maybe_load_yaml(FLAGS.input_pipeline_train)
FLAGS.input_pipeline_dev = _maybe_load_yaml(FLAGS.input_pipeline_dev)
# Load flags from config file
final_config = {}
if FLAGS.config_paths:
for config_path in FLAGS.config_paths.split(","):
config_path = config_path.strip()
if not config_path:
continue
config_path = os.path.abspath(config_path)
tf.logging.info("Loading config from %s", config_path)
with gfile.GFile(config_path.strip()) as config_file:
config_flags = yaml.load(config_file)
final_config = _deep_merge_dict(final_config, config_flags)
tf.logging.info("Final Config:\n%s", yaml.dump(final_config))
# Merge flags with config values
for flag_key, flag_value in final_config.items():
if hasattr(FLAGS, flag_key) and isinstance(getattr(FLAGS, flag_key), dict):
merged_value = _deep_merge_dict(flag_value, getattr(FLAGS, flag_key))
setattr(FLAGS, flag_key, merged_value)
elif hasattr(FLAGS, flag_key):
setattr(FLAGS, flag_key, flag_value)
else:
tf.logging.warning("Ignoring config flag: %s", flag_key)
if FLAGS.save_checkpoints_secs is None \
and FLAGS.save_checkpoints_steps is None:
FLAGS.save_checkpoints_secs = 600
tf.logging.info("Setting save_checkpoints_secs to %d",
FLAGS.save_checkpoints_secs)
if not FLAGS.output_dir:
FLAGS.output_dir = tempfile.mkdtemp()
if not FLAGS.input_pipeline_train:
raise ValueError("You must specify input_pipeline_train")
if not FLAGS.input_pipeline_dev:
raise ValueError("You must specify input_pipeline_dev")
learn_runner.run(
experiment_fn=create_experiment,
output_dir=FLAGS.output_dir,
schedule=FLAGS.schedule)
def infer(flags_tasks, model_dir, flags_model_params):
flags_tasks = _maybe_load_yaml(flags_tasks)
# Load saved training options
train_options = training_utils.TrainOptions.load(model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(
model_params, _maybe_load_yaml(flags_model_params))
model = model_cls(
params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
for tdict in flags_tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
return model, hooks
def main(_argv):
"""The entrypoint for the script"""
# Parse YAML FLAGS
FLAGS.hooks = _maybe_load_yaml(FLAGS.hooks)
FLAGS.metrics = _maybe_load_yaml(FLAGS.metrics)
FLAGS.model_params = _maybe_load_yaml(FLAGS.model_params)
FLAGS.input_pipeline_train = _maybe_load_yaml(FLAGS.input_pipeline_train)
FLAGS.input_pipeline_dev = _maybe_load_yaml(FLAGS.input_pipeline_dev)
# Load flags from config file
final_config = {}
if FLAGS.config_paths:
for config_path in FLAGS.config_paths.split(","):
config_path = config_path.strip()
if not config_path:
continue
config_path = os.path.abspath(config_path)
tf.logging.info("Loading config from %s", config_path)
with gfile.GFile(config_path.strip()) as config_file:
config_flags = yaml.load(config_file)
final_config = _deep_merge_dict(final_config, config_flags)
tf.logging.info("Final Config:\n%s", yaml.dump(final_config))
# Merge flags with config values
for flag_key, flag_value in final_config.items():
if hasattr(FLAGS, flag_key) and isinstance(getattr(FLAGS, flag_key), dict):
merged_value = _deep_merge_dict(flag_value, getattr(FLAGS, flag_key))
setattr(FLAGS, flag_key, merged_value)
else:
setattr(FLAGS, flag_key, flag_value)
if FLAGS.save_checkpoints_secs is None \
and FLAGS.save_checkpoints_steps is None:
FLAGS.save_checkpoints_secs = 600
tf.logging.info("Setting save_checkpoints_secs to %d",
FLAGS.save_checkpoints_secs)
if not FLAGS.output_dir:
FLAGS.output_dir = tempfile.mkdtemp()
learn_runner.run(
experiment_fn=create_experiment,
output_dir=FLAGS.output_dir,
schedule=FLAGS.schedule)
def __init__(self, _argv):
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
# Load saved training options
train_options = training_utils.TrainOptions.load(FLAGS.model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params,
_maybe_load_yaml(FLAGS.model_params))
self.model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
# Create the graph used for inference
self.input_data = tf.placeholder(tf.string)
self.tokens = tf.string_split(self.input_data).values
self.length = tf.size(self.tokens)
self.features = {
'source_tokens': tf.expand_dims(self.tokens, 0),
'source_len': tf.expand_dims(self.length, 0)
}
self.predictions, _, _ = self.model(features=self.features,
labels=None,
params=None)
saver = tf.train.Saver()
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
def session_init_op(_scaffold, sess):
saver.restore(sess, checkpoint_path)
tf.logging.info("Restored model from %s", checkpoint_path)
scaffold = tf.train.Scaffold(init_fn=session_init_op)
session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
self.sess = tf.train.MonitoredSession(session_creator=session_creator,
hooks=hooks)
print('Model initialized')
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
# if isinstance(FLAGS.input_pipeline, string_types):
# FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
# input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
# FLAGS.input_pipeline, mode=tf.contrib.learn.ModeKeys.INFER,
# shuffle=False, num_epochs=1)
# Load saved training options
train_options = training_utils.TrainOptions.load(FLAGS.model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(
model_params, _maybe_load_yaml(FLAGS.model_params))
model = model_cls(
params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
print (FLAGS.tasks)
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
# Create the graph used for inference
input_data = tf.placeholder(tf.string)
tokens = tf.string_split(input_data).values
length = tf.size(tokens)
# features = {'source_tokens': tf.expand_dims(tokens, 0),
# 'source_len': tf.expand_dims(length, 0)}
features = {'source_tokens': tf.stack([tokens, tokens]),
'source_len': tf.stack([length, length])}
predictions, _, _ = model(features=features, labels=None, params=None)
# predictions, _, _ = create_inference_graph(
# model=model,
# input_pipeline=input_pipeline_infer,
# batch_size=FLAGS.batch_size)
saver = tf.train.Saver()
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
def session_init_op(_scaffold, sess):
saver.restore(sess, checkpoint_path)
tf.logging.info("Restored model from %s", checkpoint_path)
scaffold = tf.train.Scaffold(init_fn=session_init_op)
session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
with tf.train.MonitoredSession(
session_creator=session_creator,
hooks=hooks) as sess:
# Run until the inputs are exhausted
# while not sess.should_stop():
# sess.run([])
output = sess.run([predictions[u'predicted_tokens']], feed_dict={input_data: [u'15 11 5 15 0 2 2 15 15 0 7 10 4 4 12 12 16 SEQUENCE_END']})
print (output[0][0])
print (u' '.join(output[0][0][:-1]))
return s
fl_tasks = _maybe_load_yaml(str(input_task_list))
fl_input_pipeline = _maybe_load_yaml(str(input_pipeline_dict))
# Load saved training options
train_options = training_utils.TrainOptions.load(model_dir_input)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
if (ARGS.beam_width != 1):
model_params["inference.beam_search.beam_width"] = ARGS.beam_width
model_params = _deep_merge_dict(model_params, _maybe_load_yaml(model_params))
model = model_cls(params=model_params, mode=tf.contrib.learn.ModeKeys.INFER)
print("========model params ==========", model_params)
def _handle_attention(attention_scores):
print(">>> Saved attention scores")
def _save_prediction_to_dict(output_string):
# http://bcho.tistory.com/1173
global decoded_string
decoded_string = output_string
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
if isinstance(FLAGS.input_pipeline, string_types):
FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
FLAGS.input_pipeline,
mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False,
num_epochs=1)
# Create output dir
if not os.path.exists(FLAGS.model_dir + '/pred'):
os.makedirs(FLAGS.model_dir + '/pred')
# Load saved training options
train_options = training_utils.TrainOptions.load(FLAGS.model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params,
_maybe_load_yaml(FLAGS.model_params))
model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
print("******", task_cls)
task = task_cls(tdict["params"],
callback_func=_save_prediction_to_dict)
hooks.append(task)
# Create the graph used for inference
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
saver = tf.train.Saver()
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
def session_init_op(_scaffold, sess):
saver.restore(sess, checkpoint_path)
tf.logging.info("Restored model from %s", checkpoint_path)
scaffold = tf.train.Scaffold(init_fn=session_init_op)
session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
with tf.train.MonitoredSession(session_creator=session_creator,
hooks=hooks) as sess:
# Run until the inputs are exhausted
while not sess.should_stop():
sess.run([])
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
if isinstance(FLAGS.input_pipeline, string_types):
FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
if isinstance(FLAGS.model_params, string_types):
FLAGS.model_params = _maybe_load_yaml(FLAGS.model_params)
if isinstance(FLAGS.models, string_types):
FLAGS.models = _maybe_load_yaml(FLAGS.models)
for mdict in FLAGS.models:
if 'params' not in mdict:
mdict['params'] = {}
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
FLAGS.input_pipeline,
mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False,
num_epochs=1)
# ---------- Load Models First to Load Model Paramerters ----------
model_variables = []
for mdict in FLAGS.models:
# Load saved training options
train_options = training_utils.TrainOptions.load(mdict['dir'])
# Get the model class
model_cls = locate(train_options.model_class) or getattr(
model_clsses, train_options.model_class)
# Load model params
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params, mdict['params'])
model_params = _deep_merge_dict(model_params, FLAGS.model_params)
# Create model
model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Create computation graph
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
# Get path to the checkpoint
checkpoint_path = mdict[
'checkpoint_path'] if 'checkpoint_path' in mdict else tf.train.latest_checkpoint(
mdict['dir'])
# Get Saver
saver = tf.train.Saver()
# Create session to load values
with tf.Session() as sess:
# Load model values from checkpoint
saver.restore(sess, checkpoint_path)
# List all variables
variables = {}
for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES):
variables[var.name] = var.eval()
model_variables.append(variables)
# Reset graph
tf.reset_default_graph()
# Create computation graph for ensemble
models = []
vocab_source = None
vocab_target = None
for n, (mdict, variables) in enumerate(zip(FLAGS.models, model_variables)):
# Load saved training options
train_options = training_utils.TrainOptions.load(mdict['dir'])
# Get the model class
model_cls = locate(train_options.model_class) or getattr(
model_clsses, train_options.model_class)
# Load model params
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params, mdict['params'])
model_params = _deep_merge_dict(model_params, FLAGS.model_params)
# Create model
model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
models.append(model)
# Predefine variables
with tf.variable_scope('model{}'.format(n)):
for name, value in variables.items():
varname = name.split(':')[0]
tf.get_variable(varname,
shape=value.shape,
initializer=tf.constant_initializer(value))
# Create computation graph
with tf.variable_scope('model{}'.format(n), reuse=True):
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
# Get vocab informatin
if 'vocab_source' in model_params:
vocab_source = vocab_source if vocab_source else model_params[
'vocab_source']
assert vocab_source == model_params[
'vocab_source'], 'Vocab Not Match'
if 'vocab_target' in model_params:
vocab_target = vocab_target if vocab_target else model_params[
'vocab_target']
assert vocab_target == model_params[
'vocab_target'], 'Vocab Not Match'
# Fill vocab info of model_params
if vocab_source:
FLAGS.model_params['vocab_source'] = vocab_source
if vocab_target:
FLAGS.model_params['vocab_target'] = vocab_target
# Create Ensemble Models
ensemble_model = EnsembleModel(models=models, params=FLAGS.model_params)
# Create Computation Graph
predictions, _, _ = create_inference_graph(ensemble_model,
input_pipeline_infer,
FLAGS.batch_size)
# DEBUG
#for var in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES):
# print(var.name)
#exit();
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
with tf.train.MonitoredSession(hooks=hooks) as sess:
# Run until the inputs are exhausted
while not sess.should_stop():
sess.run([])
def main(_argv):
"""The entrypoint for the script"""
# Parse YAML FLAGS
FLAGS.hooks = _maybe_load_yaml(FLAGS.hooks)
FLAGS.metrics = _maybe_load_yaml(FLAGS.metrics)
FLAGS.model_params = _maybe_load_yaml(FLAGS.model_params)
FLAGS.input_pipeline_train = _maybe_load_yaml(FLAGS.input_pipeline_train)
FLAGS.input_pipeline_dev = _maybe_load_yaml(FLAGS.input_pipeline_dev)
# Load flags from config file
final_config = {}
if FLAGS.config_paths:
for config_path in FLAGS.config_paths.split(","):
config_path = config_path.strip()
if not config_path:
continue
config_path = os.path.abspath(config_path)
tf.logging.info("Loading config from %s", config_path)
with gfile.GFile(config_path.strip()) as config_file:
config_flags = yaml.load(config_file)
final_config = _deep_merge_dict(final_config, config_flags)
tf.logging.info("Flags from file:\n%s", yaml.dump(final_config))
# Merge flags with config values
for flag_key, flag_value in final_config.items():
if hasattr(FLAGS, flag_key) and isinstance(getattr(FLAGS, flag_key),
dict):
merged_value = _deep_merge_dict(flag_value,
getattr(FLAGS, flag_key))
setattr(FLAGS, flag_key, merged_value)
elif hasattr(FLAGS, flag_key):
setattr(FLAGS, flag_key, flag_value)
else:
tf.logging.warning("Ignoring config flag: %s", flag_key)
if FLAGS.save_checkpoints_secs is None \
and FLAGS.save_checkpoints_steps is None:
FLAGS.save_checkpoints_secs = 600
tf.logging.info("Setting save_checkpoints_secs to %d",
FLAGS.save_checkpoints_secs)
if not FLAGS.output_dir:
FLAGS.output_dir = tempfile.mkdtemp()
if not FLAGS.input_pipeline_train:
raise ValueError("You must specify input_pipeline_train")
if not FLAGS.input_pipeline_dev:
raise ValueError("You must specify input_pipeline_dev")
###expand source_files and dev_files
train_source_files = FLAGS.input_pipeline_train["params"]["source_files"]
train_target_files = FLAGS.input_pipeline_train["params"]["target_files"]
FLAGS.input_pipeline_train["params"]["source_files"] = expand_paths(
train_source_files)
FLAGS.input_pipeline_train["params"]["target_files"] = expand_paths(
train_target_files)
dev_source_files = FLAGS.input_pipeline_dev["params"]["source_files"]
dev_target_files = FLAGS.input_pipeline_dev["params"]["target_files"]
FLAGS.input_pipeline_dev["params"]["source_files"] = expand_paths(
dev_source_files)
FLAGS.input_pipeline_dev["params"]["target_files"] = expand_paths(
dev_target_files)
tf.logging.info("now flags:")
tf.logging.info(FLAGS.__dict__["__flags"])
##get and change schedule here
config = get_run_config()
schedule = get_distributed_schedule(config)
setattr(FLAGS, "schedule", schedule)
tf.logging.warn("{} 's schdule: {}".format(config.master, schedule))
tf.logging.warn("flags:", FLAGS.__dict__["__flags"])
if FLAGS.clear_output_dir is True:
if os.path.exists(FLAGS.output_dir) is True:
shutil.rmtree(FLAGS.output_dir)
tf.logging.debug("rm output dir:{}".format(FLAGS.output_dir))
learn_runner.run(experiment_fn=create_experiment,
output_dir=FLAGS.output_dir,
schedule=FLAGS.schedule)
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
if isinstance(FLAGS.input_pipeline, string_types):
FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
FLAGS.input_pipeline,
mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False,
num_epochs=1)
# Load saved training options
train_options = training_utils.TrainOptions.load(FLAGS.model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params,
_maybe_load_yaml(FLAGS.model_params))
model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
# Create the graph used for inference
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
saver = tf.train.Saver(tf.all_variables())
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
def session_init_op(_scaffold, sess):
saver.restore(sess, checkpoint_path)
tf.logging.info("Restored model from %s", checkpoint_path)
if FLAGS.save_pb_during_infer:
save_vars = {}
for v in tf.trainable_variables():
save_vars[v.value().name] = sess.run(v)
g2 = tf.Graph()
with g2.as_default():
consts = {}
for k in save_vars.keys():
consts[k] = tf.constant(save_vars[k])
tf.import_graph_def(
sess.graph_def,
input_map={name: consts[name]
for name in consts.keys()})
tf.train.write_graph(g2.as_graph_def(), FLAGS.save_pb_dir,
'rnn.pb', False)
tf.train.write_graph(g2.as_graph_def(), FLAGS.save_pb_dir,
'rnn.txt')
tf.logging.info("Save pb down! %s", FLAGS.save_pb_dir)
scaffold = tf.train.Scaffold(init_fn=session_init_op)
session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
with tf.train.MonitoredSession(session_creator=session_creator,
hooks=hooks) as sess:
# Run until the inputs are exhausted
while not sess.should_stop():
sess.run([])
def main(_argv):
"""The entrypoint for the script"""
global FLAGS
if conf.IN_LOCAL_ENV:
# Parse YAML FLAGS
FLAGS.hooks = _maybe_load_yaml(FLAGS.hooks)
FLAGS.metrics = _maybe_load_yaml(FLAGS.metrics)
FLAGS.model_params = _maybe_load_yaml(FLAGS.model_params)
FLAGS.input_pipeline_train = _maybe_load_yaml(
FLAGS.input_pipeline_train)
FLAGS.input_pipeline_dev = _maybe_load_yaml(FLAGS.input_pipeline_dev)
# Load flags from config file
final_config = {}
if FLAGS.config_paths:
for config_path in FLAGS.config_paths.split(","):
config_path = config_path.strip()
if not config_path:
continue
config_path = os.path.abspath(config_path)
tf.logging.info("Loading config from %s", config_path)
with gfile.GFile(config_path.strip()) as config_file:
config_flags = yaml.load(config_file)
final_config = _deep_merge_dict(final_config, config_flags)
tf.logging.info("Final Config:\n%s", yaml.dump(final_config))
# Merge flags with config values
for flag_key, flag_value in final_config.items():
if hasattr(FLAGS, flag_key) and isinstance(
getattr(FLAGS, flag_key), dict):
merged_value = _deep_merge_dict(flag_value,
getattr(FLAGS, flag_key))
setattr(FLAGS, flag_key, merged_value)
elif hasattr(FLAGS, flag_key):
setattr(FLAGS, flag_key, flag_value)
else:
tf.logging.warning("Ignoring config flag: %s", flag_key)
if FLAGS.save_checkpoints_secs is None \
and FLAGS.save_checkpoints_steps is None:
FLAGS.save_checkpoints_secs = 600
tf.logging.info("Setting save_checkpoints_secs to %d",
FLAGS.save_checkpoints_secs)
#save FLAGS in json
#FLAGS._parse_flags()
print("-----------------")
print(FLAGS.__dict__)
pickle.dump(FLAGS.__dict__['__flags'], open("train_flags.pkl", "wb"))
else:
saved = pickle.load(open("train_flags.pkl", "rb"))
for k, v in saved.items():
if k == "model_output_path":
print("ignore model_output_path")
continue
setattr(FLAGS, k, v)
print("-----------------")
print(FLAGS.__dict__)
if not FLAGS.output_dir:
FLAGS.output_dir = tempfile.mkdtemp()
if not FLAGS.input_pipeline_train:
raise ValueError("You must specify input_pipeline_train")
if not FLAGS.input_pipeline_dev:
raise ValueError("You must specify input_pipeline_dev")
learn_runner.run(experiment_fn=create_experiment,
output_dir=FLAGS.output_dir,
schedule=FLAGS.schedule)
if not conf.IN_LOCAL_ENV:
if not hdfs.path.exists(FLAGS.model_output_path):
hdfs.mkdir(FLAGS.model_output_path)
print("make hdfs dir:{}".format(FLAGS.model_output_path))
print("dir:{} exists:{}".format(
FLAGS.model_output_path,
hdfs.path.exists(FLAGS.model_output_path)))
print("copying files from {} to hdfs:{}".format(
FLAGS.output_dir, FLAGS.model_output_path))
output_files = os.listdir(FLAGS.output_dir)
for f in output_files:
if f in COPY_FILE_LIST:
print("copy file:{} to hdfs:{}".format(
f, FLAGS.model_output_path))
pydoop.hdfs.put(os.path.join(FLAGS.output_dir, f),
FLAGS.model_output_path)
else:
for p in COPY_PATTERN_LIST:
if re.match(p, f):
print("copy file:{} to hdfs:{}".format(
f, FLAGS.model_output_path))
pydoop.hdfs.put(os.path.join(FLAGS.output_dir, f),
FLAGS.model_output_path)
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
if isinstance(FLAGS.input_pipeline, string_types):
FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
FLAGS.input_pipeline,
mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False,
num_epochs=1)
# Load saved training options
train_options = training_utils.TrainOptions.load(FLAGS.model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params,
_maybe_load_yaml(FLAGS.model_params))
model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
# Create the graph used for inference
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
# moving_average_variables = []
#
# for var in slim.get_model_variables():
# if 'resnet_v1_50' in var.name and 'fully_connected' not in var.name:
# moving_average_variables.append(var)
#
#
# tf_global_step = slim.get_or_create_global_step()
#
# variable_averages = tf.train.ExponentialMovingAverage(
# 0.99, tf_global_step)
#
# #get variable of the models and apply average to the concerned variables
# variables_to_restore = variable_averages.variables_to_restore(
# moving_average_variables)
#
#
# #TODO
# #current checkpoint have model/att_seq2seq/model/att_seq2seq/encode/.... for the exp moving
# #instead of model/att_seq2seq/
# #so need to rename these:
#
# def name_in_checkpoint(var):
# if 'ExponentialMovingAverage' in var:
# return var.replace('model/att_seq2seq/', 'model/att_seq2seq/model/att_seq2seq/')
# return var
#
#
# variables_to_restore = {name_in_checkpoint(k): v
# for k,v in variables_to_restore.items()}
#
#
# variables_to_restore[tf_global_step.op.name] = tf_global_step
#
# for k,v in variables_to_restore.items():
# print(k)
# print(v)
# print("#############")
# import sys
# sys.exit()
# saver = tf.train.Saver(var_list=variables_to_restore)
saver = tf.train.Saver()
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
def session_init_op(_scaffold, sess):
saver.restore(sess, checkpoint_path)
tf.logging.info("Restored model from %s", checkpoint_path)
scaffold = tf.train.Scaffold(init_fn=session_init_op)
session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
with tf.train.MonitoredSession(session_creator=session_creator,
hooks=hooks) as sess:
# Run until the inputs are exhausted
while not sess.should_stop():
sess.run([])
def main(_argv):
"""The entrypoint for the script"""
if not FLAGS.output_dir:
FLAGS.output_dir = tempfile.mkdtemp()
add_file_logger(FLAGS.output_dir)
_dump_flags()
# Parse YAML FLAGS
FLAGS.hooks = _maybe_load_yaml(FLAGS.hooks)
FLAGS.metrics = _maybe_load_yaml(FLAGS.metrics)
FLAGS.model_params = _maybe_load_yaml(FLAGS.model_params)
FLAGS.input_pipeline_train = _maybe_load_yaml(FLAGS.input_pipeline_train)
FLAGS.input_pipeline_dev = _maybe_load_yaml(FLAGS.input_pipeline_dev)
# Load flags from config file
final_config = {}
if FLAGS.config_paths:
for config_path in FLAGS.config_paths.split(","):
config_path = config_path.strip()
if not config_path:
continue
config_path = os.path.abspath(config_path)
tf.logging.info("Loading config from %s", config_path)
with gfile.GFile(config_path.strip()) as config_file:
config_flags = yaml.load(config_file)
# Update config_flags
if FLAGS.learning_rate:
if "model_params" in config_flags:
config_flags["model_params"].update(
{'optimizer.learning_rate': FLAGS.learning_rate})
if FLAGS.source_embedding:
if "model_params" in config_flags:
config_flags["model_params"].update({
'embedding.source_embedding':
FLAGS.source_embedding
})
if FLAGS.target_embedding:
if "model_params" in config_flags:
config_flags["model_params"].update({
'embedding.target_embedding':
FLAGS.target_embedding
})
"""
if FLAGS.optimizer_name:
if "model_params" in config_flags:
config_flags["model_params"].update({'optimizer.name': FLAGS.learning_rate})"""
final_config = _deep_merge_dict(final_config, config_flags)
tf.logging.info("Final Config:\n%s", yaml.dump(final_config))
# Merge flags with config values
for flag_key, flag_value in final_config.items():
if hasattr(FLAGS, flag_key) and isinstance(getattr(FLAGS, flag_key),
dict):
merged_value = _deep_merge_dict(flag_value,
getattr(FLAGS, flag_key))
setattr(FLAGS, flag_key, merged_value)
elif hasattr(FLAGS, flag_key):
setattr(FLAGS, flag_key, flag_value)
else:
tf.logging.warning("Ignoring config flag: %s", flag_key)
if FLAGS.save_checkpoints_secs is None \
and FLAGS.save_checkpoints_steps is None:
FLAGS.save_checkpoints_secs = 600
tf.logging.info("Setting save_checkpoints_secs to %d",
FLAGS.save_checkpoints_secs)
if not FLAGS.input_pipeline_train:
raise ValueError("You must specify input_pipeline_train")
if not FLAGS.input_pipeline_dev:
raise ValueError("You must specify input_pipeline_dev")
learn_runner.run(experiment_fn=create_experiment,
output_dir=FLAGS.output_dir,
schedule=FLAGS.schedule)
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
if isinstance(FLAGS.input_pipeline, string_types):
FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
FLAGS.input_pipeline, mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False, num_epochs=1)
# Load saved training options
train_options = training_utils.TrainOptions.load(FLAGS.model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(
model_params, _maybe_load_yaml(FLAGS.model_params))
model = model_cls(
params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
# Create the graph used for inference
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
saver = tf.train.Saver()
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
def session_init_op(_scaffold, sess):
saver.restore(sess, checkpoint_path)
tf.logging.info("Restored model from %s", checkpoint_path)
scaffold = tf.train.Scaffold(init_fn=session_init_op)
session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
with tf.train.MonitoredSession(
session_creator=session_creator,
hooks=hooks) as sess:
# Run until the inputs are exhausted
while not sess.should_stop():
sess.run([])
def main(_argv):
"""Program entry point.
"""
# Load flags from config file
if FLAGS.config_path:
with gfile.GFile(FLAGS.config_path) as config_file:
config_flags = yaml.load(config_file)
for flag_key, flag_value in config_flags.items():
setattr(FLAGS, flag_key, flag_value)
if isinstance(FLAGS.tasks, string_types):
FLAGS.tasks = _maybe_load_yaml(FLAGS.tasks)
if isinstance(FLAGS.input_pipeline, string_types):
FLAGS.input_pipeline = _maybe_load_yaml(FLAGS.input_pipeline)
input_pipeline_infer = input_pipeline.make_input_pipeline_from_def(
FLAGS.input_pipeline,
mode=tf.contrib.learn.ModeKeys.INFER,
shuffle=False,
num_epochs=1)
# Load saved training options
train_options = training_utils.TrainOptions.load(FLAGS.model_dir)
# Create the model
model_cls = locate(train_options.model_class) or \
getattr(models, train_options.model_class)
model_params = train_options.model_params
model_params = _deep_merge_dict(model_params,
_maybe_load_yaml(FLAGS.model_params))
model = model_cls(params=model_params,
mode=tf.contrib.learn.ModeKeys.INFER)
# Load inference tasks
hooks = []
for tdict in FLAGS.tasks:
if not "params" in tdict:
tdict["params"] = {}
task_cls = locate(tdict["class"]) or getattr(tasks, tdict["class"])
task = task_cls(tdict["params"])
hooks.append(task)
# Create the graph used for inference
predictions, _, _ = create_inference_graph(
model=model,
input_pipeline=input_pipeline_infer,
batch_size=FLAGS.batch_size)
saver = tf.train.Saver()
checkpoint_path = FLAGS.checkpoint_path
if not checkpoint_path:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
#def session_init_op(_scaffold, sess):
# saver.restore(sess, checkpoint_path)
# tf.logging.info("Restored model from %s", checkpoint_path)
#scaffold = tf.train.Scaffold(init_fn=session_init_op)
#session_creator = tf.train.ChiefSessionCreator(scaffold=scaffold)
#with tf.train.MonitoredSession(
# session_creator=session_creator,
# hooks=hooks) as sess:
# # Run until the inputs are exhausted
# while not sess.should_stop():
# sess.run([])
with tf.Session() as sess:
print("start to restore checkpoint:{} into session".format(
checkpoint_path))
saver.restore(sess, checkpoint_path)
saved_model_path = os.path.join(FLAGS.model_dir, FLAGS.export_dir)
if os.path.exists(saved_model_path):
print("remove old directory:{}".format(saved_model_path))
shutil.rmtree(saved_model_path)
print("start to export SavedModel")
builder = tf.saved_model.builder.SavedModelBuilder(saved_model_path)
builder.add_meta_graph_and_variables(sess, ["fin_biseq2seq"])
builder.save()
print("finish exporting SavedModel")