def load(self):
run_config = t2t_trainer.create_run_config(self.hp)
self.hp.add_hparam("model_dir", run_config.model_dir)
self.estimator = trainer_lib.create_estimator(
self.model,
self.hp,
run_config,
decode_hparams=self.decode_hp,
use_tpu=self.use_tpu)
self.estimator_predictor = tf.contrib.predictor.from_estimator(
self.estimator,
self.input_fn,
config=tf.ConfigProto(log_device_placement=True,
allow_soft_placement=True))
FLAGS.problem = "translate_enfr_wmt32k_rev"
self.problem = "translate_enfr_wmt32k_rev"
self.problem_name = self.problem
FLAGS.checkpoint_path = os.path.join(
os.getcwd(), "checkpoints/fren/model.ckpt-500000")
run_config = t2t_trainer.create_run_config(self.hp)
self.hp.model_dir = run_config.model_dir
self.estimator = trainer_lib.create_estimator(
self.model,
self.hp,
run_config,
decode_hparams=self.decode_hp,
use_tpu=self.use_tpu)
self.estimator_decoder_predictor = tf.contrib.predictor.from_estimator(
self.estimator,
self.input_fn,
config=tf.ConfigProto(log_device_placement=True,
allow_soft_placement=True))
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
hparams = trainer_lib.create_hparams(FLAGS.hparams_set,
FLAGS.hparams,
data_dir=FLAGS.data_dir,
problem_name=FLAGS.problem)
# set appropriate dataset-split, if flags.eval_use_test_set.
dataset_split = "test" if FLAGS.eval_use_test_set else None
dataset_kwargs = {"dataset_split": dataset_split}
eval_input_fn = hparams.problem.make_estimator_input_fn(
tf.estimator.ModeKeys.EVAL, hparams, dataset_kwargs=dataset_kwargs)
config = t2t_trainer.create_run_config(hparams)
# summary-hook in tf.estimator.EstimatorSpec requires
# hparams.model_dir to be set.
hparams.add_hparam("model_dir", config.model_dir)
estimator = trainer_lib.create_estimator(FLAGS.model,
hparams,
config,
use_tpu=FLAGS.use_tpu)
ckpt_iter = trainer_lib.next_checkpoint(hparams.model_dir,
FLAGS.eval_timeout_mins)
for ckpt_path in ckpt_iter:
predictions = estimator.evaluate(eval_input_fn,
steps=FLAGS.eval_steps,
checkpoint_path=ckpt_path)
tf.logging.info(predictions)
def create_estimator(run_config, hparams):
return trainer_lib.create_estimator(
FLAGS.model,
hparams,
run_config,
decode_hparams=decoding.decode_hparams(FLAGS.decode_hparams),
use_tpu=FLAGS.use_tpu)
def _init_env(self):
FLAGS.use_tpu = False
tf.logging.set_verbosity(tf.logging.DEBUG)
tf.logging.info("Import usr dir from %s", self._usr_dir)
if self._usr_dir != None:
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
tf.logging.info("Start to create hparams,for %s of %s", self._problem,
self._hparams_set)
self._hparams = create_hparams()
self._hparams_decode = create_decode_hparams(
extra_length=self._extra_length,
batch_size=self._batch_size,
beam_size=self._beam_size,
alpha=self._alpha,
return_beams=self._return_beams,
write_beam_scores=self._write_beam_scores)
self.estimator = trainer_lib.create_estimator(
FLAGS.model,
self._hparams,
t2t_trainer.create_run_config(self._hparams),
decode_hparams=self._hparams_decode,
use_tpu=False)
tf.logging.info("Finish intialize environment")
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
if FLAGS.score_file:
filename = os.path.expanduser(FLAGS.score_file)
if not tf.gfile.Exists(filename):
raise ValueError("The file to score doesn't exist: %s" % filename)
results = score_file(filename)
# if not FLAGS.decode_to_file:
# raise ValueError("To score a file, specify --decode_to_file for results.")
# write_file = tf.gfile.Open(os.path.expanduser(FLAGS.decode_to_file), "w")
# for sentence, score in results:
# write_file.write(sentence + "\t" + "SCORE:" + "%.6f\n" % score)
# write_file.close()
return
hp = create_hparams()
decode_hp = create_decode_hparams()
run_config = t2t_trainer.create_run_config(hp)
if FLAGS.disable_grappler_optimizations:
run_config.session_config.graph_options.rewrite_options.disable_meta_optimizer = True
# summary-hook in tf.estimator.EstimatorSpec requires
# hparams.model_dir to be set.
hp.add_hparam("model_dir", run_config.model_dir)
estimator = trainer_lib.create_estimator(FLAGS.model,
hp,
run_config,
decode_hparams=decode_hp,
use_tpu=FLAGS.use_tpu)
decode(estimator, hp, decode_hp)
def export(self, hparams):
"""Run model export for serving."""
hparams.no_data_parallelism = True
problem = hparams.problem
run_config = trainer_lib.create_run_config(model_name=self.model_name,
model_dir=self.model_dir,
num_gpus=0,
use_tpu=False)
estimator = trainer_lib.create_estimator(
self.model_name,
hparams,
run_config,
decode_hparams=decoding.decode_hparams(self.decode_hparams))
exporter = tf.estimator.FinalExporter(
"exporter", lambda: problem.serving_input_fn(hparams), as_text=True)
exporter.export(estimator,
self.export_dir,
checkpoint_path=tf.train.latest_checkpoint(self.model_dir),
eval_result=None,
is_the_final_export=True)
def __prepare_model(self, train_mode=False):
"""Prepare utilities for decoding."""
hparams = registry.hparams(self.params.hparams_set)
hparams.problem = self.problem
hparams.problem_hparams = self.problem.get_hparams(hparams)
if self.params.hparams:
tf.logging.info("Overriding hparams in %s with %s",
self.params.hparams_set,
self.params.hparams)
hparams = hparams.parse(self.params.hparams)
trainer_run_config = g2p_trainer_utils.create_run_config(hparams,
self.params)
if train_mode:
exp_fn = g2p_trainer_utils.create_experiment_fn(self.params, self.problem)
self.exp = exp_fn(trainer_run_config, hparams)
decode_hp = decoding.decode_hparams(self.params.decode_hparams)
estimator = trainer_lib.create_estimator(
self.params.model_name,
hparams,
trainer_run_config,
decode_hparams=decode_hp,
use_tpu=False)
return estimator, decode_hp, hparams
def create_estimator_fn(model_name,
hparams,
run_config,
schedule="train_and_evaluate",
decode_hparams=None
):
return trainer_lib.create_estimator(model_name, hparams, run_config, schedule, decode_hparams, False)
def t2t_decoder(problem_name, data_dir, decode_from_file, decode_to_file,
checkpoint_path):
trainer_lib.set_random_seed(FLAGS.random_seed)
hp = trainer_lib.create_hparams(FLAGS.hparams_set,
FLAGS.hparams,
data_dir=os.path.expanduser(data_dir),
problem_name=problem_name)
decode_hp = decoding.decode_hparams(FLAGS.decode_hparams)
decode_hp.shards = FLAGS.decode_shards
decode_hp.shard_id = FLAGS.worker_id
decode_in_memory = FLAGS.decode_in_memory or decode_hp.decode_in_memory
decode_hp.decode_in_memory = decode_in_memory
decode_hp.decode_to_file = decode_to_file
decode_hp.decode_reference = None
FLAGS.checkpoint_path = checkpoint_path
estimator = trainer_lib.create_estimator(FLAGS.model,
hp,
t2t_trainer.create_run_config(hp),
decode_hparams=decode_hp,
use_tpu=FLAGS.use_tpu)
decode_from_text_file(estimator,
problem_name,
decode_from_file,
hp,
decode_hp,
decode_to_file,
checkpoint_path=checkpoint_path)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
if FLAGS.score_file:
filename = os.path.expanduser(FLAGS.score_file)
if not tf.gfile.Exists(filename):
raise ValueError("The file to score doesn't exist: %s" % filename)
results = score_file(filename)
if not FLAGS.decode_to_file:
raise ValueError("To score a file, specify --decode_to_file for results.")
write_file = tf.gfile.Open(os.path.expanduser(FLAGS.decode_to_file), "w")
for score in results:
write_file.write("%.6f\n" % score)
write_file.close()
return
hp = create_hparams()
decode_hp = create_decode_hparams()
estimator = trainer_lib.create_estimator(
FLAGS.model,
hp,
t2t_trainer.create_run_config(hp),
decode_hparams=decode_hp,
use_tpu=FLAGS.use_tpu)
decode(estimator, hp, decode_hp)
def __prepare_model(self, train_mode=False):
"""Prepare utilities for decoding."""
hparams = registry.hparams(self.params.hparams_set)
hparams.problem = self.problem
hparams.problem_hparams = self.problem.get_hparams(hparams)
if self.params.hparams:
tf.logging.info("Overriding hparams in %s with %s",
self.params.hparams_set,
self.params.hparams)
hparams = hparams.parse(self.params.hparams)
trainer_run_config = g2p_trainer_utils.create_run_config(hparams,
self.params)
if train_mode:
exp_fn = g2p_trainer_utils.create_experiment_fn(self.params, self.problem)
self.exp = exp_fn(trainer_run_config, hparams)
decode_hp = decoding.decode_hparams(self.params.decode_hparams)
estimator = trainer_lib.create_estimator(
self.params.model_name,
hparams,
trainer_run_config,
decode_hparams=decode_hp,
use_tpu=False)
return estimator, decode_hp, hparams
def create_estimator(run_config, model_config):
hparams = trainer_lib.create_hparams("transformer_base_single_gpu")
# SentimentIMDBCortex subclasses SentimentIMDB
problem = SentimentIMDBCortex(list(model_config["input"]["vocab"]))
hparams.problem = problem
hparams.problem_hparams = problem.get_hparams(hparams)
# metrics specific to the sentiment problem
problem.eval_metrics = lambda: [
metrics.Metrics.ACC_TOP5,
metrics.Metrics.ACC_PER_SEQ,
metrics.Metrics.NEG_LOG_PERPLEXITY,
]
# reduce memory load
hparams.num_hidden_layers = 2
hparams.hidden_size = 32
hparams.filter_size = 32
hparams.num_heads = 2
# t2t expects these keys
hparams.warm_start_from = None
run_config.data_parallelism = None
run_config.t2t_device_info = {"num_async_replicas": 1}
return trainer_lib.create_estimator("transformer", hparams, run_config)
def create_estimator(run_config, hparams):
return trainer_lib.create_estimator(FLAGS.model,
hparams,
run_config,
decode_hparams=decoding.decode_hparams(
FLAGS.decode_hparams),
use_tpu=FLAGS.use_tpu)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
if FLAGS.score_file:
filename = os.path.expanduser(FLAGS.score_file)
if not tf.gfile.Exists(filename):
raise ValueError("The file to score doesn't exist: %s" % filename)
results = score_file(filename)
if not FLAGS.decode_to_file:
raise ValueError(
"To score a file, specify --decode_to_file for results.")
write_file = tf.gfile.Open(os.path.expanduser(FLAGS.decode_to_file),
"w")
for score in results:
write_file.write("%.6f\n" % score)
write_file.close()
return
hp = create_hparams()
decode_hp = create_decode_hparams()
estimator = trainer_lib.create_estimator(FLAGS.model,
hp,
t2t_trainer.create_run_config(hp),
decode_hparams=decode_hp,
use_tpu=FLAGS.use_tpu)
decode(estimator, hp, decode_hp)
def create_estimator(run_config, model_config):
# t2t expects these keys in run_config
run_config.data_parallelism = None
run_config.t2t_device_info = {"num_async_replicas": 1}
hparams = trainer_lib.create_hparams("transformer_base_single_gpu")
problem = SentimentIMDBCortex(
list(model_config["aggregates"]["reviews_vocab"]))
p_hparams = problem.get_hparams(hparams)
hparams.problem = problem
hparams.problem_hparams = p_hparams
problem.eval_metrics = lambda: [
metrics.Metrics.ACC_TOP5,
metrics.Metrics.ACC_PER_SEQ,
metrics.Metrics.NEG_LOG_PERPLEXITY,
]
# t2t expects this key
hparams.warm_start_from = None
# reduce memory load
hparams.num_hidden_layers = 2
hparams.hidden_size = 32
hparams.filter_size = 32
hparams.num_heads = 2
estimator = trainer_lib.create_estimator("transformer", hparams,
run_config)
return estimator
def create_new_estimator(hp,decode_hp):
estimator = trainer_lib.create_estimator(
FLAGS.model,
hp,
t2t_trainer.create_run_config(hp),
decode_hparams=decode_hp,
use_tpu=FLAGS.use_tpu)
return estimator
def run():
"""
Load Transformer model according to flags and start sampling.
:raises:
ValueError: if required flags are missing or invalid.
"""
if FLAGS.model_path is None:
raise ValueError('Required Transformer pre-trained model path.')
if FLAGS.output_dir is None:
raise ValueError('Required Midi output directory.')
if FLAGS.decode_length <= 0:
raise ValueError('Decode length must be > 0.')
problem = utils.PianoPerformanceLanguageModelProblem()
unconditional_encoders = problem.get_feature_encoders()
primer_ns = music_pb2.NoteSequence()
if FLAGS.primer_path is None:
targets = []
else:
if FLAGS.max_primer_second <= 0:
raise ValueError('Max primer second must be > 0.')
primer_ns = utils.get_primer_ns(FLAGS.primer_path,
FLAGS.max_primer_second)
targets = unconditional_encoders['targets'].encode_note_sequence(
primer_ns)
# Remove the end token from the encoded primer.
targets = targets[:-1]
if len(targets) >= FLAGS.decode_length:
raise ValueError(
'Primer has more or equal events than maximum sequence length:'
' %d >= %d; Aborting' % (len(targets), FLAGS.decode_length))
decode_length = FLAGS.decode_length - len(targets)
# Set up HParams.
hparams = trainer_lib.create_hparams(hparams_set=FLAGS.hparams_set)
trainer_lib.add_problem_hparams(hparams, problem)
hparams.num_hidden_layers = FLAGS.layers
hparams.sampling_method = FLAGS.sample
# Set up decoding HParams.
decode_hparams = decoding.decode_hparams()
decode_hparams.alpha = FLAGS.alpha
decode_hparams.beam_size = FLAGS.beam_size
# Create Estimator.
utils.LOGGER.info('Loading model')
run_config = trainer_lib.create_run_config(hparams)
estimator = trainer_lib.create_estimator(FLAGS.model_name,
hparams,
run_config,
decode_hparams=decode_hparams)
generate(estimator, unconditional_encoders, decode_length, targets,
primer_ns)
def create_estimator(run_config, hparams):
return trainer_lib.create_estimator(
FLAGS.model,
hparams,
run_config,
decode_hparams=decoding.decode_hparams(FLAGS.decode_hparams),
use_tpu=FLAGS.use_tpu,
export_saved_model_api_version=FLAGS.export_saved_model_api_version,
use_guarantee_const_getter=FLAGS.use_guarantee_const_getter)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
# sess_dir = FLAGS.sess_dir
# output_dir = os.path.expanduser(sess_dir+problem_name+'-'+model+'-'+hparams)
output_dir = FLAGS.output_dir
if FLAGS.score_file:
filename = os.path.expanduser(FLAGS.score_file)
if not tf.gfile.Exists(filename):
raise ValueError("The file to score doesn't exist: %s" % filename)
results = score_file(filename)
if not FLAGS.decode_to_file:
raise ValueError(
"To score a file, specify --decode_to_file for results.")
write_file = tf.gfile.Open(os.path.expanduser(FLAGS.decode_to_file),
"w")
for score in results:
write_file.write("%.6f\n" % score)
write_file.close()
return
hp = create_hparams()
if FLAGS.global_steps:
FLAGS.checkpoint_path = os.path.join(
FLAGS.model_dir, f"model.ckpt-{FLAGS.global_steps}")
else:
FLAGS.checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir)
# Check if already exists
dataset_split = "test" if FLAGS.split == "test" else "dev"
decode_path = os.path.join(FLAGS.model_dir,
"decode_00000") # default decoded_to_file
decode_path = FLAGS.decode_to_file if FLAGS.decode_to_file else decode_path
if os.path.isdir(decode_path):
files = os.listdir(decode_path)
for file in files:
file_name = file.split(".")[0]
file_name_to_be = f"{FLAGS.global_steps}{dataset_split}{FLAGS.test_shard:03d}"
if file_name == file_name_to_be:
print(f"Already {file_name_to_be} exists")
return
tf.reset_default_graph()
decode_hp = create_decode_hparams(decode_path, FLAGS.test_shard)
estimator = trainer_lib.create_estimator(FLAGS.model,
hp,
create_run_config(hp),
decode_hparams=decode_hp,
use_tpu=FLAGS.use_tpu)
decode(estimator, hp, decode_hp)
print("shard " + str(FLAGS.test_shard) + " completed")
def testCompatibility(self):
model = "transformer"
hp_set = "transformer_test"
problem_name = "translate_ende_wmt8k"
hp = trainer_lib.create_hparams(
hp_set, data_dir=_DATA_DIR, problem_name=problem_name)
run_config = trainer_lib.create_run_config(model, model_dir=_CKPT_DIR)
estimator = trainer_lib.create_estimator(model, hp, run_config)
for prediction in estimator.predict(self.input_fn):
self.assertEqual(prediction["outputs"].dtype, np.int32)
def testCompatibility(self):
model = "transformer"
hp_set = "transformer_test"
problem_name = "translate_ende_wmt8k"
hp = trainer_lib.create_hparams(
hp_set, data_dir=_DATA_DIR, problem_name=problem_name)
run_config = trainer_lib.create_run_config(model_dir=_CKPT_DIR)
estimator = trainer_lib.create_estimator(model, hp, run_config)
for prediction in estimator.predict(self.input_fn):
self.assertEqual(prediction["outputs"].dtype, np.int32)
def main(_):
FLAGS.decode_interactive = True
hp = create_hparams()
decode_hp = create_decode_hparams()
estimator = trainer_lib.create_estimator(FLAGS.model,
hp,
t2t_trainer.create_run_config(hp),
decode_hparams=decode_hp,
use_tpu=False)
decode(estimator, hp, decode_hp)
def main(_):
import ipdb
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
if FLAGS.score_file:
filename = os.path.expanduser(FLAGS.score_file)
if not tf.gfile.Exists(filename):
raise ValueError("The file to score doesn't exist: %s" % filename)
results = score_file(filename)
if not FLAGS.decode_to_file:
raise ValueError("To score a file, specify --decode_to_file for results.")
write_file = tf.gfile.Open(os.path.expanduser(FLAGS.decode_to_file), "w")
for score in results:
write_file.write("%.6f\n" % score)
write_file.close()
return
hp = create_hparams()
decode_hp = create_decode_hparams()
# eval_input_fn = hp.problem.make_estimator_input_fn(
# tf.estimator.ModeKeys.TRAIN, hp, dataset_kwargs={"dataset_split": "eval"})
# print(eval_input_fn)
# for foo in eval_input_fn(None, None):
# print(type(foo[0]['targets']))
# print(foo[0]['targets'].numpy())
# exit()
run_config = t2t_trainer.create_run_config(hp)
if FLAGS.disable_grappler_optimizations:
run_config.session_config.graph_options.rewrite_options.disable_meta_optimizer = True
# summary-hook in tf.estimator.EstimatorSpec requires
# hparams.model_dir to be set.
hp.add_hparam("model_dir", run_config.model_dir)
estimator = trainer_lib.create_estimator(
FLAGS.model,
hp,
run_config,
decode_hparams=decode_hp,
use_tpu=FLAGS.use_tpu)
decode(estimator, hp, decode_hp)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
if FLAGS.score_file:
filename = os.path.expanduser(FLAGS.score_file)
if not tf.gfile.Exists(filename):
raise ValueError("The file to score doesn't exist: %s" % filename)
results = score_file(filename)
if not FLAGS.decode_to_file:
raise ValueError(
"To score a file, specify --decode_to_file for results.")
write_file = tf.gfile.Open(os.path.expanduser(FLAGS.decode_to_file),
"w")
for score in results:
write_file.write("%.6f\n" % score)
write_file.close()
return
hp = create_hparams()
decode_hp = create_decode_hparams()
estimator = trainer_lib.create_estimator(FLAGS.model,
hp,
t2t_trainer.create_run_config(hp),
decode_hparams=decode_hp,
use_tpu=FLAGS.use_tpu)
decode(estimator, hp, decode_hp)
# Post-process decodings (if necessary).
if FLAGS.decode_to_file and FLAGS.output_line_prefix_tag:
decode_filename_original = FLAGS.decode_to_file
decode_filename_prefixed = "%s-%s" % (decode_filename_original,
FLAGS.output_line_prefix_tag)
tf.logging.info("Writing prefexed decodes into %s" %
decode_filename_prefixed)
# Read original lines.
with tf.gfile.Open(decode_filename_original, "r") as original_fp:
original_lines = original_fp.readlines()
# Write prefixed lines.
prefix = "<%s> " % FLAGS.output_line_prefix_tag
prefixed_fp = tf.gfile.Open(decode_filename_prefixed, "w")
for line in original_lines:
prefixed_fp.write(prefix + line)
prefixed_fp.flush()
prefixed_fp.close()
tf.logging.info("Done.")
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
hp = t2t_decoder.create_hparams()
decode_hp = t2t_decoder.create_decode_hparams()
estimator = trainer_lib.create_estimator(FLAGS.model,
hp,
t2t_trainer.create_run_config(hp),
decode_hparams=decode_hp,
use_tpu=FLAGS.use_tpu)
decode(estimator, hp, decode_hp)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
FLAGS.use_tpu = False # decoding not supported on TPU
hp = create_hparams()
decode_hp = create_decode_hparams()
estimator = trainer_lib.create_estimator(FLAGS.model,
hp,
t2t_trainer.create_run_config(hp),
decode_hparams=decode_hp,
use_tpu=False)
decode(estimator, hp, decode_hp)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
FLAGS.use_tpu = False # decoding not supported on TPU
hp = create_hparams()
decode_hp = create_decode_hparams()
estimator = trainer_lib.create_estimator(
FLAGS.model,
hp,
t2t_trainer.create_run_config(hp),
decode_hparams=decode_hp,
use_tpu=False)
decode(estimator, hp, decode_hp)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
# Fathom start
checkpoint_path = fathom_t2t_model_setup()
# Fathom end
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
if FLAGS.score_file:
filename = os.path.expanduser(FLAGS.score_file)
if not tf.gfile.Exists(filename):
raise ValueError("The file to score doesn't exist: %s" % filename)
results = score_file(filename)
if not FLAGS.decode_to_file:
raise ValueError(
"To score a file, specify --decode_to_file for results.")
write_file = tf.gfile.Open(os.path.expanduser(FLAGS.decode_to_file),
"w")
for score in results:
write_file.write("%.6f\n" % score)
write_file.close()
return
hp = create_hparams()
decode_hp = create_decode_hparams()
estimator = trainer_lib.create_estimator(FLAGS.model,
hp,
t2t_trainer.create_run_config(hp),
decode_hparams=decode_hp,
use_tpu=FLAGS.use_tpu)
decode(estimator, hp, decode_hp)
# Fathom
# This xcom is here so that tasks after decode know the local path to the
# downloaded model. Train does this same xcom echo.
# Decode, predict, and evaluate code should
# converge to use the same fathom_t2t_model_setup.
# TODO: since the truncation-boundary xcom value should be available in
# the hparams_set, we should probably have consumers access this via a
# SavedModel.hparams property rather than XCOM
echo_yaml_for_xcom_ingest({
'output-dir': os.path.dirname(checkpoint_path),
'output-file': FLAGS.decode_output_file,
'truncation-boundary': hp.max_input_seq_length
})
def __init__(self, processor_configuration):
"""Creates the Transformer estimator.
Args:
processor_configuration: A ProcessorConfiguration protobuffer with the
transformer fields populated.
"""
# Do the pre-setup tensor2tensor requires for flags and configurations.
transformer_config = processor_configuration["transformer"]
FLAGS.output_dir = transformer_config["model_dir"]
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
data_dir = os.path.expanduser(transformer_config["data_dir"])
# Create the basic hyper parameters.
self.hparams = trainer_lib.create_hparams(
transformer_config["hparams_set"],
transformer_config["hparams"],
data_dir=data_dir,
problem_name=transformer_config["problem"])
decode_hp = decoding.decode_hparams()
decode_hp.add_hparam("shards", 1)
decode_hp.add_hparam("shard_id", 0)
# Create the estimator and final hyper parameters.
self.estimator = trainer_lib.create_estimator(
transformer_config["model"],
self.hparams,
t2t_trainer.create_run_config(self.hparams),
decode_hparams=decode_hp,
use_tpu=False)
# Fetch the vocabulary and other helpful variables for decoding.
self.source_vocab = self.hparams.problem_hparams.vocabulary["inputs"]
self.targets_vocab = self.hparams.problem_hparams.vocabulary["targets"]
self.const_array_size = 10000
# Prepare the Transformer's debug data directory.
run_dirs = sorted(
glob.glob(os.path.join("/tmp/t2t_server_dump", "run_*")))
for run_dir in run_dirs:
shutil.rmtree(run_dir)
def create_estimator(run_config, model_config):
# t2t expects these keys in run_config
run_config.data_parallelism = None
run_config.t2t_device_info = {"num_async_replicas": 1}
# t2t has its own set of hyperparameters we can use
hparams = trainer_lib.create_hparams("basic_fc_small")
problem = registry.problem("image_mnist")
p_hparams = problem.get_hparams(hparams)
hparams.problem = problem
hparams.problem_hparams = p_hparams
# don't need eval_metrics
problem.eval_metrics = lambda: []
# t2t expects this key
hparams.warm_start_from = None
estimator = trainer_lib.create_estimator("basic_fc_relu", hparams, run_config)
return estimator
def __prepare_model(self):
"""Prepare utilities for decoding."""
hparams = trainer_lib.create_hparams(
hparams_set=self.params.hparams_set,
hparams_overrides_str=self.params.hparams)
trainer_run_config = g2p_trainer_utils.create_run_config(hparams,
self.params)
exp_fn = g2p_trainer_utils.create_experiment_fn(self.params, self.problem)
self.exp = exp_fn(trainer_run_config, hparams)
decode_hp = decoding.decode_hparams(self.params.decode_hparams)
decode_hp.add_hparam("shards", self.params.decode_shards)
decode_hp.add_hparam("shard_id", self.params.worker_id)
estimator = trainer_lib.create_estimator(
self.params.model_name,
hparams,
trainer_run_config,
decode_hparams=decode_hp,
use_tpu=False)
return estimator, decode_hp, hparams
def __init__(self, processor_configuration):
"""Creates the Transformer estimator.
Args:
processor_configuration: A ProcessorConfiguration protobuffer with the
transformer fields populated.
"""
# Do the pre-setup tensor2tensor requires for flags and configurations.
transformer_config = processor_configuration["transformer"]
FLAGS.output_dir = transformer_config["model_dir"]
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
data_dir = os.path.expanduser(transformer_config["data_dir"])
# Create the basic hyper parameters.
self.hparams = trainer_lib.create_hparams(
transformer_config["hparams_set"],
transformer_config["hparams"],
data_dir=data_dir,
problem_name=transformer_config["problem"])
decode_hp = decoding.decode_hparams()
decode_hp.add_hparam("shards", 1)
decode_hp.add_hparam("shard_id", 0)
# Create the estimator and final hyper parameters.
self.estimator = trainer_lib.create_estimator(
transformer_config["model"],
self.hparams,
t2t_trainer.create_run_config(self.hparams),
decode_hparams=decode_hp, use_tpu=False)
# Fetch the vocabulary and other helpful variables for decoding.
self.source_vocab = self.hparams.problem_hparams.vocabulary["inputs"]
self.targets_vocab = self.hparams.problem_hparams.vocabulary["targets"]
self.const_array_size = 10000
# Prepare the Transformer's debug data directory.
run_dirs = sorted(glob.glob(os.path.join("/tmp/t2t_server_dump", "run_*")))
for run_dir in run_dirs:
shutil.rmtree(run_dir)
def initialize(self, is_conditioned=False):
self.model_name = 'transformer'
self.hparams_set = 'transformer_tpu'
self.conditioned = is_conditioned
if self.conditioned:
self.ckpt_path = 'models/checkpoints/melody_conditioned_model_16.ckpt'
problem = MelodyToPianoPerformanceProblem()
else:
self.ckpt_path = 'models/checkpoints/unconditional_model_16.ckpt'
problem = PianoPerformanceLanguageModelProblem()
self.encoders = problem.get_feature_encoders()
# Set up hyperparams
hparams = trainer_lib.create_hparams(hparams_set=self.hparams_set)
trainer_lib.add_problem_hparams(hparams, problem)
hparams.num_hidden_layers = 16
hparams.sampling_method = 'random'
# Set up decoding hyperparams
decode_hparams = decoding.decode_hparams()
decode_hparams.alpha = 0.0
decode_hparams.beam_size = 1
if self.conditioned:
self.inputs = []
else:
self.targets = []
self.decode_length = 0
run_config = trainer_lib.create_run_config(hparams)
estimator = trainer_lib.create_estimator(
self.model_name, hparams, run_config,
decode_hparams=decode_hparams)
fnc = self.input_generation_conditional if self.conditioned else self.input_generator_unconditional
input_fn = decoding.make_input_fn_from_generator(fnc())
self.samples = estimator.predict(
input_fn, checkpoint_path=self.ckpt_path)
_ = next(self.samples)
def create_hp_and_estimator(problem_name, data_dir, checkpoint_path):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
hp = trainer_lib.create_hparams(FLAGS.hparams_set,
FLAGS.hparams,
data_dir=os.path.expanduser(data_dir),
problem_name=problem_name)
decode_hp = decoding.decode_hparams(FLAGS.decode_hparams)
decode_hp.shards = FLAGS.decode_shards
decode_hp.shard_id = FLAGS.worker_id
decode_in_memory = FLAGS.decode_in_memory or decode_hp.decode_in_memory
decode_hp.decode_in_memory = decode_in_memory
decode_hp.decode_to_file = None
decode_hp.decode_reference = None
FLAGS.checkpoint_path = checkpoint_path
estimator = trainer_lib.create_estimator(FLAGS.model,
hp,
t2t_trainer.create_run_config(hp),
decode_hparams=decode_hp,
use_tpu=FLAGS.use_tpu)
return hp, decode_hp, estimator
def music_generator(primer='erik_gnossienne',
primer_begin_buffer=10,
primer_length=90,
output_path='.',
filename='./public/output'):
SF2_PATH = './models/Yamaha-C5-Salamander-JNv5.1.sf2'
SAMPLE_RATE = 16000
# Upload a MIDI file and convert to NoteSequence.
def upload_midi():
data = list(files.upload().values())
if len(data) > 1:
print('Multiple files uploaded; using only one.')
return mm.midi_to_note_sequence(data[0])
# Decode a list of IDs.
def decode(ids, encoder):
ids = list(ids)
if text_encoder.EOS_ID in ids:
ids = ids[:ids.index(text_encoder.EOS_ID)]
return encoder.decode(ids)
model_name = 'transformer'
hparams_set = 'transformer_tpu'
ckpt_path = './models/checkpoints/unconditional_model_16.ckpt'
class PianoPerformanceLanguageModelProblem(score2perf.Score2PerfProblem):
@property
def add_eos_symbol(self):
return True
problem = PianoPerformanceLanguageModelProblem()
unconditional_encoders = problem.get_feature_encoders()
# Set up HParams.
hparams = trainer_lib.create_hparams(hparams_set=hparams_set)
trainer_lib.add_problem_hparams(hparams, problem)
hparams.num_hidden_layers = 16
hparams.sampling_method = 'random'
# Set up decoding HParams.
decode_hparams = decoding.decode_hparams()
decode_hparams.alpha = 0.0
decode_hparams.beam_size = 1
# Create Estimator.
run_config = trainer_lib.create_run_config(hparams)
estimator = trainer_lib.create_estimator(model_name,
hparams,
run_config,
decode_hparams=decode_hparams)
# These values will be changed by subsequent cells.
targets = []
decode_length = 0
# Create input generator (so we can adjust priming and
# decode length on the fly).
def input_generator():
global targets
global decode_length
while True:
yield {
'targets': np.array([targets], dtype=np.int32),
'decode_length': np.array(decode_length, dtype=np.int32)
}
# Start the Estimator, loading from the specified checkpoint.
input_fn = decoding.make_input_fn_from_generator(input_generator())
unconditional_samples = estimator.predict(input_fn,
checkpoint_path=ckpt_path)
# "Burn" one.
_ = next(unconditional_samples)
filenames = {
'C major arpeggio': './models/primers/c_major_arpeggio.mid',
'C major scale': './models/primers/c_major_scale.mid',
'Clair de Lune': './models/primers/clair_de_lune.mid',
'Classical':
'audio_midi/Classical_Piano_piano-midi.de_MIDIRip/bach/bach_846_format0.mid',
'erik_gymnopedie': 'audio_midi/erik_satie/gymnopedie_1_(c)oguri.mid',
'erik_gymnopedie_2': 'audio_midi/erik_satie/gymnopedie_2_(c)oguri.mid',
'erik_gymnopedie_3': 'audio_midi/erik_satie/gymnopedie_3_(c)oguri.mid',
'erik_gnossienne': 'audio_midi/erik_satie/gnossienne_1_(c)oguri.mid',
'erik_gnossienne_2': 'audio_midi/erik_satie/gnossienne_2_(c)oguri.mid',
'erik_gnossienne_3': 'audio_midi/erik_satie/gnossienne_3_(c)oguri.mid',
'erik_gnossienne_dery':
'audio_midi/erik_satie/gnossienne_1_(c)dery.mid',
'erik_gnossienne_dery_2':
'audio_midi/erik_satie/gnossienne_2_(c)dery.mid',
'erik_gnossienne_dery_3':
'audio_midi/erik_satie/gnossienne_3_(c)dery.mid',
'erik_gnossienne_dery_5':
'audio_midi/erik_satie/gnossienne_5_(c)dery.mid',
'erik_gnossienne_dery_6':
'audio_midi/erik_satie/gnossienne_6_(c)dery.mid',
'1': 'audio_midi/erik_satie/1.mid',
'2': 'audio_midi/erik_satie/2.mid',
'3': 'audio_midi/erik_satie/3.mid',
'4': 'audio_midi/erik_satie/4.mid',
'5': 'audio_midi/erik_satie/5.mid',
'6': 'audio_midi/erik_satie/6.mid',
'7': 'audio_midi/erik_satie/7.mid',
'8': 'audio_midi/erik_satie/8.mid',
'9': 'audio_midi/erik_satie/9.mid',
'10': 'audio_midi/erik_satie/10.mid',
}
# primer = 'C major scale'
#if primer == 'Upload your own!':
# primer_ns = upload_midi()
#else:
# # Use one of the provided primers.
# primer_ns = mm.midi_file_to_note_sequence(filenames[primer])
primer_ns = mm.midi_file_to_note_sequence(filenames[primer])
# Handle sustain pedal in the primer.
primer_ns = mm.apply_sustain_control_changes(primer_ns)
# Trim to desired number of seconds.
max_primer_seconds = primer_length
if primer_ns.total_time > max_primer_seconds:
print('Primer is longer than %d seconds, truncating.' %
max_primer_seconds)
primer_ns = mm.extract_subsequence(
primer_ns, primer_begin_buffer,
max_primer_seconds + primer_begin_buffer)
# Remove drums from primer if present.
if any(note.is_drum for note in primer_ns.notes):
print('Primer contains drums; they will be removed.')
notes = [note for note in primer_ns.notes if not note.is_drum]
del primer_ns.notes[:]
primer_ns.notes.extend(notes)
# Set primer instrument and program.
for note in primer_ns.notes:
note.instrument = 1
note.program = 0
## Play and plot the primer.
#mm.play_sequence(
# primer_ns,
# synth=mm.fluidsynth, sample_rate=SAMPLE_RATE, sf2_path=SF2_PATH)
#mm.plot_sequence(primer_ns)
mm.sequence_proto_to_midi_file(
primer_ns, join(output_path, 'primer_{}.mid'.format(filename)))
targets = unconditional_encoders['targets'].encode_note_sequence(primer_ns)
# Remove the end token from the encoded primer.
targets = targets[:-1]
decode_length = max(0, 10000 - len(targets))
if len(targets) >= 4096:
print(
'Primer has more events than maximum sequence length; nothing will be generated.'
)
# Generate sample events.
sample_ids = next(unconditional_samples)['outputs']
# Decode to NoteSequence.
midi_filename = decode(sample_ids,
encoder=unconditional_encoders['targets'])
ns = mm.midi_file_to_note_sequence(midi_filename)
print('Sample IDs: {}'.format(sample_ids))
print('Sample IDs length: {}'.format(len(sample_ids)))
print('Encoder: {}'.format(unconditional_encoders['targets']))
print('Unconditional Samples: {}'.format(unconditional_samples))
# print('{}'.format(ns))
# continuation_ns = mm.concatenate_sequences([primer_ns, ns])
continuation_ns = ns
# mm.play_sequence(
# continuation_ns,
# synth=mm.fluidsynth, sample_rate=SAMPLE_RATE, sf2_path=SF2_PATH)
# mm.plot_sequence(continuation_ns)
# try:
audio = mm.fluidsynth(continuation_ns,
sample_rate=SAMPLE_RATE,
sf2_path=SF2_PATH)
normalizer = float(np.iinfo(np.int16).max)
array_of_ints = np.array(np.asarray(audio) * normalizer, dtype=np.int16)
wavfile.write(join(output_path, filename + '.wav'), SAMPLE_RATE,
array_of_ints)
print('[+] Output stored as {}'.format(filename + '.wav'))
mm.sequence_proto_to_midi_file(
continuation_ns,
join(output_path, 'continuation_{}.mid'.format(filename)))
def _init_env(self):
FLAGS.use_tpu = False
tf.logging.set_verbosity(tf.logging.DEBUG)
tf.logging.info("Import usr dir from %s", self._usr_dir)
if self._usr_dir != None:
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
tf.logging.info("Start to create hparams,for %s of %s", self._problem,
self._hparams_set)
self._hparams = create_hparams()
self._hparams_decode = create_decode_hparams(
extra_length=self._extra_length,
batch_size=self._batch_size,
beam_size=self._beam_size,
alpha=self._alpha,
return_beams=self._return_beams,
write_beam_scores=self._write_beam_scores,
force_decode_length=self._force_decode_length)
self.estimator = trainer_lib.create_estimator(
FLAGS.model,
self._hparams,
t2t_trainer.create_run_config(self._hparams),
decode_hparams=self._hparams_decode,
use_tpu=False)
tf.logging.info("Finish intialize environment")
#######
### make input placeholder
#self._inputs_ph = tf.placeholder(dtype=tf.int32) # shape not specified,any shape
# x=tf.placeholder(dtype=tf.int32)
# x.set_shape([None, None]) # ? -> (?,?)
# x = tf.expand_dims(x, axis=[2])# -> (?,?,1)
# x = tf.to_int32(x)
# self._inputs_ph=x
#batch_inputs = tf.reshape(self._inputs_ph, [self._batch_size, -1, 1, 1])
#batch_inputs=x
###
# batch_inputs = tf.reshape(self._inputs_ph, [-1, -1, 1, 1])
#targets_ph = tf.placeholder(dtype=tf.int32)
#batch_targets = tf.reshape(targets_ph, [1, -1, 1, 1])
self.inputs_ph = tf.placeholder(tf.int32,
shape=(None, None, 1, 1),
name='inputs')
self.targets_ph = tf.placeholder(tf.int32,
shape=(None, None, None, None),
name='targets')
self.input_extra_length_ph = tf.placeholder(dtype=tf.int32, shape=[])
self._features = {
"inputs": self.inputs_ph,
"problem_choice": 0, # We run on the first problem here.
"input_space_id": self._hparams.problem_hparams.input_space_id,
"target_space_id": self._hparams.problem_hparams.target_space_id
}
### 加入 decode length 变长的
self._features['decode_length'] = self.input_extra_length_ph
## target
self._features['targets'] = self.targets_ph
## 去掉 整数的
del self._features["problem_choice"]
del self._features["input_space_id"]
del self._features["target_space_id"]
#del self._features['decode_length']
####
mode = tf.estimator.ModeKeys.EVAL
translate_model = registry.model(self._model_name)(
hparams=self._hparams,
decode_hparams=self._hparams_decode,
mode=mode)
self.predict_dict = {}
### get logit ,attention mats
self.logits, _ = translate_model(self._features) #[? ? ? 1 vocabsz]
#translate_model(features)
from visualization import get_att_mats
self.att_mats = get_att_mats(translate_model,
self._model_name) # enc, dec, encdec
### get infer
translate_model.set_mode(tf.estimator.ModeKeys.PREDICT)
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
self.outputs_scores = translate_model.infer(
features=self._features,
decode_length=self._extra_length,
beam_size=self._beam_size,
top_beams=self._beam_size,
alpha=self._alpha) #outputs 4,4,63
######
tf.logging.info("Start to init tf session")
if self._isGpu:
print('Using GPU in Decoder')
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=self._fraction)
self._sess = tf.Session(
config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
gpu_options=gpu_options))
else:
print('Using CPU in Decoder')
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0)
config = tf.ConfigProto(gpu_options=gpu_options)
config.allow_soft_placement = True
config.log_device_placement = False
self._sess = tf.Session(config=config)
with self._sess.as_default():
ckpt = saver_mod.get_checkpoint_state(self._model_dir)
saver = tf.train.Saver()
tf.logging.info("Start to restore the parameters from %s",
ckpt.model_checkpoint_path)
saver.restore(self._sess, ckpt.model_checkpoint_path)
tf.logging.info("Finish intialize environment")
def create_experiment(run_config,
hparams,
model_name,
params,
problem_instance,
data_dir,
train_steps,
eval_steps,
min_eval_frequency=2000,
eval_throttle_seconds=600,
schedule="train_and_evaluate",
export=False,
decode_hparams=None,
use_tfdbg=False,
use_dbgprofile=False,
use_validation_monitor=False,
eval_early_stopping_steps=None,
eval_early_stopping_metric=None,
eval_early_stopping_metric_delta=None,
eval_early_stopping_metric_minimize=True,
autotune=False,
use_tpu=False):
"""Create Experiment."""
# HParams
hparams.add_hparam('model_dir', params.model_dir)
hparams.add_hparam("data_dir", data_dir)
hparams.add_hparam("train_steps", train_steps)
hparams.add_hparam("eval_steps", eval_steps)
hparams.add_hparam("schedule", schedule)
add_problem_hparams(hparams, problem_instance)
# Estimator
estimator = trainer_lib.create_estimator(
model_name,
hparams,
run_config,
schedule=schedule,
decode_hparams=decode_hparams,
use_tpu=use_tpu)
# Input fns from Problem
problem = hparams.problem
train_input_fn = problem.make_estimator_input_fn(
tf.estimator.ModeKeys.TRAIN, hparams)
eval_input_fn = problem.make_estimator_input_fn(
tf.estimator.ModeKeys.EVAL, hparams)
# Export
if export:
tf.logging.warn("Exporting from the trainer is deprecated. "
"See serving/export.py.")
# Hooks
validation_monitor_kwargs = dict(
input_fn=eval_input_fn,
eval_steps=eval_steps,
every_n_steps=min_eval_frequency,
early_stopping_rounds=eval_early_stopping_steps,
early_stopping_metric=eval_early_stopping_metric,
early_stopping_metric_minimize=eval_early_stopping_metric_minimize)
dbgprofile_kwargs = {"output_dir": run_config.model_dir}
early_stopping_kwargs = dict(
events_dir=os.path.join(run_config.model_dir, "eval_continuous"),
tag=eval_early_stopping_metric,
num_plateau_steps=eval_early_stopping_steps,
plateau_decrease=eval_early_stopping_metric_minimize,
plateau_delta=eval_early_stopping_metric_delta,
every_n_steps=min_eval_frequency)
# In-process eval (and possible early stopping)
if schedule == "continuous_train_and_eval" and min_eval_frequency:
tf.logging.warn("ValidationMonitor only works with "
"--schedule=train_and_evaluate")
use_validation_monitor = (
schedule == "train_and_evaluate" and min_eval_frequency)
# Distributed early stopping
local_schedules = ["train_and_evaluate", "continuous_train_and_eval"]
use_early_stopping = (
schedule not in local_schedules and eval_early_stopping_steps)
train_hooks, eval_hooks = trainer_lib.create_hooks(
use_tfdbg=use_tfdbg,
use_dbgprofile=use_dbgprofile,
dbgprofile_kwargs=dbgprofile_kwargs,
use_validation_monitor=use_validation_monitor,
validation_monitor_kwargs=validation_monitor_kwargs,
use_early_stopping=use_early_stopping,
early_stopping_kwargs=early_stopping_kwargs)
train_hooks += t2t_model.T2TModel.get_train_hooks(model_name)
eval_hooks += t2t_model.T2TModel.get_eval_hooks(model_name)
train_hooks = tf.contrib.learn.monitors.replace_monitors_with_hooks(
train_hooks, estimator)
eval_hooks = tf.contrib.learn.monitors.replace_monitors_with_hooks(
eval_hooks, estimator)
train_spec = tf.estimator.TrainSpec(
train_input_fn, max_steps=train_steps, hooks=train_hooks)
eval_spec = tf.estimator.EvalSpec(
eval_input_fn,
steps=eval_steps,
hooks=eval_hooks,
start_delay_secs=0 if hparams.schedule == "evaluate" else 120,
throttle_secs=eval_throttle_seconds)
if autotune:
hooks_kwargs = {"train_monitors": train_hooks, "eval_hooks": eval_hooks}
return tf.contrib.learn.Experiment(
estimator=estimator,
train_input_fn=train_input_fn,
eval_input_fn=eval_input_fn,
train_steps=train_steps,
eval_steps=eval_steps,
min_eval_frequency=min_eval_frequency,
train_steps_per_iteration=min(min_eval_frequency, train_steps),
eval_delay_secs=0 if schedule == "evaluate" else 120,
**hooks_kwargs if not use_tpu else {})
return trainer_lib.T2TExperiment(estimator, hparams, train_spec, eval_spec,
use_validation_monitor, decode_hparams)
