def testModel(self):
# HParams
hparams = trainer_lib.create_hparams(
"transformer_tiny", data_dir=self.data_dir, problem_name="tiny_algo")
# Dataset
problem = hparams.problem
dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN, self.data_dir)
dataset = dataset.repeat(None).padded_batch(10, dataset.output_shapes)
features = dataset.make_one_shot_iterator().get_next()
features = problem_lib.standardize_shapes(features)
# Model
model = registry.model("transformer")(hparams, tf.estimator.ModeKeys.TRAIN)
logits, losses = model(features)
self.assertTrue("training" in losses)
loss = losses["training"]
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
logits_val, loss_val = sess.run([logits, loss])
logits_shape = list(logits_val.shape)
logits_shape[1] = None
self.assertAllEqual(logits_shape, [10, None, 1, 1, 4])
self.assertEqual(loss_val.shape, tuple())
def testT2TModelRegistration(self):
@registry.register_model
class MyModel1(t2t_model.T2TModel):
pass
model = registry.model("my_model1")
self.assertTrue(model is MyModel1)
def testNamedRegistration(self):
@registry.register_model("model2")
class MyModel1(t2t_model.T2TModel):
pass
model = registry.model("model2")
self.assertTrue(model is MyModel1)
def testNonT2TModelRegistration(self):
@registry.register_model
def model_fn():
pass
model = registry.model("model_fn")
self.assertTrue(model is model_fn)
def nth_model(n):
"""Build the model for the n-th problem, plus some added variables."""
model_class = registry.model(model)(
hparams,
mode,
hparams.problems[n],
n,
dp,
devices.ps_devices(all_workers=True),
decode_hparams=decode_hparams)
if mode == tf.estimator.ModeKeys.PREDICT:
return model_class.infer(
features,
beam_size=decode_hp.beam_size,
top_beams=(decode_hp.beam_size if decode_hp.return_beams else 1),
alpha=decode_hp.alpha,
decode_length=decode_hp.extra_length)
# In distributed mode, we build graph for problem=0 and problem=worker_id.
skipping_is_on = hparams.problem_choice == "distributed" and is_training
problem_worker_id = worker_id % len(hparams.problems)
skip_this_one = n != 0 and n % worker_replicas != problem_worker_id
# On worker 0 also build graph for problems <= 1.
# TODO(lukaszkaiser): why is this hack needed for variables init? Repair.
skip_this_one = skip_this_one and (worker_id != 0 or n > 1)
if eval_run_autoregressive and mode == tf.estimator.ModeKeys.EVAL:
logits, losses_dict = model_class.eval_autoregressive(features)
else:
logits, losses_dict = model_class(
features, skip=(skipping_is_on and skip_this_one))
with tf.variable_scope("losses_avg"):
total_loss, ops = 0.0, []
for loss_key, loss_value in six.iteritems(losses_dict):
loss_name = "problem_%d/%s_loss" % (n, loss_key)
loss_moving_avg = tf.get_variable(
loss_name, initializer=100.0, trainable=False)
loss_variable_names.append(loss_name)
ops.append(
loss_moving_avg.assign(loss_moving_avg * 0.9 + loss_value * 0.1))
total_loss += loss_value
try: # Total loss avg might be reused or not, we try both.
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
# Total loss was already constructed on input.
loss_moving_avg = tf.get_variable("problem_%d/total_loss" % n)
except ValueError:
loss_moving_avg = tf.get_variable(
"problem_%d/total_loss" % n, initializer=100.0, trainable=False)
ops.append(
loss_moving_avg.assign(loss_moving_avg * 0.9 + total_loss * 0.1))
with tf.variable_scope("train_stats"): # Count steps for this problem.
problem_steps = tf.get_variable(
"problem_%d_steps" % n, initializer=0, trainable=False)
ops.append(problem_steps.assign_add(1))
with tf.control_dependencies(ops): # Make sure the ops run.
# Ensure the loss is a scalar here.
total_loss = tf.reshape(total_loss, [], name="total_loss_control_id")
return [total_loss, logits]
def score_file(filename):
"""Score each line in a file and return the scores."""
# Prepare model.
hparams = create_hparams()
encoders = registry.problem(FLAGS.problem).feature_encoders(FLAGS.data_dir)
has_inputs = "inputs" in encoders
# Prepare features for feeding into the model.
if has_inputs:
inputs_ph = tf.placeholder(dtype=tf.int32) # Just length dimension.
batch_inputs = tf.reshape(inputs_ph, [1, -1, 1, 1]) # Make it 4D.
targets_ph = tf.placeholder(dtype=tf.int32) # Just length dimension.
batch_targets = tf.reshape(targets_ph, [1, -1, 1, 1]) # Make it 4D.
features = {
"inputs": batch_inputs,
"targets": batch_targets,
} if has_inputs else {"targets": batch_targets}
# Prepare the model and the graph when model runs on features.
model = registry.model(FLAGS.model)(hparams, tf.estimator.ModeKeys.EVAL)
_, losses = model(features)
saver = tf.train.Saver()
with tf.Session() as sess:
# Load weights from checkpoint.
ckpts = tf.train.get_checkpoint_state(FLAGS.output_dir)
ckpt = ckpts.model_checkpoint_path
saver.restore(sess, ckpt)
# Run on each line.
with tf.gfile.Open(filename) as f:
lines = f.readlines()
results = []
for line in lines:
tab_split = line.split("\t")
if len(tab_split) > 2:
raise ValueError("Each line must have at most one tab separator.")
if len(tab_split) == 1:
targets = tab_split[0].strip()
else:
targets = tab_split[1].strip()
inputs = tab_split[0].strip()
# Run encoders and append EOS symbol.
targets_numpy = encoders["targets"].encode(
targets) + [text_encoder.EOS_ID]
if has_inputs:
inputs_numpy = encoders["inputs"].encode(inputs) + [text_encoder.EOS_ID]
# Prepare the feed.
feed = {
inputs_ph: inputs_numpy,
targets_ph: targets_numpy
} if has_inputs else {targets_ph: targets_numpy}
# Get the score.
np_loss = sess.run(losses["training"], feed)
results.append(np_loss)
return results
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
# Create hparams
hparams = create_hparams()
hparams.force_full_predict = True
batch_size = hparams.batch_size
# Iterating over dev/test partition of the data.
# Change the data partition if necessary.
dataset = registry.problem(FLAGS.problem).dataset(
tf.estimator.ModeKeys.PREDICT,
shuffle_files=False,
hparams=hparams)
dataset = dataset.apply(tf.contrib.data.batch_and_drop_remainder(batch_size))
data = dataset.make_one_shot_iterator().get_next()
input_data = dict((k, data[k]) for k in data.keys() if k.startswith("input"))
# Creat model
model_cls = registry.model(FLAGS.model)
model = model_cls(hparams, tf.estimator.ModeKeys.PREDICT)
prediction_ops = model.infer(input_data)
# Confusion Matrix
nr = hparams.problem.num_rewards
cm_per_frame = np.zeros((nr, nr), dtype=np.uint64)
cm_next_frame = np.zeros((nr, nr), dtype=np.uint64)
saver = tf.train.Saver()
with tf.train.SingularMonitoredSession() as sess:
# Load latest checkpoint
ckpt = tf.train.get_checkpoint_state(FLAGS.output_dir).model_checkpoint_path
saver.restore(sess.raw_session(), ckpt)
counter = 0
while not sess.should_stop():
counter += 1
if counter % 1 == 0:
print(counter)
# Predict next frames
rew_pd, rew_gt = sess.run(
[prediction_ops["target_reward"], data["target_reward"]])
for i in range(batch_size):
cm_next_frame[rew_gt[i, 0, 0], rew_pd[i, 0, 0]] += 1
for gt, pd in zip(rew_gt[i], rew_pd[i]):
cm_per_frame[gt, pd] += 1
print_confusion_matrix("Per-frame Confusion Matrix", cm_per_frame)
print_confusion_matrix("Next-frame Confusion Matrix", cm_next_frame)
def __init__(self,
hparams,
mode=tf.estimator.ModeKeys.TRAIN,
problem_hparams=None,
data_parallelism=None,
decode_hparams=None):
assert hparams.distill_phase in ["train", "distill"]
if hparams.distill_phase == "train" and hparams.teacher_learning_rate:
hparams.learning_rate = hparams.teacher_learning_rate
elif hparams.distill_phase == "distill" and hparams.student_learning_rate:
hparams.learning_rate = hparams.student_learning_rate
self.teacher_hparams = registry.hparams(hparams.teacher_hparams)
self.teacher_model = registry.model(
hparams.teacher_model)(self.teacher_hparams, mode, problem_hparams,
data_parallelism, decode_hparams)
self.student_hparams = registry.hparams(hparams.student_hparams)
self.student_model = registry.model(
hparams.student_model)(self.student_hparams, mode, problem_hparams,
data_parallelism, decode_hparams)
super(Distillation, self).__init__(hparams, mode, problem_hparams,
data_parallelism, decode_hparams)
def testSingleTrainStepCall(self):
"""Illustrate how to run a T2T model in a raw session."""
# Set model name, hparams, problems as would be set on command line.
model_name = "transformer"
FLAGS.hparams_set = "transformer_test"
FLAGS.problems = "tiny_algo"
data_dir = "/tmp" # Used only when a vocab file or such like is needed.
# Create the problem object, hparams, placeholders, features dict.
encoders = registry.problem(FLAGS.problems).feature_encoders(data_dir)
hparams = trainer_utils.create_hparams(FLAGS.hparams_set, data_dir)
trainer_utils.add_problem_hparams(hparams, FLAGS.problems)
# Now set a mode and create the model.
mode = tf.estimator.ModeKeys.TRAIN
model = registry.model(model_name)(hparams, mode)
# Create placeholder for features and make them batch-sized.
inputs_ph = tf.placeholder(dtype=tf.int32) # Just length dimension.
batch_inputs = tf.reshape(inputs_ph, [1, -1, 1, 1]) # Make it 4D.
targets_ph = tf.placeholder(dtype=tf.int32) # Just length dimension.
batch_targets = tf.reshape(targets_ph, [1, -1, 1, 1]) # Make it 4D.
features = {
"inputs": batch_inputs,
"targets": batch_targets,
"target_space_id": tf.constant(hparams.problems[0].target_space_id)
}
# Call the model.
predictions, _ = model(features)
nvars = len(tf.trainable_variables())
model(features) # Call again and check that reuse works.
self.assertEqual(nvars, len(tf.trainable_variables()))
# Having the graph, let's run it on some data.
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
inputs = "0 1 0"
targets = "0 1 0"
# Encode from raw string to numpy input array using problem encoders.
inputs_numpy = encoders["inputs"].encode(inputs)
targets_numpy = encoders["targets"].encode(targets)
# Feed the encoded inputs and targets and run session.
feed = {inputs_ph: inputs_numpy, targets_ph: targets_numpy}
np_predictions = sess.run(predictions, feed)
# Check that the result has the correct shape: batch x length x vocab_size
# where, for us, batch = 1, length = 3, vocab_size = 4.
self.assertEqual(np_predictions.shape, (1, 3, 1, 1, 4))
def make_estimator_model_fn(model_name,
hparams,
decode_hparams=None,
use_tpu=False):
model_cls = registry.model(model_name)
def wrapping_model_fn(features, labels, mode, params=None, config=None):
return model_cls.estimator_model_fn(
hparams,
features,
labels,
mode,
config=config,
params=params,
decode_hparams=decode_hparams,
use_tpu=use_tpu)
return wrapping_model_fn
def get_mnist_random_output(self, model_name, hparams_set=None,
mode=tf.estimator.ModeKeys.TRAIN):
hparams_set = hparams_set or model_name
x = np.random.random_integers(0, high=255, size=(1, 28, 28, 1))
y = np.random.random_integers(0, high=9, size=(1, 1))
features = {
"targets": tf.constant(x, dtype=tf.int32),
"inputs": tf.constant(y, dtype=tf.int32),
}
hparams = trainer_lib.create_hparams(
hparams_set, problem_name="image_mnist_rev", data_dir=".")
model = registry.model(model_name)(hparams, mode)
tf.train.create_global_step()
logits, _ = model(features)
with self.test_session() as session:
session.run(tf.global_variables_initializer())
res = session.run(logits)
return res
def build_model(hparams_set, model_name, data_dir, problem_name, beam_size=1):
"""Build the graph required to fetch the attention weights.
Args:
hparams_set: HParams set to build the model with.
model_name: Name of model.
data_dir: Path to directory containing training data.
problem_name: Name of problem.
beam_size: (Optional) Number of beams to use when decoding a translation.
If set to 1 (default) then greedy decoding is used.
Returns:
Tuple of (
inputs: Input placeholder to feed in ids to be translated.
targets: Targets placeholder to feed to translation when fetching
attention weights.
samples: Tensor representing the ids of the translation.
att_mats: Tensors representing the attention weights.
)
"""
hparams = trainer_lib.create_hparams(
hparams_set, data_dir=data_dir, problem_name=problem_name)
translate_model = registry.model(model_name)(
hparams, tf.estimator.ModeKeys.EVAL)
inputs = tf.placeholder(tf.int32, shape=(1, None, 1, 1), name='inputs')
targets = tf.placeholder(tf.int32, shape=(1, None, 1, 1), name='targets')
translate_model({
'inputs': inputs,
'targets': targets,
})
# Must be called after building the training graph, so that the dict will
# have been filled with the attention tensors. BUT before creating the
# inference graph otherwise the dict will be filled with tensors from
# inside a tf.while_loop from decoding and are marked unfetchable.
att_mats = get_att_mats(translate_model)
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
samples = translate_model.infer({
'inputs': inputs,
}, beam_size=beam_size)['outputs']
return inputs, targets, samples, att_mats
def __init__(self, environment_spec, length, other_hparams):
"""Batch of environments inside the TensorFlow graph."""
del other_hparams
self.length = length
initial_frames_problem = environment_spec.initial_frames_problem
self._min_reward = initial_frames_problem.min_reward
self._num_frames = environment_spec.video_num_input_frames
self._intrinsic_reward_scale = environment_spec.intrinsic_reward_scale
model_hparams = trainer_lib.create_hparams(
FLAGS.hparams_set, problem_name=FLAGS.problem)
model_hparams.force_full_predict = True
self._model = registry.model(FLAGS.model)(
model_hparams, tf.estimator.ModeKeys.PREDICT)
_, self.action_shape, self.action_dtype = get_action_space(environment_spec)
hparams = HParams(video_num_input_frames=
environment_spec.video_num_input_frames,
video_num_target_frames=
environment_spec.video_num_target_frames,
environment_spec=environment_spec)
if environment_spec.simulation_random_starts:
dataset = initial_frames_problem.dataset(tf.estimator.ModeKeys.TRAIN,
FLAGS.data_dir,
shuffle_files=True,
hparams=hparams)
dataset = dataset.shuffle(buffer_size=100)
else:
dataset = initial_frames_problem.dataset(tf.estimator.ModeKeys.TRAIN,
FLAGS.data_dir,
shuffle_files=False,
hparams=hparams).take(1)
dataset = dataset.map(lambda x: x["inputs"]).repeat()
self.history_buffer = HistoryBuffer(dataset, self.length)
shape = (self.length, initial_frames_problem.frame_height,
initial_frames_problem.frame_width,
initial_frames_problem.num_channels)
self._observ = tf.Variable(tf.zeros(shape, tf.float32), trainable=False)
def encode_env_frames(problem_name, ae_problem_name, autoencoder_path,
epoch_data_dir):
"""Encode all frames from problem_name and write out as ae_problem_name."""
with tf.Graph().as_default():
ae_hparams = trainer_lib.create_hparams("autoencoder_discrete_pong",
problem_name=problem_name)
problem = ae_hparams.problem
model = registry.model("autoencoder_ordered_discrete")(
ae_hparams, tf.estimator.ModeKeys.EVAL)
ae_problem = registry.problem(ae_problem_name)
ae_training_paths = ae_problem.training_filepaths(epoch_data_dir, 10, True)
ae_eval_paths = ae_problem.dev_filepaths(epoch_data_dir, 1, True)
skip_train = False
skip_eval = False
for path in ae_training_paths:
if tf.gfile.Exists(path):
skip_train = True
break
for path in ae_eval_paths:
if tf.gfile.Exists(path):
skip_eval = True
break
# Encode train data
if not skip_train:
dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN, epoch_data_dir,
shuffle_files=False, output_buffer_size=100,
preprocess=False)
encode_dataset(model, dataset, problem, ae_hparams, autoencoder_path,
ae_training_paths)
# Encode eval data
if not skip_eval:
dataset = problem.dataset(tf.estimator.ModeKeys.EVAL, epoch_data_dir,
shuffle_files=False, output_buffer_size=100,
preprocess=False)
encode_dataset(model, dataset, problem, ae_hparams, autoencoder_path,
ae_eval_paths)
def testMultipleTargetModalities(self):
# Use existing hparams and override target modality.
hparams = trainer_lib.create_hparams(
"transformer_tiny", data_dir=algorithmic.TinyAlgo.data_dir,
problem_name="tiny_algo")
# Manually turn off sharing. It is not currently supported for multitargets.
hparams.shared_embedding_and_softmax_weights = 0 # pylint: disable=line-too-long
hparams.problem_hparams.modality = {
"targets": hparams.problem_hparams.modality["targets"],
"targets_A": hparams.problem_hparams.modality["targets"],
"targets_B": hparams.problem_hparams.modality["targets"],
}
hparams.problem._hparams = hparams.problem_hparams
# Dataset
problem = hparams.problem
dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN,
algorithmic.TinyAlgo.data_dir)
dataset = dataset.repeat(None).padded_batch(10, dataset.output_shapes)
features = dataset.make_one_shot_iterator().get_next()
features = data_reader.standardize_shapes(features)
features["targets_A"] = features["targets_B"] = features["targets"]
# Model
model = registry.model("transformer")(hparams, tf.estimator.ModeKeys.TRAIN)
def body(args, mb=model.body):
out = mb(args)
return {"targets": out, "targets_A": out, "targets_B": out}
model.body = body
logits, losses = model(features)
self.assertTrue("training" in losses)
loss = losses["training"]
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
sess.run([logits, loss])
def __init__(self, environment_lambda, length):
"""Batch of environments inside the TensorFlow graph."""
self.length = length
initialization_env = environment_lambda()
hparams = trainer_lib.create_hparams(FLAGS.hparams_set,
problem_name=FLAGS.problem,
data_dir="UNUSED")
hparams.force_full_predict = True
self._model = registry.model(FLAGS.model)(
hparams, tf.estimator.ModeKeys.PREDICT)
self.action_space = initialization_env.action_space
self.action_shape = list(initialization_env.action_space.shape)
self.action_dtype = tf.int32
if hasattr(initialization_env.env, "get_starting_data"):
starting_observations, _, _ = initialization_env.env.get_starting_data(
)
obs_1 = starting_observations[0]
obs_2 = starting_observations[1]
else:
# Ancient method for environments not supporting get_starting_data
initialization_env.reset()
skip_frames = 20
for _ in range(skip_frames):
initialization_env.step(0)
obs_1 = initialization_env.step(0)[0]
obs_2 = initialization_env.step(0)[0]
self.frame_1 = tf.expand_dims(tf.cast(obs_1, tf.float32), 0)
self.frame_2 = tf.expand_dims(tf.cast(obs_2, tf.float32), 0)
shape = (self.length, ) + initialization_env.observation_space.shape
# TODO(blazej0) - make more generic - make higher number of
# and make it compatibile with NUMBER_OF_HISTORY_FRAMES
# previous observations possible.
self._observ = tf.Variable(tf.zeros(shape, tf.float32),
trainable=False)
self._prev_observ = tf.Variable(tf.zeros(shape, tf.float32),
trainable=False)
def testMultipleTargetModalities(self):
# Use existing hparams and override target modality.
hparams = trainer_lib.create_hparams(
"transformer_tiny", data_dir=algorithmic.TinyAlgo.data_dir,
problem_name="tiny_algo")
# Manually turn off sharing. It is not currently supported for multitargets.
hparams.shared_embedding_and_softmax_weights = 0 # pylint: disable=line-too-long
hparams.problem_hparams.modality = {
"targets": hparams.problem_hparams.modality["targets"],
"targets_A": hparams.problem_hparams.modality["targets"],
"targets_B": hparams.problem_hparams.modality["targets"],
}
hparams.problem._hparams = hparams.problem_hparams
# Dataset
problem = hparams.problem
dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN,
algorithmic.TinyAlgo.data_dir)
dataset = dataset.repeat(None).padded_batch(10, dataset.output_shapes)
features = dataset.make_one_shot_iterator().get_next()
features = problem_lib.standardize_shapes(features)
features["targets_A"] = features["targets_B"] = features["targets"]
# Model
model = registry.model("transformer")(hparams, tf.estimator.ModeKeys.TRAIN)
def body(args, mb=model.body):
out = mb(args)
return {"targets": out, "targets_A": out, "targets_B": out}
model.body = body
logits, losses = model(features)
self.assertTrue("training" in losses)
loss = losses["training"]
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
sess.run([logits, loss])
def __init__(self, *args, **kwargs):
model_args = copy.deepcopy(args)
model_kwargs = copy.deepcopy(kwargs)
super(Glue, self).__init__(*args, **kwargs)
try:
self._glue_symbol = self._hparams.glue_symbol
except:
self._glue_symbol = 2
if hasattr(model_args[0], "problem_hparams"):
model_args[0].problem_hparams = None
model_kwargs["problem_hparams"] = None
model_class = registry.model(self._hparams.glue_model)
if self._hparams.glue_is_lm:
class ModelWithoutInput(model_class):
# This hack is necessary because has_input() always returns
# true if no problem hparams are provided
@property
def has_input(self):
return False
self._glue_model = ModelWithoutInput(*model_args, **model_kwargs)
else:
self._glue_model = model_class(*model_args, **model_kwargs)
def __init__(self,
environment_lambda,
length,
problem,
simulation_random_starts=False,
intrinsic_reward_scale=0.):
"""Batch of environments inside the TensorFlow graph."""
self.length = length
self._min_reward = problem.min_reward
self._num_frames = problem.num_input_frames
self._intrinsic_reward_scale = intrinsic_reward_scale
initialization_env = environment_lambda()
hparams = trainer_lib.create_hparams(FLAGS.hparams_set,
problem_name=FLAGS.problem)
hparams.force_full_predict = True
self._model = registry.model(FLAGS.model)(
hparams, tf.estimator.ModeKeys.PREDICT)
self.action_space = initialization_env.action_space
self.action_shape = list(initialization_env.action_space.shape)
self.action_dtype = tf.int32
if simulation_random_starts:
dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN,
FLAGS.data_dir,
shuffle_files=True)
else:
dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN,
FLAGS.data_dir,
shuffle_files=False).take(1)
dataset = dataset.map(lambda x: x["inputs"]).repeat()
self.history_buffer = HistoryBuffer(dataset, self.length)
shape = (self.length, problem.frame_height, problem.frame_width,
problem.num_channels)
self._observ = tf.Variable(tf.zeros(shape, tf.float32),
trainable=False)
def testMultipleTargetModalities(self):
# HParams
hparams = trainer_lib.create_hparams("transformer_tiny",
data_dir=self.data_dir,
problem_name="tiny_algo")
tm = hparams.problem.get_hparams().target_modality
hparams.problem.get_hparams().target_modality = {
"targets": tm,
"A": tm,
"B": tm
}
# Dataset
problem = hparams.problem
dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN, self.data_dir)
dataset = dataset.repeat(None).padded_batch(10, dataset.output_shapes)
features = dataset.make_one_shot_iterator().get_next()
features = problem_lib.standardize_shapes(features)
features["A"] = features["B"] = features["targets"]
# Model
model = registry.model("transformer")(hparams,
tf.estimator.ModeKeys.TRAIN)
def body(args, mb=model.body):
out = mb(args)
return {"targets": out, "A": out, "B": out}
model.body = body
logits, losses = model(features)
self.assertTrue("training" in losses)
loss = losses["training"]
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
sess.run([logits, loss])
def main(_):
# Fetch the problem
wmt_problem = problems.problem(FLAGS.problem)
# Declare the path we need
data_dir = FLAGS.data_dir
checkpoint_dir = FLAGS.model_dir
ckpt_name = FLAGS.problem
# ckpt_dir = tf.train.latest_checkpoint(os.path.join(checkpoint_dir, ckpt_name))
ckpt_dir = tf.train.latest_checkpoint(checkpoint_dir)
# Create hparams and the model
model_name = FLAGS.model
hparams_set = FLAGS.hparams_set
hparams = trainer_lib.create_hparams(hparams_set,
data_dir=data_dir,
problem_name=FLAGS.problem)
# Get the encoders from the problem
encoders = wmt_problem.feature_encoders(data_dir)
translate_model = registry.model(model_name)(hparams, Modes.EVAL)
sys.stdout.write('> ')
sys.stdout.flush()
sentence_en = sys.stdin.readline().strip()
while sentence_en:
if sentence_en == 'q':
print("Close this process")
break
outputs = translate(encoders, translate_model, ckpt_dir, sentence_en)
print(outputs)
print('> ', end='')
sys.stdout.flush()
sentence_en = sys.stdin.readline()
def __init__(self,
reward_range,
observation_space,
action_space,
frame_stack_size,
initial_frame_chooser,
batch_size,
model_name,
model_hparams,
model_dir,
intrinsic_reward_scale=0.0):
"""Batch of environments inside the TensorFlow graph."""
super(SimulatedBatchEnv, self).__init__(observation_space,
action_space)
self.batch_size = batch_size
self._min_reward = reward_range[0]
self._num_frames = frame_stack_size
self._intrinsic_reward_scale = intrinsic_reward_scale
model_hparams = copy.copy(model_hparams)
problem = DummyWorldModelProblem(action_space, reward_range)
trainer_lib.add_problem_hparams(model_hparams, problem)
model_hparams.force_full_predict = True
self._model = registry.model(model_name)(model_hparams,
tf.estimator.ModeKeys.PREDICT)
self.history_buffer = HistoryBuffer(initial_frame_chooser,
self.observ_shape,
self.observ_dtype,
self._num_frames, self.batch_size)
self._observ = tf.Variable(tf.zeros((batch_size, ) + self.observ_shape,
self.observ_dtype),
trainable=False)
self._model_dir = model_dir
def initialize_model(problem_name,
data_dir,
hparam_set,
hparams,
model_name,
ckpt_dir,
split=Modes.TRAIN):
"""Returns an initialized model, dataset iterator and hparams."""
tf.reset_default_graph()
# create hparams and get glyphazzn problem definition
hparams = trainer_lib.create_hparams(hparam_set,
hparams,
data_dir=data_dir,
problem_name=problem_name)
problem = registry.problem(problem_name)
# get model definition
ModelClass = registry.model(model_name)
model = ModelClass(hparams,
mode=Modes.PREDICT,
problem_hparams=hparams.problem_hparams)
# create dataset iterator from problem definition
dataset = problem.dataset(Modes.PREDICT,
dataset_split=split,
data_dir=data_dir,
shuffle_files=False,
hparams=hparams).batch(1)
iterator = tfe.Iterator(dataset)
# finalize/initialize model
# creates ops to be initialized
output, extra_losses = model(iterator.next())
model.initialize_from_ckpt(ckpt_dir) # initializes ops
return model, iterator, hparams
def testMultipleTargetModalities(self):
# HParams
hparams = trainer_lib.create_hparams(
"transformer_tiny", data_dir=self.data_dir, problem_name="tiny_algo")
tm = hparams.problem.get_hparams().target_modality
hparams.problem.get_hparams().target_modality = {
"targets": tm,
"A": tm,
"B": tm
}
# Dataset
problem = hparams.problem
dataset = problem.dataset(tf.estimator.ModeKeys.TRAIN, self.data_dir)
dataset = dataset.repeat(None).padded_batch(10, dataset.output_shapes)
features = dataset.make_one_shot_iterator().get_next()
features = problem_lib.standardize_shapes(features)
features["A"] = features["B"] = features["targets"]
# Model
model = registry.model("transformer")(hparams, tf.estimator.ModeKeys.TRAIN)
def body(args, mb=model.body):
out = mb(args)
return {"targets": out, "A": out, "B": out}
model.body = body
logits, losses = model(features)
self.assertTrue("training" in losses)
loss = losses["training"]
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
sess.run([logits, loss])
def __init__(
self, reward_range, observation_space, action_space, frame_stack_size,
frame_height, frame_width, initial_frame_chooser, batch_size, model_name,
model_hparams, model_dir, intrinsic_reward_scale=0.0
):
"""Batch of environments inside the TensorFlow graph."""
super(SimulatedBatchEnv, self).__init__(observation_space, action_space)
self.batch_size = batch_size
self._min_reward = reward_range[0]
self._num_frames = frame_stack_size
self._intrinsic_reward_scale = intrinsic_reward_scale
model_hparams = copy.copy(model_hparams)
problem = DummyWorldModelProblem(action_space, reward_range,
frame_height, frame_width)
trainer_lib.add_problem_hparams(model_hparams, problem)
model_hparams.force_full_predict = True
self._model = registry.model(model_name)(
model_hparams, tf.estimator.ModeKeys.PREDICT
)
self.history_buffer = HistoryBuffer(
initial_frame_chooser, self.observ_shape, self.observ_dtype,
self._num_frames, self.batch_size
)
self._observ = tf.Variable(
tf.zeros((batch_size,) + self.observ_shape, self.observ_dtype),
trainable=False
)
self._reset_model = tf.get_variable(
"reset_model", [], trainable=False, initializer=tf.zeros_initializer())
self._model_dir = model_dir
# -*- coding: utf-8 -*-
"""
@author: 代码医生工作室
@公众号：xiangyuejiqiren （内有更多优秀文章及学习资料）
@来源: <深度学习之TensorFlow工程化项目实战>配套代码 （700+页）
@配套代码技术支持：bbs.aianaconda.com (有问必答)
"""
#6-19
import tensorflow as tf
from tensor2tensor import models
from tensor2tensor.utils import t2t_model
from tensor2tensor.utils import registry
print(len(registry.list_models()), registry.list_models())
print(registry.model('transformer'))
print(len(registry.list_hparams()), registry.list_hparams('transformer'))
print(registry.hparams('transformer_base_v1'))
def __init__(self,
reward_range,
observation_space,
action_space,
frame_stack_size,
frame_height,
frame_width,
initial_frame_chooser,
batch_size,
model_name,
model_hparams,
model_dir,
intrinsic_reward_scale=0.0,
sim_video_dir=None):
"""Batch of environments inside the TensorFlow graph."""
super(SimulatedBatchEnv, self).__init__(observation_space,
action_space)
self._ffmpeg_works = common_video.ffmpeg_works()
self.batch_size = batch_size
self._min_reward = reward_range[0]
self._num_frames = frame_stack_size
self._intrinsic_reward_scale = intrinsic_reward_scale
self._episode_counter = tf.get_variable("episode_counter",
initializer=tf.zeros(
(), dtype=tf.int32),
trainable=False,
dtype=tf.int32)
if sim_video_dir:
self._video_every_epochs = 100
self._video_dir = sim_video_dir
self._video_writer = None
self._video_counter = 0
tf.gfile.MakeDirs(self._video_dir)
self._video_condition = tf.equal(
self._episode_counter.read_value() % self._video_every_epochs,
0)
else:
self._video_condition = tf.constant(False, dtype=tf.bool, shape=())
model_hparams = copy.copy(model_hparams)
problem = DummyWorldModelProblem(action_space, reward_range,
frame_height, frame_width)
trainer_lib.add_problem_hparams(model_hparams, problem)
model_hparams.force_full_predict = True
self._model = registry.model(model_name)(model_hparams,
tf.estimator.ModeKeys.PREDICT)
self.history_buffer = HistoryBuffer(initial_frame_chooser,
self.observ_shape,
self.observ_dtype,
self._num_frames, self.batch_size)
self._observ = tf.Variable(tf.zeros((batch_size, ) + self.observ_shape,
self.observ_dtype),
trainable=False)
self._reset_model = tf.get_variable("reset_model", [],
trainable=False,
initializer=tf.zeros_initializer())
self._model_dir = model_dir
def get_policy(observations, hparams, action_space,
distributional_size=1, epoch=-1):
"""Get a policy network.
Args:
observations: observations
hparams: parameters
action_space: action space
distributional_size: optional number of buckets for distributional RL
epoch: optional epoch number
Returns:
Tuple (action logits, value).
"""
if not isinstance(action_space, gym.spaces.Discrete):
raise ValueError("Expecting discrete action space.")
obs_shape = common_layers.shape_list(observations)
(frame_height, frame_width) = obs_shape[2:4]
# TODO(afrozm): We have these dummy problems mainly for hparams, so cleanup
# when possible and do this properly.
if hparams.policy_problem_name == "dummy_policy_problem_ttt":
tf.logging.info("Using DummyPolicyProblemTTT for the policy.")
policy_problem = tic_tac_toe_env.DummyPolicyProblemTTT()
else:
tf.logging.info("Using DummyPolicyProblem for the policy.")
policy_problem = DummyPolicyProblem(action_space, frame_height, frame_width)
trainer_lib.add_problem_hparams(hparams, policy_problem)
hparams.force_full_predict = True
model = registry.model(hparams.policy_network)(
hparams, tf.estimator.ModeKeys.TRAIN
)
try:
num_target_frames = hparams.video_num_target_frames
except AttributeError:
num_target_frames = 1
target_value_shape_suffix = [num_target_frames]
if distributional_size > 1:
target_value_shape_suffix = [num_target_frames, distributional_size]
features = {
"inputs": observations,
"epoch": tf.constant(epoch + 1),
"input_action": tf.zeros(obs_shape[:2] + [1], dtype=tf.int32),
"input_reward": tf.zeros(obs_shape[:2] + [1], dtype=tf.int32),
"targets": tf.zeros(obs_shape[:1] + [num_target_frames] + obs_shape[2:]),
"target_action": tf.zeros(
obs_shape[:1] + [num_target_frames, 1], dtype=tf.int32),
"target_reward": tf.zeros(
obs_shape[:1] + [num_target_frames, 1], dtype=tf.int32),
"target_policy": tf.zeros(
obs_shape[:1] + [num_target_frames] + [action_space.n]),
"target_value": tf.zeros(
obs_shape[:1] + target_value_shape_suffix)
}
model.distributional_value_size = max(distributional_size, 1)
model.use_epochs = hparams.use_epochs
with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
t2t_model.create_dummy_vars()
(targets, _) = model(features)
target_values = targets["target_value"][:, 0]
if distributional_size > 1:
target_values = targets["target_value"][:, :]
return (targets["target_policy"][:, 0, :], target_values)
def _init_env(self):
FLAGS.use_tpu = False
tf.logging.set_verbosity(tf.logging.DEBUG)
tf.logging.info("Import usr dir from %s", self._usr_dir)
if self._usr_dir != None:
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
tf.logging.info("Start to create hparams,for %s of %s", self._problem,
self._hparams_set)
# 获取 模型参数
self._hparams = create_hparams()
# 获取 decode用的参数
self._hparams_decode = create_decode_hparams(
extra_length=self._extra_length,
batch_size=self._batch_size,
beam_size=self._beam_size,
alpha=self._alpha,
return_beams=self._return_beams,
write_beam_scores=self._write_beam_scores,
force_decode_length=self._force_decode_length)
# self.estimator = trainer_lib.create_estimator(
# FLAGS.model,
# self._hparams,
# t2t_trainer.create_run_config(self._hparams),
# decode_hparams=self._hparams_decode,
# use_tpu=False)
tf.logging.info("Finish intialize environment")
#######
### make input placeholder
self._inputs_ph = tf.placeholder(
dtype=tf.int32) # shape not specified,any shape
x = tf.placeholder(dtype=tf.int32)
x.set_shape([None, None]) # ? -> (?,?)
x = tf.expand_dims(x, axis=[2]) # -> (?,?,1)
x = tf.to_int32(x)
self._inputs_ph = x
#batch_inputs = tf.reshape(self._inputs_ph, [self._batch_size, -1, 1, 1])
batch_inputs = x
###
# batch_inputs = tf.reshape(self._inputs_ph, [-1, -1, 1, 1])
#targets_ph = tf.placeholder(dtype=tf.int32)
#batch_targets = tf.reshape(targets_ph, [1, -1, 1, 1])
self._features = {
"inputs": batch_inputs,
"problem_choice": 0, # We run on the first problem here.
"input_space_id": self._hparams.problem_hparams.input_space_id,
"target_space_id": self._hparams.problem_hparams.target_space_id
}
### 加入 decode length 变长的 分类时候没用
self.input_extra_length_ph = tf.placeholder(dtype=tf.int32)
self._features['decode_length'] = self.input_extra_length_ph
## target
self._targets_ph = tf.placeholder(tf.int32,
shape=(None, None, None, None),
name='targets')
self._features['targets'] = self._targets_ph
target_pretend = np.zeros((1, 1, 1, 1))
## 去掉 整数的
del self._features["problem_choice"]
del self._features["input_space_id"]
del self._features["target_space_id"]
del self._features['decode_length']
####
#mode = tf.estimator.ModeKeys.PREDICT # affect last_only t2t_model._top_single ,[1,?,1,512]->[1,1,1,1,64]
# if self.predict_or_eval=='EVAL':
# mode = tf.estimator.ModeKeys.EVAL # affect last_only t2t_model._top_single ,[1,?,1,512]->[1,?,1,1,64]
# # estimator_spec = model_builder.model_fn(self._model_name, features, mode, self._hparams,
# # problem_names=[self._problem], decode_hparams=self._hparams_dc)
# if self.predict_or_eval=='PREDICT':
# mode = tf.estimator.ModeKeys.PREDICT
if self.predict_or_eval == 'and':
mode = tf.estimator.ModeKeys.EVAL
###########
# registry.model
############
translate_model = registry.model(self._model_name)(
hparams=self._hparams,
decode_hparams=self._hparams_decode,
mode=mode)
self.predict_dict = {}
### get logit ,EVAL mode
self.logits, _ = translate_model(self._features)
### get infer result ,PREDICT mode
translate_model.set_mode(tf.estimator.ModeKeys.PREDICT)
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
self.outputs_scores = translate_model.infer(
features=self._features,
decode_length=50,
beam_size=self._beam_size,
top_beams=self._beam_size,
alpha=self._alpha)
######
tf.logging.info("Start to init tf session")
if self._isGpu:
print('Using GPU in Decoder')
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=self._fraction)
self._sess = tf.Session(
config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
gpu_options=gpu_options))
else:
print('Using CPU in Decoder')
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0)
config = tf.ConfigProto(gpu_options=gpu_options)
config.allow_soft_placement = True
config.log_device_placement = False
self._sess = tf.Session(config=config)
with self._sess.as_default():
ckpt = saver_mod.get_checkpoint_state(self._model_dir)
saver = tf.train.Saver()
tf.logging.info("Start to restore the parameters from %s",
ckpt.model_checkpoint_path)
saver.restore(self._sess, ckpt.model_checkpoint_path)
tf.logging.info("Finish intialize environment")
def _init_env(self):
FLAGS.use_tpu = False
tf.logging.set_verbosity(tf.logging.DEBUG)
tf.logging.info("Import usr dir from %s", self._usr_dir)
if self._usr_dir != None:
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
tf.logging.info("Start to create hparams,for %s of %s", self._problem,
self._hparams_set)
self._hparams = create_hparams()
self._hparams_decode = create_decode_hparams(
extra_length=self._extra_length,
batch_size=self._batch_size,
beam_size=self._beam_size,
alpha=self._alpha,
return_beams=self._return_beams,
write_beam_scores=self._write_beam_scores,
force_decode_length=self._force_decode_length)
self.estimator = trainer_lib.create_estimator(
FLAGS.model,
self._hparams,
t2t_trainer.create_run_config(self._hparams),
decode_hparams=self._hparams_decode,
use_tpu=False)
tf.logging.info("Finish intialize environment")
#######
### make input placeholder
#self._inputs_ph = tf.placeholder(dtype=tf.int32) # shape not specified,any shape
# x=tf.placeholder(dtype=tf.int32)
# x.set_shape([None, None]) # ? -> (?,?)
# x = tf.expand_dims(x, axis=[2])# -> (?,?,1)
# x = tf.to_int32(x)
# self._inputs_ph=x
#batch_inputs = tf.reshape(self._inputs_ph, [self._batch_size, -1, 1, 1])
#batch_inputs=x
###
# batch_inputs = tf.reshape(self._inputs_ph, [-1, -1, 1, 1])
#targets_ph = tf.placeholder(dtype=tf.int32)
#batch_targets = tf.reshape(targets_ph, [1, -1, 1, 1])
self.inputs_ph = tf.placeholder(tf.int32,
shape=(None, None, 1, 1),
name='inputs')
self.targets_ph = tf.placeholder(tf.int32,
shape=(None, None, None, None),
name='targets')
self.input_extra_length_ph = tf.placeholder(dtype=tf.int32, shape=[])
self._features = {
"inputs": self.inputs_ph,
"problem_choice": 0, # We run on the first problem here.
"input_space_id": self._hparams.problem_hparams.input_space_id,
"target_space_id": self._hparams.problem_hparams.target_space_id
}
### 加入 decode length 变长的
self._features['decode_length'] = self.input_extra_length_ph
## target
self._features['targets'] = self.targets_ph
## 去掉 整数的
del self._features["problem_choice"]
del self._features["input_space_id"]
del self._features["target_space_id"]
#del self._features['decode_length']
####
mode = tf.estimator.ModeKeys.EVAL
translate_model = registry.model(self._model_name)(
hparams=self._hparams,
decode_hparams=self._hparams_decode,
mode=mode)
self.predict_dict = {}
### get logit ,attention mats
self.logits, _ = translate_model(self._features) #[? ? ? 1 vocabsz]
#translate_model(features)
from visualization import get_att_mats
self.att_mats = get_att_mats(translate_model,
self._model_name) # enc, dec, encdec
### get infer
translate_model.set_mode(tf.estimator.ModeKeys.PREDICT)
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
self.outputs_scores = translate_model.infer(
features=self._features,
decode_length=self._extra_length,
beam_size=self._beam_size,
top_beams=self._beam_size,
alpha=self._alpha) #outputs 4,4,63
######
tf.logging.info("Start to init tf session")
if self._isGpu:
print('Using GPU in Decoder')
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=self._fraction)
self._sess = tf.Session(
config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
gpu_options=gpu_options))
else:
print('Using CPU in Decoder')
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0)
config = tf.ConfigProto(gpu_options=gpu_options)
config.allow_soft_placement = True
config.log_device_placement = False
self._sess = tf.Session(config=config)
with self._sess.as_default():
ckpt = saver_mod.get_checkpoint_state(self._model_dir)
saver = tf.train.Saver()
tf.logging.info("Start to restore the parameters from %s",
ckpt.model_checkpoint_path)
saver.restore(self._sess, ckpt.model_checkpoint_path)
tf.logging.info("Finish intialize environment")
def benchmark(self, ckpt_dir, outer_steps=100, inner_steps=1000):
"""Run repeatedly on dummy data to benchmark inference."""
# Turn off Grappler optimizations.
options = {"disable_meta_optimizer": True}
tf.config.optimizer.set_experimental_options(options)
# Create the model outside the loop body.
hparams = registry.hparams(self.hparams_set)
hparams_lib.add_problem_hparams(hparams, self.problem_name)
model_cls = registry.model(self.model_name)
model = model_cls(hparams, tf.estimator.ModeKeys.EVAL)
# Run only the model body (no data pipeline) on device.
feature_shape = [
hparams.batch_size, 3 * self.image_size * self.image_size
]
features = {"targets": tf.zeros(feature_shape, dtype=tf.int32)}
# Call the model once to initialize the variables. Note that
# this should never execute.
with tf.variable_scope(self.model_name) as vso:
transformed_features = model.bottom(features)
with tf.variable_scope("body") as vsi:
body_out = model.body(transformed_features)
logits = model.top(body_out, features)
model.loss(logits, features)
def call_model(features):
with tf.variable_scope(vso, reuse=tf.AUTO_REUSE):
transformed_features = model.bottom(features)
with tf.variable_scope(vsi, reuse=tf.AUTO_REUSE):
body_out = model.body(transformed_features)
logits = model.top(body_out, features)
return model.loss(logits, features)
# Run the function body in a loop to amortize session overhead.
loop_index = tf.zeros([], dtype=tf.int32)
initial_loss = (tf.zeros([]), tf.zeros([]))
def loop_cond(idx, _):
return tf.less(idx, tf.constant(inner_steps, dtype=tf.int32))
def loop_body(idx, _):
return idx + 1, call_model(features)
benchmark_op = tf.while_loop(loop_cond,
loop_body, [loop_index, initial_loss],
parallel_iterations=1,
back_prop=False)
session_config = tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=False, per_process_gpu_memory_fraction=0.95))
run_metadata = tf.RunMetadata()
with tf.Session(config=session_config) as sess:
self.restore_model(sess, ckpt_dir)
tps = []
for idx in range(outer_steps):
start_time = time.time()
sess.run(benchmark_op, run_metadata=run_metadata)
elapsed_time = time.time() - start_time
tps.append(inner_steps * hparams.batch_size * (64 * 64 * 3) /
elapsed_time)
logging.error("Iterations %d processed %f TPS.", idx, tps[-1])
# Skip the first iteration where all the setup and allocation happens.
tps = np.asarray(tps[1:])
logging.error("Mean/Std/Max/Min throughput = %f / %f / %f / %f",
np.mean(tps), np.std(tps), tps.max(), tps.min())
def __init__(self,
src_vocab_size,
trg_vocab_size,
model_name,
problem_name,
hparams_set_name,
t2t_usr_dir,
checkpoint_dir,
t2t_unk_id=None,
single_cpu_thread=False,
max_terminal_id=-1,
pop_id=-1):
"""Creates a new simultaneous T2T predictor. The constructor prepares
the TensorFlow session for predict_next() calls. This includes:
- Load hyper parameters from the given set (hparams)
- Update registry, load T2T model
- Create TF placeholders for source sequence and target prefix
- Create computation graph for computing log probs.
- Create a MonitoredSession object, which also handles
restoring checkpoints.
Args:
src_vocab_size (int): Source vocabulary size.
trg_vocab_size (int): Target vocabulary size.
model_name (string): T2T model name.
problem_name (string): T2T problem name.
hparams_set_name (string): T2T hparams set name.
t2t_usr_dir (string): See --t2t_usr_dir in tensor2tensor.
checkpoint_dir (string): Path to the T2T checkpoint
directory. The predictor will load
the top most checkpoint in the
`checkpoints` file.
t2t_unk_id (int): If set, use this ID to get UNK scores. If
None, UNK is always scored with -inf.
single_cpu_thread (bool): If true, prevent tensorflow from
doing multithreading.
max_terminal_id (int): If positive, maximum terminal ID. Needs to
be set for syntax-based T2T models.
pop_id (int): If positive, ID of the POP or closing bracket symbol.
Needs to be set for syntax-based T2T models.
"""
super(SimT2TPredictor_v2, self).__init__(t2t_usr_dir, checkpoint_dir,
t2t_unk_id, single_cpu_thread)
self.consumed = []
self.src_sentence = []
self.pop_id = pop_id
self.max_terminal_id = max_terminal_id
self.previous_encode = -1
self.previous_decode = -1
predictor_graph = tf.Graph()
with predictor_graph.as_default() as g:
hparams = self._create_hparams(src_vocab_size, trg_vocab_size,
hparams_set_name, problem_name)
p_hparams = hparams.problems[0]
self._inputs_var = tf.placeholder(dtype=tf.int32,
shape=[None],
name="sgnmt_inputs")
self._targets_var = tf.placeholder(dtype=tf.int32,
shape=[None],
name="sgnmt_targets")
features = {
"problem_choice": tf.constant(0),
"input_space_id": tf.constant(p_hparams.input_space_id),
"target_space_id": tf.constant(p_hparams.target_space_id),
"inputs": expand_input_dims_for_t2t(self._inputs_var),
"targets": expand_input_dims_for_t2t(self._targets_var)
}
model = registry.model(model_name)(
hparams, tf.estimator.ModeKeys.PREDICT, hparams.problems[0], 0,
devices.data_parallelism(),
devices.ps_devices(all_workers=True))
sharded_logits, _ = model.model_fn(features)
self._log_probs = log_prob_from_logits(sharded_logits[0])
self._encoder_output = model.encoder_output
self._encoder_decoder_attention_bias = model.attention_bias
self._decoder_output = model.decoder_output
self.mon_sess = self.create_session()
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
# Create hparams
hparams = create_hparams()
hparams.force_full_predict = True
hparams.scheduled_sampling_k = -1
# Params
num_agents = 1 # TODO(mbz): fix the code for more agents
num_steps = FLAGS.num_steps
num_actions = hparams.problem.num_actions
frame_shape = hparams.problem.frame_shape
resized_frame = hparams.preprocess_resize_frames is not None
if resized_frame:
frame_shape = hparams.preprocess_resize_frames
frame_shape += [hparams.problem.num_channels]
dataset = registry.problem(FLAGS.problem).dataset(
tf.estimator.ModeKeys.TRAIN, shuffle_files=True, hparams=hparams)
dataset = dataset.apply(
tf.contrib.data.batch_and_drop_remainder(num_agents))
data = dataset.make_one_shot_iterator().get_next()
# Setup input placeholders
input_size = [num_agents, hparams.video_num_input_frames]
placeholders = {
"inputs": tf.placeholder(tf.float32, input_size + frame_shape),
"input_action": tf.placeholder(tf.int64, input_size + [1]),
"input_reward": tf.placeholder(tf.int64, input_size + [1]),
}
# Creat model
model_cls = registry.model(FLAGS.model)
model = model_cls(hparams, tf.estimator.ModeKeys.PREDICT)
prediction_ops = model.infer(placeholders)
states_q = Queue(maxsize=hparams.video_num_input_frames)
actions_q = Queue(maxsize=hparams.video_num_input_frames)
rewards_q = Queue(maxsize=hparams.video_num_input_frames)
all_qs = (states_q, actions_q, rewards_q)
writer = common_video.WholeVideoWriter(fps=10,
output_path=FLAGS.output_gif)
saver = tf.train.Saver()
with tf.train.SingularMonitoredSession() as sess:
# Load latest checkpoint
ckpt = tf.train.get_checkpoint_state(
FLAGS.output_dir).model_checkpoint_path
saver.restore(sess.raw_session(), ckpt)
# get init frames from the dataset
data_np = sess.run(data)
frames = np.split(data_np["inputs"], hparams.video_num_input_frames, 1)
for frame in frames:
frame = np.squeeze(frame, 1)
states_q.put(frame)
writer.write(frame[0].astype(np.uint8))
actions = np.split(data_np["input_action"],
hparams.video_num_input_frames, 1)
for action in actions:
actions_q.put(np.squeeze(action, 1))
rewards = np.split(data_np["input_reward"],
hparams.video_num_input_frames, 1)
for reward in rewards:
rewards_q.put(np.squeeze(reward, 1))
for step in range(num_steps):
print(">>>>>>> ", step)
random_actions = np.random.randint(num_actions - 1)
random_actions = np.expand_dims(random_actions, 0)
random_actions = np.tile(random_actions, (num_agents, 1))
# Shape inputs and targets
inputs, input_action, input_reward = (np.stack(list(q.queue),
axis=1)
for q in all_qs)
# Predict next frames
feed = {
placeholders["inputs"]: inputs,
placeholders["input_action"]: input_action,
placeholders["input_reward"]: input_reward,
}
predictions = sess.run(prediction_ops, feed_dict=feed)
predicted_states = predictions["targets"][:, 0]
predicted_reward = predictions["target_reward"][:, 0]
# Update queues
new_data = (predicted_states, random_actions, predicted_reward)
for q, d in zip(all_qs, new_data):
q.get()
q.put(d.copy())
writer.write(np.round(predicted_states[0]).astype(np.uint8))
video = writer.finish()
writer.save_to_disk(video)
x_y = iterator.get_next()
x = tf.cast(x_y['inputs'], tf.int32)
y = tf.cast(x_y['targets'], tf.int32)
inputs = {
"inputs": tf.expand_dims(tf.expand_dims(x, 2), 3),
"target_space_id": p_hparams.target_space_id,
"targets": tf.expand_dims(tf.expand_dims(y, 2), 3)
}
# version 1.3.0 introduced changes in the API
from pkg_resources import get_distribution as get_version
t2t_version = int(''.join(get_version('tensor2tensor').version.split('.')[:2]))
is_t2t130_or_greater = t2t_version >= 13
if is_t2t130_or_greater:
translate_model = registry.model(model_name)(hparams, mode)
inputs['target_space_id'] = tf.convert_to_tensor(inputs['target_space_id'])
logits, losses = translate_model(inputs)
logits = tf.squeeze(logits, [2, 3])
else:
from tensor2tensor.models.transformer import Transformer
translate_model = Transformer(hparams, mode, p_hparams)
translate_model._hparams.problems[0].target_modality = SymbolModality(
hparams, encoders['targets'].vocab_size)
sharded_logits, losses = translate_model.model_fn(features=inputs)
logits = tf.concat(sharded_logits, 0)
logits = tf.squeeze(logits, [2, 3])
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
if args.training:
preds = tf.to_int32(tf.arg_max(logits, dimension=-1))
def testUnknownModel(self):
with self.assertRaisesRegexp(LookupError, "never registered"):
registry.model("not_registered")
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
# Create hparams
hparams = trainer_lib.create_hparams(
FLAGS.hparams_set,
FLAGS.hparams,
data_dir=os.path.expanduser(FLAGS.data_dir),
problem_name=FLAGS.problem)
hparams.force_full_predict = True
hparams.scheduled_sampling_k = -1
# Params
num_agents = 1 # TODO(mbz): fix the code for more agents
num_steps = FLAGS.num_steps
if hasattr(hparams.problem, "num_actions"):
num_actions = hparams.problem.num_actions
else:
num_actions = None
frame_shape = hparams.problem.frame_shape
resized_frame = hparams.preprocess_resize_frames is not None
if resized_frame:
frame_shape = hparams.preprocess_resize_frames
frame_shape += [hparams.problem.num_channels]
dataset = registry.problem(FLAGS.problem).dataset(
tf.estimator.ModeKeys.TRAIN,
shuffle_files=True,
data_dir=os.path.expanduser(FLAGS.data_dir),
hparams=hparams)
dataset = dataset.apply(tf.contrib.data.batch_and_drop_remainder(num_agents))
data = dataset.make_one_shot_iterator().get_next()
# Setup input placeholders
input_size = [num_agents, hparams.video_num_input_frames]
if num_actions is None:
placeholders = {
"inputs": tf.placeholder(tf.float32, input_size + frame_shape)
}
else:
placeholders = {
"inputs": tf.placeholder(tf.float32, input_size + frame_shape),
"input_action": tf.placeholder(tf.int64, input_size + [1]),
"input_reward": tf.placeholder(tf.int64, input_size + [1]),
"reset_internal_states": tf.placeholder(tf.float32, []),
}
# Create model.
model_cls = registry.model(FLAGS.model)
model = model_cls(hparams, tf.estimator.ModeKeys.PREDICT)
prediction_ops = model.infer(placeholders)
states_q = Queue(maxsize=hparams.video_num_input_frames)
actions_q = Queue(maxsize=hparams.video_num_input_frames)
rewards_q = Queue(maxsize=hparams.video_num_input_frames)
if num_actions is not None:
all_qs = [states_q, actions_q, rewards_q]
else:
all_qs = [states_q]
writer = common_video.WholeVideoWriter(
fps=FLAGS.fps, output_path=FLAGS.output_gif)
saver = tf.train.Saver(tf.trainable_variables())
with tf.train.SingularMonitoredSession() as sess:
# Load latest checkpoint
ckpt = tf.train.get_checkpoint_state(FLAGS.output_dir).model_checkpoint_path
saver.restore(sess.raw_session(), ckpt)
# get init frames from the dataset
data_np = sess.run(data)
frames = np.split(data_np["inputs"], hparams.video_num_input_frames, 1)
for frame in frames:
frame = np.squeeze(frame, 1)
states_q.put(frame)
writer.write(frame[0].astype(np.uint8))
if num_actions is not None:
actions = np.split(data_np["input_action"],
hparams.video_num_input_frames, 1)
for action in actions:
actions_q.put(np.squeeze(action, 1))
rewards = np.split(data_np["input_reward"],
hparams.video_num_input_frames, 1)
for reward in rewards:
rewards_q.put(np.squeeze(reward, 1))
for step in range(num_steps):
print(">>>>>>> ", step)
if num_actions is not None:
random_actions = np.random.randint(num_actions-1)
random_actions = np.expand_dims(random_actions, 0)
random_actions = np.tile(random_actions, (num_agents, 1))
# Shape inputs and targets
inputs, input_action, input_reward = (
np.stack(list(q.queue), axis=1) for q in all_qs)
else:
assert len(all_qs) == 1
q = all_qs[0]
elems = list(q.queue)
# Need to adjust shapes sometimes.
for i, e in enumerate(elems):
if len(e.shape) < 4:
elems[i] = np.expand_dims(e, axis=0)
inputs = np.stack(elems, axis=1)
# Predict next frames
if num_actions is None:
feed = {placeholders["inputs"]: inputs}
else:
feed = {
placeholders["inputs"]: inputs,
placeholders["input_action"]: input_action,
placeholders["input_reward"]: input_reward,
placeholders["reset_internal_states"]: float(step == 0),
}
predictions = sess.run(prediction_ops, feed_dict=feed)
if num_actions is None:
predicted_states = predictions[:, 0]
else:
predicted_states = predictions["targets"][:, 0]
predicted_reward = predictions["target_reward"][:, 0]
# Update queues
if num_actions is None:
new_data = (predicted_states)
else:
new_data = (predicted_states, random_actions, predicted_reward)
for q, d in zip(all_qs, new_data):
q.get()
q.put(d.copy())
writer.write(np.round(predicted_states[0]).astype(np.uint8))
writer.finish_to_disk()
def t2t_score_file(filename):
"""
Score each line in a file and return the scores.
:param str filename: T2T checkpoint
"""
# Prepare model.
hparams = create_t2t_hparams()
encoders = registry.problem(FLAGS_problem).feature_encoders(FLAGS_data_dir)
# Prepare features for feeding into the model.
inputs_ph = tf.placeholder(dtype=tf.int32, shape=(None, None)) # Just length dimension.
targets_ph = tf.placeholder(dtype=tf.int32, shape=(None, None)) # Just length dimension.
features = {
"inputs": inputs_ph,
"targets": targets_ph,
}
# Prepare the model and the graph when model runs on features.
model = registry.model(FLAGS_model)(hparams, tf.estimator.ModeKeys.EVAL)
assert isinstance(model, tensor2tensor.models.transformer.Transformer)
# final_output: tensor of logits with shape [batch_size, O, P, body_output_size.
# losses: either single loss as a scalar, a list, a tensor (to be averaged)
# or a dictionary of losses.
final_output, losses = model(features)
assert isinstance(losses, dict)
saver = tf.train.Saver()
sess = tf.Session()
# Load weights from checkpoint.
ckpts = tf.train.get_checkpoint_state(FLAGS_output_dir)
ckpt = ckpts.model_checkpoint_path
saver.restore(sess, ckpt)
# writer = tf.summary.FileWriter('logs', sess.graph)
# writer.close()
# Run on each line.
results = []
for line in open(filename):
tab_split = line.split("\t")
if len(tab_split) > 2:
raise ValueError("Each line must have at most one tab separator.")
assert len(tab_split) == 2
targets = tab_split[1].strip()
inputs = tab_split[0].strip()
# Run encoders and append EOS symbol.
targets_numpy = encoders["targets"].encode(targets) + [text_encoder.EOS_ID]
inputs_numpy = encoders["inputs"].encode(inputs) + [text_encoder.EOS_ID]
# Prepare the feed.
feed = {
inputs_ph: [inputs_numpy],
targets_ph: [targets_numpy]
}
np_res = sess.run({"losses": losses, "final_output": final_output}, feed_dict=feed)
pprint(np_res)
tvars = tf.trainable_variables()
print('t2t inputs_ph:', inputs_ph, inputs_numpy)
print('t2t targets_ph:', targets_ph, targets_numpy)
return sess, tvars, inputs_ph, targets_ph, losses
def __init__(self,
src_vocab_size,
trg_vocab_size,
model_name,
problem_name,
hparams_set_name,
t2t_usr_dir,
checkpoint_dir,
t2t_unk_id=None,
n_cpu_threads=-1,
max_terminal_id=-1,
pop_id=-1):
"""Creates a new T2T predictor. The constructor prepares the
TensorFlow session for predict_next() calls. This includes:
- Load hyper parameters from the given set (hparams)
- Update registry, load T2T model
- Create TF placeholders for source sequence and target prefix
- Create computation graph for computing log probs.
- Create a MonitoredSession object, which also handles
restoring checkpoints.
Args:
src_vocab_size (int): Source vocabulary size.
trg_vocab_size (int): Target vocabulary size.
model_name (string): T2T model name.
problem_name (string): T2T problem name.
hparams_set_name (string): T2T hparams set name.
t2t_usr_dir (string): See --t2t_usr_dir in tensor2tensor.
checkpoint_dir (string): Path to the T2T checkpoint
directory. The predictor will load
the top most checkpoint in the
`checkpoints` file.
t2t_unk_id (int): If set, use this ID to get UNK scores. If
None, UNK is always scored with -inf.
n_cpu_threads (int): Number of TensorFlow CPU threads.
max_terminal_id (int): If positive, maximum terminal ID. Needs to
be set for syntax-based T2T models.
pop_id (int): If positive, ID of the POP or closing bracket symbol.
Needs to be set for syntax-based T2T models.
"""
super(T2TPredictor, self).__init__(t2t_usr_dir,
checkpoint_dir,
src_vocab_size,
trg_vocab_size,
t2t_unk_id,
n_cpu_threads,
max_terminal_id,
pop_id)
if not model_name or not problem_name or not hparams_set_name:
logging.fatal(
"Please specify t2t_model, t2t_problem, and t2t_hparams_set!")
raise AttributeError
self.consumed = []
self.src_sentence = []
predictor_graph = tf.Graph()
with predictor_graph.as_default() as g:
hparams = trainer_lib.create_hparams(hparams_set_name)
self._add_problem_hparams(hparams, problem_name)
translate_model = registry.model(model_name)(
hparams, tf.estimator.ModeKeys.PREDICT)
self._inputs_var = tf.placeholder(dtype=tf.int32, shape=[None],
name="sgnmt_inputs")
self._targets_var = tf.placeholder(dtype=tf.int32, shape=[None],
name="sgnmt_targets")
features = {"inputs": expand_input_dims_for_t2t(self._inputs_var),
"targets": expand_input_dims_for_t2t(self._targets_var)}
translate_model.prepare_features_for_infer(features)
translate_model._fill_problem_hparams_features(features)
logits, _ = translate_model(features)
logits = tf.squeeze(logits, [0, 1, 2, 3])
self._log_probs = log_prob_from_logits(logits)
self.mon_sess = self.create_session()
def nth_model(n):
"""Build the model for the n-th problem, plus some added variables."""
model_class = registry.model(model)(
hparams, mode, hparams.problems[n], n, dp,
devices.ps_devices(all_workers=True))
if mode == tf.estimator.ModeKeys.PREDICT:
return model_class.infer(
features,
beam_size=decode_hp.beam_size,
top_beams=(decode_hp.beam_size
if decode_hp.return_beams else 1),
last_position_only=decode_hp.use_last_position_only,
alpha=decode_hp.alpha,
decode_length=decode_hp.extra_length)
# In distributed mode, we build graph for problem=0 and problem=worker_id.
skipping_is_on = hparams.problem_choice == "distributed" and is_training
problem_worker_id = worker_id % len(hparams.problems)
skip_this_one = n != 0 and n % worker_replicas != problem_worker_id
# On worker 0 also build graph for problems <= 1.
# TODO(lukaszkaiser): why is this hack needed for variables init? Repair.
skip_this_one = skip_this_one and (worker_id != 0 or n > 1)
if eval_run_autoregressive and mode == tf.estimator.ModeKeys.EVAL:
sharded_logits, losses_dict = model_class.eval_autoregressive(
features)
else:
sharded_logits, losses_dict = model_class.model_fn(
features, skip=(skipping_is_on and skip_this_one))
with tf.variable_scope("losses_avg"):
total_loss, ops = 0.0, []
for loss_key, loss_value in six.iteritems(losses_dict):
loss_name = "problem_%d/%s_loss" % (n, loss_key)
loss_moving_avg = tf.get_variable(loss_name,
initializer=100.0,
trainable=False)
loss_variable_names.append(loss_name)
ops.append(
loss_moving_avg.assign(loss_moving_avg * 0.9 +
loss_value * 0.1))
total_loss += loss_value
try: # Total loss avg might be reused or not, we try both.
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
# Total loss was already constructed on input.
loss_moving_avg = tf.get_variable("problem_%d/total_loss" %
n)
except ValueError:
loss_moving_avg = tf.get_variable("problem_%d/total_loss" % n,
initializer=100.0,
trainable=False)
ops.append(
loss_moving_avg.assign(loss_moving_avg * 0.9 +
total_loss * 0.1))
with tf.variable_scope("train_stats"): # Count steps for this problem.
problem_steps = tf.get_variable("problem_%d_steps" % n,
initializer=0,
trainable=False)
ops.append(problem_steps.assign_add(1))
with tf.control_dependencies(ops): # Make sure the ops run.
# Ensure the loss is a scalar here.
total_loss = tf.reshape(total_loss, [],
name="total_loss_control_id")
return [total_loss, tf.concat(sharded_logits, 0)]
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
trainer_lib.set_random_seed(FLAGS.random_seed)
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
# Create hparams
hparams = trainer_lib.create_hparams(FLAGS.hparams_set,
FLAGS.hparams,
data_dir=os.path.expanduser(
FLAGS.data_dir),
problem_name=FLAGS.problem)
hparams.force_full_predict = True
hparams.scheduled_sampling_k = -1
# Params
num_agents = 1 # TODO(mbz): fix the code for more agents
num_steps = FLAGS.num_steps
if hasattr(hparams.problem, "num_actions"):
num_actions = hparams.problem.num_actions
else:
num_actions = None
frame_shape = hparams.problem.frame_shape
resized_frame = hparams.preprocess_resize_frames is not None
if resized_frame:
frame_shape = hparams.preprocess_resize_frames
frame_shape += [hparams.problem.num_channels]
dataset = registry.problem(FLAGS.problem).dataset(
tf.estimator.ModeKeys.TRAIN,
shuffle_files=True,
data_dir=os.path.expanduser(FLAGS.data_dir),
hparams=hparams)
dataset = dataset.batch(num_agents, drop_remainder=True)
data = dataset.make_one_shot_iterator().get_next()
# Setup input placeholders
input_size = [num_agents, hparams.video_num_input_frames]
if num_actions is None:
placeholders = {
"inputs": tf.placeholder(tf.float32, input_size + frame_shape)
}
else:
placeholders = {
"inputs": tf.placeholder(tf.float32, input_size + frame_shape),
"input_action": tf.placeholder(tf.int64, input_size + [1]),
"input_reward": tf.placeholder(tf.int64, input_size + [1]),
"reset_internal_states": tf.placeholder(tf.float32, []),
}
# Create model.
model_cls = registry.model(FLAGS.model)
model = model_cls(hparams, tf.estimator.ModeKeys.PREDICT)
prediction_ops = model.infer(placeholders)
states_q = Queue(maxsize=hparams.video_num_input_frames)
actions_q = Queue(maxsize=hparams.video_num_input_frames)
rewards_q = Queue(maxsize=hparams.video_num_input_frames)
if num_actions is not None:
all_qs = [states_q, actions_q, rewards_q]
else:
all_qs = [states_q]
writer = common_video.WholeVideoWriter(fps=FLAGS.fps,
output_path=FLAGS.output_gif)
saver = tf.train.Saver(tf.trainable_variables())
with tf.train.SingularMonitoredSession() as sess:
# Load latest checkpoint
ckpt = tf.train.get_checkpoint_state(
FLAGS.output_dir).model_checkpoint_path
saver.restore(sess.raw_session(), ckpt)
# get init frames from the dataset
data_np = sess.run(data)
frames = np.split(data_np["inputs"], hparams.video_num_input_frames, 1)
for frame in frames:
frame = np.squeeze(frame, 1)
states_q.put(frame)
writer.write(frame[0].astype(np.uint8))
if num_actions is not None:
actions = np.split(data_np["input_action"],
hparams.video_num_input_frames, 1)
for action in actions:
actions_q.put(np.squeeze(action, 1))
rewards = np.split(data_np["input_reward"],
hparams.video_num_input_frames, 1)
for reward in rewards:
rewards_q.put(np.squeeze(reward, 1))
for step in range(num_steps):
print(">>>>>>> ", step)
if num_actions is not None:
random_actions = np.random.randint(num_actions - 1)
random_actions = np.expand_dims(random_actions, 0)
random_actions = np.tile(random_actions, (num_agents, 1))
# Shape inputs and targets
inputs, input_action, input_reward = (np.stack(list(q.queue),
axis=1)
for q in all_qs)
else:
assert len(all_qs) == 1
q = all_qs[0]
elems = list(q.queue)
# Need to adjust shapes sometimes.
for i, e in enumerate(elems):
if len(e.shape) < 4:
elems[i] = np.expand_dims(e, axis=0)
inputs = np.stack(elems, axis=1)
# Predict next frames
if num_actions is None:
feed = {placeholders["inputs"]: inputs}
else:
feed = {
placeholders["inputs"]: inputs,
placeholders["input_action"]: input_action,
placeholders["input_reward"]: input_reward,
placeholders["reset_internal_states"]: float(step == 0),
}
predictions = sess.run(prediction_ops, feed_dict=feed)
if num_actions is None:
predicted_states = predictions[:, 0]
else:
predicted_states = predictions["targets"][:, 0]
predicted_reward = predictions["target_reward"][:, 0]
# Update queues
if num_actions is None:
new_data = (predicted_states)
else:
new_data = (predicted_states, random_actions, predicted_reward)
for q, d in zip(all_qs, new_data):
q.get()
q.put(d.copy())
writer.write(np.round(predicted_states[0]).astype(np.uint8))
writer.finish_to_disk()
def __init__(self,
t2t_usr_dir,
src_vocab_size,
trg_vocab_size,
model_name,
problem_name,
hparams_set_name,
checkpoint_dir,
t2t_unk_id=None,
single_cpu_thread=False):
"""Creates a new T2T predictor. The constructor prepares the
TensorFlow session for predict_next() calls. This includes:
- Load hyper parameters from the given set (hparams)
- Update registry, load T2T model
- Create TF placeholders for source sequence and target pefix
- Create computation graph for computing log probs.
- Create a MonitoredSession object, which also handles
restoring checkpoints.
Args:
t2t_usr_dir (string): See --t2t_usr_dir in tensor2tensor.
src_vocab_size (int): Source vocabulary size.
trg_vocab_size (int): Target vocabulary size.
model_name (string): T2T model name.
problem_name (string): T2T problem name.
hparams_set_name (string): T2T hparams set name.
checkpoint_dir (string): Path to the T2T checkpoint
directory. The predictor will load
the top most checkpoint in the
`checkpoints` file.
t2t_unk_id (int): If set, use this ID to get UNK scores. If
None, UNK is always scored with -inf.
single_cpu_thread (bool): If true, prevent tensorflow from
doing multithreading.
"""
super(T2TPredictor, self).__init__(t2t_usr_dir, checkpoint_dir,
t2t_unk_id, single_cpu_thread)
self.consumed = []
self.src_sentence = []
predictor_graph = tf.Graph()
with predictor_graph.as_default() as g:
hparams = self._create_hparams(src_vocab_size, trg_vocab_size,
hparams_set_name, problem_name)
p_hparams = hparams.problems[0]
self._inputs_var = tf.placeholder(dtype=tf.int32,
shape=[None],
name="sgnmt_inputs")
self._targets_var = tf.placeholder(dtype=tf.int32,
shape=[None],
name="sgnmt_targets")
def expand_input_dims_for_t2t(t):
t = tf.expand_dims(t, 0) # Because of batch_size
t = tf.expand_dims(t, -1) # Because of modality
t = tf.expand_dims(t, -1) # Because of random reason X
return t
features = {
"problem_choice": tf.constant(0),
"input_space_id": tf.constant(p_hparams.input_space_id),
"target_space_id": tf.constant(p_hparams.target_space_id),
"inputs": expand_input_dims_for_t2t(self._inputs_var),
"targets": expand_input_dims_for_t2t(self._targets_var)
}
model = registry.model(model_name)(
hparams, tf.estimator.ModeKeys.PREDICT, hparams.problems[0], 0,
devices.data_parallelism(),
devices.ps_devices(all_workers=True))
sharded_logits, _ = model.model_fn(features,
last_position_only=True)
self._log_probs = log_prob_from_logits(sharded_logits[0])
self.mon_sess = self.create_session()
def __init__(self,
src_vocab_size,
trg_vocab_size,
model_name,
problem_name,
hparams_set_name,
t2t_usr_dir,
checkpoint_dir,
t2t_unk_id=None,
n_cpu_threads=-1,
max_terminal_id=-1,
pop_id=-1):
"""Creates a new document-level T2T predictor. See
T2TPredictor.__init__().
Args:
src_vocab_size (int): Source vocabulary size.
trg_vocab_size (int): Target vocabulary size.
model_name (string): T2T model name.
problem_name (string): T2T problem name.
hparams_set_name (string): T2T hparams set name.
t2t_usr_dir (string): See --t2t_usr_dir in tensor2tensor.
checkpoint_dir (string): Path to the T2T checkpoint
directory. The predictor will load
the top most checkpoint in the
`checkpoints` file.
t2t_unk_id (int): If set, use this ID to get UNK scores. If
None, UNK is always scored with -inf.
n_cpu_threads (int): Number of TensorFlow CPU threads.
max_terminal_id (int): If positive, maximum terminal ID. Needs to
be set for syntax-based T2T models.
pop_id (int): If positive, ID of the POP or closing bracket symbol.
Needs to be set for syntax-based T2T models.
"""
super(SegT2TPredictor, self).__init__(t2t_usr_dir,
checkpoint_dir,
src_vocab_size,
trg_vocab_size,
t2t_unk_id,
n_cpu_threads,
max_terminal_id,
pop_id)
if not model_name or not problem_name or not hparams_set_name:
logging.fatal(
"Please specify t2t_model, t2t_problem, and t2t_hparams_set!")
raise AttributeError
self.begin_margin = 4
self.end_margin = 1
self.max_sentences = self.begin_margin + self.end_margin
self.max_sentences = 10000
predictor_graph = tf.Graph()
with predictor_graph.as_default() as g:
hparams = trainer_lib.create_hparams(hparams_set_name)
self._add_problem_hparams(hparams, problem_name)
translate_model = registry.model(model_name)(
hparams, tf.estimator.ModeKeys.PREDICT)
self._inputs_var = tf.placeholder(dtype=tf.int32, shape=[None],
name="sgnmt_inputs")
self._targets_var = tf.placeholder(dtype=tf.int32, shape=[None],
name="sgnmt_targets")
self._inputs_seg_var = tf.placeholder(dtype=tf.int32, shape=[None],
name="sgnmt_inputs_seg")
self._targets_seg_var = tf.placeholder(dtype=tf.int32, shape=[None],
name="sgnmt_targets_seg")
self._inputs_pos_var = tf.placeholder(dtype=tf.int32, shape=[None],
name="sgnmt_inputs_pos")
self._targets_pos_var = tf.placeholder(dtype=tf.int32, shape=[None],
name="sgnmt_targets_pos")
features = {
"inputs": expand_input_dims_for_t2t(self._inputs_var),
"targets": expand_input_dims_for_t2t(self._targets_var),
"inputs_seg": tf.expand_dims(self._inputs_seg_var, 0),
"targets_seg": tf.expand_dims(self._targets_seg_var, 0),
"inputs_pos": tf.expand_dims(self._inputs_pos_var, 0),
"targets_pos": tf.expand_dims(self._targets_pos_var, 0)
}
translate_model.prepare_features_for_infer(features)
translate_model._fill_problem_hparams_features(features)
logits, _ = translate_model(features)
logits = tf.squeeze(logits, [0, 1, 2, 3])
self._log_probs = log_prob_from_logits(logits)
self.mon_sess = self.create_session()
def testUnknownModel(self):
with self.assertRaisesRegexp(LookupError, "never registered"):
registry.model("not_registered")
def _init_env(self):
FLAGS.use_tpu = False
tf.logging.set_verbosity(tf.logging.DEBUG)
tf.logging.info("Import usr dir from %s", self._usr_dir)
if self._usr_dir != None:
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
tf.logging.info("Start to create hparams,for %s of %s", self._problem,
self._hparams_set)
self._hparams = create_hparams()
self._hparams_decode = create_decode_hparams(
extra_length=self._extra_length,
batch_size=self._batch_size,
beam_size=self._beam_size,
alpha=self._alpha,
return_beams=self._return_beams,
write_beam_scores=self._write_beam_scores,
force_decode_length=self._force_decode_length)
# self.estimator_spec = t2t_model.T2TModel.make_estimator_model_fn(
# self._model_name, self._hparams, decode_hparams=self._hparams_decode, use_tpu=False)
self.estimator = trainer_lib.create_estimator(
FLAGS.model,
self._hparams,
t2t_trainer.create_run_config(self._hparams),
decode_hparams=self._hparams_decode,
use_tpu=False)
tf.logging.info("Finish intialize environment")
#######
### make input placeholder
self._inputs_ph = tf.placeholder(
dtype=tf.int32) # shape not specified,any shape
x = tf.placeholder(dtype=tf.int32)
x.set_shape([None, None]) # ? -> (?,?)
x = tf.expand_dims(x, axis=[2]) # -> (?,?,1)
x = tf.to_int32(x)
self._inputs_ph = x
#batch_inputs = tf.reshape(self._inputs_ph, [self._batch_size, -1, 1, 1])
batch_inputs = x
###
# batch_inputs = tf.reshape(self._inputs_ph, [-1, -1, 1, 1])
#targets_ph = tf.placeholder(dtype=tf.int32)
#batch_targets = tf.reshape(targets_ph, [1, -1, 1, 1])
#self.inputs_ph = tf.placeholder(tf.int32, shape=(None, None, 1, 1), name='inputs')
#self.targets_ph = tf.placeholder(tf.int32, shape=(None, None, None, None), name='targets')
self.inputs_ph = tf.placeholder(tf.int32,
shape=(None, None, 1, 1),
name='inputs')
self.targets_ph = tf.placeholder(tf.int32,
shape=(None, None, 1, 1),
name='targets')
self.targets_ph_2 = tf.placeholder(tf.int32,
shape=(None, None, 1, 1),
name='targets')
self._features = {
"inputs": self.inputs_ph,
"problem_choice": 0, # We run on the first problem here.
"input_space_id": self._hparams.problem_hparams.input_space_id,
"target_space_id": self._hparams.problem_hparams.target_space_id
}
### 加入 decode length 变长的
self.input_extra_length_ph = tf.placeholder(dtype=tf.int32)
self._features['decode_length'] = self.input_extra_length_ph
## target
#self._targets_ph= tf.placeholder(tf.int32, shape=(None, None, None, None), name='targets')
self._features['targets'] = self.targets_ph
self._features['targets2'] = self.targets_ph_2
target_pretend = np.zeros((1, 1, 1, 1))
## 去掉 整数的
del self._features["problem_choice"]
del self._features["input_space_id"]
del self._features["target_space_id"]
del self._features['decode_length']
####
#mode = tf.estimator.ModeKeys.PREDICT # affect last_only t2t_model._top_single ,[1,?,1,512]->[1,1,1,1,64]
# if self.predict_or_eval=='EVAL':
# mode = tf.estimator.ModeKeys.EVAL # affect last_only t2t_model._top_single ,[1,?,1,512]->[1,?,1,1,64]
# # estimator_spec = model_builder.model_fn(self._model_name, features, mode, self._hparams,
# # problem_names=[self._problem], decode_hparams=self._hparams_dc)
# if self.predict_or_eval=='PREDICT':
# mode = tf.estimator.ModeKeys.PREDICT
if self.predict_or_eval == 'and':
mode = tf.estimator.ModeKeys.TRAIN
###########
# registry.model
############
translate_model = registry.model(self._model_name)(
hparams=self._hparams,
decode_hparams=self._hparams_decode,
mode=mode)
self.predict_dict = {}
# if self.predict_or_eval == 'EVAL':
# self.logits,_=translate_model(self._features)
# self.predict_dict['scores']=self.logits
#
# if self.predict_or_eval == 'PREDICT':
#
# self.predict_dict=translate_model.infer(features=self._features,
# decode_length=50,
# beam_size=1,
# top_beams=1)
# print ''
if self.predict_or_eval == 'and':
### get logit EVAL mode
#self._features['targets'] = [[self._targets_ph]] # function body()
self.logits, self.ret2 = translate_model(self._features)
##################
## model_fn fetch logits FAIL : key not found
#############
# logits,_=translate_model.model_fn(self._features)
# self._beam_result = model_i._fast_decode(self._features, decode_length=5, beam_size=10, top_beams=10,
# alpha=0.6) #fail
# self._beam_result = model_i._beam_decode(self._features,
# decode_length=5,
# beam_size=self._beam_size,
# top_beams=self._beam_size,
# alpha=0.6)
##########
# logits,_=model_i.model_fn(self._features)
# assert len(logits.shape) == 5
# logits = tf.squeeze(logits, [2, 3])
# # Compute the log probabilities
# from tensor2tensor.layers import common_layers
# self.log_probs = common_layers.log_prob_from_logits(logits)
######
#self._predictions = self._predictions_dict["outputs"]
# self._scores=predictions_dict['scores'] not return when greedy search
tf.logging.info("Start to init tf session")
if self._isGpu:
print('Using GPU in Decoder')
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=self._fraction)
self._sess = tf.Session(
config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
gpu_options=gpu_options))
else:
print('Using CPU in Decoder')
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0)
config = tf.ConfigProto(gpu_options=gpu_options)
config.allow_soft_placement = True
config.log_device_placement = False
self._sess = tf.Session(config=config)
with self._sess.as_default():
#ckpt = saver_mod.get_checkpoint_state(self._model_dir)
self._sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
#train_handle = sess.run(train_iterator.string_handle())
#dev_handle = sess.run(dev_iterator.string_handle())
if os.path.exists(os.path.join(self._model_dir, "checkpoint")):
saver.restore(self._sess,
tf.train.latest_checkpoint(self._model_dir))
global_step = max(sess.run(translate_model.global_step), 1)
for _ in tqdm(range(global_step, 1000 + 1)):
global_step = sess.run(translate_model.global_step) + 1
loss, train_op = sess.run([self.ret2, model.train_op],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
writer.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
_, summ = evaluate_batch(model, config.val_num_batches,
train_eval_file, sess, "train",
handle, train_handle)
for s in summ:
writer.add_summary(s, global_step)
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1,
dev_eval_file, sess, "dev", handle, dev_handle)
dev_f1 = metrics["f1"]
dev_em = metrics["exact_match"]
if dev_f1 < best_f1 and dev_em < best_em:
patience += 1
if patience > config.early_stop:
break
else:
patience = 0
best_em = max(best_em, dev_em)
best_f1 = max(best_f1, dev_f1)
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
filename = os.path.join(
config.save_dir, "model_{}.ckpt".format(global_step))
saver.save(sess, filename)
def model_fn(features, labels, mode, params, config):
"""Model fn."""
del params
hparams = copy.deepcopy(hp)
problem_hp = hparams.problems[0]
orig_features = features
# Instantiate model and retrieve modalities. Note that autoregressive models
# have no input modality.
model_class = registry.model(model)(hparams, mode, problem_hp)
input_modality = problem_hp.input_modality.get("inputs")
target_modality = problem_hp.target_modality
# Transform features
transformed_features = {}
if input_modality is not None:
transformed_features["inputs"] = input_modality.bottom(
features["inputs"])
transformed_features["targets"] = target_modality.targets_bottom(
features["targets"])
transformed_features["problem_choice"] = tf.constant(0)
transformed_features["input_space_id"] = tf.constant(
problem_hp.input_space_id)
transformed_features["target_space_id"] = tf.constant(
problem_hp.target_space_id)
# Model construction
outputs = model_class.model_fn_body(transformed_features)
logits = target_modality.top(outputs, labels)
# Ensure the length is known statically
shape = [None] * logits.get_shape().ndims
shape[1] = hparams.max_length
logits.set_shape(logits.get_shape().merge_with(shape))
# Loss
loss_num, loss_den = target_modality.loss(logits, labels)
loss = loss_num / tf.maximum(1.0, loss_den)
if mode == tf.estimator.ModeKeys.EVAL:
problem = hp.problem_instances[0]
eval_metrics_fn = create_eval_metrics_fn(problem)
_remove_summaries()
return tf.contrib.tpu.TPUEstimatorSpec(
mode,
eval_metrics=(eval_metrics_fn,
[logits, orig_features["targets"]]),
loss=loss)
assert mode == tf.estimator.ModeKeys.TRAIN
# Learning rate
num_shards = config.tpu_config.num_shards
lr = hparams.learning_rate * model_builder.learning_rate_decay(
hparams, num_worker_replicas=num_shards)
lr /= math.sqrt(float(num_shards))
# Optimizer
opt = model_builder.ConditionalOptimizer(hparams.optimizer, lr,
hparams)
if use_tpu:
opt = tf.contrib.tpu.CrossShardOptimizer(opt)
# Optimize
gradients = opt.compute_gradients(loss, tf.trainable_variables())
if hparams.clip_grad_norm:
gradients = _clip_gradients_by_norm(gradients,
hparams.clip_grad_norm)
train_op = opt.apply_gradients(
gradients, global_step=tf.train.get_or_create_global_step())
with tf.control_dependencies([train_op]):
train_op = tf.identity(loss)
_remove_summaries()
return tf.contrib.tpu.TPUEstimatorSpec(mode,
loss=loss,
train_op=train_op)
tf.gfile.MakeDirs(train_dir)
tf.gfile.MakeDirs(checkpoint_dir)
gs_ckpt_dir = "gs://tensor2tensor-checkpoints/"
problem_name = "librispeech_clean"
asr_problem = problems.problem(problem_name)
encoders = asr_problem.feature_encoders(None)
model_name = "transformer"
hparams_set = "transformer_librispeech_tpu"
hparams = trainer_lib.create_hparams(hparams_set,
data_dir=data_dir,
problem_name=problem_name)
asr_model = registry.model(model_name)(hparams, Modes.PREDICT)
def encode(x):
waveforms = encoders["waveforms"].encode(x)
encoded_dict = asr_problem.preprocess_example(
{
"waveforms": waveforms,
"targets": []
}, Modes.PREDICT, hparams)
return {
"inputs": tf.expand_dims(encoded_dict["inputs"], 0),
"targets": tf.expand_dims(encoded_dict["targets"], 0)
}
def score_file(filename):
"""Score each line in a file and return the scores."""
# Prepare model.
hparams = create_hparams()
encoders = registry.problem(FLAGS.problem).feature_encoders(FLAGS.data_dir)
has_inputs = "inputs" in encoders
# Prepare features for feeding into the model.
if has_inputs:
inputs_ph = tf.placeholder(dtype=tf.int32) # Just length dimension.
batch_inputs = tf.reshape(inputs_ph, [1, -1, 1, 1]) # Make it 4D.
targets_ph = tf.placeholder(dtype=tf.int32) # Just length dimension.
batch_targets = tf.reshape(targets_ph, [1, -1, 1, 1]) # Make it 4D.
if has_inputs:
features = {"inputs": batch_inputs, "targets": batch_targets}
else:
features = {"targets": batch_targets}
# Prepare the model and the graph when model runs on features.
model = registry.model(FLAGS.model)(hparams, tf.estimator.ModeKeys.EVAL)
_, losses = model(features)
saver = tf.train.Saver()
with tf.Session() as sess:
# Load weights from checkpoint.
ckpts = tf.train.get_checkpoint_state(FLAGS.output_dir)
ckpt = ckpts.model_checkpoint_path
saver.restore(sess, ckpt)
# Run on each line.
with tf.gfile.Open(filename) as f:
lines = f.readlines()
results = []
for ix, line in enumerate(lines):
if ix % 10000 == 0:
print('id: {}'.format(ix))
tab_split = line.split("\t")
if len(tab_split) > 2:
raise ValueError(
"Each line must have at most one tab separator.")
if len(tab_split) == 1:
targets = tab_split[0].strip()
else:
targets = tab_split[1].strip()
inputs = tab_split[0].strip()
# Run encoders and append EOS symbol.
targets_numpy = encoders["targets"].encode(targets) + [
text_encoder.EOS_ID
]
if has_inputs:
inputs_numpy = encoders["inputs"].encode(inputs) + [
text_encoder.EOS_ID
]
# Prepare the feed.
if has_inputs:
feed = {inputs_ph: inputs_numpy, targets_ph: targets_numpy}
else:
feed = {targets_ph: targets_numpy}
# Get the score.
np_loss = sess.run(losses["training"], feed)
results.append(np_loss)
return results
def score_file(filename):
"""Score each line in a file and return the scores."""
# Prepare model.
hparams = create_hparams()
encoders = registry.problem(FLAGS.problem).feature_encoders(FLAGS.data_dir)
has_inputs = "inputs" in encoders
# Prepare features for feeding into the model.
if has_inputs:
inputs_ph = tf.placeholder(dtype=tf.int32) # Just length dimension.
batch_inputs = tf.reshape(inputs_ph, [1, -1, 1, 1]) # Make it 4D.
targets_ph = tf.placeholder(dtype=tf.int32) # Just length dimension.
batch_targets = tf.reshape(targets_ph, [1, -1, 1, 1]) # Make it 4D.
if has_inputs:
features = {"inputs": batch_inputs, "targets": batch_targets}
else:
features = {"targets": batch_targets}
# Prepare the model and the graph when model runs on features.
model = registry.model(FLAGS.model)(hparams, tf.estimator.ModeKeys.EVAL)
_, losses = model(features)
saver = tf.train.Saver()
with tf.Session() as sess: # SOLUTION SOLUTION SOLUTION SOLUTION
# Load weights from checkpoint.
if FLAGS.checkpoint_path is None:
ckpts = tf.train.get_checkpoint_state(FLAGS.output_dir)
ckpt = ckpts.model_checkpoint_path
else:
ckpt = FLAGS.checkpoint_path
saver.restore(sess, ckpt)
# # DELETE THAT Run the language model on each line.
# with tf.gfile.Open(filename) as f:
# lines = f.readlines()
DIR_FOR_DECODE = os.environ['DIR_FOR_DECODE']
DIR_TO_DECODE = os.environ['DIR_TO_DECODE']
# DIR_FOR_DECODE = 'rule-cleaned-summaries'
# DIR_TO_DECODE = 'lm-cleaned-summaries'
for file in os.listdir(DIR_FOR_DECODE):
file_path = os.path.join(DIR_FOR_DECODE, file)
cleaned_text = []
with tf.gfile.Open(file_path) as f:
lines = f.readlines()
for line in lines:
# print and clean the line
print(line)
targets = reformat(line.strip())
# run the word elimination model
results = word_elimination(
[(targets, None)
], # pass the line as a list not as a string
encoders,
targets_ph,
sess,
losses,
top_k=5,
min_sent_length=10)
# drop duplicates and sort the list
results = sorted(dict(results).items(),
key=operator.itemgetter(1))
# save best sentences
best_sent = reformat2(results[0][0])
cleaned_text.append(best_sent)
# save corrected text
with open(os.path.join(DIR_TO_DECODE, file),
'w',
encoding='utf-8') as new_file:
new_file.write("\n".join(cleaned_text))
# # word incrementation
# vocab = [word.strip() for word in targets.split("_")]
# results = word_incrementation([(vocab[0], None)],
# encoders,
# targets_ph,
# sess,
# losses,
# max_sent_length=len(vocab),
# vocab=vocab)
#
# # drop duplicates and sort the list
# results = sorted(dict(results).items(), key=operator.itemgetter(1))
#
# # save results
# write_file = tf.gfile.Open(os.path.expanduser(FLAGS.decode_to_file), "a")
# write_file.write('COMPRESS: {}'.format(line))
# for sentence, score in results[:15]:
# write_file.write(sentence + "\t" + "SCORE:" + "%.6f\n" % score)
return results
"""Input str to features dict, ready for inference"""
inputs = encoders["inputs"].encode(input_str) + [1] # add EOS id
batch_inputs = tf.reshape(inputs, [1, -1, 1]) # Make it 3D.
return {"inputs": batch_inputs}
def decode(integers):
"""List of ints to str"""
integers = list(np.squeeze(integers))
if 1 in integers:
integers = integers[:integers.index(1)]
return encoders["inputs"].decode(np.squeeze(integers))
# Create hparams and the model
model_name = "transformer"
hparams_set = "transformer_base"
hparams = tpu_trainer_lib.create_hparams(hparams_set, data_dir=ckpt_path, problem_name="translate_ende_wmt32k")
# NOTE: Only create the model once when restoring from a checkpoint; it's a
# Layer and so subsequent instantiations will have different variable scopes
# that will not match the checkpoint.
translate_model = registry.model(model_name)(hparams, Modes.EVAL)
with open(fin_name, encoding='utf-8') as fin, open(fout_name, mode='w', encoding='utf-8') as fout, tfe.restore_variables_on_create(ckpt_path):
for inputs in fin:
encoded_inputs = encode(inputs.strip())
model_output = translate_model.infer(encoded_inputs, decode_length=100)
res = decode(model_output)
print(res, file=fout)
fout.flush()
def __init__(self,
src_vocab_size,
trg_vocab_size,
model_name,
problem_name,
hparams_set_name,
trg_test_file,
beam_size,
t2t_usr_dir,
checkpoint_dir,
t2t_unk_id=None,
n_cpu_threads=-1,
max_terminal_id=-1,
pop_id=-1):
"""Creates a new edit T2T predictor. This constructor is
similar to the constructor of T2TPredictor but creates a
different computation graph which retrieves scores at each
target position, not only the last one.
Args:
src_vocab_size (int): Source vocabulary size.
trg_vocab_size (int): Target vocabulary size.
model_name (string): T2T model name.
problem_name (string): T2T problem name.
hparams_set_name (string): T2T hparams set name.
trg_test_file (string): Path to a plain text file with
initial target sentences. Can be empty.
beam_size (int): Determines how many substitutions and
insertions are considered at each position.
t2t_usr_dir (string): See --t2t_usr_dir in tensor2tensor.
checkpoint_dir (string): Path to the T2T checkpoint
directory. The predictor will load
the top most checkpoint in the
`checkpoints` file.
t2t_unk_id (int): If set, use this ID to get UNK scores. If
None, UNK is always scored with -inf.
n_cpu_threads (int): Number of TensorFlow CPU threads.
max_terminal_id (int): If positive, maximum terminal ID. Needs to
be set for syntax-based T2T models.
pop_id (int): If positive, ID of the POP or closing bracket symbol.
Needs to be set for syntax-based T2T models.
"""
super(EditT2TPredictor, self).__init__(t2t_usr_dir,
checkpoint_dir,
src_vocab_size,
trg_vocab_size,
t2t_unk_id,
n_cpu_threads,
max_terminal_id,
pop_id)
if not model_name or not problem_name or not hparams_set_name:
logging.fatal(
"Please specify t2t_model, t2t_problem, and t2t_hparams_set!")
raise AttributeError
if trg_vocab_size >= EditT2TPredictor.POS_FACTOR:
logging.fatal("Target vocabulary size (%d) must be less than %d!"
% (trg_vocab_size, EditT2TPredictor.POS_FACTOR))
raise AttributeError
self.beam_size = max(1, beam_size // 10) + 1
self.batch_size = 2048 # TODO(fstahlberg): Move to config
self.initial_trg_sentences = None
if trg_test_file:
self.initial_trg_sentences = []
with open(trg_test_file) as f:
for line in f:
self.initial_trg_sentences.append(utils.oov_to_unk(
[int(w) for w in line.strip().split()] + [utils.EOS_ID],
self.trg_vocab_size, self._t2t_unk_id))
predictor_graph = tf.Graph()
with predictor_graph.as_default() as g:
hparams = trainer_lib.create_hparams(hparams_set_name)
self._add_problem_hparams(hparams, problem_name)
translate_model = registry.model(model_name)(
hparams, tf.estimator.ModeKeys.EVAL)
self._inputs_var = tf.placeholder(dtype=tf.int32, shape=[None],
name="sgnmt_inputs")
self._targets_var = tf.placeholder(dtype=tf.int32, shape=[None, None],
name="sgnmt_targets")
shp = tf.shape(self._targets_var)
bsz = shp[0]
inputs = tf.tile(tf.expand_dims(self._inputs_var, 0), [bsz, 1])
features = {"inputs": expand_input_dims_for_t2t(inputs,
batched=True),
"targets": expand_input_dims_for_t2t(self._targets_var,
batched=True)}
translate_model.prepare_features_for_infer(features)
translate_model._fill_problem_hparams_features(features)
logits, _ = translate_model(features)
logits = tf.squeeze(logits, [2, 3])
self._log_probs = log_prob_from_logits(logits)
diag_logits = gather_2d(logits, tf.expand_dims(tf.range(bsz), 1))
self._diag_log_probs = log_prob_from_logits(diag_logits)
no_pad = tf.cast(tf.not_equal(
self._targets_var, text_encoder.PAD_ID), tf.float32)
flat_bsz = shp[0] * shp[1]
word_scores = gather_2d(
tf.reshape(self._log_probs, [flat_bsz, -1]),
tf.reshape(self._targets_var, [flat_bsz, 1]))
word_scores = tf.reshape(word_scores, (shp[0], shp[1])) * no_pad
self._sentence_scores = tf.reduce_sum(word_scores, -1)
self.mon_sess = self.create_session()
def build_model(hparams_set, model_name, data_dir, problem_name, return_beams,beam_size,custom_problem_type,force_decode_len):
"""Build the graph required to fetch the attention weights.
Args:
hparams_set: HParams set to build the model with.
model_name: Name of model.
data_dir: Path to directory containing training data.
problem_name: Name of problem.
beam_size: (Optional) Number of beams to use when decoding a translation.
If set to 1 (default) then greedy decoding is used.
Returns:
Tuple of (
inputs: Input placeholder to feed in ids to be translated.
targets: Targets placeholder to feed to translation when fetching
attention weights.
samples: Tensor representing the ids of the translation.
att_mats: Tensors representing the attention weights.
)
"""
hparams = trainer_lib.create_hparams(
hparams_set, data_dir=data_dir, problem_name=problem_name)
hparams.add_hparam("force_decode_length", True)
##
from problem_util_yr.t2t162.ProblemDecoder_predict import create_decode_hparams
hparams_decode = create_decode_hparams(extra_length=111,
batch_size=1,
beam_size=beam_size,
alpha=0.4,
return_beams=return_beams,
write_beam_scores=False,
force_decode_length=force_decode_len)
###
translate_model = registry.model(model_name)(
hparams=hparams,decode_hparams=hparams_decode,mode=tf.estimator.ModeKeys.EVAL)
inputs = tf.placeholder(tf.int32, shape=(1, None, 1, 1), name='inputs')
targets = tf.placeholder(tf.int32, shape=(1, None, 1, 1), name='targets')
input_extra_length_ph = tf.placeholder(dtype=tf.int32, shape=[])
# translate_model([{
# 'inputs': inputs,
# 'targets': targets,
# }]) ##########t2t_model.py call function ->
features={
'inputs': inputs,
'targets': targets,
'decode_length':input_extra_length_ph
}
translate_model(features)
# translate_model({
# 'inputs': [inputs],
# 'targets': [targets],
# }) # univeral_transformer
# Must be called after building the training graph, so that the dict will
# have been filled with the attention tensors. BUT before creating the
# inference graph otherwise the dict will be filled with tensors from
# inside a tf.while_loop from decoding and are marked unfetchable.
att_mats = get_att_mats(translate_model,custom_problem_type,model_name)
# with tf.variable_scope(tf.get_variable_scope(), reuse=True):
# samples = translate_model.infer(features, beam_size=beam_size)['outputs']
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
samples = translate_model.infer(features, beam_size=beam_size,top_beams=beam_size)
return inputs, targets, input_extra_length_ph,samples, att_mats
"""Input str to features dict, ready for inference"""
inputs = encoders["inputs"].encode(input_str) + [1] # add EOS id
batch_inputs = tf.reshape(inputs, [1, -1, 1]) # Make it 3D.
return {"inputs": batch_inputs}
def decode(integers):
"""List of ints to str"""
integers = list(np.squeeze(integers))
if 1 in integers:
integers = integers[:integers.index(1)]
return encoders["inputs"].decode(np.squeeze(integers))
#Predict
hparams = trainer_lib.create_hparams(HPARAMS, data_dir=DATA_DIR, problem_name=PROBLEM)
translate_model = registry.model(MODEL)(hparams, Modes.PREDICT)
#inputs = "那是一条狗"
#ref = "Đó là một con chó" ## this just a reference for evaluate the quality of the traduction
#outputs = translate(inputs)
with open("test.zh", "r", encoding="utf8") as f:
lineList = f.readlines()
outputs = []
print(len(lineList))
i = 0
for line in lineList:
print(i)
outputs.append(translate(line))
i = i + 1
with open("outputs.vi", "w+", encoding="utf8") as fo:
for output in outputs:
def model(name):
return registry.model(name)
def model(name): return registry.model(name)
