def _testBatchExamples(self):
tf.set_random_seed(1)
tmp_dir = self.get_temp_dir()
(_, tmp_file_path) = tempfile.mkstemp(dir=tmp_dir)
tmp_file_name = os.path.basename(tmp_file_path)
# Generate a file with 100 examples, n-th example of length n + 1.
def test_generator():
for i in xrange(100):
yield {"inputs": [i + 1 for _ in xrange(i + 1)], "targets": [i + 1]}
filenames = generator_utils.train_data_filenames(tmp_file_name, tmp_dir, 1)
generator_utils.generate_files(test_generator(), filenames)
self.assertTrue(tf.gfile.Exists(tmp_file_path + "-train-00000-of-00001"))
examples_train = data_reader.examples_queue([tmp_file_path + "*"], {
"inputs": tf.VarLenFeature(tf.int64),
"targets": tf.VarLenFeature(tf.int64)
}, True)
batch_train = data_reader.batch_examples(examples_train, 4)
examples_eval = data_reader.examples_queue([tmp_file_path + "*"], {
"inputs": tf.VarLenFeature(tf.int64),
"targets": tf.VarLenFeature(tf.int64)
}, False)
batch_eval = data_reader.batch_examples(examples_eval, 2)
session, coord = tf.Session(), tf.train.Coordinator()
with session.as_default():
tf.train.start_queue_runners(coord=coord)
# Evaluation data comes in the same order as in the file.
# The first batch will be inputs=[[1, 0], [2, 2]], targets=[[1], [2]].
examples = session.run(batch_eval)
self.assertAllClose(examples["inputs"], np.array([[1, 0], [2, 2]]))
self.assertAllClose(examples["targets"], np.array([[1], [2]]))
# Check the second batch too.
examples = session.run(batch_eval)
self.assertAllClose(examples["inputs"],
np.array([[3, 3, 3, 0], [4, 4, 4, 4]]))
self.assertAllClose(examples["targets"], np.array([[3], [4]]))
# Training data is shuffled but shouldn't have too many pads.
for _ in xrange(10):
examples = session.run(batch_train)
inputs = examples["inputs"]
# Only 3 out of 4 examples in a batch have padding zeros at all.
pad_per_example = (inputs.size - np.count_nonzero(inputs)) // 3
# Default bucketing is in steps of 8 until 64 and 32 later.
if int(max(examples["targets"])) < 64:
self.assertLess(pad_per_example, 8)
else:
self.assertLess(pad_per_example, 32)
# Clean up.
coord.request_stop()
coord.join()
os.remove(tmp_file_path + "-train-00000-of-00001")
os.remove(tmp_file_path)
def _testBatchExamples(self):
tf.set_random_seed(1)
tmp_dir = self.get_temp_dir()
(_, tmp_file_path) = tempfile.mkstemp(dir=tmp_dir)
tmp_file_name = os.path.basename(tmp_file_path)
# Generate a file with 100 examples, n-th example of length n + 1.
def test_generator():
for i in xrange(100):
yield {"inputs": [i + 1 for _ in xrange(i + 1)], "targets": [i + 1]}
generator_utils.generate_files(test_generator(), tmp_file_name, tmp_dir)
self.assertTrue(tf.gfile.Exists(tmp_file_path + "-00000-of-00001"))
examples_train = data_reader.examples_queue([tmp_file_path + "*"], {
"inputs": tf.VarLenFeature(tf.int64),
"targets": tf.VarLenFeature(tf.int64)
}, True)
batch_train = data_reader.batch_examples(examples_train, 4)
examples_eval = data_reader.examples_queue([tmp_file_path + "*"], {
"inputs": tf.VarLenFeature(tf.int64),
"targets": tf.VarLenFeature(tf.int64)
}, False)
batch_eval = data_reader.batch_examples(examples_eval, 2)
session, coord = tf.Session(), tf.train.Coordinator()
with session.as_default():
tf.train.start_queue_runners(coord=coord)
# Evaluation data comes in the same order as in the file.
# The first batch will be inputs=[[1, 0], [2, 2]], targets=[[1], [2]].
examples = session.run(batch_eval)
self.assertAllClose(examples["inputs"], np.array([[1, 0], [2, 2]]))
self.assertAllClose(examples["targets"], np.array([[1], [2]]))
# Check the second batch too.
examples = session.run(batch_eval)
self.assertAllClose(examples["inputs"],
np.array([[3, 3, 3, 0], [4, 4, 4, 4]]))
self.assertAllClose(examples["targets"], np.array([[3], [4]]))
# Training data is shuffled but shouldn't have too many pads.
for _ in xrange(10):
examples = session.run(batch_train)
inputs = examples["inputs"]
# Only 3 out of 4 examples in a batch have padding zeros at all.
pad_per_example = (inputs.size - np.count_nonzero(inputs)) // 3
# Default bucketing is in steps of 8 until 64 and 32 later.
if int(max(examples["targets"])) < 64:
self.assertLess(pad_per_example, 8)
else:
self.assertLess(pad_per_example, 32)
# Clean up.
coord.request_stop()
coord.join()
os.remove(tmp_file_path + "-00000-of-00001")
os.remove(tmp_file_path)
def input_fn():
"""Supplies input to our model.
This function supplies input to our model, where this input is a
function of the mode. For example, we supply different data if
we're performing training versus evaluation.
Returns:
A tuple consisting of 1) a dictionary of tensors whose keys are
the feature names, and 2) a tensor of target labels if the mode
is not INFER (and None, otherwise).
Raises:
ValueError: if one of the parameters has an unsupported value.
"""
problem_count, batches = len(hparams.problems), []
with tf.name_scope("input_reader"):
for n in xrange(problem_count):
if fixed_problem is not None and n != fixed_problem:
continue
problem_instance = hparams.problem_instances[n]
p_hparams = hparams.problems[n]
with tf.name_scope("problem_%d" % n):
with tf.device("/cpu:0"): # Input reading on CPU
capacity = p_hparams.max_expected_batch_size_per_shard
capacity *= num_datashards
examples = data_reader.input_pipeline(
problem_instance, data_file_patterns
and data_file_patterns[n], capacity, mode, hparams)
feature_map = data_reader.batch_examples(
examples,
data_reader.hparams_to_batching_scheme(
hparams,
shard_multiplier=num_datashards,
drop_long_sequences=(
mode == tf.contrib.learn.ModeKeys.TRAIN
or hparams.eval_drop_long_sequences),
length_multiplier=(
p_hparams.batch_size_multiplier)))
# Reverse inputs and targets features if the problem was reversed.
if problem_instance is not None:
problem_instance.maybe_reverse_features(feature_map)
problem_instance.maybe_copy_features(feature_map)
else:
if p_hparams.was_reversed:
inputs = feature_map["inputs"]
targets = feature_map["targets"]
feature_map["inputs"] = targets
feature_map["targets"] = inputs
# Use the inputs as the targets if the problem is a copy problem.
if p_hparams.was_copy:
feature_map["targets"] = feature_map["inputs"]
# Ensure inputs and targets are proper rank.
while len(feature_map["inputs"].get_shape()) != 4:
feature_map["inputs"] = tf.expand_dims(
feature_map["inputs"], axis=-1)
while len(feature_map["targets"].get_shape()) != 4:
feature_map["targets"] = tf.expand_dims(
feature_map["targets"], axis=-1)
batches.append(
(feature_map["inputs"], feature_map["targets"],
tf.constant(n), tf.constant(p_hparams.input_space_id),
tf.constant(p_hparams.target_space_id)))
# We choose which problem to process.
loss_moving_avgs = [] # Need loss moving averages for that.
for n in xrange(problem_count):
with tf.variable_scope("losses_avg"):
loss_moving_avgs.append(
tf.get_variable("problem_%d/total_loss" % n,
initializer=100.0,
trainable=False))
tf.get_variable("problem_%d/training_loss" % n,
initializer=100.0,
trainable=False)
tf.get_variable("problem_%d/extra_loss" % n,
initializer=100.0,
trainable=False)
if fixed_problem is None:
if (hparams.problem_choice == "uniform"
or mode != tf.contrib.learn.ModeKeys.TRAIN):
problem_choice = tf.random_uniform([],
maxval=problem_count,
dtype=tf.int32)
elif hparams.problem_choice == "adaptive":
loss_moving_avgs = tf.stack(loss_moving_avgs)
problem_choice = tf.multinomial(
tf.reshape(loss_moving_avgs, [1, -1]), 1)
problem_choice = tf.to_int32(tf.squeeze(problem_choice))
elif hparams.problem_choice == "distributed":
assert worker_replicas >= problem_count
assert worker_replicas % problem_count == 0
problem_choice = tf.to_int32(worker_id % problem_count)
else:
raise ValueError(
"Value of hparams.problem_choice is %s and must be "
"one of [uniform, adaptive, distributed]" %
hparams.problem_choice)
# Inputs and targets conditional on problem_choice.
rand_inputs, rand_target, choice, inp_id, tgt_id = cond_on_index(
lambda n: batches[n], problem_choice, 0, problem_count - 1)
else:
problem_choice = tf.constant(fixed_problem)
# Take the only constructed batch, which is the fixed_problem.
rand_inputs, rand_target, choice, inp_id, tgt_id = batches[0]
# Set shapes so the ranks are clear.
rand_inputs.set_shape([None, None, None, None])
rand_target.set_shape([None, None, None, None])
choice.set_shape([])
inp_id.set_shape([])
tgt_id.set_shape([])
# Forced shape obfuscation is necessary for inference.
if mode == tf.contrib.learn.ModeKeys.INFER:
rand_inputs._shape = tf.TensorShape([None, None, None, None]) # pylint: disable=protected-access
rand_target._shape = tf.TensorShape([None, None, None, None]) # pylint: disable=protected-access
# Final feature map.
rand_feature_map = {
"inputs": rand_inputs,
"problem_choice": choice,
"input_space_id": inp_id,
"target_space_id": tgt_id
}
if mode == tf.contrib.learn.ModeKeys.INFER:
rand_feature_map["infer_targets"] = rand_target
rand_target = None
return rand_feature_map, rand_target