def gpu_valid_size(example):
drop_long_sequences = is_training or hparams.eval_drop_long_sequences
return data_reader.example_valid_size(
example,
hparams.min_length,
max_length
if drop_long_sequences else 10**9
)
def testLengthFilter(self):
max_len = 15
dataset = data_reader.read_examples(self.problem, self.filepatterns[0],
32)
dataset = dataset.filter(
lambda ex: data_reader.example_valid_size(ex, max_len))
examples = dataset.make_one_shot_iterator().get_next()
with tf.train.MonitoredSession() as sess:
ex_lens = []
for _ in xrange(max_len):
ex_lens.append(len(sess.run(examples)["inputs"]))
self.assertAllEqual(list(range(1, max_len + 1)), sorted(ex_lens))
def testLengthFilter(self):
max_len = 15
dataset = self.problem.dataset(
tf.estimator.ModeKeys.TRAIN, data_dir=self.data_dir)
dataset = dataset.filter(
lambda ex: data_reader.example_valid_size(ex, 0, max_len))
examples = dataset.make_one_shot_iterator().get_next()
with tf.train.MonitoredSession() as sess:
ex_lens = []
for _ in xrange(max_len):
ex_lens.append(len(sess.run(examples)["inputs"]))
self.assertAllEqual(list(range(1, max_len + 1)), sorted(ex_lens))
def testLengthFilter(self):
max_len = 15
dataset = self.problem.dataset(tf.estimator.ModeKeys.TRAIN,
data_dir=self.data_dir)
dataset = dataset.filter(
lambda ex: data_reader.example_valid_size(ex, 0, max_len))
examples = dataset.make_one_shot_iterator().get_next()
with tf.train.MonitoredSession() as sess:
ex_lens = []
for _ in xrange(max_len):
ex_lens.append(len(sess.run(examples)["inputs"]))
self.assertAllEqual(list(range(1, max_len + 1)), sorted(ex_lens))
def tpu_valid_size(example):
return data_reader.example_valid_size(
example,
hparams.min_length,
max_length
)
def _valid_size(example):
return data_reader.example_valid_size(
example, batching_scheme["min_length"],
batching_scheme["max_length"])
def gpu_valid_size(example):
drop_long_sequences = is_training or hparams.eval_drop_long_sequences
return data_reader.example_valid_size(example, hparams.min_length,
max_length
if drop_long_sequences else 10**9)
def tpu_valid_size(example):
return data_reader.example_valid_size(example, hparams.min_length,
max_length)
