def gpu_valid_size(example): drop_long_sequences = is_training or hparams.eval_drop_long_sequences return data_reader.example_valid_size( example, hparams.min_length, max_length if drop_long_sequences else 10**9 )
def testLengthFilter(self): max_len = 15 dataset = data_reader.read_examples(self.problem, self.filepatterns[0], 32) dataset = dataset.filter( lambda ex: data_reader.example_valid_size(ex, max_len)) examples = dataset.make_one_shot_iterator().get_next() with tf.train.MonitoredSession() as sess: ex_lens = [] for _ in xrange(max_len): ex_lens.append(len(sess.run(examples)["inputs"])) self.assertAllEqual(list(range(1, max_len + 1)), sorted(ex_lens))
def testLengthFilter(self): max_len = 15 dataset = self.problem.dataset( tf.estimator.ModeKeys.TRAIN, data_dir=self.data_dir) dataset = dataset.filter( lambda ex: data_reader.example_valid_size(ex, 0, max_len)) examples = dataset.make_one_shot_iterator().get_next() with tf.train.MonitoredSession() as sess: ex_lens = [] for _ in xrange(max_len): ex_lens.append(len(sess.run(examples)["inputs"])) self.assertAllEqual(list(range(1, max_len + 1)), sorted(ex_lens))
def testLengthFilter(self): max_len = 15 dataset = self.problem.dataset(tf.estimator.ModeKeys.TRAIN, data_dir=self.data_dir) dataset = dataset.filter( lambda ex: data_reader.example_valid_size(ex, 0, max_len)) examples = dataset.make_one_shot_iterator().get_next() with tf.train.MonitoredSession() as sess: ex_lens = [] for _ in xrange(max_len): ex_lens.append(len(sess.run(examples)["inputs"])) self.assertAllEqual(list(range(1, max_len + 1)), sorted(ex_lens))
def tpu_valid_size(example): return data_reader.example_valid_size( example, hparams.min_length, max_length )
def _valid_size(example): return data_reader.example_valid_size( example, batching_scheme["min_length"], batching_scheme["max_length"])
def gpu_valid_size(example): drop_long_sequences = is_training or hparams.eval_drop_long_sequences return data_reader.example_valid_size(example, hparams.min_length, max_length if drop_long_sequences else 10**9)
def tpu_valid_size(example): return data_reader.example_valid_size(example, hparams.min_length, max_length)