def _makeDataset(self,
inputter,
data_file,
metadata=None,
dataset_size=1,
shapes=None):
if metadata is not None:
inputter.initialize(metadata)
self.assertEqual(dataset_size, inputter.get_dataset_size(data_file))
dataset = inputter.make_dataset(data_file)
dataset = dataset.map(
lambda *arg: inputter.process(item_or_tuple(arg)))
dataset = dataset.padded_batch(
1, padded_shapes=data.get_padded_shapes(dataset))
iterator = dataset.make_initializable_iterator()
tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS,
iterator.initializer)
next_element = iterator.get_next()
if shapes is not None:
for features in (next_element,
inputter.get_serving_input_receiver().features):
self.assertNotIn("raw", features)
for field, shape in six.iteritems(shapes):
self.assertIn(field, features)
self.assertAllEqual(shape,
features[field].get_shape().as_list())
transformed = inputter.transform_data(next_element)
return next_element, transformed
def _makeDataset(self,
inputter,
data_file,
data_config=None,
dataset_size=1,
shapes=None):
if data_config is not None:
inputter.initialize(data_config)
self.assertEqual(inputter.get_dataset_size(data_file), dataset_size)
dataset = inputter.make_dataset(data_file)
eager_features = inputter.make_features(iter(dataset).next(),
training=True)
eager_features = tf.nest.map_structure(lambda t: tf.expand_dims(t, 0),
eager_features)
dataset = dataset.map(lambda *arg: inputter.make_features(
item_or_tuple(arg), training=True))
dataset = dataset.apply(dataset_util.batch_dataset(1))
features = iter(dataset).next()
if shapes is not None:
self._checkFeatures(features, shapes)
self._checkFeatures(eager_features, shapes)
keep = inputter.keep_for_training(features)
self.assertIs(keep.dtype, tf.bool)
inputs = inputter(features, training=True)
if not isinstance(inputter, inputters.ExampleInputter):
self._testServing(inputter)
return self.evaluate((features, inputs))
def _makeDataset(self, inputter, data_file, metadata=None, dataset_size=1, shapes=None):
if metadata is not None:
inputter.initialize(metadata)
self.assertEqual(dataset_size, inputter.get_dataset_size(data_file))
dataset = inputter.make_dataset(data_file)
dataset = dataset.map(lambda *arg: inputter.process(item_or_tuple(arg)))
dataset = dataset.padded_batch(1, padded_shapes=data.get_padded_shapes(dataset))
if compat.is_tf2():
iterator = None
features = iter(dataset).next()
else:
iterator = dataset.make_initializable_iterator()
features = iterator.get_next()
if shapes is not None:
all_features = [features]
if not compat.is_tf2() and not inputter.is_target:
all_features.append(inputter.get_serving_input_receiver().features)
for f in all_features:
for field, shape in six.iteritems(shapes):
self.assertIn(field, f)
self.assertTrue(f[field].shape.is_compatible_with(shape))
inputs = inputter.make_inputs(features, training=True)
if not compat.is_tf2():
with self.test_session() as sess:
sess.run(tf.tables_initializer())
sess.run(tf.global_variables_initializer())
sess.run(iterator.initializer)
return self.evaluate((features, inputs))
def _first_element(inputter, data_file, metadata):
inputter.initialize(metadata)
dataset = inputter.make_dataset(data_file)
iterator = dataset.make_initializable_iterator()
tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer)
next_element = iterator.get_next()
data = inputter.process(next_element)
for key, value in data.items():
data[key] = tf.expand_dims(value, 0)
transformed = inputter.transform_data(data)
return data, transformed
def _first_element(inputter, data_file, metadata):
inputter.initialize(metadata)
dataset = inputter.make_dataset(data_file)
iterator = dataset.make_initializable_iterator()
tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer)
next_element = iterator.get_next()
data = inputter.process(next_element)
for key, value in six.iteritems(data):
data[key] = tf.expand_dims(value, 0)
transformed = inputter.transform_data(data)
return data, transformed
def _first_element(inputter, data_file, metadata=None):
if metadata is not None:
inputter.initialize(metadata)
dataset = inputter.make_dataset(data_file)
iterator = dataset.make_initializable_iterator()
tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, iterator.initializer)
next_element = iterator.get_next()
data = inputter.process(next_element)
data_in = {}
for key, value in six.iteritems(data):
value = tf.expand_dims(value, 0)
value.set_shape([None] + inputter.padded_shapes[key])
data_in[key] = value
transformed = inputter.transform_data(data_in)
return data, transformed
def _makeDataset(self,
inputter,
data_file,
data_config=None,
dataset_size=1,
shapes=None):
if data_config is not None:
inputter.initialize(data_config)
dataset = inputter.make_dataset(data_file)
dataset = dataset.map(lambda *arg: inputter.make_features(
item_or_tuple(arg), training=True))
dataset = dataset.apply(dataset_util.batch_dataset(1))
features = iter(dataset).next()
if shapes is not None:
self._checkFeatures(features, shapes)
inputs = inputter(features, training=True)
return self.evaluate((features, inputs))
def testSequenceRecordBatch(self):
vectors = [
np.random.rand(3, 2),
np.random.rand(6, 2),
np.random.rand(1, 2),
]
record_file = os.path.join(self.get_temp_dir(), "data.records")
record_inputter.create_sequence_records(vectors, record_file)
inputter = record_inputter.SequenceRecordInputter(2)
dataset = inputter.make_dataset(record_file)
dataset = dataset.batch(3)
dataset = dataset.map(inputter.make_features)
features = next(iter(dataset))
lengths = features["length"]
tensors = features["tensor"]
self.assertAllEqual(lengths, [3, 6, 1])
for length, tensor, expected_vector in zip(lengths, tensors, vectors):
self.assertAllClose(tensor[:length], expected_vector)
