def testNestedInputtersWithFlatDataFiles(self):
inputters = inputter.ParallelInputter(
[
record_inputter.SequenceRecordInputter(10),
record_inputter.SequenceRecordInputter(10),
],
reducer=reducer.SumReducer(),
)
inputters = inputter.ParallelInputter(
[
record_inputter.SequenceRecordInputter(10),
inputters,
],
reducer=reducer.ConcatReducer(),
)
self.assertListEqual(inputters._structure(), [None, [None, None]])
empty_file = os.path.join(self.get_temp_dir(), "test.txt")
with open(empty_file, "w"):
pass
with self.assertRaises(ValueError):
inputters.make_inference_dataset([empty_file, empty_file],
batch_size=2)
inputters.make_inference_dataset([empty_file, empty_file, empty_file],
batch_size=2)
def testSequenceRecord(self):
vector = np.array([[0.2, 0.3], [0.4, 0.5]], dtype=np.float32)
writer = tf.python_io.TFRecordWriter(record_file)
record_inputter.write_sequence_record(vector, writer)
writer.close()
inputter = record_inputter.SequenceRecordInputter()
data, transformed = _first_element(inputter, record_file)
input_receiver = inputter.get_serving_input_receiver()
self.assertIn("length", data)
self.assertIn("tensor", data)
self.assertIn("length", input_receiver.features)
self.assertIn("tensor", input_receiver.features)
self.assertAllEqual([None, None, 2], transformed.get_shape().as_list())
self.assertAllEqual([None, None, 2], input_receiver.features["tensor"].get_shape().as_list())
with self.test_session() as sess:
sess.run(tf.tables_initializer())
data, transformed = sess.run([data, transformed])
self.assertEqual(2, data["length"])
self.assertAllEqual(vector, data["tensor"])
self.assertAllEqual([vector], transformed)
def testSequenceRecordWithCompression(self):
vector = np.array([[0.2, 0.3], [0.4, 0.5]], dtype=np.float32)
compression = "GZIP"
record_file = os.path.join(self.get_temp_dir(), "data.records")
record_file = record_inputter.create_sequence_records(
[vector], record_file, compression=compression)
inputter = record_inputter.SequenceRecordInputter(2)
dataset = inputter.make_inference_dataset(record_file, batch_size=1)
iterator = iter(dataset)
self.assertAllEqual(next(iterator)["tensor"].numpy()[0], vector)
def testSequenceRecord(self):
vector = np.array([[0.2, 0.3], [0.4, 0.5]], dtype=np.float32)
record_file = os.path.join(self.get_temp_dir(), "data.records")
writer = compat.tf_compat(v2="io.TFRecordWriter", v1="python_io.TFRecordWriter")(record_file)
record_inputter.write_sequence_record(vector, writer)
writer.close()
inputter = record_inputter.SequenceRecordInputter()
features, transformed = self._makeDataset(
inputter,
record_file,
shapes={"tensor": [None, None, 2], "length": [None]})
self.assertEqual([2], features["length"])
self.assertAllEqual([vector], features["tensor"])
self.assertAllEqual([vector], transformed)
def testSequenceRecord(self):
vector = np.array([[0.2, 0.3], [0.4, 0.5]], dtype=np.float32)
record_file = os.path.join(self.get_temp_dir(), "data.records")
record_inputter.create_sequence_records([vector], record_file)
inputter = record_inputter.SequenceRecordInputter(2)
features, transformed = self._makeDataset(
inputter,
record_file,
dataset_size=None,
shapes={"tensor": [None, None, 2], "length": [None]},
)
self.assertEqual([2], features["length"])
self.assertAllEqual([vector], features["tensor"])
self.assertAllEqual([vector], transformed)
def testSequenceRecord(self):
vector = np.array([[0.2, 0.3], [0.4, 0.5]], dtype=np.float32)
record_file = os.path.join(self.get_temp_dir(), "data.records")
writer = tf.python_io.TFRecordWriter(record_file)
record_inputter.write_sequence_record(vector, writer)
writer.close()
inputter = record_inputter.SequenceRecordInputter()
features, transformed = self._makeDataset(
inputter,
record_file,
shapes={"tensor": [None, None, 2], "length": [None]})
with self.test_session() as sess:
sess.run(tf.tables_initializer())
features, transformed = sess.run([features, transformed])
self.assertEqual([2], features["length"])
self.assertAllEqual([vector], features["tensor"])
self.assertAllEqual([vector], transformed)
def testSequenceRecordBatch(self):
vectors = [
np.random.rand(3, 2),
np.random.rand(6, 2),
np.random.rand(1, 2),
]
record_file = os.path.join(self.get_temp_dir(), "data.records")
record_inputter.create_sequence_records(vectors, record_file)
inputter = record_inputter.SequenceRecordInputter(2)
dataset = inputter.make_dataset(record_file)
dataset = dataset.batch(3)
dataset = dataset.map(inputter.make_features)
features = next(iter(dataset))
lengths = features["length"]
tensors = features["tensor"]
self.assertAllEqual(lengths, [3, 6, 1])
for length, tensor, expected_vector in zip(lengths, tensors, vectors):
self.assertAllClose(tensor[:length], expected_vector)
