def testNestedInputtersWithFlatDataFiles(self): inputters = inputter.ParallelInputter( [ record_inputter.SequenceRecordInputter(10), record_inputter.SequenceRecordInputter(10), ], reducer=reducer.SumReducer(), ) inputters = inputter.ParallelInputter( [ record_inputter.SequenceRecordInputter(10), inputters, ], reducer=reducer.ConcatReducer(), ) self.assertListEqual(inputters._structure(), [None, [None, None]]) empty_file = os.path.join(self.get_temp_dir(), "test.txt") with open(empty_file, "w"): pass with self.assertRaises(ValueError): inputters.make_inference_dataset([empty_file, empty_file], batch_size=2) inputters.make_inference_dataset([empty_file, empty_file, empty_file], batch_size=2)
def testSequenceRecord(self): vector = np.array([[0.2, 0.3], [0.4, 0.5]], dtype=np.float32) writer = tf.python_io.TFRecordWriter(record_file) record_inputter.write_sequence_record(vector, writer) writer.close() inputter = record_inputter.SequenceRecordInputter() data, transformed = _first_element(inputter, record_file) input_receiver = inputter.get_serving_input_receiver() self.assertIn("length", data) self.assertIn("tensor", data) self.assertIn("length", input_receiver.features) self.assertIn("tensor", input_receiver.features) self.assertAllEqual([None, None, 2], transformed.get_shape().as_list()) self.assertAllEqual([None, None, 2], input_receiver.features["tensor"].get_shape().as_list()) with self.test_session() as sess: sess.run(tf.tables_initializer()) data, transformed = sess.run([data, transformed]) self.assertEqual(2, data["length"]) self.assertAllEqual(vector, data["tensor"]) self.assertAllEqual([vector], transformed)
def testSequenceRecordWithCompression(self): vector = np.array([[0.2, 0.3], [0.4, 0.5]], dtype=np.float32) compression = "GZIP" record_file = os.path.join(self.get_temp_dir(), "data.records") record_file = record_inputter.create_sequence_records( [vector], record_file, compression=compression) inputter = record_inputter.SequenceRecordInputter(2) dataset = inputter.make_inference_dataset(record_file, batch_size=1) iterator = iter(dataset) self.assertAllEqual(next(iterator)["tensor"].numpy()[0], vector)
def testSequenceRecord(self): vector = np.array([[0.2, 0.3], [0.4, 0.5]], dtype=np.float32) record_file = os.path.join(self.get_temp_dir(), "data.records") writer = compat.tf_compat(v2="io.TFRecordWriter", v1="python_io.TFRecordWriter")(record_file) record_inputter.write_sequence_record(vector, writer) writer.close() inputter = record_inputter.SequenceRecordInputter() features, transformed = self._makeDataset( inputter, record_file, shapes={"tensor": [None, None, 2], "length": [None]}) self.assertEqual([2], features["length"]) self.assertAllEqual([vector], features["tensor"]) self.assertAllEqual([vector], transformed)
def testSequenceRecord(self): vector = np.array([[0.2, 0.3], [0.4, 0.5]], dtype=np.float32) record_file = os.path.join(self.get_temp_dir(), "data.records") record_inputter.create_sequence_records([vector], record_file) inputter = record_inputter.SequenceRecordInputter(2) features, transformed = self._makeDataset( inputter, record_file, dataset_size=None, shapes={"tensor": [None, None, 2], "length": [None]}, ) self.assertEqual([2], features["length"]) self.assertAllEqual([vector], features["tensor"]) self.assertAllEqual([vector], transformed)
def testSequenceRecord(self): vector = np.array([[0.2, 0.3], [0.4, 0.5]], dtype=np.float32) record_file = os.path.join(self.get_temp_dir(), "data.records") writer = tf.python_io.TFRecordWriter(record_file) record_inputter.write_sequence_record(vector, writer) writer.close() inputter = record_inputter.SequenceRecordInputter() features, transformed = self._makeDataset( inputter, record_file, shapes={"tensor": [None, None, 2], "length": [None]}) with self.test_session() as sess: sess.run(tf.tables_initializer()) features, transformed = sess.run([features, transformed]) self.assertEqual([2], features["length"]) self.assertAllEqual([vector], features["tensor"]) self.assertAllEqual([vector], transformed)
def testSequenceRecordBatch(self): vectors = [ np.random.rand(3, 2), np.random.rand(6, 2), np.random.rand(1, 2), ] record_file = os.path.join(self.get_temp_dir(), "data.records") record_inputter.create_sequence_records(vectors, record_file) inputter = record_inputter.SequenceRecordInputter(2) dataset = inputter.make_dataset(record_file) dataset = dataset.batch(3) dataset = dataset.map(inputter.make_features) features = next(iter(dataset)) lengths = features["length"] tensors = features["tensor"] self.assertAllEqual(lengths, [3, 6, 1]) for length, tensor, expected_vector in zip(lengths, tensors, vectors): self.assertAllClose(tensor[:length], expected_vector)