def __init__(self, dataset_name, dataset_ver, split, epochs): self.data_files = locate_data(dataset_name, dataset_ver, split) self.features = { "label": db.FixedLenFeature([], db.int64), "image": db.FixedLenFeature([], db.string), } self.epochs = epochs
def test_parsing_single_record_with_uint8_alternative_to_string_raise(self): features_alternative = { 'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([3, 32, 64], db.uint8) } parser = db.RecordParser(features_alternative) self.assertIsNotNone(parser) with self.assertRaises(RuntimeError) as context: for record, image_gt in zip(self.records, self.images_gt): shape, data = parser.parse_single_example(record) self.assertEqual('Key: data. Number of uint8 values != expected. Values size: 3072 but output shape: [3, 32, 64].', context.exception.args[0])
def test_parsing_single_record_with_uint8_alternative_to_string(self): features_alternative = { 'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([3, 32, 32], db.uint8) } parser = db.RecordParser(features_alternative) self.assertIsNotNone(parser) for record, image_gt in zip(self.records, self.images_gt): shape, data = parser.parse_single_example(record) self.assertTrue(np.all(shape == [3, 32, 32])) self.assertTrue(np.all(data == image_gt))
def test_parsing_single_record_inplace(self): features = { 'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([], db.string)} parser = db.RecordParser(features) self.assertIsNotNone(parser) shape = np.zeros(3, dtype=np.int64) data = np.asarray([bytes()], dtype=object) for record, image_gt in zip(self.records, self.images_gt): parser.parse_single_example_inplace(record, [shape, data], 0) image = np.frombuffer(data[0], dtype=np.uint8).reshape(shape) self.assertTrue(np.all(image == image_gt))
def test_parsing_single_record(self): features = { 'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([], db.string) } parser = db.RecordParser(features) self.assertIsNotNone(parser) for record, image_gt in zip(self.records, self.images_gt): shape, data = parser.parse_single_example(record) self.assertTrue(np.all(shape == [3, 32, 32])) self.assertTrue(np.all( np.frombuffer(data[0], dtype=np.uint8).reshape(3, 32, 32) == image_gt ))
def reset(self, lod, batch_size): assert lod in self.filenames.keys() self.current_filenames = self.filenames[lod] self.batch_size = batch_size img_size = 2**lod self.features = { 'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([3, img_size, img_size], db.uint8) } buffer_size = self.buffer_size_b // (3 * img_size * img_size) self.iterator = db.ParsedTFRecordsDatasetIterator( self.current_filenames, self.features, self.batch_size, buffer_size, seed=np.uint64(time.time() * 1000))
def reading_tf_records_from_dareblopy(): features = {'data': db.FixedLenFeature([3, 256, 256], db.uint8)} iterator = db.data_loader(db.ParsedTFRecordsDatasetIterator( filenames, features, batch_size, 64), worker_count=6) records = [] for batch in iterator: records += batch
def reading_tf_records_from_dareblopy_withoutdecoding(): features = {'data': db.FixedLenFeature([], db.string)} iterator = db.data_loader(db.ParsedTFRecordsDatasetIterator( filenames, features, batch_size, 128), worker_count=6) records = [] for batch in iterator: records += batch
def test_dataset_iterator(self): features = { 'data': db.FixedLenFeature([3, 32, 32], db.uint8) } iterator = db.ParsedTFRecordsDatasetIterator(['test_utils/test-small-r00.tfrecords'], features, 32, buffer_size=1) images = np.concatenate([x[0] for x in iterator], axis=0) self.assertTrue(np.all(images == self.images_gt))
def test_parsing_records_in_batch(self): features_alternative = { 'data': db.FixedLenFeature([3, 32, 32], db.uint8) } parser = db.RecordParser(features_alternative, False) self.assertIsNotNone(parser) data = parser.parse_example(self.records)[0] self.assertTrue(np.all(data == self.images_gt))
def test_ParsedTFRecordsDatasetIterator(): features = { #'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([3, 256, 256], db.uint8) } iterator = db.ParsedTFRecordsDatasetIterator(filenames, features, 32, 64) records = [] for batch in iterator: records += batch
def reset(self, lod, batch_size): assert lod in self.filenames.keys() self.current_filenames = self.filenames[lod] self.batch_size = batch_size img_size = 2**lod if self.needs_labels: self.features = { # 'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([self.channels, img_size, img_size], db.uint8), 'label': db.FixedLenFeature([], db.int64) } else: self.features = { # 'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([self.channels, img_size, img_size], db.uint8) } buffer_size = self.buffer_size_b // (self.channels * img_size * img_size) if self.seed is None: seed = np.uint64(time.time() * 1000) else: seed = self.seed self.logger.info('!' * 80) self.logger.info( '! Seed is used for to shuffle data in TFRecordsDataset!') self.logger.info('!' * 80) self.iterator = db.ParsedTFRecordsDatasetIterator( self.current_filenames, self.features, self.batch_size, buffer_size, seed=seed)
def test_parsing_single_record_inplace_with_uint8(self): features_alternative = { 'data': db.FixedLenFeature([3, 32, 32], db.uint8) } parser = db.RecordParser(features_alternative, False) self.assertIsNotNone(parser) data = np.zeros([3, 32, 32], dtype=np.uint8) for record, image_gt in zip(self.records, self.images_gt): parser.parse_single_example_inplace(record, [data], 0) self.assertTrue(np.all(data == image_gt))
def test_parsing_single_record_raise2(self): features = { 'data': db.FixedLenFeature([], db.int64) } parser = db.RecordParser(features) self.assertIsNotNone(parser) with self.assertRaises(RuntimeError) as context: for record, image_gt in zip(self.records, self.images_gt): shape, data = parser.parse_single_example(record) self.assertEqual('Feature: data. Data types don\'t match. Expected type: int64, Feature is: string.', context.exception.args[0])
def test_parsing_single_record_raise(self): features = { 'does_not_exist': db.FixedLenFeature([], db.string) } parser = db.RecordParser(features) self.assertIsNotNone(parser) with self.assertRaises(RuntimeError) as context: for record, image_gt in zip(self.records, self.images_gt): shape, data = parser.parse_single_example(record) self.assertEqual('Feature does_not_exist is required but could not be found.', context.exception.args[0])
def test_yielder_randomized_parallel(): features = { #'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([3, 256, 256], db.uint8) } parser = db.RecordParser(features, True) record_yielder = db.ParsedRecordYielderRandomized( parser, filenames, 64, 1, 0) records = [] while True: try: records += record_yielder.next_n(32) except StopIteration: break