def reading_tf_records_from_dareblopy(): features = {'data': db.FixedLenFeature([3, 256, 256], db.uint8)} iterator = db.data_loader(db.ParsedTFRecordsDatasetIterator( filenames, features, batch_size, 64), worker_count=6) records = [] for batch in iterator: records += batch
def reading_tf_records_from_dareblopy_withoutdecoding(): features = {'data': db.FixedLenFeature([], db.string)} iterator = db.data_loader(db.ParsedTFRecordsDatasetIterator( filenames, features, batch_size, 128), worker_count=6) records = [] for batch in iterator: records += batch
def test_dataset_iterator(self): features = { 'data': db.FixedLenFeature([3, 32, 32], db.uint8) } iterator = db.ParsedTFRecordsDatasetIterator(['test_utils/test-small-r00.tfrecords'], features, 32, buffer_size=1) images = np.concatenate([x[0] for x in iterator], axis=0) self.assertTrue(np.all(images == self.images_gt))
def test_ParsedTFRecordsDatasetIterator(): features = { #'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([3, 256, 256], db.uint8) } iterator = db.ParsedTFRecordsDatasetIterator(filenames, features, 32, 64) records = [] for batch in iterator: records += batch
def reset(self, lod, batch_size): assert lod in self.filenames.keys() self.current_filenames = self.filenames[lod] self.batch_size = batch_size img_size = 2 ** lod self.features = { # 'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([3, img_size, img_size], db.uint8) } buffer_size = self.buffer_size_b // (3 * img_size * img_size) self.iterator = db.ParsedTFRecordsDatasetIterator(self.current_filenames, self.features, self.batch_size, buffer_size, seed=np.uint64(time.time() * 1000))
def __iter__(self): iters = [] for _ in range(self.epochs): # Note this is shuffled by default loader = db.ParsedTFRecordsDatasetIterator( self.data_files, self.features, batch_size=1, buffer_size=32, seed=randint(0, 10000), ) decoded_iter = map(self.preprocess, loader) iters.append(decoded_iter) return chain(*iters)
def reset(self, lod, batch_size): assert lod in self.filenames.keys() self.current_filenames = self.filenames[lod] self.batch_size = batch_size img_size = 2**lod if self.needs_labels: self.features = { # 'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([self.channels, img_size, img_size], db.uint8), 'label': db.FixedLenFeature([], db.int64) } else: self.features = { # 'shape': db.FixedLenFeature([3], db.int64), 'data': db.FixedLenFeature([self.channels, img_size, img_size], db.uint8) } buffer_size = self.buffer_size_b // (self.channels * img_size * img_size) if self.seed is None: seed = np.uint64(time.time() * 1000) else: seed = self.seed self.logger.info('!' * 80) self.logger.info( '! Seed is used for to shuffle data in TFRecordsDataset!') self.logger.info('!' * 80) self.iterator = db.ParsedTFRecordsDatasetIterator( self.current_filenames, self.features, self.batch_size, buffer_size, seed=seed)