def reading_tf_records_from_dareblopy():
features = {'data': db.FixedLenFeature([3, 256, 256], db.uint8)}
iterator = db.data_loader(db.ParsedTFRecordsDatasetIterator(
filenames, features, batch_size, 64),
worker_count=6)
records = []
for batch in iterator:
records += batch
def reading_tf_records_from_dareblopy_withoutdecoding():
features = {'data': db.FixedLenFeature([], db.string)}
iterator = db.data_loader(db.ParsedTFRecordsDatasetIterator(
filenames, features, batch_size, 128),
worker_count=6)
records = []
for batch in iterator:
records += batch
def test_dataset_iterator(self):
features = {
'data': db.FixedLenFeature([3, 32, 32], db.uint8)
}
iterator = db.ParsedTFRecordsDatasetIterator(['test_utils/test-small-r00.tfrecords'],
features, 32, buffer_size=1)
images = np.concatenate([x[0] for x in iterator], axis=0)
self.assertTrue(np.all(images == self.images_gt))
def test_ParsedTFRecordsDatasetIterator():
features = {
#'shape': db.FixedLenFeature([3], db.int64),
'data': db.FixedLenFeature([3, 256, 256], db.uint8)
}
iterator = db.ParsedTFRecordsDatasetIterator(filenames, features, 32,
64)
records = []
for batch in iterator:
records += batch
def reset(self, lod, batch_size):
assert lod in self.filenames.keys()
self.current_filenames = self.filenames[lod]
self.batch_size = batch_size
img_size = 2 ** lod
self.features = {
# 'shape': db.FixedLenFeature([3], db.int64),
'data': db.FixedLenFeature([3, img_size, img_size], db.uint8)
}
buffer_size = self.buffer_size_b // (3 * img_size * img_size)
self.iterator = db.ParsedTFRecordsDatasetIterator(self.current_filenames, self.features, self.batch_size, buffer_size, seed=np.uint64(time.time() * 1000))
def __iter__(self):
iters = []
for _ in range(self.epochs):
# Note this is shuffled by default
loader = db.ParsedTFRecordsDatasetIterator(
self.data_files,
self.features,
batch_size=1,
buffer_size=32,
seed=randint(0, 10000),
)
decoded_iter = map(self.preprocess, loader)
iters.append(decoded_iter)
return chain(*iters)
def reset(self, lod, batch_size):
assert lod in self.filenames.keys()
self.current_filenames = self.filenames[lod]
self.batch_size = batch_size
img_size = 2**lod
if self.needs_labels:
self.features = {
# 'shape': db.FixedLenFeature([3], db.int64),
'data':
db.FixedLenFeature([self.channels, img_size, img_size],
db.uint8),
'label':
db.FixedLenFeature([], db.int64)
}
else:
self.features = {
# 'shape': db.FixedLenFeature([3], db.int64),
'data':
db.FixedLenFeature([self.channels, img_size, img_size],
db.uint8)
}
buffer_size = self.buffer_size_b // (self.channels * img_size *
img_size)
if self.seed is None:
seed = np.uint64(time.time() * 1000)
else:
seed = self.seed
self.logger.info('!' * 80)
self.logger.info(
'! Seed is used for to shuffle data in TFRecordsDataset!')
self.logger.info('!' * 80)
self.iterator = db.ParsedTFRecordsDatasetIterator(
self.current_filenames,
self.features,
self.batch_size,
buffer_size,
seed=seed)
