def prepare(self):
self.meta = self.prepare_meta(self.lmdb_paths)
if self.unpack is None:
self.unpack = self.default_unpack
# The fetcher is supposed to be initialized in the
# sub-processes, or it will cause CRC Error.
self.fetcher = None
self.data_ids = self.meta.get('data_ids', self.meta.get('data_id', []))
if self.debug:
self.data_ids = self.data_ids[:32]
self.num_samples = len(self.data_ids)
# prepare lmdb environments
for path in self.lmdb_paths:
path = os.path.join(path, '')
env = lmdb.open(path, max_dbs=1, lock=False)
db_image = env.open_db('image'.encode())
self.envs.append(env)
self.txns[path] = env.begin(db=db_image)
if is_main():
print(self.num_samples, 'images found')
return self
def read(self, nori_path):
path = self.hash(nori_path)
while not self.connection.exists(os.path.join(path, '__keys__')):
warnings.warn('Cache not found in redis:' + str(path))
if is_main():
warnings.warn('Now may try to load and save meta from file')
return None
else:
time.sleep(2)
return RedisMeta(self.connection, path)
def save(self, nori_path, meta):
if not is_main():
return False
path = self.hash(nori_path)
for key, value_list in meta.items():
pipe = self.connection.pipeline()
pipe.sadd(os.path.join(path, '__keys__'), key)
for value in value_list:
pipe.lpush(os.path.join(path, key), value)
pipe.execute()
return True
def prepare(self):
self.meta = self.prepare_meta(self.file_paths)
if self.unpack is None:
self.unpack = self.default_unpack
self.data_ids = self.meta.get('data_ids', self.meta.get('data_id', []))
if self.debug:
self.data_ids = self.data_ids[:32]
self.num_samples = len(self.data_ids)
if is_main():
print(self.num_samples, 'images found')
return self
def prepare(self):
self.meta = self.prepare_meta(self.nori_paths)
if self.unpack is None:
self.unpack = self.defulat_unpack
# The fetcher is supposed to be initialized in the
# sub-processes, or it will cause CRC Error.
self.fetcher = None
self.data_ids = self.meta.get('data_ids', self.meta.get('data_id', []))
if self.debug:
self.data_ids = self.data_ids[:32]
self.num_samples = len(self.data_ids)
if is_main():
print(self.num_samples, 'images found')
return self