def test_stat(self):
_, env = testlib.temp_env()
db1 = env.open_db(B('db1'))
db2 = env.open_db(B('db2'))
txn = lmdb.Transaction(env)
for db in db1, db2:
stat = txn.stat(db)
for k in 'psize', 'depth', 'branch_pages', 'overflow_pages',\
'entries':
assert isinstance(stat[k], INT_TYPES), k
assert stat[k] >= 0
assert stat['entries'] == 0
txn = lmdb.Transaction(env, write=True)
txn.put(B('a'), B('b'), db=db1)
txn.commit()
txn = lmdb.Transaction(env)
stat = txn.stat(db1)
assert stat['entries'] == 1
stat = txn.stat(db2)
assert stat['entries'] == 0
txn.abort()
self.assertRaises(Exception, lambda: env.stat(db1))
env.close()
self.assertRaises(Exception, lambda: env.stat(db1))
def test_readonly(self):
_, env = testlib.temp_env()
txn = lmdb.Transaction(env)
# Read txn can't write.
self.assertRaises(lmdb.ReadonlyError,
lambda: txn.put(B('a'), B('')))
txn.abort()
def test_buffers(self):
_, env = testlib.temp_env()
txn = lmdb.Transaction(env, write=True, buffers=True)
assert txn.put(B('a'), B('a'))
b = txn.get(B('a'))
assert b is not None
assert len(b) == 1
assert not isinstance(b, type(B('')))
txn.commit()
txn = lmdb.Transaction(env, buffers=False)
b = txn.get(B('a'))
assert b is not None
assert len(b) == 1
assert isinstance(b, type(B('')))
txn.abort()
def collect_data_parallel(self):
im_filenames = glob.glob(osp.join(self.im_dir, '*.jpg'))
block_size = 4000
n_blocks = np.ceil(len(im_filenames) / float(block_size)).astype(np.int)
ge = GISTExtractor(width=256, height=256)
im = cv2.imread(im_filenames[0])
gist_feat_size = ge.extract_gist(im).shape[0]
pool = Pool(initializer=pool_init, initargs=(256, 256))
txn = self.db.begin(write=True)
for b in xrange(n_blocks):
print '##### Block {:d} / {:d}'.format(b, n_blocks)
im_fs = im_filenames[b*block_size : (b+1)*block_size]
gist_feats = np.zeros((len(im_fs), gist_feat_size), dtype=np.float)
chunksize = len(im_fs) / 4
for idx, feat in enumerate(pool.imap(gist_wrapper, im_fs, chunksize)):
gist_feats[idx, :] = feat
for im_name, feat in zip(im_fs, gist_feats):
key = '{:s}/{:s}'.format(self.city, im_name.split('/')[-1][:-4])
txn.put(key, feat.tostring())
txn.commit()
txn = lmdb.Transaction(env=self.db, write=True)
# terminate
pool.close()
pool.join()
txn.commit()
self.db.close()
def test_bind_db(self):
_, env = testlib.temp_env()
main = env.open_db(None)
sub = env.open_db(B('db1'))
txn = lmdb.Transaction(env, write=True, db=sub)
assert txn.put(B('b'), B('')) # -> sub
assert txn.put(B('a'), B(''), db=main) # -> main
txn.commit()
txn = lmdb.Transaction(env)
assert txn.get(B('a')) == B('')
assert txn.get(B('b')) is None
assert txn.get(B('a'), db=sub) is None
assert txn.get(B('b'), db=sub) == B('')
txn.abort()
def _commit(self, write: bool):
# Write cached changes
for path, node in self._set_cache.items():
self._txn.put(path.encode(), node.serialize())
self._txn.commit()
self._txn = lmdb.Transaction(self._env,
write=write,
buffers=self.node_type.uses_buffers())
self.current_txn_size = 0
def __init__(self, database, write=False, buffers=False):
self._transactions = []
self._coc = [database.node]
self._tid = None
self._replicated = False
self._db = database
self._txn = lmdb.Transaction(self._db.env,
write=write,
buffers=buffers)
def begin_adding_samples_to_lmdb(self) -> None:
"""
A function that needs to be called before adding samples to the buffer, in case of using an lmdb buffer, so that
a transaction is created.
"""
if self.save_buffer_on_disk:
if not self._is_lmdb_env_created():
self.reset_lmdb_database()
self._txn = lmdb.Transaction(self._lmdb_env, write=True)
def test_parent(self):
_, env = testlib.temp_env()
parent = lmdb.Transaction(env, write=True)
parent.put(B('a'), B('a'))
child = lmdb.Transaction(env, write=True, parent=parent)
assert child.get(B('a')) == B('a')
assert child.put(B('a'), B('b'))
child.abort()
# put() should have rolled back
assert parent.get(B('a')) == B('a')
child = lmdb.Transaction(env, write=True, parent=parent)
assert child.put(B('a'), B('b'))
child.commit()
# put() should be visible
assert parent.get(B('a')) == B('b')
def __init__(self, **kwargs):
self.env = lmdb.open(kwargs['file_path'])
self.txn = lmdb.Transaction(self.env)
self.cur = self.txn.cursor()
self.num_items = self.env.stat()['entries']
self.labels = None
super(LmdbLoader, self).__init__(**kwargs)
assert self.batch_size < self.num_items, "Length Batch " + str(
n) + " > " + str(self.num_items)
def test_bind_db_methods(self):
_, env = testlib.temp_env()
maindb = env.open_db(None)
db1 = env.open_db(B('d1'))
txn = lmdb.Transaction(env, write=True, db=db1)
assert txn.put(B('a'), B('d1'))
assert txn.get(B('a'), db=db1) == B('d1')
assert txn.get(B('a'), db=maindb) is None
assert txn.replace(B('a'), B('d11')) == B('d1')
assert txn.pop(B('a')) == B('d11')
assert txn.put(B('a'), B('main'), db=maindb, overwrite=False)
assert not txn.delete(B('a'))
txn.abort()
def _export(self,
target_dir,
dataset_name='train',
map_size=1024 * 1024 * 1024 * 1024):
log.info('Converting %s data', dataset_name)
env = lmdb.open(os.path.join(target_dir, dataset_name),
map_size=map_size)
count = 0
txn = lmdb.Transaction(env, write=True)
for image in getattr(self.widerface, '{}_set'.format(dataset_name))():
key, datum = self._convert(image)
txn.put(key, datum.SerializeToString())
if count % 1000 == 0:
#Commit db
txn.commit()
txn = lmdb.Transaction(env, write=True)
log.info('Processed %s files.', count)
count += 1
#write the last batch
if count % 1000 != 0:
txn.commit()
log.info('Processed %s files.', count)
def __init__(self,
node_type: type(SerializableNode),
lmdb_dir: str,
set_cache_size: int = 48,
get_cache_size: int = 36):
super().__init__(node_type)
self.lmdb_dir = lmdb_dir
# writemap=True and map_async=True increase speed slightly
not_macos = platform != "darwin" # OS X doesn't support sparse files, so these just break things
self._env = lmdb.open(self.lmdb_dir,
map_size=1024**4,
writemap=not_macos,
map_async=not_macos)
self._txn = lmdb.Transaction(self._env,
write=True,
buffers=node_type.uses_buffers())
self._set_cache = FIFOCache(set_cache_size)
self._get_cache = FIFOCache(get_cache_size)
self.current_txn_size = 0
def close(self):
not_macos = platform != "darwin" # OS X doesn't support sparse files, so these just break things24 ** 3, writemap=not_macos,
self._env = lmdb.open(self.lmdb_dir,
map_size=1024**4,
writemap=not_macos,
map_async=not_macos)
for i, (path, node) in enumerate(self._store.items()):
if i % self.max_txn_size == 0:
print("Starting/Committing")
if i != 0:
self._txn.commit()
self._txn = lmdb.Transaction(
self._env,
write=True,
buffers=self._node_type.uses_buffers())
if i % 10000 == 0:
print(i)
self._txn.put(path.encode(), node.serialize())
self._txn.put(b'_root_path', path.split("/")[0].encode())
print("Finished dumping to DB")
self._txn.commit()
def for_reading(self) -> ReadingTransaction:
# TODO: Parent transactions
with lmdb.Transaction(self.environ) as tx:
yield ReadingTransaction(self, tx)
def test_begin_write(self):
_, env = testlib.temp_env()
txn = lmdb.Transaction(env, write=True)
# Write txn can write.
assert txn.put(B('a'), B(''))
txn.commit()
def test_closed(self):
_, env = testlib.temp_env()
env.close()
self.assertRaises(Exception, lambda: lmdb.Transaction(env))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.tx = lmdb.Transaction(env, write=False)
db_path = options.dbpath
if db_path == None:
parser.print_help()
sys.exit(1)
startTime = time.time()
# env == db coz max_dbs=0
env = lmdb.Environment(db_path,
map_size=24 * (1023**3),
subdir=False,
readonly=False,
create=False,
max_dbs=0,
lock=False)
txn = lmdb.Transaction(env, db=None, write=True)
linecount = 0
while 1:
try:
line = sys.stdin.readline()
except KeyboardInterrupt:
break
if not line: break
try:
line = line.strip()
except:
continue
if not line: continue
linecount += 1
if linecount % 1000 == 0:
def begin(self):
return lmdb.Transaction(self._ctx.env)
def __init__(self, env):
self.env = env
self._handle = lmdb.Transaction(self.env._handle, buffers=True)
def __enter__(self):
assert (self._txn is None)
self._txn = lmdb.Transaction(self._env, write=self._write)
self.attach()
return self
def test_parent_readonly(self):
_, env = testlib.temp_env()
parent = lmdb.Transaction(env)
# Nonsensical.
self.assertRaises(lmdb.InvalidParameterError,
lambda: lmdb.Transaction(env, parent=parent))
def for_writing(self) -> WritingTransaction:
# TODO: Parent transactions
with lmdb.Transaction(self.environ) as tx:
yield WritingTransaction(self, tx)
if options.verbose == 1: VERBOSE = 1
db_path = options.dbpath
if db_path == None:
parser.print_help()
sys.exit(1)
# env == db coz max_dbs=0
env = lmdb.Environment(db_path,
map_size=24 * (1023**3),
subdir=False,
readonly=True,
create=False,
max_dbs=0,
lock=False)
txn = lmdb.Transaction(env, db=None, write=False)
startTime = time.time()
linecount = 0
while 1:
try:
line = sys.stdin.readline()
except KeyboardInterrupt:
break
if not line: break
try:
line = line.strip()
except:
continue
if not line: continue
def __enter__(self):
self._txn = lmdb.Transaction(env, write=self._write)
return self