def __init__(self, db, queue_size=1000, dumper=None):
"""
The db can be the db file name and also can be the lmdb db object
which is already been opened.
So, it can be a 'str' and also can be a db object
The queue_size is the size of the queue
dumper: If set, the content of the input will be converted to
string by the dumper function, e.g. pickle.dumps or yaml.dump or str
If not set, it will be assumed the content already been string type
"""
# Call the parents __init__ first
super(DbReadThread, self).__init__()
if type(db) == str:
self.db = lmdb_tools.open_ro(db)
self.need_close = True
else:
self.db = db
self.need_close = False
# Create the queue
self.inqueue = Queue.Queue(maxsize=queue_size)
self.outqueue = Queue.Queue(maxsize=queue_size)
# When set to True, the thread will keep runing
# When set to False, the thread will exit
self._thread_state = True
self.dumper = dumper
# Indicate whether the IO is balance
self.io_balance = 0
def load_keylist_from_db(db_file):
"""
Load the keys from a lmdb file, return the keys as a list
"""
log.info('Loading keys from \033[0;33m%s\033[0m' % db_file)
db = lmdb_tools.open_ro(db_file)
rst_list = lmdb_tools.get_keylist(db)
lmdb_tools.close(db)
log.info('Finish. loaded \033[0;32m%d\033[0m lines' % len(rst_list))
return rst_list
def load_dict_from_db(db_file, operation=None):
"""
Open a db and load the content into a dict. close the db after finish
The operation defines how to convert the value back to its original format
"""
log.info('Loading from \033[0;33m%s\033[0m' % db_file)
db = lmdb_tools.open_ro(db_file)
rst_dict = lmdb_tools.read_dict(db, operation)
lmdb_tools.close(db)
log.info('Finish. loaded \033[0;32m%d\033[0m lines' % len(rst_dict))
return rst_dict
def append_db(db_file_src, db_file_dst):
"""
Append all contains of db_file_src to db_file_dst
"""
db_src = lmdb_tools.open_ro(db_file_src)
log.info('Src Db \033[0;33m%s\033[0m size: \033[0;32m%d\033[0m' %
(db_file_src, lmdb_tools.get_entries(db_src)))
db_dst = lmdb_tools.open(db_file_dst, append=True)
log.info('Dst Db \033[0;33m%s\033[0m size: \033[0;32m%d\033[0m' %
(db_file_dst, lmdb_tools.get_entries(db_dst)))
with db_src.begin(write=False) as txn_src:
with txn_src.cursor() as cur:
for key, val in cur:
with db_dst.begin(write=True) as txn_dst:
txn_dst.put(key, val)
log.info('Append Finished. Dst Size: \033[0;32m%d\033[0m' %
lmdb_tools.get_entries(db_dst))
lmdb_tools.close(db_src)
lmdb_tools.close(db_dst)
def __init__(self, db_file, readonly=True, echo=True, append=False):
self.readonly = readonly
self.db_file = db_file
self._echo = echo
self._warn = True
# If max commit interval time is 60 sec, it the commit interval
# is more than it, the buffer will be commited.
self._max_commit_interval = 60
# The timer
self._timer = timer_lib.timer(start=True)
# Define the dumper of the key and value
self._key_dumper = None
self._val_dumper = None
self._key_parser = None
self._val_parser = None
# The size of the commit buffer
self._buf_size = 100
self._buf_counter = 0
self._cur = None
self._iter = None
if readonly:
self.db = lmdb_tools.open_ro(db_file)
else:
self.db = lmdb_tools.open(db_file, append=append)
if self.db is None:
log.error('\033[01;31mERROR\033[0m: Can not open the \
db file \033[32m%s\033[0m' % db_file)
return
self._txn = self.db.begin(write=not self.readonly)
if echo:
log.info('Open %s, size: \033[01;31m%d\033[0m' %
(db_file, self.get_entries()))
def open_ro(db_file):
log.info('Opening lmdb file: \033[0;33m%s\033[0m for read only' % db_file)
db = lmdb_tools.open_ro(db_file)
log.info('Db size: \033[0;32m%d\033[0m' % lmdb_tools.get_entries(db))
return db