def delete_keylist_from_db(db_file, key_list):
"""
Open the db file, delete the keys in the key list
"""
db = lmdb_tools.open(db_file, append=True)
log.info('Deleting entries from \033[0;33m%s\033[0m, \
Db size: \033[0;32m%d\033[0m, list size: \033[0;32m%d\033[0m' %
(db_file, lmdb_tools.get_entries(db), len(key_list)))
lmdb_tools.delete_keylist(db, key_list)
log.info('Finish. DB size: \033[0;32m%d\033[0m' %
lmdb_tools.get_entries(db))
lmdb_tools.close(db)
def append_db(db_file_src, db_file_dst):
"""
Append all contains of db_file_src to db_file_dst
"""
db_src = lmdb_tools.open_ro(db_file_src)
log.info('Src Db \033[0;33m%s\033[0m size: \033[0;32m%d\033[0m' %
(db_file_src, lmdb_tools.get_entries(db_src)))
db_dst = lmdb_tools.open(db_file_dst, append=True)
log.info('Dst Db \033[0;33m%s\033[0m size: \033[0;32m%d\033[0m' %
(db_file_dst, lmdb_tools.get_entries(db_dst)))
with db_src.begin(write=False) as txn_src:
with txn_src.cursor() as cur:
for key, val in cur:
with db_dst.begin(write=True) as txn_dst:
txn_dst.put(key, val)
log.info('Append Finished. Dst Size: \033[0;32m%d\033[0m' %
lmdb_tools.get_entries(db_dst))
lmdb_tools.close(db_src)
lmdb_tools.close(db_dst)
def save_dict_to_db(db_file, key_dict, operation=None, append=False):
"""
Open a new db (if exist, ask whether to delete the old one)
save the dict to db and close.
NOTE:
1. If the key is not string, use str to convert it
2. If the val is not string, try str first, if not work, use pickle
"""
log.info('Saving dict to \033[0;33m%s\033[0m, the size of dict \
is \033[0;32m%d\033[0m' % (db_file, len(key_dict)))
db = lmdb_tools.open(db_file, append)
try:
lmdb_tools.write_dict(db, key_dict, operation)
except:
try:
lmdb_tools.write_dict_str(db, key_dict)
except:
lmdb_tools.write_dict_pickle(db, key_dict)
log.info('Finished, the Db size: \033[0;32m%d\033[0m' %
lmdb_tools.get_entries(db))
lmdb_tools.close(db)
def open_ro(db_file):
log.info('Opening lmdb file: \033[0;33m%s\033[0m for read only' % db_file)
db = lmdb_tools.open_ro(db_file)
log.info('Db size: \033[0;32m%d\033[0m' % lmdb_tools.get_entries(db))
return db
def get_entries(self):
# self.commit()
return lmdb_tools.get_entries(self.db)
def get_db_entries(self):
return lmdb_tools.get_entries(self.db)