def test_key_value_store_verify_compatibility_with_existent_store(self):
leveldb_store = KeyValueStore.new(
"file://./existent_db",
store_type=KeyValueStore.STORE_TYPE_LEVELDB,
create_if_missing=False)
plyvel_store = KeyValueStore.new(
"file://./existent_db_copy",
store_type=KeyValueStore.STORE_TYPE_PLYVEL,
create_if_missing=False)
leveldb_count = 0
for key, value in leveldb_store.Iterator():
plyvel_value = plyvel_store.get(bytes(key))
self.assertEqual(value, plyvel_value)
leveldb_count += 1
if (leveldb_count % 1000000) == 0:
utils.logger.spam(f"leveldb count={leveldb_count}")
plyvel_count = 0
for key, value in plyvel_store.Iterator():
leveldb_value = leveldb_store.get(key)
self.assertEqual(value, leveldb_value)
plyvel_count += 1
if (plyvel_count % 1000000) == 0:
utils.logger.spam(f"plyvel count={plyvel_count}")
self.assertEqual(leveldb_count, plyvel_count)
utils.logger.debug(
f"count leveldb={leveldb_count}, plyvel={plyvel_count}")
def _new_store(self, uri, store_type=None, create_if_missing=True):
try:
if store_type == KeyValueStore.STORE_TYPE_DICT:
from loopchain.store.key_value_store_dict import KeyValueStoreDict
utils.logger.info(f"New KeyValueStore. store_type={store_type}, uri={uri}")
return KeyValueStoreDict()
return KeyValueStore.new(uri, store_type=store_type, create_if_missing=create_if_missing)
except KeyValueStoreError as e:
utils.logger.spam(f"Doesn't need to clean the store. uri={uri}, e={e}")
return KeyValueStore.new(uri, store_type=store_type, create_if_missing=True)
def init_default_key_value_store(store_identity) -> Tuple[KeyValueStore, str]:
"""init default key value store
:param store_identity: identity for store
:return: KeyValueStore, store_path
"""
if not os.path.exists(conf.DEFAULT_STORAGE_PATH):
os.makedirs(conf.DEFAULT_STORAGE_PATH, exist_ok=True)
store_path = os.path.join(conf.DEFAULT_STORAGE_PATH, 'db_' + store_identity)
logger.spam(f"utils:init_default_key_value_store ({store_identity})")
retry_count = 0
store = None
while store is None and retry_count < conf.MAX_RETRY_CREATE_DB:
try:
uri = f"file://{store_path}"
store = KeyValueStore.new(uri, create_if_missing=True)
except KeyValueStoreError as e:
logging.error(f"KeyValueStoreError: {e}")
logger.debug(f"retry_count: {retry_count}, uri: {uri}")
traceback.print_exc()
retry_count += 1
if store is None:
logging.error("Fail! Create key value store")
raise KeyValueStoreError(f"Fail to create key value store. path={store_path}")
return store, store_path
def test_key_value_store_verify_compatibility(self):
test_items = self._get_test_items(5)
leveldb_store = self._new_store(
"file://./key_value_store_test_verify_leveldb_and_plyvel",
store_type=KeyValueStore.STORE_TYPE_LEVELDB,
create_if_missing=True)
for key, value in test_items.items():
leveldb_store.put(key, value)
plyvel_store = KeyValueStore.new(
"file://./key_value_store_test_verify_leveldb_and_plyvel",
store_type=KeyValueStore.STORE_TYPE_PLYVEL,
create_if_missing=False)
leveldb_bin = bytes()
for key, value in leveldb_store.Iterator():
utils.logger.spam(f"leveldb iterator: key={key}, value={value}")
self.assertEqual(value, plyvel_store.get(bytes(key)))
leveldb_bin += key + value
plyvel_bin = bytes()
for key, value in plyvel_store.Iterator():
plyvel_bin += key + value
utils.logger.debug(f"leveldb binary: {leveldb_bin}")
utils.logger.debug(f"plyvel binary: {plyvel_bin}")
self.assertEqual(leveldb_bin, plyvel_bin)
plyvel_store.close()
leveldb_store.destroy_store()
def init_default_key_value_store(store_id: str) -> KeyValueStore:
"""init default key value store
:param store_id: new identity of key-value store
:return: KeyValueStore, store_path
"""
if not os.path.exists(conf.DEFAULT_STORAGE_PATH):
os.makedirs(conf.DEFAULT_STORAGE_PATH, exist_ok=True)
store_path = os.path.join(conf.DEFAULT_STORAGE_PATH, f'db_{store_id}')
logger.info(f"store_id={store_id}")
retry_count = 0
store = None
uri = f"file://{store_path}"
while store is None and retry_count < conf.MAX_RETRY_CREATE_DB:
try:
store = KeyValueStore.new(uri, create_if_missing=True)
except KeyValueStoreError as e:
logging.exception(f"KeyValueStore create failed: {e!r}")
logger.debug(f"retry_count: {retry_count}, uri: {uri}")
retry_count += 1
if store is None:
logging.error("Fail! Create key value store")
raise KeyValueStoreError(
f"Fail to create key value store. path={store_path}")
return store
def make_key_value_store(store_identity="") -> Optional[KeyValueStore]:
store_default_path = './' + (store_identity,
"db_test")[store_identity == ""]
store_path = store_default_path
store = None
retry_count = 0
while store is None and retry_count < conf.MAX_RETRY_CREATE_DB:
try:
uri = f"file://{store_path}"
store = KeyValueStore.new(uri, create_if_missing=True)
logging.debug(f"make key value store uri: {uri}")
except KeyValueStoreError:
store_path = store_default_path + str(retry_count)
retry_count += 1
return store
def init_default_key_value_store(old_store_id: str,
store_id: str) -> KeyValueStore:
"""init default key value store
:param old_store_id: old identity of key-value store
:param store_id: new identity of key-value store
:return: KeyValueStore, store_path
"""
if not os.path.exists(conf.DEFAULT_STORAGE_PATH):
os.makedirs(conf.DEFAULT_STORAGE_PATH, exist_ok=True)
db_dirname = f'db_{store_id}'
# FIXME : remove rename_db_dir() after all applied, maybe next release
rename_db_dir(f"db_{old_store_id}", db_dirname)
store_path = os.path.join(conf.DEFAULT_STORAGE_PATH, db_dirname)
logger.info(f"init_default_key_value_store() store_id={store_id}")
retry_count = 0
store = None
uri = f"file://{store_path}"
while store is None and retry_count < conf.MAX_RETRY_CREATE_DB:
try:
store = KeyValueStore.new(uri, create_if_missing=True)
except KeyValueStoreError as e:
logging.error(f"KeyValueStoreError: {e}")
logger.debug(f"retry_count: {retry_count}, uri: {uri}")
traceback.print_exc()
retry_count += 1
if store is None:
logging.error("Fail! Create key value store")
raise KeyValueStoreError(
f"Fail to create key value store. path={store_path}")
return store