class KyotoIndex(BinaryIndex):
''' Kyoto Cabinet index.
Notably this uses a B+ tree for the index and thus one can
traverse from one key forwards and backwards, which supports
the coming Store synchronisation processes.
'''
NAME = 'kyoto'
SUFFIX = 'kct'
def __init__(self, nmdbpathbase):
super().__init__(nmdbpathbase)
self._kyoto = None
@classmethod
def is_supported(cls):
''' Test whether this index class is supported by the Python environment.
'''
# pylint: disable=import-error,unused-import,import-outside-toplevel
try:
import kyotocabinet
except ImportError:
return False
return True
def startup(self):
''' Open the index.
'''
# pylint: disable=import-error,import-outside-toplevel
from kyotocabinet import DB
self._kyoto = DB()
self._kyoto.open(self.path, DB.OWRITER | DB.OCREATE)
def shutdown(self):
''' Close the index.
'''
self._kyoto.close()
self._kyoto = None
def flush(self):
''' Flush pending updates to the index.
'''
try:
self._kyoto.synchronize(hard=False)
except TypeError:
self._kyoto.synchronize()
def __len__(self):
return self._kyoto.count()
def __contains__(self, key):
return self._kyoto.check(key) >= 0
def __getitem__(self, key):
binary_entry = self._kyoto.get(key)
if binary_entry is None:
raise KeyError(key)
return binary_entry
def __setitem__(self, key, binary_entry):
self._kyoto[key] = binary_entry
def keys(self, *, start_hashcode=None):
''' Generator yielding the keys from the index
in order starting with optional `start_hashcode`.
Parameters:
* `start_hashcode`: the starting key; if missing or `None`,
iteration starts with the first key in the index
'''
cursor = self._kyoto.cursor()
if start_hashcode is not None:
cursor.jump(start_hashcode)
yield cursor.get_key()
while cursor.step():
yield cursor.get_key()
cursor.disable()
sorted_keys = keys
__iter__ = keys
class KyotoCabinetGraph(BaseGraph):
def __init__(self, path):
# create the database object
self._path = path
self._db = DB()
# open the database
if not self._db.open(path, DB.OREADER | DB.OWRITER | DB.OCREATE):
raise GrapheekDataKyotoCabinetInitFailureException(
str(self._db.error()))
super(KyotoCabinetGraph, self).__init__()
self._ensure_prepared()
self._closed = False
# Start method overriding :
def _db_close(self):
if not self._closed:
self._db.close()
def _transaction_begin(self):
self._db.begin_transaction()
return True
def _transaction_commit(self, txn):
self._db.end_transaction(True)
def _transaction_rollback(self, txn):
self._db.end_transaction(False)
def _has_key(self, key):
return self._db.check(key) >= 0
def _get(self, txn, key):
raw_data = self._db.get(key)
if raw_data is None:
return UNDEFINED # Not returning None, as None is a valid value
return msgpack.loads(raw_data, encoding='utf8')
def _bulk_get(self, txn, keys):
result = {}
key_raw_datas = self._db.get_bulk(keys)
for key, raw_data in list(key_raw_datas.items()):
if PYTHON2: # pragma : no cover
k = key
else: # pragma : no cover
k = str(key, encoding='utf8')
result[k] = msgpack.loads(raw_data, encoding='utf8')
return result
def _set(self, txn, key, value):
res = self._db.set(key, msgpack.dumps(value, encoding='utf8'))
if not (res): # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
def _bulk_set(self, txn, updates):
dic = {}
for key, value in list(updates.items()):
dic[key] = msgpack.dumps(value, encoding='utf8')
res = self._db.set_bulk(dic)
if res == -1: # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
def _remove(self, txn, key):
# Contrary to LocalMemoryGraph implementation, it is not needed to wrap
# key removal in try.. except because KyotoCabinet only send "False"
# when key does not exist
# Thus ... _removemethod is idempotent (cf LocalMemoryGraph _remove method comment)
self._db.remove(key)
def _bulk_remove(self, txn, keys):
res = self._db.remove_bulk(list(keys))
if res == -1: # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
def _remove_prefix(self, txn, prefix):
keys = self._db.match_prefix(prefix)
self._db.remove_bulk(keys)
# overriding list methods
# looks like a bucket of hacks, and yes indeed it is :)
# btw, it REALLY improves performance if we compare to default implementation which,
# in the case of KyotoCabinet would involve msgpack deserialization followed by a serialization
def _init_lst(self, txn, key):
res = self._db.set(key, '')
if not (res): # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
def _get_lst(self, txn, key):
value = self._db.get(key)
if value is None:
return UNDEFINED
# look _append_to_lst code below to understand why a split is done
# And why resulting list is sliced from 1
if PYTHON2: # pragma : no cover
return list(map(int, value.split('|')[1:]))
return list(map(
int,
str(value, encoding='utf8').split('|')[1:])) # pragma : no cover
def _set_lst(self, txn, key, values):
newval = '|'.join([str(value) for value in values])
res = self._db.set(key, '|' + newval)
if not (res): # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
def _bulk_get_lst(self, txn, keys):
dic_values = self._db.get_bulk(keys)
results = []
for key in keys:
if PYTHON2: # pragma : no cover
values = dic_values.get(key, UNDEFINED)
else: # pragma : no cover
values = dic_values.get(bytes(key, encoding='utf8'), UNDEFINED)
if values == UNDEFINED:
results.append([])
else:
if PYTHON2: # pragma : no cover
results.append(list(map(int, values.split('|')[1:])))
else: # pragma : no cover
results.append(
list(
map(int,
str(values, encoding='utf8').split('|')[1:])))
return results
def _append_to_lst(self, txn, key, value):
self._db.append(key, '|' + str(value))
def _bulk_append_to_lst(self, txn, key, values):
newval = '|'.join([str(value) for value in values])
self._db.append(key, '|' + newval)
def _remove_from_lst(self, txn, key, value):
old = self._db.get(key)
if not PYTHON2: # pragma : no cover
old = str(old, encoding='utf8')
# Caution : we are only removing ONE occurence
# This is voluntary
# For instance, it lst contains neighbour node, we need to remove only one occurence
# cause current entity and neighbour node can be linked multiple time
new = old.replace('|%s' % value, '', 1)
if new == old:
raise ValueError("list.remove(x): x not in list")
res = self._db.set(key, new)
if not (res): # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
def _bulk_remove_from_lst(self, txn, key, values):
assert (len(values))
old = self._db.get(key)
if PYTHON2: # pragma : no cover
new = old
else: # pragma : no cover
new = str(old, encoding='utf8')
for value in values:
new = new.replace('|%s' % value, '', 1)
if new == old: # pragma : no cover
raise ValueError("list.remove(x): x not in list")
res = self._db.set(key, new)
if not (res): # pragma : no cover
raise GrapheekDataKyotoCabinetException(
'KyotoCabinet : error while saving')
return res
