def __init__(self, host, port, id_litteral=DEFAULT_ID_LITTERAL,
count_key=DEFAULT_COUNT_KEY):
if not _validate_host(host):
raise DatabaseError("Invalid hostname")
if not _validate_port(port):
raise DatabaseError("Invalid port")
if not _validate_string_litteral(id_litteral):
raise DatabaseError("Invalid id_litteral")
if not _validate_string_litteral(count_key):
raise DatabaseError("Invalid count_key")
self._cli = Client(host, port)
self._id_litteral = id_litteral
self._count_key = count_key
self.select(DEFAULT_COLLECTION)
def __init__(self, ttl=None, batchsize=1000, **kwargs):
self.ttl = ttl
self.batchsize = batchsize
self.client = Client(**kwargs)
class SSDBNodeStorage(NodeStorage):
"""
A SSDB-based backend for storing node data.
>>> SSDBNodeStorage(
... host='localhost',
... port=8888,
... socket_timeout=2,
... ttl=60*60*24*30,
... batchsize=10000,
... )
"""
def __init__(self, ttl=None, batchsize=1000, **kwargs):
self.ttl = ttl
self.batchsize = batchsize
self.client = Client(**kwargs)
def delete(self, id):
"""
>>> nodestore.delete('key1')
"""
self.client.delete(uid(id))
def delete_multi(self, id_list):
"""
Delete multiple nodes.
>>> delete_multi(['key1', 'key2'])
"""
self.client.multi_del(*uids(id_list))
def get(self, id):
"""
>>> data = nodestore.get('key1')
>>> print data
"""
return loads(self.client.get(uid(id)))
def get_multi(self, id_list):
"""
>>> data_map = nodestore.get_multi(['key1', 'key2')
>>> print 'key1', data_map['key1']
>>> print 'key2', data_map['key2']
"""
# with self.client.pipeline() as pipe:
# for id in id_list: pipe.get(uid(id))
# values = pipe.execute()
# return dict(zip(id_list, [loads(v) for v in values]))
kv_list = self.client.multi_get(*uids(id_list))
return defaultdict(lambda: None, map(lambda k, v: (b64id(k), loads(v)), *[iter(kv_list)]*2))
def set(self, id, data):
"""
>>> nodestore.set('key1', {'foo': 'bar'})
"""
if self.ttl:
self.client.setx(uid(id), dumps(data), self.ttl)
else:
self.client.set(uid(id), dumps(data))
def set_multi(self, values):
"""
>>> nodestore.set_multi({
>>> 'key1': {'foo': 'bar'},
>>> 'key2': {'foo': 'baz'},
>>> })
"""
# kv_list = [item for sublist in iteritems(values) for item in sublist]
if self.ttl:
# ssdb has no multi_setx, use piped setx instead
with self.client.pipeline() as pipe:
for id, data in iteritems(values):
pipe.setx(uid(id), dumps(data), self.ttl)
pipe.execute()
# TODO Benchmark against multi:set + expire loop
# for id, data in iteritems(values):
# self.client.setx(uid(id), dumps(data), self.ttl)
else:
kv_list = reduce(lambda l, (k, v): add(l, (uid(k), dumps(v))), iteritems(values), ())
self.client.multi_set(*kv_list)
def generate_id(self):
# ids are 32 byte base64, but stored as 24 byte binary
return b64encode(tsbin(time()) + uuid4().bytes)
def cleanup(self, cutoff_timestamp):
# this is the smallest possible timestamp (1970-01-01)
start = tsbin(0)
# mktime truncates milliseconds, add them back manually
cutoff = tsbin(mktime(cutoff_timestamp.timetuple()) + cutoff_timestamp.microsecond/1e6)
while True:
keys = self.client.keys(start, cutoff, self.batchsize)
count = len(keys)
# print "Cleanup %016d -> %016d %d / %d" % (bints(start), bints(cutoff), count, self.batchsize)
if count == 0: break
self.client.multi_del(*keys)
if count < self.batchsize: break
start = keys[-1]
class SSDBClient(object):
def __init__(self, host, port, id_litteral=DEFAULT_ID_LITTERAL,
count_key=DEFAULT_COUNT_KEY):
if not _validate_host(host):
raise DatabaseError("Invalid hostname")
if not _validate_port(port):
raise DatabaseError("Invalid port")
if not _validate_string_litteral(id_litteral):
raise DatabaseError("Invalid id_litteral")
if not _validate_string_litteral(count_key):
raise DatabaseError("Invalid count_key")
self._cli = Client(host, port)
self._id_litteral = id_litteral
self._count_key = count_key
self.select(DEFAULT_COLLECTION)
def _get_coll_count(self):
return '{}|count'.format(self.coll)
def _get_coll_types(self):
return '{}|types'.format(self.coll)
def _get_current_id(self, unchange=False):
if not self._cli.hexists(self._coll_count, self._count_key):
if unchange:
return None
self._cli.hset(self._coll_count,
self._count_key,
DEFAULT_ID_START)
return DEFAULT_ID_START
if unchange:
return int(self._cli.hget(self._coll_count, self._count_key))
return int(self._cli.hincr(self._coll_count, self._count_key))
def _get_obj_from_scan(self, scan, this_id):
obj = {}
for j in xrange(len(scan)/2):
this_key = scan[2*j]
this_type = _get_type(self._cli.hget(self._coll_types, this_key))
length_unkey = len('{}|{}|'.format(self._id_litteral, this_id))
this_key = this_key[length_unkey:]
this_value = this_type(scan[2*j+1])
obj[this_key] = this_value
return obj
def _check_id(self, id):
objs = self.find_all({}, {}, {}, [])
id_list = [obj[self._id_litteral] for obj in objs]
return id in id_list
def _save_document(self, obj, id):
for key, value in obj.iteritems():
if isinstance(value, unicode):
value = value.encode('utf-8')
key_db = '{}|{}|{}'.format(self._id_litteral,
id,
key)
self._cli.hset(self.coll, key_db, str(value))
self._cli.hset(self._coll_types, key_db, str(type(value)))
return id
def select(self, coll):
if not _validate_string_litteral(coll):
raise DatabaseError("Invalid collection name")
self.coll = coll
self._coll_count = self._get_coll_count()
self._coll_types = self._get_coll_types()
return self.coll
def collection(self):
return self.coll
def insert(self, obj):
if not _validate_insert(obj, id_litteral=self._id_litteral):
raise DocumentError("Document must be a dictionary and Document must have a specific structure and Document can not contain the '{}' field".format(self._id_litteral))
current_id = self._get_current_id()
obj[self._id_litteral] = int(current_id)
return self._save_document(obj, obj[self._id_litteral])
def find_all_count(self, query, projection, search, sort, page_size=0, page_num=0):
if not _validate_query(query):
raise QueryError("Query must be a dictionary and Query must have a specific structure")
if not _validate_projection(projection):
raise ProjectionError("Projection must be a dictionary and Projection must have a specific structure")
if not _validate_search(search):
raise SearchError("Search query must be a dictionary and Search query must have a specific structure")
if not _validate_sort(sort):
raise SortError("Sort query must be a dictionary and Sort query must have a specific structure")
if not _validate_page_value(page_size):
raise PageSizeError("Page Size must be a integer greater then or equal 0")
if not _validate_page_value(page_num):
raise PageNumError("Page Num must be a integer greater then or equal 0")
search = _search_processing(search)
for field in search.iterkeys():
if field in query:
query[field].update(search[field])
else:
query[field] = search[field]
objs = []
last_id = self._get_current_id(unchange=True)
if not last_id:
return objs, 0
for i in xrange(last_id):
this_id = i + 1
key_start = '{}|{}'.format(self._id_litteral, this_id)
key_end = '{}|{}||'.format(self._id_litteral, this_id)
scan = self._cli.hscan(self.coll, key_start, key_end, -1)
obj = self._get_obj_from_scan(scan, this_id)
if obj and _is_match_query(obj, query):
objs.append(_filter_field(obj, projection, id_litteral=self._id_litteral))
if sort:
objs = _sorted_processing(objs, sort)
count = len(objs)
if page_size != 0:
el_start = page_size*page_num
el_end = el_start + page_size
objs = objs[el_start:el_end]
return objs, count
def find_all(self, query, projection, search, sort, page_size=0, page_num=0):
return self.find_all_count(query, projection, search, sort, page_size, page_num)[0]
def find_one(self, query, projection):
objs = self.find_all(query, projection, {}, [])
if query and len(objs) == 1:
obj = objs[0]
else:
obj = None
return obj
def update(self, obj):
if not _validate_update(obj, id_litteral=self._id_litteral):
raise DocumentError("Document must be a dictionary and Document must have a specific structure and Document must have the '{}' field".format(self._id_litteral))
if not self._check_id(obj[self._id_litteral]):
raise DocumentError("'{}' field of Document is undefined in current collection".format(self._id_litteral))
return self._save_document(obj, obj[self._id_litteral])
def remove(self, query):
objs = self.find_all(query, {}, {}, [])
for obj in objs:
id = obj[self._id_litteral]
for key in obj.iterkeys():
key_db = '{}|{}|{}'.format(self._id_litteral,
id,
key)
self._cli.hdel(self.coll, key_db)
self._cli.hdel(self._coll_types, key_db)
return len(objs)
def count(self):
return self.find_all_count({}, {}, {}, [])[1]
def clear(self):
count = self.count()
self._cli.hclear(self.coll)
self._cli.hclear(self._coll_types)
return count