Beispiel #1
0
 def __init__(self, host, port, id_litteral=DEFAULT_ID_LITTERAL,
              count_key=DEFAULT_COUNT_KEY):
     if not _validate_host(host):
         raise DatabaseError("Invalid hostname")
     if not _validate_port(port):
         raise DatabaseError("Invalid port")
     if not _validate_string_litteral(id_litteral):
         raise DatabaseError("Invalid id_litteral")
     if not _validate_string_litteral(count_key):
         raise DatabaseError("Invalid count_key")
     self._cli = Client(host, port)
     self._id_litteral = id_litteral
     self._count_key = count_key
     self.select(DEFAULT_COLLECTION)
 def __init__(self, ttl=None, batchsize=1000, **kwargs):
     self.ttl = ttl
     self.batchsize = batchsize
     self.client = Client(**kwargs)
class SSDBNodeStorage(NodeStorage):
    """
    A SSDB-based backend for storing node data.
    >>> SSDBNodeStorage(
    ...     host='localhost',
    ...     port=8888,
    ...     socket_timeout=2,
    ...     ttl=60*60*24*30,
    ...     batchsize=10000,
    ... )
    """

    def __init__(self, ttl=None, batchsize=1000, **kwargs):
        self.ttl = ttl
        self.batchsize = batchsize
        self.client = Client(**kwargs)

    def delete(self, id):
        """
        >>> nodestore.delete('key1')
        """
        self.client.delete(uid(id))

    def delete_multi(self, id_list):
        """
        Delete multiple nodes.

        >>> delete_multi(['key1', 'key2'])
        """
        self.client.multi_del(*uids(id_list))

    def get(self, id):
        """
        >>> data = nodestore.get('key1')
        >>> print data
        """
        return loads(self.client.get(uid(id)))

    def get_multi(self, id_list):
        """
        >>> data_map = nodestore.get_multi(['key1', 'key2')
        >>> print 'key1', data_map['key1']
        >>> print 'key2', data_map['key2']
        """
        # with self.client.pipeline() as pipe:
        #     for id in id_list: pipe.get(uid(id))
        #     values = pipe.execute()
        # return dict(zip(id_list, [loads(v) for v in values]))
        kv_list = self.client.multi_get(*uids(id_list))
        return defaultdict(lambda: None, map(lambda k, v: (b64id(k), loads(v)), *[iter(kv_list)]*2))

    def set(self, id, data):
        """
        >>> nodestore.set('key1', {'foo': 'bar'})
        """
        if self.ttl:
            self.client.setx(uid(id), dumps(data), self.ttl)
        else:
            self.client.set(uid(id), dumps(data))

    def set_multi(self, values):
        """
        >>> nodestore.set_multi({
        >>>     'key1': {'foo': 'bar'},
        >>>     'key2': {'foo': 'baz'},
        >>> })
        """
        # kv_list = [item for sublist in iteritems(values) for item in sublist]
        if self.ttl:
            # ssdb has no multi_setx, use piped setx instead
            with self.client.pipeline() as pipe:
                for id, data in iteritems(values):
                    pipe.setx(uid(id), dumps(data), self.ttl)
                pipe.execute()

            # TODO Benchmark against multi:set + expire loop
            # for id, data in iteritems(values):
            #     self.client.setx(uid(id), dumps(data), self.ttl)
        else:
            kv_list = reduce(lambda l, (k, v): add(l, (uid(k), dumps(v))), iteritems(values), ())
            self.client.multi_set(*kv_list)

    def generate_id(self):
        # ids are 32 byte base64, but stored as 24 byte binary
        return b64encode(tsbin(time()) + uuid4().bytes)

    def cleanup(self, cutoff_timestamp):
        # this is the smallest possible timestamp (1970-01-01)
        start  = tsbin(0)
        # mktime truncates milliseconds, add them back manually
        cutoff = tsbin(mktime(cutoff_timestamp.timetuple()) + cutoff_timestamp.microsecond/1e6)

        while True:
            keys = self.client.keys(start, cutoff, self.batchsize)
            count = len(keys)
            # print "Cleanup %016d -> %016d %d / %d" % (bints(start), bints(cutoff), count, self.batchsize)
            if count == 0: break
            self.client.multi_del(*keys)
            if count < self.batchsize: break
            start = keys[-1]
Beispiel #4
0
class SSDBClient(object):

    def __init__(self, host, port, id_litteral=DEFAULT_ID_LITTERAL,
                 count_key=DEFAULT_COUNT_KEY):
        if not _validate_host(host):
            raise DatabaseError("Invalid hostname")
        if not _validate_port(port):
            raise DatabaseError("Invalid port")
        if not _validate_string_litteral(id_litteral):
            raise DatabaseError("Invalid id_litteral")
        if not _validate_string_litteral(count_key):
            raise DatabaseError("Invalid count_key")
        self._cli = Client(host, port)
        self._id_litteral = id_litteral
        self._count_key = count_key
        self.select(DEFAULT_COLLECTION)

    def _get_coll_count(self):
        return '{}|count'.format(self.coll)

    def _get_coll_types(self):
        return '{}|types'.format(self.coll)

    def _get_current_id(self, unchange=False):
        if not self._cli.hexists(self._coll_count, self._count_key):
            if unchange:
                return None
            self._cli.hset(self._coll_count,
                           self._count_key,
                           DEFAULT_ID_START)
            return DEFAULT_ID_START
        if unchange:
            return int(self._cli.hget(self._coll_count, self._count_key))
        return int(self._cli.hincr(self._coll_count, self._count_key))

    def _get_obj_from_scan(self, scan, this_id):
        obj = {}
        for j in xrange(len(scan)/2):
            this_key = scan[2*j]
            this_type = _get_type(self._cli.hget(self._coll_types, this_key))
            length_unkey = len('{}|{}|'.format(self._id_litteral, this_id))
            this_key = this_key[length_unkey:]
            this_value = this_type(scan[2*j+1])
            obj[this_key] = this_value
        return obj

    def _check_id(self, id):
        objs = self.find_all({}, {}, {}, [])
        id_list = [obj[self._id_litteral] for obj in objs]
        return id in id_list

    def _save_document(self, obj, id):
        for key, value in obj.iteritems():
            if isinstance(value, unicode):
                value = value.encode('utf-8')
            key_db = '{}|{}|{}'.format(self._id_litteral,
                                       id,
                                       key)
            self._cli.hset(self.coll, key_db, str(value))
            self._cli.hset(self._coll_types, key_db, str(type(value)))
        return id

    def select(self, coll):
        if not _validate_string_litteral(coll):
            raise DatabaseError("Invalid collection name")
        self.coll = coll
        self._coll_count = self._get_coll_count()
        self._coll_types = self._get_coll_types()
        return self.coll

    def collection(self):
        return self.coll

    def insert(self, obj):
        if not _validate_insert(obj, id_litteral=self._id_litteral):
            raise DocumentError("Document must be a dictionary and Document must have a specific structure and Document can not contain the '{}' field".format(self._id_litteral))
        current_id = self._get_current_id()
        obj[self._id_litteral] = int(current_id)
        return self._save_document(obj, obj[self._id_litteral])

    def find_all_count(self, query, projection, search, sort, page_size=0, page_num=0):
        if not _validate_query(query):
            raise QueryError("Query must be a dictionary and Query must have a specific structure")
        if not _validate_projection(projection):
            raise ProjectionError("Projection must be a dictionary and Projection must have a specific structure")
        if not _validate_search(search):
            raise SearchError("Search query must be a dictionary and Search query must have a specific structure")
        if not _validate_sort(sort):
            raise SortError("Sort query must be a dictionary and Sort query must have a specific structure")
        if not _validate_page_value(page_size):
            raise PageSizeError("Page Size must be a integer greater then or equal 0")
        if not _validate_page_value(page_num):
            raise PageNumError("Page Num must be a integer greater then or equal 0")
        search = _search_processing(search)
        for field in search.iterkeys():
            if field in query:
                query[field].update(search[field])
            else:
                query[field] = search[field]
        objs = []
        last_id = self._get_current_id(unchange=True)
        if not last_id:
            return objs, 0
        for i in xrange(last_id):
            this_id = i + 1
            key_start = '{}|{}'.format(self._id_litteral, this_id)
            key_end = '{}|{}||'.format(self._id_litteral, this_id)
            scan = self._cli.hscan(self.coll, key_start, key_end, -1)
            obj = self._get_obj_from_scan(scan, this_id)
            if obj and _is_match_query(obj, query):
                objs.append(_filter_field(obj, projection, id_litteral=self._id_litteral))
        if sort:
            objs = _sorted_processing(objs, sort)
        count = len(objs)
        if page_size != 0:
            el_start = page_size*page_num
            el_end = el_start + page_size
            objs = objs[el_start:el_end]
        return objs, count

    def find_all(self, query, projection, search, sort, page_size=0, page_num=0):
        return self.find_all_count(query, projection, search, sort, page_size, page_num)[0]

    def find_one(self, query, projection):
        objs = self.find_all(query, projection, {}, [])
        if query and len(objs) == 1:
            obj = objs[0]
        else:
            obj = None
        return obj

    def update(self, obj):
        if not _validate_update(obj, id_litteral=self._id_litteral):
            raise DocumentError("Document must be a dictionary and Document must have a specific structure and Document must have the '{}' field".format(self._id_litteral))
        if not self._check_id(obj[self._id_litteral]):
            raise DocumentError("'{}' field of Document is undefined in current collection".format(self._id_litteral))
        return self._save_document(obj, obj[self._id_litteral])

    def remove(self, query):
        objs = self.find_all(query, {}, {}, [])
        for obj in objs:
            id = obj[self._id_litteral]
            for key in obj.iterkeys():
                key_db = '{}|{}|{}'.format(self._id_litteral,
                                           id,
                                           key)
                self._cli.hdel(self.coll, key_db)
                self._cli.hdel(self._coll_types, key_db)
        return len(objs)

    def count(self):
        return self.find_all_count({}, {}, {}, [])[1]

    def clear(self):
        count = self.count()
        self._cli.hclear(self.coll)
        self._cli.hclear(self._coll_types)
        return count