示例#1
0
    def __init__(self, url:str, database:str, max_buffer_size:int = 1000,
                                              buffer_latency:timedelta = timedelta(milliseconds=50),
                                              **kwargs):
        """
        Acquire a connection with the CouchDB server and initialise the
        buffering queue

        @param   url              CouchDB server URL
        @param   database         Database name
        @param   max_buffer_size  Maximum buffer size (no. of documents)
        @param   buffer_latency   Buffer latency before discharge
        @kwargs  Additional constructor parameters to
                 pycouchdb.client.Server should be passed through here
        """
        self._db = SofterCouchDB(url, database, **kwargs)

        self._designs = []
        self._designs_dirty = False

        # Batch action to DB method mapping
        self._batch_methods = {
            Actions.Upsert: self._db.save_bulk,
            Actions.Delete: self._db.delete_bulk
        }

        # Setup database action buffer and queue
        self._buffer = Buffer(max_buffer_size, buffer_latency)
        self._buffer.add_listener(self._batch)
示例#2
0
class Sofabed(object):
    """ Buffered, append-optimised CouchDB interface """
    def __init__(self, url:str, database:str, max_buffer_size:int = 1000,
                                              buffer_latency:timedelta = timedelta(milliseconds=50),
                                              **kwargs):
        """
        Acquire a connection with the CouchDB server and initialise the
        buffering queue

        @param   url              CouchDB server URL
        @param   database         Database name
        @param   max_buffer_size  Maximum buffer size (no. of documents)
        @param   buffer_latency   Buffer latency before discharge
        @kwargs  Additional constructor parameters to
                 pycouchdb.client.Server should be passed through here
        """
        self._db = SofterCouchDB(url, database, **kwargs)

        self._designs = []
        self._designs_dirty = False

        # Batch action to DB method mapping
        self._batch_methods = {
            Actions.Upsert: self._db.save_bulk,
            Actions.Delete: self._db.delete_bulk
        }

        # Setup database action buffer and queue
        self._buffer = Buffer(max_buffer_size, buffer_latency)
        self._buffer.add_listener(self._batch)

    def _batch(self, broadcast:BatchListenerT):
        """
        Perform a batch action against the database

        @param   broadcast  Broadcast data pushed by the buffer
        """
        action, docs = broadcast
        to_batch = deepcopy(docs)

        # To avoid conflicts, we must merge in the revision IDs of
        # existing documents
        document_ids = [doc['_id'] for doc in to_batch]
        revision_ids = {
            query_row['id']: query_row['value']['rev']
            for query_row in self._db.all(keys=document_ids, include_docs=False)
            if 'error' not in query_row
        }

        for doc in to_batch:
            doc_id = doc['_id']
            if doc_id in revision_ids:
                doc['_rev'] = revision_ids[doc_id]

        try:
            _ = self._batch_methods[action](to_batch, transaction=True)

        except (UnresponsiveCouchDB, Conflict):
            self._buffer.requeue(action, docs)

    def fetch(self, key:str, revision:Optional[str] = None) -> Optional[dict]:
        """
        Get a database document by its ID and, optionally, revision

        @param   key       Document ID
        @param   revision  Revision ID
        @return  Database document (or None, if not found)
        """
        try:
            if not revision:
                output = self._db.get(key)

            else:
                output = next((
                    doc
                    for doc in self._db.revisions(key)
                    if  doc['_rev'] == revision
                ), None)

        except NotFound:
            output = None

        return output

    def upsert(self, data:dict, key:Optional[str] = None):
        """
        Upsert document, via the upsert buffer and queue

        @param   data  Document data
        @param   key   Document ID

        NOTE If the document ID is not provided and the document data
        does not contain an '_id' member, then a key will be generated;
        revisions IDs (_rev) are stripped out; and any other CouchDB
        reserved keys (i.e., prefixed with an underscore) will raise an
        InvalidCouchDBKey exception
        """
        if '_rev' in data:
            del data['_rev']

        if any(key.startswith('_') for key in data.keys() if key != '_id'):
            raise InvalidCouchDBKey

        self._buffer.append({'_id':key or uuid4().hex, **data})

    def delete(self, key:str):
        """
        Delete document from CouchDB, via the deletion buffer and queue

        @param   key  Document ID
        """
        doc = self.fetch(key)

        if doc:
            # Remove all CouchDB keys except _id
            to_delete = {
                key: value
                for key, value in doc.items()
                if  key == '_id'
                or  not key.startswith('_')
            }
            self._buffer.remove(to_delete)

    def query(self, design:str, view:str, wrapper:Optional[Callable[[dict], Any]] = None, **kwargs) -> Generator:
        """
        Query a predefined view

        @param   design   Design document name
        @param   view     View name
        @param   wrapper  Wrapper function applied over result rows
        @kwargs  Query string options for CouchDB
        @return  Results generator
        """
        # Check view exists
        doc = self.fetch('_design/{}'.format(design))
        if not doc or 'views' not in doc or view not in doc['views']:
            raise NotFound

        view_name = '{}/{}'.format(design, view)
        return self._db.query(view_name, wrapper=wrapper, **kwargs)

    def create_design(self, name:str) -> _DesignDocument:
        """
        Append a new design document

        @param   name  Design document name
        @return  The design document
        """
        new_design = _DesignDocument(self._db, name)
        self._designs.append(new_design)
        self._designs_dirty = True
        return new_design

    def get_design(self, name:str) -> Optional[_DesignDocument]:
        """
        Get an in-memory design document by name

        @param   name  Design document name
        @return  The design document (None, if not found)
        """
        design_id = '_design/{}'.format(name)
        return next((
            design
            for design in self._designs
            if  design.design_id == design_id
        ), None)

    def commit_designs(self):
        """ Commit all design documents to the database """
        if self._designs_dirty:
            for design in self._designs:
                design._commit()
            self._designs_dirty = False