def __init__(self, url:str, database:str, max_buffer_size:int = 1000, buffer_latency:timedelta = timedelta(milliseconds=50), **kwargs): """ Acquire a connection with the CouchDB server and initialise the buffering queue @param url CouchDB server URL @param database Database name @param max_buffer_size Maximum buffer size (no. of documents) @param buffer_latency Buffer latency before discharge @kwargs Additional constructor parameters to pycouchdb.client.Server should be passed through here """ self._db = SofterCouchDB(url, database, **kwargs) self._designs = [] self._designs_dirty = False # Batch action to DB method mapping self._batch_methods = { Actions.Upsert: self._db.save_bulk, Actions.Delete: self._db.delete_bulk } # Setup database action buffer and queue self._buffer = Buffer(max_buffer_size, buffer_latency) self._buffer.add_listener(self._batch)
class Sofabed(object): """ Buffered, append-optimised CouchDB interface """ def __init__(self, url:str, database:str, max_buffer_size:int = 1000, buffer_latency:timedelta = timedelta(milliseconds=50), **kwargs): """ Acquire a connection with the CouchDB server and initialise the buffering queue @param url CouchDB server URL @param database Database name @param max_buffer_size Maximum buffer size (no. of documents) @param buffer_latency Buffer latency before discharge @kwargs Additional constructor parameters to pycouchdb.client.Server should be passed through here """ self._db = SofterCouchDB(url, database, **kwargs) self._designs = [] self._designs_dirty = False # Batch action to DB method mapping self._batch_methods = { Actions.Upsert: self._db.save_bulk, Actions.Delete: self._db.delete_bulk } # Setup database action buffer and queue self._buffer = Buffer(max_buffer_size, buffer_latency) self._buffer.add_listener(self._batch) def _batch(self, broadcast:BatchListenerT): """ Perform a batch action against the database @param broadcast Broadcast data pushed by the buffer """ action, docs = broadcast to_batch = deepcopy(docs) # To avoid conflicts, we must merge in the revision IDs of # existing documents document_ids = [doc['_id'] for doc in to_batch] revision_ids = { query_row['id']: query_row['value']['rev'] for query_row in self._db.all(keys=document_ids, include_docs=False) if 'error' not in query_row } for doc in to_batch: doc_id = doc['_id'] if doc_id in revision_ids: doc['_rev'] = revision_ids[doc_id] try: _ = self._batch_methods[action](to_batch, transaction=True) except (UnresponsiveCouchDB, Conflict): self._buffer.requeue(action, docs) def fetch(self, key:str, revision:Optional[str] = None) -> Optional[dict]: """ Get a database document by its ID and, optionally, revision @param key Document ID @param revision Revision ID @return Database document (or None, if not found) """ try: if not revision: output = self._db.get(key) else: output = next(( doc for doc in self._db.revisions(key) if doc['_rev'] == revision ), None) except NotFound: output = None return output def upsert(self, data:dict, key:Optional[str] = None): """ Upsert document, via the upsert buffer and queue @param data Document data @param key Document ID NOTE If the document ID is not provided and the document data does not contain an '_id' member, then a key will be generated; revisions IDs (_rev) are stripped out; and any other CouchDB reserved keys (i.e., prefixed with an underscore) will raise an InvalidCouchDBKey exception """ if '_rev' in data: del data['_rev'] if any(key.startswith('_') for key in data.keys() if key != '_id'): raise InvalidCouchDBKey self._buffer.append({'_id':key or uuid4().hex, **data}) def delete(self, key:str): """ Delete document from CouchDB, via the deletion buffer and queue @param key Document ID """ doc = self.fetch(key) if doc: # Remove all CouchDB keys except _id to_delete = { key: value for key, value in doc.items() if key == '_id' or not key.startswith('_') } self._buffer.remove(to_delete) def query(self, design:str, view:str, wrapper:Optional[Callable[[dict], Any]] = None, **kwargs) -> Generator: """ Query a predefined view @param design Design document name @param view View name @param wrapper Wrapper function applied over result rows @kwargs Query string options for CouchDB @return Results generator """ # Check view exists doc = self.fetch('_design/{}'.format(design)) if not doc or 'views' not in doc or view not in doc['views']: raise NotFound view_name = '{}/{}'.format(design, view) return self._db.query(view_name, wrapper=wrapper, **kwargs) def create_design(self, name:str) -> _DesignDocument: """ Append a new design document @param name Design document name @return The design document """ new_design = _DesignDocument(self._db, name) self._designs.append(new_design) self._designs_dirty = True return new_design def get_design(self, name:str) -> Optional[_DesignDocument]: """ Get an in-memory design document by name @param name Design document name @return The design document (None, if not found) """ design_id = '_design/{}'.format(name) return next(( design for design in self._designs if design.design_id == design_id ), None) def commit_designs(self): """ Commit all design documents to the database """ if self._designs_dirty: for design in self._designs: design._commit() self._designs_dirty = False