def __init__(self, database, unserializer=None): log.Logger.__init__(self, database) log.LogProxy.__init__(self, database) self._database = IDatabaseDriver(database) self._serializer = json.Serializer(sort_keys=True, force_unicode=True) self._unserializer = (unserializer or common.CouchdbUnserializer()) # listner_id -> doc_ids self._listeners = dict() self._change_cb = None # Changed to use a normal dictionary. # It will grow boundless up to the number of documents # modified by the connection. It is a kind of memory leak # done to temporarily resolve the problem of notifications # received after the expiration time due to reconnection # killing agents. self._known_revisions = {} # {DOC_ID: (REV_INDEX, REV_HASH)} # If the counter of current tasks on database which can produce # a new revision self._update_lock_counter = 0 # Unlocked callbacks self._unlocked_callbacks = set() # set([doc_id, rev]), This is used to trigger the asynchronous hook # of the document upgrade only ones self._upgrades_ran = set()
def __init__(self, database, unserializer=None): log.Logger.__init__(self, database) log.LogProxy.__init__(self, database) self._database = IDatabaseDriver(database) self._serializer = serialization.json.Serializer() self._unserializer = (unserializer or serialization.json.PaisleyUnserializer()) # listner_id -> doc_ids self._listeners = dict() self._change_cb = None # Changed to use a normal dictionary. # It will grow boundless up to the number of documents # modified by the connection. It is a kind of memory leak # done to temporarily resolve the problem of notifications # received after the expiration time due to reconnection # killing agents. self._known_revisions = {} # {DOC_ID: (REV_INDEX, REV_HASH)}
class Connection(log.Logger, log.LogProxy): '''API for agency to call against the database.''' type_name = 'db-connection' implements(IDatabaseClient, ITimeProvider, IRevisionStore, ISerializable) def __init__(self, database, unserializer=None): log.Logger.__init__(self, database) log.LogProxy.__init__(self, database) self._database = IDatabaseDriver(database) self._serializer = json.Serializer(sort_keys=True, force_unicode=True) self._unserializer = (unserializer or common.CouchdbUnserializer()) # listner_id -> doc_ids self._listeners = dict() self._change_cb = None # Changed to use a normal dictionary. # It will grow boundless up to the number of documents # modified by the connection. It is a kind of memory leak # done to temporarily resolve the problem of notifications # received after the expiration time due to reconnection # killing agents. self._known_revisions = {} # {DOC_ID: (REV_INDEX, REV_HASH)} # If the counter of current tasks on database which can produce # a new revision self._update_lock_counter = 0 # Unlocked callbacks self._unlocked_callbacks = set() # set([doc_id, rev]), This is used to trigger the asynchronous hook # of the document upgrade only ones self._upgrades_ran = set() ### IRevisionStore ### @property def known_revisions(self): return self._known_revisions @property def analyzes_locked(self): return self._update_lock_counter > 0 def wait_unlocked(self, callback): self._unlocked_callbacks.add(callback) ### private used for locking and unlocking the updates ### def _lock_notifications(self): self._update_lock_counter += 1 def _unlock_notifications(self): assert self._update_lock_counter > 0, "Lock value dropped below 0!" self._update_lock_counter -= 1 if self._update_lock_counter == 0: u = self._unlocked_callbacks self._unlocked_callbacks = set() for callback in u: callback() ### ITimeProvider ### def get_time(self): return time.time() ### IDatabaseClient ### @property def database(self): return self._database @serialization.freeze_tag('IDatabaseClient.create_database') def create_database(self): return self._database.create_db() @serialization.freeze_tag('IDatabaseClient.save_document') @defer.inlineCallbacks def save_document(self, doc): assert IDocument.providedBy(doc) or isinstance(doc, dict), repr(doc) try: self._lock_notifications() serialized = self._serializer.convert(doc) if IDocument.providedBy(doc): following_attachments = dict( (name, attachment) for name, attachment in doc.get_attachments().iteritems() if not attachment.saved) doc_id = doc.doc_id else: following_attachments = dict() doc_id = doc.get('_id') resp = yield self._database.save_doc(serialized, doc_id, following_attachments) self._update_id_and_rev(resp, doc) for attachment in following_attachments.itervalues(): attachment.set_saved() # now process all the documents which have been registered to # be saved together with this document if IDocument.providedBy(doc): while doc.links.to_save: to_link, linker_roles, linkee_roles = ( doc.links.to_save.pop(0)) to_link.links.create(doc=doc, linker_roles=linker_roles, linkee_roles=linkee_roles) yield self.save_document(to_link) defer.returnValue(doc) finally: self._unlock_notifications() @serialization.freeze_tag('IDatabaseClient.get_attachment_body') def get_attachment_body(self, attachment): d = self._database.get_attachment(attachment.doc_id, attachment.name) return d @serialization.freeze_tag('IDatabaseClient.get_document') def get_document(self, doc_id, raw=False, **extra): d = self._database.open_doc(doc_id, **extra) if not raw: d.addCallback(self.unserialize_document) d.addCallback(self._notice_doc_revision) return d @serialization.freeze_tag('IDatabaseClient.update_document') def update_document(self, _doc, _method, *args, **kwargs): return self.update_document_ex(_doc, _method, args, kwargs) @serialization.freeze_tag('IDatabaseClient.update_document_ex') def update_document_ex(self, doc, _method, args=tuple(), keywords=dict()): if not IDocument.providedBy(doc): d = self.get_document(doc) else: d = defer.succeed(doc) d.addCallback(self._iterate_on_update, _method, args, keywords) return d @serialization.freeze_tag('IDatabaseClient.get_revision') def get_revision(self, doc_id): # FIXME: this could be done by lightweight HEAD request d = self._database.open_doc(doc_id) d.addCallback(lambda doc: doc['_rev']) return d @serialization.freeze_tag('IDatabaseClient.reload_database') def reload_document(self, doc): assert IDocument.providedBy(doc), \ "Incorrect type: %r, expected IDocument" % (type(doc), ) return self.get_document(doc.doc_id) @serialization.freeze_tag('IDatabaseClient.delete_document') def delete_document(self, doc): if IDocument.providedBy(doc): body = { "_id": doc.doc_id, "_rev": doc.rev, "_deleted": True, ".type": unicode(doc.type_name)} for field in type(doc)._fields: if field.meta('keep_deleted'): body[field.serialize_as] = getattr(doc, field.name) elif isinstance(doc, dict): body = { "_id": doc["_id"], "_rev": doc["_rev"], "_deleted": True} else: raise ValueError(repr(doc)) serialized = self._serializer.convert(body) self._lock_notifications() d = self._database.save_doc(serialized, body["_id"]) d.addCallback(self._update_id_and_rev, doc) d.addBoth(defer.bridge_param, self._unlock_notifications) return d @serialization.freeze_tag('IDatabaseClient.copy_document') def copy_document(self, doc_or_id, destination_id, rev=None): if isinstance(doc_or_id, (str, unicode)): doc_id = doc_or_id elif IDocument.providedBy(doc_or_id): doc_id = doc_or_id.doc_id elif isinstance(doc_or_id, dict): doc_id = doc_or_id['_id'] else: raise TypeError(type(doc_or_id)) if not doc_id: raise ValueError("Cannot determine doc id from %r" % (doc_or_id, )) return self._database.copy_doc(doc_id, destination_id, rev) @serialization.freeze_tag('IDatabaseClient.changes_listener') def changes_listener(self, filter_, callback, **kwargs): assert callable(callback) r = RevisionAnalytic(self, callback) d = self._database.listen_changes(filter_, r.on_change, kwargs) def set_listener_id(l_id, filter_): self._listeners[l_id] = filter_ return l_id d.addCallback(set_listener_id, filter_) return d @serialization.freeze_tag('IDatabaseClient.cancel_listener') @journal.named_side_effect('IDatabaseClient.cancel_listener') def cancel_listener(self, filter_): for l_id, listener_filter in self._listeners.items(): if ((IViewFactory.providedBy(listener_filter) and filter_ is listener_filter) or (isinstance(listener_filter, (list, tuple)) and (filter_ in listener_filter))): self._cancel_listener(l_id) @serialization.freeze_tag('IDatabaseClient.query_view') def query_view(self, factory, parse_results=True, **options): factory = IViewFactory(factory) d = self._database.query_view(factory, **options) if parse_results: d.addCallback(self._parse_view_results, factory, options) return d @serialization.freeze_tag('IDatabaseClient.disconnect') @journal.named_side_effect('IDatabaseClient.disconnect') def disconnect(self): for l_id in self._listeners.keys(): self._cancel_listener(l_id) @serialization.freeze_tag('IDatabaseClient.get_update_seq') def get_update_seq(self): return self._database.get_update_seq() @serialization.freeze_tag('IDatabaseClient.get_changes') def get_changes(self, filter_=None, limit=None, since=0): if IViewFactory.providedBy(filter_): filter_ = ViewFilter(filter_, params=dict()) elif filter_ is not None: raise ValueError("%r should provide IViewFacory" % (filter_, )) return self._database.get_changes(filter_, limit, since) @serialization.freeze_tag('IDatabaseClient.bulk_get') def bulk_get(self, doc_ids, consume_errors=True): def parse_bulk_response(resp): assert isinstance(resp, dict), repr(resp) assert 'rows' in resp, repr(resp) result = list() for doc_id, row in zip(doc_ids, resp['rows']): if 'error' in row or 'deleted' in row['value']: if not consume_errors: result.append(NotFoundError(doc_id)) else: self.debug("Bulk get parser consumed error row: %r", row) else: result.append(row['doc']) return self.unserialize_list_of_documents(result) d = self._database.bulk_get(doc_ids) d.addCallback(parse_bulk_response) return d ### public methods used by update and replication mechanism ### def get_database_tag(self): ''' Each feat database has a unique tag which identifies it. Thanks to it the mechanism cleaning up the update logs make the difference between the changes done locally and remotely. The condition for cleaning those up is different. ''' def parse_response(doc): self._database_tag = doc['tag'] return self._database_tag def create_new(fail): fail.trap(NotFoundError) doc = {'_id': doc_id, 'tag': unicode(uuid.uuid1())} return self.save_document(doc) def conflict_handler(fail): fail.trap(ConflictError) return self.get_database_tag() if not hasattr(self, '_database_tag'): doc_id = u'_local/database_tag' d = self.get_document(doc_id) d.addErrback(create_new) d.addErrback(conflict_handler) d.addCallback(parse_response) return d else: return defer.succeed(self._database_tag) ### ISerializable Methods ### def snapshot(self): return None ### private def _cancel_listener(self, lister_id): self._database.cancel_listener(lister_id) try: del(self._listeners[lister_id]) except KeyError: self.warning('Tried to remove nonexistining listener id %r.', lister_id) def _parse_view_results(self, rows, factory, options): ''' rows here should be a list of tuples: - (key, value) for reduce views - (key, value, id) for nonreduce views without include docs - (key, value, id, doc) for nonreduce with with include docs ''' kwargs = dict() kwargs['reduced'] = factory.use_reduce and options.get('reduce', True) kwargs['include_docs'] = options.get('include_docs', False) # Lines below pass extra arguments to the parsing function if they # are expected. These arguments are bound method unserialize() and # unserialize_list(). They methods perform the magic of parsing and # upgrading if necessary the loaded documents. spec = inspect.getargspec(factory.parse_view_result) if 'unserialize' in spec.args: kwargs['unserialize'] = self.unserialize_document if 'unserialize_list' in spec.args: kwargs['unserialize_list'] = self.unserialize_list_of_documents return factory.parse_view_result(rows, **kwargs) def _update_id_and_rev(self, resp, doc): if IDocument.providedBy(doc): doc.doc_id = unicode(resp.get('id', None)) doc.rev = unicode(resp.get('rev', None)) self._notice_doc_revision(doc) else: doc['_id'] = unicode(resp.get('id', None)) doc['_rev'] = unicode(resp.get('rev', None)) return doc def _notice_doc_revision(self, doc): if IDocument.providedBy(doc): doc_id = doc.doc_id rev = doc.rev else: doc_id = doc['_id'] rev = doc['_rev'] self.log('Storing knowledge about doc rev. ID: %r, REV: %r', doc_id, rev) self._known_revisions[doc_id] = _parse_doc_revision(rev) return doc ### parsing of the documents ### def unserialize_document(self, raw): doc = self._unserializer.convert(raw) if IVersionedDocument.providedBy(doc): if doc.has_migrated: d = defer.succeed(doc) key = (doc.doc_id, doc.rev) if key not in self._upgrades_ran: # Make sure that the connection instance only triggers # once asychronous upgrade. This minimizes the amount # of possible conflicts when fetching old document # version more than once. self._upgrades_ran.add(key) for handler, context in doc.get_asynchronous_actions(): if handler.use_custom_registry: conn = Connection(self._database, handler.unserializer) else: conn = self d.addCallback(defer.keep_param, defer.inject_param, 1, handler.asynchronous_hook, conn, context) d.addErrback(self.handle_immediate_failure, handler.asynchronous_hook, context) d.addCallback(self.save_document) d.addErrback(self.handle_unserialize_failure, raw) return d return doc def handle_immediate_failure(self, fail, hook, context): error.handle_failure(self, fail, 'Failed calling %r with context %r. ', hook, context) return fail def handle_unserialize_failure(self, fail, raw): type_name = raw.get('.type') version = raw.get('.version') if fail.check(ConflictError): self.debug('Got conflict error when trying to upgrade the ' 'document: %s version: %s. Refetching it.', type_name, version) # probably we've already upgraded it concurrently return self.get_document(raw['_id']) error.handle_failure(self, fail, 'Asynchronous hooks of ' 'the upgrade failed. Raising NotMigratable') return failure.Failure(NotMigratable((type_name, version, ))) def unserialize_list_of_documents(self, list_of_raw): result = list() defers = list() for raw in list_of_raw: if isinstance(raw, Exception): # Exceptions are simply preserved. This behaviour is # optionally used by bulk_get(). d = raw else: d = self.unserialize_document(raw) if isinstance(d, defer.Deferred): result.append(None) index = len(result) - 1 d.addCallback(defer.inject_param, 1, result.__setitem__, index) defers.append(d) else: result.append(d) if defers: d = defer.DeferredList(defers, consumeErrors=True) d.addCallback(defer.override_result, result) return d else: return result ### private parts of update_document subroutine ### def _iterate_on_update(self, _document, _method, args, keywords): if IDocument.providedBy(_document): doc_id = _document.doc_id rev = _document.rev else: doc_id = _document['_id'] rev = _document['_rev'] try: result = _method(_document, *args, **keywords) except ResignFromModifying: return _document if result is None: d = self.delete_document(_document) else: d = self.save_document(result) if (IDocument.providedBy(_document) and _document.conflict_resolution_strategy == ConflictResolutionStrategy.merge): update_log = document.UpdateLog( handler=_method, args=args, keywords=keywords, rev_from=rev, timestamp=time.time()) d.addCallback(lambda doc: defer.DeferredList([defer.succeed(doc), self.get_database_tag(), self.get_update_seq()])) d.addCallback(self._log_update, update_log) d.addErrback(self._errback_on_update, doc_id, _method, args, keywords) return d def _log_update(self, ((_s1, doc), (_s2, tag), (_s3, seq)), update_log): update_log.rev_to = doc.rev update_log.owner_id = doc.doc_id update_log.seq_num = seq update_log.partition_tag = tag d = self.save_document(update_log) d.addCallback(defer.override_result, doc) return d
class Connection(log.Logger, log.LogProxy): '''API for agency to call against the database.''' type_name = 'db-connection' implements(IDatabaseClient, ITimeProvider, IRevisionStore, ISerializable) def __init__(self, database, unserializer=None): log.Logger.__init__(self, database) log.LogProxy.__init__(self, database) self._database = IDatabaseDriver(database) self._serializer = serialization.json.Serializer() self._unserializer = (unserializer or serialization.json.PaisleyUnserializer()) # listner_id -> doc_ids self._listeners = dict() self._change_cb = None # Changed to use a normal dictionary. # It will grow boundless up to the number of documents # modified by the connection. It is a kind of memory leak # done to temporarily resolve the problem of notifications # received after the expiration time due to reconnection # killing agents. self._known_revisions = {} # {DOC_ID: (REV_INDEX, REV_HASH)} ### IRevisionStore ### @property def known_revisions(self): return self._known_revisions ### ITimeProvider ### def get_time(self): return time.time() ### IDatabaseClient ### @serialization.freeze_tag('IDatabaseClient.create_database') def create_database(self): return self._database.create_db() @serialization.freeze_tag('IDatabaseClient.save_document') @defer.inlineCallbacks def save_document(self, doc): doc = IDocument(doc) serialized = self._serializer.convert(doc) resp = yield self._database.save_doc(serialized, doc.doc_id) self._update_id_and_rev(resp, doc) for name, attachment in doc.get_attachments().iteritems(): if not attachment.saved: resp = yield self._database.save_attachment( doc.doc_id, doc.rev, attachment) self._update_id_and_rev(resp, doc) attachment.set_saved() defer.returnValue(doc) @serialization.freeze_tag('IDatabaseClient.get_attachment_body') def get_attachment_body(self, attachment): d = self._database.get_attachment(attachment.doc_id, attachment.name) return d @serialization.freeze_tag('IDatabaseClient.get_document') def get_document(self, doc_id): d = self._database.open_doc(doc_id) d.addCallback(self._unserializer.convert) d.addCallback(self._notice_doc_revision) return d @serialization.freeze_tag('IDatabaseClient.get_revision') def get_revision(self, doc_id): # FIXME: this could be done by lightweight HEAD request d = self._database.open_doc(doc_id) d.addCallback(lambda doc: doc['_rev']) return d @serialization.freeze_tag('IDatabaseClient.reload_database') def reload_document(self, doc): assert IDocument.providedBy(doc), \ "Incorrect type: %r, expected IDocument" % (type(doc), ) return self.get_document(doc.doc_id) @serialization.freeze_tag('IDatabaseClient.delete_document') def delete_document(self, doc): assert isinstance(doc, document.Document), type(doc) d = self._database.delete_doc(doc.doc_id, doc.rev) d.addCallback(self._update_id_and_rev, doc) return d @serialization.freeze_tag('IDatabaseClient.changes_listener') def changes_listener(self, filter_, callback, **kwargs): assert callable(callback) r = RevisionAnalytic(self, callback) d = self._database.listen_changes(filter_, r.on_change, kwargs) def set_listener_id(l_id, filter_): self._listeners[l_id] = filter_ return l_id d.addCallback(set_listener_id, filter_) return d @serialization.freeze_tag('IDatabaseClient.cancel_listener') @journal.named_side_effect('IDatabaseClient.cancel_listener') def cancel_listener(self, filter_): for l_id, listener_filter in self._listeners.items(): if ((IViewFactory.providedBy(listener_filter) and filter_ is listener_filter) or (isinstance(listener_filter, (list, tuple)) and (filter_ in listener_filter))): self._cancel_listener(l_id) @serialization.freeze_tag('IDatabaseClient.query_view') def query_view(self, factory, **options): factory = IViewFactory(factory) d = self._database.query_view(factory, **options) d.addCallback(self._parse_view_results, factory, options) return d @serialization.freeze_tag('IDatabaseClient.disconnect') @journal.named_side_effect('IDatabaseClient.disconnect') def disconnect(self): if hasattr(self, '_query_cache'): self._query_cache.empty() for l_id in self._listeners.keys(): self._cancel_listener(l_id) @serialization.freeze_tag('IDatabaseClient.get_update_seq') def get_update_seq(self): return self._database.get_update_seq() @serialization.freeze_tag('IDatabaseClient.get_changes') def get_changes(self, filter_=None, limit=None, since=0): if IViewFactory.providedBy(filter_): filter_ = ViewFilter(filter_, params=dict()) elif filter_ is not None: raise ValueError("%r should provide IViewFacory" % (filter_, )) return self._database.get_changes(filter_, limit, since) @serialization.freeze_tag('IDatabaseClient.bulk_get') def bulk_get(self, doc_ids, consume_errors=True): def parse_bulk_response(resp): assert isinstance(resp, dict), repr(resp) assert 'rows' in resp, repr(resp) result = list() for doc_id, row in zip(doc_ids, resp['rows']): if 'error' in row or 'deleted' in row['value']: if not consume_errors: result.append(NotFoundError(doc_id)) else: self.debug("Bulk get parser consumed error row: %r", row) else: result.append(self._unserializer.convert(row['doc'])) return result d = self._database.bulk_get(doc_ids) d.addCallback(parse_bulk_response) return d ### public method used by query mechanism ### def get_query_cache(self, create=True): '''Called by methods inside feat.database.query module to obtain the query cache. @param create: C{bool} if True cache will be initialized if it doesnt exist yet, returns None otherwise ''' if not hasattr(self, '_query_cache'): if create: self._query_cache = query.Cache(self) else: return None return self._query_cache ### ISerializable Methods ### def snapshot(self): return None ### private def _cancel_listener(self, lister_id): self._database.cancel_listener(lister_id) try: del(self._listeners[lister_id]) except KeyError: self.warning('Tried to remove nonexistining listener id %r.', lister_id) def _parse_view_results(self, rows, factory, options): ''' rows here should be a list of tuples: - (key, value) for reduce views - (key, value, id) for nonreduce views without include docs - (key, value, id, doc) for nonreduce with with include docs ''' reduced = factory.use_reduce and options.get('reduce', True) include_docs = options.get('include_docs', False) return factory.parse_view_result(rows, reduced, include_docs) def _update_id_and_rev(self, resp, doc): doc.doc_id = unicode(resp.get('id', None)) doc.rev = unicode(resp.get('rev', None)) self._notice_doc_revision(doc) return doc def _notice_doc_revision(self, doc): self.log('Storing knowledge about doc rev. ID: %r, REV: %r', doc.doc_id, doc.rev) self._known_revisions[doc.doc_id] = _parse_doc_revision(doc.rev) return doc