def __init__(self, database, unserializer=None):
log.Logger.__init__(self, database)
log.LogProxy.__init__(self, database)
self._database = IDatabaseDriver(database)
self._serializer = json.Serializer(sort_keys=True, force_unicode=True)
self._unserializer = (unserializer or common.CouchdbUnserializer())
# listner_id -> doc_ids
self._listeners = dict()
self._change_cb = None
# Changed to use a normal dictionary.
# It will grow boundless up to the number of documents
# modified by the connection. It is a kind of memory leak
# done to temporarily resolve the problem of notifications
# received after the expiration time due to reconnection
# killing agents.
self._known_revisions = {} # {DOC_ID: (REV_INDEX, REV_HASH)}
# If the counter of current tasks on database which can produce
# a new revision
self._update_lock_counter = 0
# Unlocked callbacks
self._unlocked_callbacks = set()
# set([doc_id, rev]), This is used to trigger the asynchronous hook
# of the document upgrade only ones
self._upgrades_ran = set()
def __init__(self, database, unserializer=None):
log.Logger.__init__(self, database)
log.LogProxy.__init__(self, database)
self._database = IDatabaseDriver(database)
self._serializer = serialization.json.Serializer()
self._unserializer = (unserializer or
serialization.json.PaisleyUnserializer())
# listner_id -> doc_ids
self._listeners = dict()
self._change_cb = None
# Changed to use a normal dictionary.
# It will grow boundless up to the number of documents
# modified by the connection. It is a kind of memory leak
# done to temporarily resolve the problem of notifications
# received after the expiration time due to reconnection
# killing agents.
self._known_revisions = {} # {DOC_ID: (REV_INDEX, REV_HASH)}
class Connection(log.Logger, log.LogProxy):
'''API for agency to call against the database.'''
type_name = 'db-connection'
implements(IDatabaseClient, ITimeProvider, IRevisionStore, ISerializable)
def __init__(self, database, unserializer=None):
log.Logger.__init__(self, database)
log.LogProxy.__init__(self, database)
self._database = IDatabaseDriver(database)
self._serializer = json.Serializer(sort_keys=True, force_unicode=True)
self._unserializer = (unserializer or common.CouchdbUnserializer())
# listner_id -> doc_ids
self._listeners = dict()
self._change_cb = None
# Changed to use a normal dictionary.
# It will grow boundless up to the number of documents
# modified by the connection. It is a kind of memory leak
# done to temporarily resolve the problem of notifications
# received after the expiration time due to reconnection
# killing agents.
self._known_revisions = {} # {DOC_ID: (REV_INDEX, REV_HASH)}
# If the counter of current tasks on database which can produce
# a new revision
self._update_lock_counter = 0
# Unlocked callbacks
self._unlocked_callbacks = set()
# set([doc_id, rev]), This is used to trigger the asynchronous hook
# of the document upgrade only ones
self._upgrades_ran = set()
### IRevisionStore ###
@property
def known_revisions(self):
return self._known_revisions
@property
def analyzes_locked(self):
return self._update_lock_counter > 0
def wait_unlocked(self, callback):
self._unlocked_callbacks.add(callback)
### private used for locking and unlocking the updates ###
def _lock_notifications(self):
self._update_lock_counter += 1
def _unlock_notifications(self):
assert self._update_lock_counter > 0, "Lock value dropped below 0!"
self._update_lock_counter -= 1
if self._update_lock_counter == 0:
u = self._unlocked_callbacks
self._unlocked_callbacks = set()
for callback in u:
callback()
### ITimeProvider ###
def get_time(self):
return time.time()
### IDatabaseClient ###
@property
def database(self):
return self._database
@serialization.freeze_tag('IDatabaseClient.create_database')
def create_database(self):
return self._database.create_db()
@serialization.freeze_tag('IDatabaseClient.save_document')
@defer.inlineCallbacks
def save_document(self, doc):
assert IDocument.providedBy(doc) or isinstance(doc, dict), repr(doc)
try:
self._lock_notifications()
serialized = self._serializer.convert(doc)
if IDocument.providedBy(doc):
following_attachments = dict(
(name, attachment) for name, attachment
in doc.get_attachments().iteritems()
if not attachment.saved)
doc_id = doc.doc_id
else:
following_attachments = dict()
doc_id = doc.get('_id')
resp = yield self._database.save_doc(serialized, doc_id,
following_attachments)
self._update_id_and_rev(resp, doc)
for attachment in following_attachments.itervalues():
attachment.set_saved()
# now process all the documents which have been registered to
# be saved together with this document
if IDocument.providedBy(doc):
while doc.links.to_save:
to_link, linker_roles, linkee_roles = (
doc.links.to_save.pop(0))
to_link.links.create(doc=doc, linker_roles=linker_roles,
linkee_roles=linkee_roles)
yield self.save_document(to_link)
defer.returnValue(doc)
finally:
self._unlock_notifications()
@serialization.freeze_tag('IDatabaseClient.get_attachment_body')
def get_attachment_body(self, attachment):
d = self._database.get_attachment(attachment.doc_id, attachment.name)
return d
@serialization.freeze_tag('IDatabaseClient.get_document')
def get_document(self, doc_id, raw=False, **extra):
d = self._database.open_doc(doc_id, **extra)
if not raw:
d.addCallback(self.unserialize_document)
d.addCallback(self._notice_doc_revision)
return d
@serialization.freeze_tag('IDatabaseClient.update_document')
def update_document(self, _doc, _method, *args, **kwargs):
return self.update_document_ex(_doc, _method, args, kwargs)
@serialization.freeze_tag('IDatabaseClient.update_document_ex')
def update_document_ex(self, doc, _method, args=tuple(), keywords=dict()):
if not IDocument.providedBy(doc):
d = self.get_document(doc)
else:
d = defer.succeed(doc)
d.addCallback(self._iterate_on_update, _method, args, keywords)
return d
@serialization.freeze_tag('IDatabaseClient.get_revision')
def get_revision(self, doc_id):
# FIXME: this could be done by lightweight HEAD request
d = self._database.open_doc(doc_id)
d.addCallback(lambda doc: doc['_rev'])
return d
@serialization.freeze_tag('IDatabaseClient.reload_database')
def reload_document(self, doc):
assert IDocument.providedBy(doc), \
"Incorrect type: %r, expected IDocument" % (type(doc), )
return self.get_document(doc.doc_id)
@serialization.freeze_tag('IDatabaseClient.delete_document')
def delete_document(self, doc):
if IDocument.providedBy(doc):
body = {
"_id": doc.doc_id,
"_rev": doc.rev,
"_deleted": True,
".type": unicode(doc.type_name)}
for field in type(doc)._fields:
if field.meta('keep_deleted'):
body[field.serialize_as] = getattr(doc, field.name)
elif isinstance(doc, dict):
body = {
"_id": doc["_id"],
"_rev": doc["_rev"],
"_deleted": True}
else:
raise ValueError(repr(doc))
serialized = self._serializer.convert(body)
self._lock_notifications()
d = self._database.save_doc(serialized, body["_id"])
d.addCallback(self._update_id_and_rev, doc)
d.addBoth(defer.bridge_param, self._unlock_notifications)
return d
@serialization.freeze_tag('IDatabaseClient.copy_document')
def copy_document(self, doc_or_id, destination_id, rev=None):
if isinstance(doc_or_id, (str, unicode)):
doc_id = doc_or_id
elif IDocument.providedBy(doc_or_id):
doc_id = doc_or_id.doc_id
elif isinstance(doc_or_id, dict):
doc_id = doc_or_id['_id']
else:
raise TypeError(type(doc_or_id))
if not doc_id:
raise ValueError("Cannot determine doc id from %r" % (doc_or_id, ))
return self._database.copy_doc(doc_id, destination_id, rev)
@serialization.freeze_tag('IDatabaseClient.changes_listener')
def changes_listener(self, filter_, callback, **kwargs):
assert callable(callback)
r = RevisionAnalytic(self, callback)
d = self._database.listen_changes(filter_, r.on_change, kwargs)
def set_listener_id(l_id, filter_):
self._listeners[l_id] = filter_
return l_id
d.addCallback(set_listener_id, filter_)
return d
@serialization.freeze_tag('IDatabaseClient.cancel_listener')
@journal.named_side_effect('IDatabaseClient.cancel_listener')
def cancel_listener(self, filter_):
for l_id, listener_filter in self._listeners.items():
if ((IViewFactory.providedBy(listener_filter) and
filter_ is listener_filter) or
(isinstance(listener_filter, (list, tuple)) and
(filter_ in listener_filter))):
self._cancel_listener(l_id)
@serialization.freeze_tag('IDatabaseClient.query_view')
def query_view(self, factory, parse_results=True, **options):
factory = IViewFactory(factory)
d = self._database.query_view(factory, **options)
if parse_results:
d.addCallback(self._parse_view_results, factory, options)
return d
@serialization.freeze_tag('IDatabaseClient.disconnect')
@journal.named_side_effect('IDatabaseClient.disconnect')
def disconnect(self):
for l_id in self._listeners.keys():
self._cancel_listener(l_id)
@serialization.freeze_tag('IDatabaseClient.get_update_seq')
def get_update_seq(self):
return self._database.get_update_seq()
@serialization.freeze_tag('IDatabaseClient.get_changes')
def get_changes(self, filter_=None, limit=None, since=0):
if IViewFactory.providedBy(filter_):
filter_ = ViewFilter(filter_, params=dict())
elif filter_ is not None:
raise ValueError("%r should provide IViewFacory" % (filter_, ))
return self._database.get_changes(filter_, limit, since)
@serialization.freeze_tag('IDatabaseClient.bulk_get')
def bulk_get(self, doc_ids, consume_errors=True):
def parse_bulk_response(resp):
assert isinstance(resp, dict), repr(resp)
assert 'rows' in resp, repr(resp)
result = list()
for doc_id, row in zip(doc_ids, resp['rows']):
if 'error' in row or 'deleted' in row['value']:
if not consume_errors:
result.append(NotFoundError(doc_id))
else:
self.debug("Bulk get parser consumed error row: %r",
row)
else:
result.append(row['doc'])
return self.unserialize_list_of_documents(result)
d = self._database.bulk_get(doc_ids)
d.addCallback(parse_bulk_response)
return d
### public methods used by update and replication mechanism ###
def get_database_tag(self):
'''
Each feat database has a unique tag which identifies it. Thanks to it
the mechanism cleaning up the update logs make the difference between
the changes done locally and remotely. The condition for cleaning
those up is different.
'''
def parse_response(doc):
self._database_tag = doc['tag']
return self._database_tag
def create_new(fail):
fail.trap(NotFoundError)
doc = {'_id': doc_id, 'tag': unicode(uuid.uuid1())}
return self.save_document(doc)
def conflict_handler(fail):
fail.trap(ConflictError)
return self.get_database_tag()
if not hasattr(self, '_database_tag'):
doc_id = u'_local/database_tag'
d = self.get_document(doc_id)
d.addErrback(create_new)
d.addErrback(conflict_handler)
d.addCallback(parse_response)
return d
else:
return defer.succeed(self._database_tag)
### ISerializable Methods ###
def snapshot(self):
return None
### private
def _cancel_listener(self, lister_id):
self._database.cancel_listener(lister_id)
try:
del(self._listeners[lister_id])
except KeyError:
self.warning('Tried to remove nonexistining listener id %r.',
lister_id)
def _parse_view_results(self, rows, factory, options):
'''
rows here should be a list of tuples:
- (key, value) for reduce views
- (key, value, id) for nonreduce views without include docs
- (key, value, id, doc) for nonreduce with with include docs
'''
kwargs = dict()
kwargs['reduced'] = factory.use_reduce and options.get('reduce', True)
kwargs['include_docs'] = options.get('include_docs', False)
# Lines below pass extra arguments to the parsing function if they
# are expected. These arguments are bound method unserialize() and
# unserialize_list(). They methods perform the magic of parsing and
# upgrading if necessary the loaded documents.
spec = inspect.getargspec(factory.parse_view_result)
if 'unserialize' in spec.args:
kwargs['unserialize'] = self.unserialize_document
if 'unserialize_list' in spec.args:
kwargs['unserialize_list'] = self.unserialize_list_of_documents
return factory.parse_view_result(rows, **kwargs)
def _update_id_and_rev(self, resp, doc):
if IDocument.providedBy(doc):
doc.doc_id = unicode(resp.get('id', None))
doc.rev = unicode(resp.get('rev', None))
self._notice_doc_revision(doc)
else:
doc['_id'] = unicode(resp.get('id', None))
doc['_rev'] = unicode(resp.get('rev', None))
return doc
def _notice_doc_revision(self, doc):
if IDocument.providedBy(doc):
doc_id = doc.doc_id
rev = doc.rev
else:
doc_id = doc['_id']
rev = doc['_rev']
self.log('Storing knowledge about doc rev. ID: %r, REV: %r',
doc_id, rev)
self._known_revisions[doc_id] = _parse_doc_revision(rev)
return doc
### parsing of the documents ###
def unserialize_document(self, raw):
doc = self._unserializer.convert(raw)
if IVersionedDocument.providedBy(doc):
if doc.has_migrated:
d = defer.succeed(doc)
key = (doc.doc_id, doc.rev)
if key not in self._upgrades_ran:
# Make sure that the connection instance only triggers
# once asychronous upgrade. This minimizes the amount
# of possible conflicts when fetching old document
# version more than once.
self._upgrades_ran.add(key)
for handler, context in doc.get_asynchronous_actions():
if handler.use_custom_registry:
conn = Connection(self._database,
handler.unserializer)
else:
conn = self
d.addCallback(defer.keep_param, defer.inject_param, 1,
handler.asynchronous_hook, conn, context)
d.addErrback(self.handle_immediate_failure,
handler.asynchronous_hook, context)
d.addCallback(self.save_document)
d.addErrback(self.handle_unserialize_failure, raw)
return d
return doc
def handle_immediate_failure(self, fail, hook, context):
error.handle_failure(self, fail,
'Failed calling %r with context %r. ',
hook, context)
return fail
def handle_unserialize_failure(self, fail, raw):
type_name = raw.get('.type')
version = raw.get('.version')
if fail.check(ConflictError):
self.debug('Got conflict error when trying to upgrade the '
'document: %s version: %s. Refetching it.',
type_name, version)
# probably we've already upgraded it concurrently
return self.get_document(raw['_id'])
error.handle_failure(self, fail, 'Asynchronous hooks of '
'the upgrade failed. Raising NotMigratable')
return failure.Failure(NotMigratable((type_name, version, )))
def unserialize_list_of_documents(self, list_of_raw):
result = list()
defers = list()
for raw in list_of_raw:
if isinstance(raw, Exception):
# Exceptions are simply preserved. This behaviour is
# optionally used by bulk_get().
d = raw
else:
d = self.unserialize_document(raw)
if isinstance(d, defer.Deferred):
result.append(None)
index = len(result) - 1
d.addCallback(defer.inject_param, 1,
result.__setitem__, index)
defers.append(d)
else:
result.append(d)
if defers:
d = defer.DeferredList(defers, consumeErrors=True)
d.addCallback(defer.override_result, result)
return d
else:
return result
### private parts of update_document subroutine ###
def _iterate_on_update(self, _document, _method, args, keywords):
if IDocument.providedBy(_document):
doc_id = _document.doc_id
rev = _document.rev
else:
doc_id = _document['_id']
rev = _document['_rev']
try:
result = _method(_document, *args, **keywords)
except ResignFromModifying:
return _document
if result is None:
d = self.delete_document(_document)
else:
d = self.save_document(result)
if (IDocument.providedBy(_document) and
_document.conflict_resolution_strategy ==
ConflictResolutionStrategy.merge):
update_log = document.UpdateLog(
handler=_method,
args=args,
keywords=keywords,
rev_from=rev,
timestamp=time.time())
d.addCallback(lambda doc:
defer.DeferredList([defer.succeed(doc),
self.get_database_tag(),
self.get_update_seq()]))
d.addCallback(self._log_update, update_log)
d.addErrback(self._errback_on_update, doc_id,
_method, args, keywords)
return d
def _log_update(self, ((_s1, doc), (_s2, tag), (_s3, seq)), update_log):
update_log.rev_to = doc.rev
update_log.owner_id = doc.doc_id
update_log.seq_num = seq
update_log.partition_tag = tag
d = self.save_document(update_log)
d.addCallback(defer.override_result, doc)
return d
class Connection(log.Logger, log.LogProxy):
'''API for agency to call against the database.'''
type_name = 'db-connection'
implements(IDatabaseClient, ITimeProvider, IRevisionStore, ISerializable)
def __init__(self, database, unserializer=None):
log.Logger.__init__(self, database)
log.LogProxy.__init__(self, database)
self._database = IDatabaseDriver(database)
self._serializer = serialization.json.Serializer()
self._unserializer = (unserializer or
serialization.json.PaisleyUnserializer())
# listner_id -> doc_ids
self._listeners = dict()
self._change_cb = None
# Changed to use a normal dictionary.
# It will grow boundless up to the number of documents
# modified by the connection. It is a kind of memory leak
# done to temporarily resolve the problem of notifications
# received after the expiration time due to reconnection
# killing agents.
self._known_revisions = {} # {DOC_ID: (REV_INDEX, REV_HASH)}
### IRevisionStore ###
@property
def known_revisions(self):
return self._known_revisions
### ITimeProvider ###
def get_time(self):
return time.time()
### IDatabaseClient ###
@serialization.freeze_tag('IDatabaseClient.create_database')
def create_database(self):
return self._database.create_db()
@serialization.freeze_tag('IDatabaseClient.save_document')
@defer.inlineCallbacks
def save_document(self, doc):
doc = IDocument(doc)
serialized = self._serializer.convert(doc)
resp = yield self._database.save_doc(serialized, doc.doc_id)
self._update_id_and_rev(resp, doc)
for name, attachment in doc.get_attachments().iteritems():
if not attachment.saved:
resp = yield self._database.save_attachment(
doc.doc_id, doc.rev, attachment)
self._update_id_and_rev(resp, doc)
attachment.set_saved()
defer.returnValue(doc)
@serialization.freeze_tag('IDatabaseClient.get_attachment_body')
def get_attachment_body(self, attachment):
d = self._database.get_attachment(attachment.doc_id, attachment.name)
return d
@serialization.freeze_tag('IDatabaseClient.get_document')
def get_document(self, doc_id):
d = self._database.open_doc(doc_id)
d.addCallback(self._unserializer.convert)
d.addCallback(self._notice_doc_revision)
return d
@serialization.freeze_tag('IDatabaseClient.get_revision')
def get_revision(self, doc_id):
# FIXME: this could be done by lightweight HEAD request
d = self._database.open_doc(doc_id)
d.addCallback(lambda doc: doc['_rev'])
return d
@serialization.freeze_tag('IDatabaseClient.reload_database')
def reload_document(self, doc):
assert IDocument.providedBy(doc), \
"Incorrect type: %r, expected IDocument" % (type(doc), )
return self.get_document(doc.doc_id)
@serialization.freeze_tag('IDatabaseClient.delete_document')
def delete_document(self, doc):
assert isinstance(doc, document.Document), type(doc)
d = self._database.delete_doc(doc.doc_id, doc.rev)
d.addCallback(self._update_id_and_rev, doc)
return d
@serialization.freeze_tag('IDatabaseClient.changes_listener')
def changes_listener(self, filter_, callback, **kwargs):
assert callable(callback)
r = RevisionAnalytic(self, callback)
d = self._database.listen_changes(filter_, r.on_change, kwargs)
def set_listener_id(l_id, filter_):
self._listeners[l_id] = filter_
return l_id
d.addCallback(set_listener_id, filter_)
return d
@serialization.freeze_tag('IDatabaseClient.cancel_listener')
@journal.named_side_effect('IDatabaseClient.cancel_listener')
def cancel_listener(self, filter_):
for l_id, listener_filter in self._listeners.items():
if ((IViewFactory.providedBy(listener_filter) and
filter_ is listener_filter) or
(isinstance(listener_filter, (list, tuple)) and
(filter_ in listener_filter))):
self._cancel_listener(l_id)
@serialization.freeze_tag('IDatabaseClient.query_view')
def query_view(self, factory, **options):
factory = IViewFactory(factory)
d = self._database.query_view(factory, **options)
d.addCallback(self._parse_view_results, factory, options)
return d
@serialization.freeze_tag('IDatabaseClient.disconnect')
@journal.named_side_effect('IDatabaseClient.disconnect')
def disconnect(self):
if hasattr(self, '_query_cache'):
self._query_cache.empty()
for l_id in self._listeners.keys():
self._cancel_listener(l_id)
@serialization.freeze_tag('IDatabaseClient.get_update_seq')
def get_update_seq(self):
return self._database.get_update_seq()
@serialization.freeze_tag('IDatabaseClient.get_changes')
def get_changes(self, filter_=None, limit=None, since=0):
if IViewFactory.providedBy(filter_):
filter_ = ViewFilter(filter_, params=dict())
elif filter_ is not None:
raise ValueError("%r should provide IViewFacory" % (filter_, ))
return self._database.get_changes(filter_, limit, since)
@serialization.freeze_tag('IDatabaseClient.bulk_get')
def bulk_get(self, doc_ids, consume_errors=True):
def parse_bulk_response(resp):
assert isinstance(resp, dict), repr(resp)
assert 'rows' in resp, repr(resp)
result = list()
for doc_id, row in zip(doc_ids, resp['rows']):
if 'error' in row or 'deleted' in row['value']:
if not consume_errors:
result.append(NotFoundError(doc_id))
else:
self.debug("Bulk get parser consumed error row: %r",
row)
else:
result.append(self._unserializer.convert(row['doc']))
return result
d = self._database.bulk_get(doc_ids)
d.addCallback(parse_bulk_response)
return d
### public method used by query mechanism ###
def get_query_cache(self, create=True):
'''Called by methods inside feat.database.query module to obtain
the query cache.
@param create: C{bool} if True cache will be initialized if it doesnt
exist yet, returns None otherwise
'''
if not hasattr(self, '_query_cache'):
if create:
self._query_cache = query.Cache(self)
else:
return None
return self._query_cache
### ISerializable Methods ###
def snapshot(self):
return None
### private
def _cancel_listener(self, lister_id):
self._database.cancel_listener(lister_id)
try:
del(self._listeners[lister_id])
except KeyError:
self.warning('Tried to remove nonexistining listener id %r.',
lister_id)
def _parse_view_results(self, rows, factory, options):
'''
rows here should be a list of tuples:
- (key, value) for reduce views
- (key, value, id) for nonreduce views without include docs
- (key, value, id, doc) for nonreduce with with include docs
'''
reduced = factory.use_reduce and options.get('reduce', True)
include_docs = options.get('include_docs', False)
return factory.parse_view_result(rows, reduced, include_docs)
def _update_id_and_rev(self, resp, doc):
doc.doc_id = unicode(resp.get('id', None))
doc.rev = unicode(resp.get('rev', None))
self._notice_doc_revision(doc)
return doc
def _notice_doc_revision(self, doc):
self.log('Storing knowledge about doc rev. ID: %r, REV: %r',
doc.doc_id, doc.rev)
self._known_revisions[doc.doc_id] = _parse_doc_revision(doc.rev)
return doc
