def get(self, *q_objs, **query):
"""Retrieve the the matching object raising
:class:`~mongoengine.queryset.MultipleObjectsReturned` or
`DocumentName.MultipleObjectsReturned` exception if multiple results
and :class:`~mongoengine.queryset.DoesNotExist` or
`DocumentName.DoesNotExist` if no results are found.
"""
queryset = self.clone()
queryset = queryset.order_by().limit(2)
queryset = queryset.filter(*q_objs, **query)
future = get_future(self)
def _get_cb(done_future):
docs = done_future.result()
if len(docs) < 1:
msg = ("%s matching query does not exist." %
queryset._document._class_name)
future.set_exception(queryset._document.DoesNotExist(msg))
elif len(docs) > 1:
msg = 'More than 1 item returned'
future.set_exception(
queryset._document.MultipleObjectsReturned(msg))
else:
future.set_result(docs[0])
list_future = queryset.to_list(length=2)
list_future.add_done_callback(_get_cb) # pragma no cover
return future
def aggregate_sum(self, field):
"""Sum over the values of the specified field.
:param field: the field to sum over; use dot-notation to refer to
embedded document fields
This method is more performant than the regular `sum`, because it uses
the aggregation framework instead of map-reduce.
"""
cursor = self._document._get_collection().aggregate([{
'$match':
self._query
}, {
'$group': {
'_id': 'sum',
'total': {
'$sum': '$' + field
}
}
}])
fn_future = cursor.fetch_next
future = get_future(self)
def sum_cb(fn_future):
if fn_future.result():
doc = cursor.next_object()
r = doc['total']
else:
r = 0
future.set_result(r)
fn_future.add_done_callback(sum_cb)
return future
def test_get_future(self):
class TestClass:
@classmethod
def _get_db(cls):
db = Mock()
db._framework = asyncio_framework
return db
future = metaprogramming.get_future(TestClass())
self.assertIsInstance(future, Future)
def sum(self, field):
"""Sum over the values of the specified field.
:param field: the field to sum over; use dot-notation to refer to
embedded document fields
"""
map_func = """
function() {
var path = '{{~%(field)s}}'.split('.'),
field = this;
for (p in path) {
if (typeof field != 'undefined')
field = field[path[p]];
else
break;
}
if (field && field.constructor == Array) {
field.forEach(function(item) {
emit(1, item||0);
});
} else if (typeof field != 'undefined') {
emit(1, field||0);
}
}
""" % dict(field=field)
reduce_func = Code("""
function(key, values) {
var sum = 0;
for (var i in values) {
sum += values[i];
}
return sum;
}
""")
mr_future = self.inline_map_reduce(map_func, reduce_func)
future = get_future(self)
def sum_cb(mr_future):
results = mr_future.result()
for result in results:
r = result.value
break
else:
r = 0
future.set_result(r)
mr_future.add_done_callback(sum_cb)
return future
def modify(self, query={}, **update):
"""Perform an atomic update of the document in the database and reload
the document object using updated version.
Returns True if the document has been updated or False if the document
in the database doesn't match the query.
.. note:: All unsaved changes that have been made to the document are
rejected if the method returns True.
:param query: the update will be performed only if the document in the
database matches the query
:param update: Django-style update keyword arguments
"""
if self.pk is None:
raise InvalidDocumentError(
"The document does not have a primary key.")
id_field = self._meta["id_field"]
query = query.copy() if isinstance(query,
dict) else query.to_query(self)
if id_field not in query:
query[id_field] = self.pk
elif query[id_field] != self.pk:
msg = "Invalid document modify query: "
msg += "it must modify only this document."
raise InvalidQueryError(msg)
updated_future = self._qs(**query).modify(new=True, **update)
ret_future = get_future(self)
def updated_cb(updated_future):
try:
updated = updated_future.result()
if updated is None:
ret_future.set_result(False)
return
for field in self._fields_ordered:
setattr(self, field, self._reload(field, updated[field]))
self._changed_fields = updated._changed_fields
self._created = False
ret_future.set_result(True)
return
except Exception as e:
ret_future.set_exception(e)
updated_future.add_done_callback(updated_cb)
return ret_future
def inline_map_reduce(self, map_f, reduce_f, **mr_kwargs):
"""Perform a map/reduce query using the current query spec
and ordering. While ``map_reduce`` respects ``QuerySet`` chaining,
it must be the last call made, as it does not return a maleable
``QuerySet``.
:param map_f: map function, as :class:`~bson.code.Code` or string
:param reduce_f: reduce function, as
:class:`~bson.code.Code` or string
:param mr_kwargs: Arguments for mongodb map_reduce
see: https://docs.mongodb.com/manual/reference/command/mapReduce/
for more information
Returns a generator of MapReduceDocument with the map/reduce results.
.. note::
This method only works with inline map/reduce. If you want to
send the output to a collection use
:meth:`~SanicMongo.queryset.Queryset.map_reduce`.
"""
queryset = self.clone()
if mr_kwargs.get('out') and mr_kwargs.get('out') != 'inline':
msg = 'inline_map_reduce only supports inline output. '
msg += 'To send the result to a collection use map_reduce'
raise OperationError(msg)
map_f = self._get_code(map_f)
reduce_f = self._get_code(reduce_f)
mr_kwargs.update({'query': queryset._query})
if mr_kwargs.get('finalize'):
mr_kwargs['finalize'] = self._get_code(mr_kwargs['finalize'])
mr_future = queryset._collection.inline_map_reduce(
map_f, reduce_f, **mr_kwargs)
future = get_future(self)
def inline_mr_cb(result_future):
result = result_future.result()
gen = (MapReduceDocument(queryset._document, queryset._collection,
doc['_id'], doc['value'])
for doc in result)
future.set_result(gen)
mr_future.add_done_callback(inline_mr_cb)
return future
def insert(self, doc_or_docs, load_bulk=True, write_concern=None):
"""bulk insert documents
:param doc_or_docs: a document or list of documents to be inserted
:param load_bulk (optional): If True returns the list of document
instances
:param write_concern: Extra keyword arguments are passed down to
:meth:`~pymongo.collection.Collection.insert`
which will be used as options for the resultant
``getLastError`` command. For example,
``insert(..., {w: 2, fsync: True})`` will wait until at least
two servers have recorded the write and will force an fsync on
each server being written to.
By default returns document instances, set ``load_bulk`` to False to
return just ``ObjectIds``
"""
super_insert = BaseQuerySet.insert
async_in_bulk = self.in_bulk
# this sync method is not really sync, it uses motor sockets and
# greenlets events, but looks like sync, so...
sync_in_bulk = functools.partial(self.in_bulk.__wrapped__, self)
insert_future = get_future(self)
with MonkeyPatcher() as patcher:
# here we change the method with the async api for the method
# with a sync api so I don't need to rewrite the mongoengine
# method.
patcher.patch_item(self, 'in_bulk', sync_in_bulk, undo=False)
future = asynchronize(super_insert)(self,
doc_or_docs,
load_bulk=load_bulk,
write_concern=write_concern)
def cb(future):
try:
result = future.result()
insert_future.set_result(result)
except Exception as e:
insert_future.set_exception(e)
finally:
patcher.patch_item(self,
'in_bulk',
async_in_bulk,
undo=False)
future.add_done_callback(cb)
return insert_future
def replace(self, file_obj, **kwargs):
del_future = self.delete()
ret_future = get_future(self)
def del_cb(del_future):
put_future = self.put(file_obj, **kwargs)
def put_cb(put_future):
result = put_future.result()
ret_future.set_result(result)
put_future.add_done_callback(put_cb)
del_future.add_done_callback(del_cb)
return ret_future
def upsert_one(self, write_concern=None, **update):
"""Overwrite or add the first document matched by the query.
:param write_concern: Extra keyword arguments are passed down which
will be used as options for the resultant
``getLastError`` command. For example,
``save(..., write_concern={w: 2, fsync: True}, ...)`` will
wait until at least two servers have recorded the write and
will force an fsync on the primary server.
:param update: Django-style update keyword arguments
:returns the new or overwritten document
"""
update_future = self.update(multi=False,
upsert=True,
write_concern=write_concern,
full_result=True,
**update)
upsert_future = get_future(self)
def update_cb(update_future):
try:
result = update_future.result()
if result['updatedExisting']:
document_future = self.first()
else:
document_future = self._document.objects.with_id(
result['upserted'])
def doc_cb(document_future):
try:
result = document_future.result()
upsert_future.set_result(result)
except Exception as e:
upsert_future.set_exception(e)
document_future.add_done_callback(doc_cb)
except Exception as e:
upsert_future.set_exception(e)
update_future.add_done_callback(update_cb)
return upsert_future
def first(self):
"""Retrieve the first object matching the query.
"""
queryset = self.clone()
first_future = queryset[0]
future = get_future(self)
def first_cb(first_future):
try:
result = first_future.result()
future.set_result(result)
except IndexError:
result = None
future.set_result(result)
except Exception as e:
future.set_exception(e)
first_future.add_done_callback(first_cb)
return future
def aggregate_average(self, field):
"""Average over the values of the specified field.
:param field: the field to average over; use dot-notation to refer to
embedded document fields
This method is more performant than the regular `average`, because it
uses the aggregation framework instead of map-reduce.
"""
cursor = self._document._get_collection().aggregate([{
'$match':
self._query
}, {
'$group': {
'_id': 'avg',
'total': {
'$avg': '$' + field
}
}
}])
fn_future = cursor.fetch_next
future = get_future(self)
def fetch_next_cb(fn_future):
result = fn_future.result()
if result:
doc = cursor.next_object()
avg = doc['total']
else:
avg = 0
future.set_result(avg)
fn_future.add_done_callback(fetch_next_cb)
return future
def to_list(self, length=100):
"""Returns a list of the current documents in the queryset.
:param length: maximum number of documents to return for this call."""
list_future = get_future(self)
def _to_list_cb(future):
# Transforms mongo's raw documents into
# SanicMongo documents
docs_list = future.result()
final_list = [
self._document._from_son(
d,
_auto_dereference=self._auto_dereference,
only_fields=self.only_fields) for d in docs_list
]
list_future.set_result(final_list)
cursor = self._cursor
future = cursor.to_list(length)
future.add_done_callback(_to_list_cb)
return list_future
def _check_delete_rules(self, doc, queryset, cascade_refs, write_concern):
"""Checks the delete rules for documents being deleted in a queryset.
Raises an exception if any document has a DENY rule."""
delete_rules = doc._meta.get('delete_rules') or {}
# Check for DENY rules before actually deleting/nullifying any other
# references
for rule_entry in delete_rules:
document_cls, field_name = rule_entry
if document_cls._meta.get('abstract'):
continue
rule = doc._meta['delete_rules'][rule_entry]
if rule == DENY and document_cls.objects(**{
field_name + '__in': self
}).count() > 0:
msg = ("Could not delete document (%s.%s refers to it)" %
(document_cls.__name__, field_name))
raise OperationError(msg)
ret_future = get_future(self)
# We need to set result for the future if there's no rules otherwise
# the callbacks will never be called.
if not delete_rules:
ret_future.set_result(None)
for rule_entry in delete_rules:
document_cls, field_name = rule_entry
if document_cls._meta.get('abstract'):
continue
rule = doc._meta['delete_rules'][rule_entry]
if rule == CASCADE:
cascade_refs = set() if cascade_refs is None else cascade_refs
for ref in queryset:
cascade_refs.add(ref.id)
ref_q = document_cls.objects(**{
field_name + '__in': self,
'id__nin': cascade_refs
})
ref_q_count_future = ref_q.count()
def count_cb(count_future):
try:
count = count_future.result()
if count > 0:
del_future = ref_q.delete(
write_concern=write_concern,
cascade_refs=cascade_refs)
def del_cb(del_future):
try:
r = del_future.result()
ret_future.set_result(r)
except Exception as e:
ret_future.set_exception(e)
del_future.add_done_callback(del_cb)
except Exception as e:
ret_future.set_exception(e)
ref_q_count_future.add_done_callback(count_cb)
elif rule in (NULLIFY, PULL):
if rule == NULLIFY:
updatekw = {'unset__%s' % field_name: 1}
else:
updatekw = {'pull_all__%s' % field_name: self}
update_future = document_cls.objects(**{
field_name + '__in': self
}).update(write_concern=write_concern, **updatekw)
def update_cb(update_future):
try:
result = update_future.result()
ret_future.set_result(result)
except Exception as e:
ret_future.set_exception(e)
update_future.add_done_callback(update_cb)
return ret_future
def average(self, field):
"""Average over the values of the specified field.
:param field: the field to average over; use dot-notation to refer to
embedded document fields
"""
map_func = """
function() {
var path = '{{~%(field)s}}'.split('.'),
field = this;
for (p in path) {
if (typeof field != 'undefined')
field = field[path[p]];
else
break;
}
if (field && field.constructor == Array) {
field.forEach(function(item) {
emit(1, {t: item||0, c: 1});
});
} else if (typeof field != 'undefined') {
emit(1, {t: field||0, c: 1});
}
}
""" % dict(field=field)
reduce_func = Code("""
function(key, values) {
var out = {t: 0, c: 0};
for (var i in values) {
var value = values[i];
out.t += value.t;
out.c += value.c;
}
return out;
}
""")
finalize_func = Code("""
function(key, value) {
return value.t / value.c;
}
""")
future = get_future(self)
mr_future = self.inline_map_reduce(map_func,
reduce_func,
finalize=finalize_func)
def average_cb(mr_future):
results = mr_future.result()
for result in results:
average = result.value
break
else:
average = 0
future.set_result(average)
mr_future.add_done_callback(average_cb)
return future
def item_frequencies(self, field, normalize=False):
"""Returns a dictionary of all items present in a field across
the whole queried set of documents, and their corresponding frequency.
This is useful for generating tag clouds, or searching documents.
.. note::
Can only do direct simple mappings and cannot map across
:class:`~mongoengine.fields.ReferenceField` or
:class:`~mongoengine.fields.GenericReferenceField` for more complex
counting a manual map reduce call would is required.
If the field is a :class:`~mongoengine.fields.ListField`,
the items within each list will be counted individually.
:param field: the field to use
:param normalize: normalize the results so they add to 1.0
:param map_reduce: Use map_reduce over exec_js
"""
map_func = """
function() {
var path = '{{~%(field)s}}'.split('.');
var field = this;
for (p in path) {
if (typeof field != 'undefined')
field = field[path[p]];
else
break;
}
if (field && field.constructor == Array) {
field.forEach(function(item) {
emit(item, 1);
});
} else if (typeof field != 'undefined') {
emit(field, 1);
} else {
emit(null, 1);
}
}
""" % dict(field=field)
reduce_func = """
function(key, values) {
var total = 0;
var valuesSize = values.length;
for (var i=0; i < valuesSize; i++) {
total += parseInt(values[i], 10);
}
return total;
}
"""
mr_future = self.inline_map_reduce(map_func, reduce_func)
future = get_future(self)
def item_frequencies_cb(mr_future):
values = mr_future.result()
frequencies = {}
for f in values:
key = f.key
if isinstance(key, float):
if int(key) == key:
key = int(key)
frequencies[key] = int(f.value)
if normalize:
count = sum(frequencies.values())
frequencies = dict([(k, float(v) / count)
for k, v in list(frequencies.items())])
future.set_result(frequencies)
mr_future.add_done_callback(item_frequencies_cb)
return future
def delete(self,
write_concern=None,
_from_doc_delete=False,
cascade_refs=None):
"""Deletes the documents matched by the query.
:param write_concern: Extra keyword arguments are passed down which
will be used as options for the resultant
``getLastError`` command. For example,
``save(..., write_concern={w: 2, fsync: True}, ...)`` will
wait until at least two servers have recorded the write and
will force an fsync on the primary server.
:param _from_doc_delete: True when called from document delete
therefore signals will have been triggered so don't loop.
:returns number of deleted documents
"""
queryset = self.clone()
doc = queryset._document
if write_concern is None:
write_concern = {}
# Handle deletes where skips or limits have been applied or
# there is an untriggered delete signal
has_delete_signal = signals.signals_available and (
signals.pre_delete.has_receivers_for(self._document)
or signals.post_delete.has_receivers_for(self._document))
call_document_delete = (queryset._skip or queryset._limit
or has_delete_signal) and not _from_doc_delete
if call_document_delete:
async_method = asynchronize(self._document_delete)
return async_method(queryset, write_concern)
dr_future = self._check_delete_rules(doc, queryset, cascade_refs,
write_concern)
ret_future = get_future(self)
def dr_cb(dr_future):
"""callback for _check_delete_rules future"""
try:
dr_future.result()
remove_future = queryset._collection.remove(
queryset._query, **write_concern)
def r_cb(remove_future):
"""Callback for _collection.remove"""
try:
result = remove_future.result()
if result:
ret_future.set_result(result.get('n'))
except Exception as e:
ret_future.set_exception(e)
remove_future.add_done_callback(r_cb)
except Exception as e:
ret_future.set_exception(e)
dr_future.add_done_callback(dr_cb)
return ret_future
