def _infer_geometry(value):
"""Helper method that tries to infer the $geometry shape for a
given value.
"""
if isinstance(value, dict):
if '$geometry' in value:
return value
elif 'coordinates' in value and 'type' in value:
return {'$geometry': value}
raise InvalidQueryError('Invalid $geometry dictionary should have '
'type and coordinates keys')
elif isinstance(value, (list, set)):
# TODO: shouldn't we test value[0][0][0][0] to see if it is MultiPolygon?
# TODO: should both TypeError and IndexError be alike interpreted?
try:
value[0][0][0]
return {'$geometry': {'type': 'Polygon', 'coordinates': value}}
except (TypeError, IndexError):
pass
try:
value[0][0]
return {'$geometry': {'type': 'LineString', 'coordinates': value}}
except (TypeError, IndexError):
pass
try:
value[0]
return {'$geometry': {'type': 'Point', 'coordinates': value}}
except (TypeError, IndexError):
pass
raise InvalidQueryError('Invalid $geometry data. Can be either a '
'dictionary or (nested) lists of coordinate(s)')
def _infer_geometry(value):
"""Helper method that tries to infer the $geometry shape for a given value"""
if isinstance(value, dict):
if "$geometry" in value:
return value
elif 'coordinates' in value and 'type' in value:
return {"$geometry": value}
raise InvalidQueryError("Invalid $geometry dictionary should have "
"type and coordinates keys")
elif isinstance(value, (list, set)):
try:
value[0][0][0]
return {"$geometry": {"type": "Polygon", "coordinates": value}}
except:
pass
try:
value[0][0]
return {"$geometry": {"type": "LineString", "coordinates": value}}
except:
pass
try:
value[0]
return {"$geometry": {"type": "Point", "coordinates": value}}
except:
pass
raise InvalidQueryError(
"Invalid $geometry data. Can be either a dictionary "
"or (nested) lists of coordinate(s)")
def _infer_geometry(value):
"""Helper method that tries to infer the $geometry shape for a
given value.
"""
if isinstance(value, dict):
if "$geometry" in value:
return value
elif "coordinates" in value and "type" in value:
return {"$geometry": value}
raise InvalidQueryError(
"Invalid $geometry dictionary should have type and coordinates keys"
)
elif isinstance(value, (list, set)):
# TODO: shouldn't we test value[0][0][0][0] to see if it is MultiPolygon?
try:
value[0][0][0]
return {"$geometry": {"type": "Polygon", "coordinates": value}}
except (TypeError, IndexError):
pass
try:
value[0][0]
return {"$geometry": {"type": "LineString", "coordinates": value}}
except (TypeError, IndexError):
pass
try:
value[0]
return {"$geometry": {"type": "Point", "coordinates": value}}
except (TypeError, IndexError):
pass
raise InvalidQueryError("Invalid $geometry data. Can be either a "
"dictionary or (nested) lists of coordinate(s)")
def purge_execution_output_objects(logger, timestamp, action_ref=None):
"""
Purge action executions output objects.
:param timestamp: Objects older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
:param action_ref: Only delete objects for the provided actions.
:type action_ref: ``str``
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging action execution output objects older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
filters = {}
filters['timestamp__lt'] = timestamp
if action_ref:
filters['action_ref'] = action_ref
try:
deleted_count = ActionExecutionOutput.delete_by_query(**filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution output instances: %s'
'Please contact support.' % (filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of execution output models failed for query with filters: %s.',
filters)
else:
logger.info('Deleted %s execution output objects' % (deleted_count))
def purge_trigger_instances(logger, timestamp):
"""
:param timestamp: Trigger instances older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging trigger instances older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
query_filters = {'occurrence_time__lt': isotime.parse(timestamp)}
try:
deleted_count = TriggerInstance.delete_by_query(**query_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete trigger instances: %s'
'Please contact support.' % (query_filters, str(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deleting instances using query_filters %s failed.', query_filters)
else:
logger.info('Deleted %s trigger instance objects' % (deleted_count))
# Print stats
logger.info('All trigger instance models older than timestamp %s were deleted.', timestamp)
def purge_workflow_executions(logger, timestamp, purge_incomplete=False):
"""
Purge workflow execution output objects.
:param timestamp: Exections older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
:param purge_incomplete: True to also delete executions which are not in a done state.
:type purge_incomplete: ``bool``
"""
if not timestamp:
raise ValueError("Specify a valid timestamp to purge.")
logger.info("Purging workflow executions older than timestamp: %s" %
timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ"))
filters = {}
if purge_incomplete:
filters["start_timestamp__lt"] = timestamp
else:
filters["end_timestamp__lt"] = timestamp
filters["start_timestamp__lt"] = timestamp
filters["status"] = {"$in": DONE_STATES}
exec_filters = copy.copy(filters)
# 1. Delete Workflow Execution objects
try:
# Note: We call list() on the query set object because it's lazyily evaluated otherwise
# to_delete_execution_dbs = list(WorkflowExecution.query(only_fields=['id'],
# no_dereference=True,
# **exec_filters))
deleted_count = WorkflowExecution.delete_by_query(**exec_filters)
except InvalidQueryError as e:
msg = ("Bad query (%s) used to delete workflow execution instances: %s"
"Please contact support." % (exec_filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception(
"Deletion of workflow execution models failed for query with filters: %s.",
exec_filters,
)
else:
logger.info("Deleted %s workflow execution objects" % deleted_count)
zombie_execution_instances = len(
WorkflowExecution.query(only_fields=["id"],
no_dereference=True,
**exec_filters))
if zombie_execution_instances > 0:
logger.error("Zombie workflow execution instances left: %d.",
zombie_execution_instances)
# Print stats
logger.info(
"All workflow execution models older than timestamp %s were deleted.",
timestamp,
)
def purge_rule_enforcements(logger, timestamp):
"""
:param timestamp: Rule enforcement instances older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
"""
if not timestamp:
raise ValueError("Specify a valid timestamp to purge.")
logger.info("Purging rule enforcements older than timestamp: %s" %
timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ"))
query_filters = {"enforced_at__lt": isotime.parse(timestamp)}
try:
deleted_count = RuleEnforcement.delete_by_query(**query_filters)
except InvalidQueryError as e:
msg = ("Bad query (%s) used to delete rule enforcements: %s"
"Please contact support." % (
query_filters,
six.text_type(e),
))
raise InvalidQueryError(msg)
except:
logger.exception(
"Deleting rule enforcements using query_filters %s failed.",
query_filters)
else:
logger.info("Deleted %s rule enforcement objects" % (deleted_count))
# Print stats
logger.info(
"All rule enforcement models older than timestamp %s were deleted.",
timestamp)
def purge_trigger_instances(logger, timestamp):
"""
:param timestamp: Trigger instances older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging trigger instances older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
query_filters = {'occurrence_time__lt': isotime.parse(timestamp)}
# TODO: Update this code to return statistics on deleted objects once we
# upgrade to newer version of MongoDB where delete_by_query actually returns
# some data
try:
TriggerInstance.delete_by_query(**query_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete trigger instances: %s'
'Please contact support.' % (query_filters, str(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deleting instances using query_filters %s failed.',
query_filters)
# Print stats
logger.info(
'All trigger instance models older than timestamp %s were deleted.',
timestamp)
async def delete(request):
try:
car = Car.objects(VIN=request.match_info['vin'])
if not car.delete():
raise InvalidQueryError('Unknown car with this VIN')
return web.json_response({'msg': 'Car deleted'}, status=204)
except Exception as e:
return web.json_response({'error': str(e)}, status=400)
async def update_car(request):
car_data = await request.json()
try:
if car_data.get('VIN'):
raise InvalidQueryError('Field VIN is Unchanged')
Car.objects(VIN=request.match_info['vin']).update(**car_data)
return web.json_response({'msg': 'Car updated'}, status=201)
except Exception as e:
return web.json_response({'msg': str(e)}, status=400)
def modify(self, query={}, **update):
"""Perform an atomic update of the document in the database and reload
the document object using updated version.
Returns True if the document has been updated or False if the document
in the database doesn't match the query.
.. note:: All unsaved changes that have been made to the document are
rejected if the method returns True.
:param query: the update will be performed only if the document in the
database matches the query
:param update: Django-style update keyword arguments
"""
if self.pk is None:
raise InvalidDocumentError(
"The document does not have a primary key.")
id_field = self._meta["id_field"]
query = query.copy() if isinstance(query,
dict) else query.to_query(self)
if id_field not in query:
query[id_field] = self.pk
elif query[id_field] != self.pk:
msg = "Invalid document modify query: "
msg += "it must modify only this document."
raise InvalidQueryError(msg)
updated_future = self._qs(**query).modify(new=True, **update)
ret_future = get_future(self)
def updated_cb(updated_future):
try:
updated = updated_future.result()
if updated is None:
ret_future.set_result(False)
return
for field in self._fields_ordered:
setattr(self, field, self._reload(field, updated[field]))
self._changed_fields = updated._changed_fields
self._created = False
ret_future.set_result(True)
return
except Exception as e:
ret_future.set_exception(e)
updated_future.add_done_callback(updated_cb)
return ret_future
def _query_conjunction(self, queries):
"""Merges query dicts - effectively &ing them together.
"""
query_ops = set()
combined_query = {}
for query in queries:
ops = set(query.keys())
# Make sure that the same operation isn't applied more than once
# to a single field
intersection = ops.intersection(query_ops)
if intersection:
msg = 'Duplicate query conditions: '
raise InvalidQueryError(msg + ', '.join(intersection))
query_ops.update(ops)
combined_query.update(copy.deepcopy(query))
return combined_query
def modify(self, query=None, **update):
"""Perform an atomic update of the document in the database and reload
the document object using updated version.
Returns True if the document has been updated or False if the document
in the database doesn't match the query.
.. note:: All unsaved changes that have been made to the document are
rejected if the method returns True.
:param query: the update will be performed only if the document in the
database matches the query
:param update: Django-style update keyword arguments
"""
if query is None:
query = {}
if self.pk is None:
raise InvalidDocumentError(
'The document does not have a primary key.')
id_field = self._meta['id_field']
query = query.copy() if isinstance(query,
dict) else query.to_query(self)
if id_field not in query:
query[id_field] = self.pk
elif query[id_field] != self.pk:
raise InvalidQueryError(
'Invalid document modify query: it must modify only this document.'
)
# Need to add shard key to query, or you get an error
query.update(self._object_key)
updated = self._qs(**query).modify(new=True, **update)
if updated is None:
return False
for field in self._fields_ordered:
setattr(self, field, self._reload(field, updated[field]))
self._changed_fields = updated._changed_fields
self._created = False
return True
def update(_doc_cls=None, **update):
"""Transform an update spec from Django-style format to Mongo format.
"""
mongo_update = {}
for key, value in update.items():
if key == "__raw__":
mongo_update.update(value)
continue
parts = key.split('__')
# Check for an operator and transform to mongo-style if there is
op = None
if parts[0] in UPDATE_OPERATORS:
op = parts.pop(0)
# Convert Pythonic names to Mongo equivalents
if op in ('push_all', 'pull_all'):
op = op.replace('_all', 'All')
elif op == 'dec':
# Support decrement by flipping a positive value's sign
# and using 'inc'
op = 'inc'
if value > 0:
value = -value
elif op == 'add_to_set':
op = 'addToSet'
elif op == 'set_on_insert':
op = "setOnInsert"
match = None
if parts[-1] in COMPARISON_OPERATORS:
match = parts.pop()
if _doc_cls:
# Switch field names to proper names [set in Field(name='foo')]
try:
fields = _doc_cls._lookup_field(parts)
except Exception, e:
raise InvalidQueryError(e)
parts = []
cleaned_fields = []
for field in fields:
append_field = True
if isinstance(field, basestring):
# Convert the S operator to $
if field == 'S':
field = '$'
parts.append(field)
append_field = False
else:
parts.append(field.db_field)
if append_field:
cleaned_fields.append(field)
# Convert value to proper value
field = cleaned_fields[-1]
if op in (None, 'set', 'push', 'pull'):
if field.required or value is not None:
value = field.prepare_query_value(op, value)
elif op in ('pushAll', 'pullAll'):
value = [field.prepare_query_value(op, v) for v in value]
elif op == 'addToSet':
if isinstance(value, (list, tuple, set)):
value = [field.prepare_query_value(op, v) for v in value]
elif field.required or value is not None:
value = field.prepare_query_value(op, value)
if match:
match = '$' + match
value = {match: value}
key = '.'.join(parts)
if not op:
raise InvalidQueryError("Updates must supply an operation "
"eg: set__FIELD=value")
if 'pull' in op and '.' in key:
# Dot operators don't work on pull operations
# it uses nested dict syntax
if op == 'pullAll':
raise InvalidQueryError("pullAll operations only support "
"a single field depth")
parts.reverse()
for key in parts:
value = {key: value}
elif op == 'addToSet' and isinstance(value, list):
value = {key: {"$each": value}}
else:
value = {key: value}
key = '$' + op
if key not in mongo_update:
mongo_update[key] = value
elif key in mongo_update and isinstance(mongo_update[key], dict):
mongo_update[key].update(value)
def query(_doc_cls=None, **kwargs):
"""Transform a query from Django-style format to Mongo format."""
mongo_query = {}
merge_query = defaultdict(list)
for key, value in sorted(kwargs.items()):
if key == '__raw__':
mongo_query.update(value)
continue
parts = key.rsplit('__')
indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()]
parts = [part for part in parts if not part.isdigit()]
# Check for an operator and transform to mongo-style if there is
op = None
if len(parts) > 1 and parts[-1] in MATCH_OPERATORS:
op = parts.pop()
# Allow to escape operator-like field name by __
if len(parts) > 1 and parts[-1] == '':
parts.pop()
negate = False
if len(parts) > 1 and parts[-1] == 'not':
parts.pop()
negate = True
if _doc_cls:
# Switch field names to proper names [set in Field(name='foo')]
try:
fields = _doc_cls._lookup_field(parts)
except Exception as e:
raise InvalidQueryError(e)
parts = []
CachedReferenceField = _import_class('CachedReferenceField')
GenericReferenceField = _import_class('GenericReferenceField')
cleaned_fields = []
for field in fields:
append_field = True
if isinstance(field, six.string_types):
parts.append(field)
append_field = False
# is last and CachedReferenceField
elif isinstance(field, CachedReferenceField) and fields[-1] == field:
parts.append('%s._id' % field.db_field)
else:
parts.append(field.db_field)
if append_field:
cleaned_fields.append(field)
# Convert value to proper value
field = cleaned_fields[-1]
singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not']
singular_ops += STRING_OPERATORS
if op in singular_ops:
if isinstance(field, six.string_types):
if (op in STRING_OPERATORS and
isinstance(value, six.string_types)):
StringField = _import_class('StringField')
value = StringField.prepare_query_value(op, value)
else:
value = field
else:
value = field.prepare_query_value(op, value)
if isinstance(field, CachedReferenceField) and value:
value = value['_id']
elif op in ('in', 'nin', 'all', 'near') and not isinstance(value, dict):
# Raise an error if the in/nin/all/near param is not iterable.
value = _prepare_query_for_iterable(field, op, value)
# If we're querying a GenericReferenceField, we need to alter the
# key depending on the value:
# * If the value is a DBRef, the key should be "field_name._ref".
# * If the value is an ObjectId, the key should be "field_name._ref.$id".
if isinstance(field, GenericReferenceField):
if isinstance(value, DBRef):
parts[-1] += '._ref'
elif isinstance(value, ObjectId):
parts[-1] += '._ref.$id'
# if op and op not in COMPARISON_OPERATORS:
if op:
if op in GEO_OPERATORS:
value = _geo_operator(field, op, value)
elif op in ('match', 'elemMatch'):
ListField = _import_class('ListField')
EmbeddedDocumentField = _import_class('EmbeddedDocumentField')
if (
isinstance(value, dict) and
isinstance(field, ListField) and
isinstance(field.field, EmbeddedDocumentField)
):
value = query(field.field.document_type, **value)
else:
value = field.prepare_query_value(op, value)
value = {'$elemMatch': value}
elif op in CUSTOM_OPERATORS:
NotImplementedError('Custom method "%s" has not '
'been implemented' % op)
elif op not in STRING_OPERATORS:
value = {'$' + op: value}
if negate:
value = {'$not': value}
for i, part in indices:
parts.insert(i, part)
key = '.'.join(parts)
if op is None or key not in mongo_query:
mongo_query[key] = value
elif key in mongo_query:
if isinstance(mongo_query[key], dict):
mongo_query[key].update(value)
# $max/minDistance needs to come last - convert to SON
value_dict = mongo_query[key]
if ('$maxDistance' in value_dict or '$minDistance' in value_dict) and \
('$near' in value_dict or '$nearSphere' in value_dict):
value_son = SON()
for k, v in value_dict.iteritems():
if k == '$maxDistance' or k == '$minDistance':
continue
value_son[k] = v
# Required for MongoDB >= 2.6, may fail when combining
# PyMongo 3+ and MongoDB < 2.6
near_embedded = False
for near_op in ('$near', '$nearSphere'):
if isinstance(value_dict.get(near_op), dict) and (
IS_PYMONGO_3 or get_connection().max_wire_version > 1):
value_son[near_op] = SON(value_son[near_op])
if '$maxDistance' in value_dict:
value_son[near_op][
'$maxDistance'] = value_dict['$maxDistance']
if '$minDistance' in value_dict:
value_son[near_op][
'$minDistance'] = value_dict['$minDistance']
near_embedded = True
if not near_embedded:
if '$maxDistance' in value_dict:
value_son['$maxDistance'] = value_dict['$maxDistance']
if '$minDistance' in value_dict:
value_son['$minDistance'] = value_dict['$minDistance']
mongo_query[key] = value_son
else:
# Store for manually merging later
merge_query[key].append(value)
# The queryset has been filter in such a way we must manually merge
for k, v in merge_query.items():
merge_query[k].append(mongo_query[k])
del mongo_query[k]
if isinstance(v, list):
value = [{k: val} for val in v]
if '$and' in mongo_query.keys():
mongo_query['$and'].extend(value)
else:
mongo_query['$and'] = value
return mongo_query
def update(_doc_cls=None, **update):
"""Transform an update spec from Django-style format to Mongo format.
"""
mongo_update = {}
for key, value in update.items():
if key == "__raw__":
mongo_update.update(value)
continue
parts = key.split('__')
# Check for an operator and transform to mongo-style if there is
op = None
if parts[0] in UPDATE_OPERATORS:
op = parts.pop(0)
# Convert Pythonic names to Mongo equivalents
if op in ('push_all', 'pull_all'):
op = op.replace('_all', 'All')
elif op == 'dec':
# Support decrement by flipping a positive value's sign
# and using 'inc'
op = 'inc'
if value > 0:
value = -value
elif op == 'add_to_set':
op = 'addToSet'
elif op == 'set_on_insert':
op = "setOnInsert"
match = None
if parts[-1] in COMPARISON_OPERATORS:
match = parts.pop()
if _doc_cls:
# Switch field names to proper names [set in Field(name='foo')]
try:
fields = _doc_cls._lookup_field(parts)
except Exception, e:
raise InvalidQueryError(e)
parts = []
cleaned_fields = []
appended_sub_field = False
for field in fields:
append_field = True
if isinstance(field, basestring):
# Convert the S operator to $
if field == 'S':
field = '$'
parts.append(field)
append_field = False
else:
parts.append(field.db_field)
if append_field:
appended_sub_field = False
cleaned_fields.append(field)
if hasattr(field, 'field'):
cleaned_fields.append(field.field)
appended_sub_field = True
# Convert value to proper value
if appended_sub_field:
field = cleaned_fields[-2]
else:
field = cleaned_fields[-1]
GeoJsonBaseField = _import_class("GeoJsonBaseField")
if isinstance(field, GeoJsonBaseField):
value = field.to_mongo(value)
if op in (None, 'set', 'push', 'pull'):
if field.required or value is not None:
value = field.prepare_query_value(op, value)
elif op in ('pushAll', 'pullAll'):
value = [field.prepare_query_value(op, v) for v in value]
elif op in ('addToSet', 'setOnInsert'):
if isinstance(value, (list, tuple, set)):
value = [field.prepare_query_value(op, v) for v in value]
elif field.required or value is not None:
value = field.prepare_query_value(op, value)
elif op == "unset":
value = 1
if match:
match = '$' + match
value = {match: value}
key = '.'.join(parts)
if not op:
raise InvalidQueryError("Updates must supply an operation "
"eg: set__FIELD=value")
if 'pull' in op and '.' in key:
# Dot operators don't work on pull operations
# unless they point to a list field
# Otherwise it uses nested dict syntax
if op == 'pullAll':
raise InvalidQueryError("pullAll operations only support "
"a single field depth")
# Look for the last list field and use dot notation until there
field_classes = [c.__class__ for c in cleaned_fields]
field_classes.reverse()
ListField = _import_class('ListField')
if ListField in field_classes:
# Join all fields via dot notation to the last ListField
# Then process as normal
last_listField = len(cleaned_fields) - field_classes.index(
ListField)
key = ".".join(parts[:last_listField])
parts = parts[last_listField:]
parts.insert(0, key)
parts.reverse()
for key in parts:
value = {key: value}
elif op == 'addToSet' and isinstance(value, list):
value = {key: {"$each": value}}
else:
value = {key: value}
key = '$' + op
if key not in mongo_update:
mongo_update[key] = value
elif key in mongo_update and isinstance(mongo_update[key], dict):
mongo_update[key].update(value)
def query(_doc_cls=None, **query):
"""Transform a query from Django-style format to Mongo format.
"""
mongo_query = {}
merge_query = defaultdict(list)
for key, value in sorted(query.items()):
if key == "__raw__":
mongo_query.update(value)
continue
parts = key.rsplit('__')
indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()]
parts = [part for part in parts if not part.isdigit()]
# Check for an operator and transform to mongo-style if there is
op = None
if len(parts) > 1 and parts[-1] in MATCH_OPERATORS:
op = parts.pop()
# Allw to escape operator-like field name by __
if len(parts) > 1 and parts[-1] == "":
parts.pop()
negate = False
if len(parts) > 1 and parts[-1] == 'not':
parts.pop()
negate = True
if _doc_cls:
# Switch field names to proper names [set in Field(name='foo')]
try:
fields = _doc_cls._lookup_field(parts)
except Exception, e:
raise InvalidQueryError(e)
parts = []
CachedReferenceField = _import_class('CachedReferenceField')
cleaned_fields = []
for field in fields:
append_field = True
if isinstance(field, basestring):
parts.append(field)
append_field = False
# is last and CachedReferenceField
elif isinstance(field, CachedReferenceField) and fields[-1] == field:
parts.append('%s._id' % field.db_field)
else:
parts.append(field.db_field)
if append_field:
cleaned_fields.append(field)
# Convert value to proper value
field = cleaned_fields[-1]
singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not']
singular_ops += STRING_OPERATORS
if op in singular_ops:
if isinstance(field, basestring):
if (op in STRING_OPERATORS and
isinstance(value, basestring)):
StringField = _import_class('StringField')
value = StringField.prepare_query_value(op, value)
else:
value = field
else:
value = field.prepare_query_value(op, value)
if isinstance(field, CachedReferenceField) and value:
value = value['_id']
elif op in ('in', 'nin', 'all', 'near') and not isinstance(value, dict):
# 'in', 'nin' and 'all' require a list of values
value = [field.prepare_query_value(op, v) for v in value]
# if op and op not in COMPARISON_OPERATORS:
if op:
if op in GEO_OPERATORS:
value = _geo_operator(field, op, value)
elif op in CUSTOM_OPERATORS:
if op in ('elem_match', 'match'):
value = field.prepare_query_value(op, value)
value = {"$elemMatch": value}
else:
NotImplementedError("Custom method '%s' has not "
"been implemented" % op)
elif op not in STRING_OPERATORS:
value = {'$' + op: value}
if negate:
value = {'$not': value}
for i, part in indices:
parts.insert(i, part)
key = '.'.join(parts)
if op is None or key not in mongo_query:
mongo_query[key] = value
elif key in mongo_query:
if key in mongo_query and isinstance(mongo_query[key], dict):
mongo_query[key].update(value)
# $max/minDistance needs to come last - convert to SON
value_dict = mongo_query[key]
if ('$maxDistance' in value_dict or '$minDistance' in value_dict) and \
('$near' in value_dict or '$nearSphere' in value_dict):
value_son = SON()
for k, v in value_dict.iteritems():
if k == '$maxDistance' or k == '$minDistance':
continue
value_son[k] = v
# Required for MongoDB >= 2.6, may fail when combining
# PyMongo 3+ and MongoDB < 2.6
near_embedded = False
for near_op in ('$near', '$nearSphere'):
if isinstance(value_dict.get(near_op), dict) and (
IS_PYMONGO_3 or get_connection().max_wire_version > 1):
value_son[near_op] = SON(value_son[near_op])
if '$maxDistance' in value_dict:
value_son[near_op][
'$maxDistance'] = value_dict['$maxDistance']
if '$minDistance' in value_dict:
value_son[near_op][
'$minDistance'] = value_dict['$minDistance']
near_embedded = True
if not near_embedded:
if '$maxDistance' in value_dict:
value_son['$maxDistance'] = value_dict['$maxDistance']
if '$minDistance' in value_dict:
value_son['$minDistance'] = value_dict['$minDistance']
mongo_query[key] = value_son
else:
# Store for manually merging later
merge_query[key].append(value)
def update(_doc_cls=None, **update):
"""Transform an update spec from Django-style format to Mongo
format.
"""
mongo_update = {}
for key, value in update.items():
if key == '__raw__':
mongo_update.update(value)
continue
parts = key.split('__')
# if there is no operator, default to 'set'
if len(parts) < 3 and parts[0] not in UPDATE_OPERATORS:
parts.insert(0, 'set')
# Check for an operator and transform to mongo-style if there is
op = None
if parts[0] in UPDATE_OPERATORS:
op = parts.pop(0)
# Convert Pythonic names to Mongo equivalents
if op in ('push_all', 'pull_all'):
op = op.replace('_all', 'All')
elif op == 'dec':
# Support decrement by flipping a positive value's sign
# and using 'inc'
op = 'inc'
value = -value
elif op == 'add_to_set':
op = 'addToSet'
elif op == 'set_on_insert':
op = 'setOnInsert'
match = None
if parts[-1] in COMPARISON_OPERATORS:
match = parts.pop()
# Allow to escape operator-like field name by __
if len(parts) > 1 and parts[-1] == '':
parts.pop()
if _doc_cls:
# Switch field names to proper names [set in Field(name='foo')]
try:
fields = _doc_cls._lookup_field(parts)
except Exception as e:
raise InvalidQueryError(e)
parts = []
cleaned_fields = []
appended_sub_field = False
for field in fields:
append_field = True
if isinstance(field, six.string_types):
# Convert the S operator to $
if field == 'S':
field = '$'
parts.append(field)
append_field = False
else:
parts.append(field.db_field)
if append_field:
appended_sub_field = False
cleaned_fields.append(field)
if hasattr(field, 'field'):
cleaned_fields.append(field.field)
appended_sub_field = True
# Convert value to proper value
if appended_sub_field:
field = cleaned_fields[-2]
else:
field = cleaned_fields[-1]
GeoJsonBaseField = _import_class('GeoJsonBaseField')
if isinstance(field, GeoJsonBaseField):
value = field.to_mongo(value)
if op == 'pull':
if field.required or value is not None:
if match == 'in' and not isinstance(value, dict):
value = _prepare_query_for_iterable(field, op, value)
else:
value = field.prepare_query_value(op, value)
elif op == 'push' and isinstance(value, (list, tuple, set)):
value = [field.prepare_query_value(op, v) for v in value]
elif op in (None, 'set', 'push'):
if field.required or value is not None:
value = field.prepare_query_value(op, value)
elif op in ('pushAll', 'pullAll'):
value = [field.prepare_query_value(op, v) for v in value]
elif op in ('addToSet', 'setOnInsert'):
if isinstance(value, (list, tuple, set)):
value = [field.prepare_query_value(op, v) for v in value]
elif field.required or value is not None:
value = field.prepare_query_value(op, value)
elif op == 'unset':
value = 1
if match:
match = '$' + match
value = {match: value}
key = '.'.join(parts)
if not op:
raise InvalidQueryError('Updates must supply an operation '
'eg: set__FIELD=value')
if 'pull' in op and '.' in key:
# Dot operators don't work on pull operations
# unless they point to a list field
# Otherwise it uses nested dict syntax
if op == 'pullAll':
raise InvalidQueryError('pullAll operations only support '
'a single field depth')
# Look for the last list field and use dot notation until there
field_classes = [c.__class__ for c in cleaned_fields]
field_classes.reverse()
ListField = _import_class('ListField')
EmbeddedDocumentListField = _import_class('EmbeddedDocumentListField')
if ListField in field_classes or EmbeddedDocumentListField in field_classes:
# Join all fields via dot notation to the last ListField or EmbeddedDocumentListField
# Then process as normal
if ListField in field_classes:
_check_field = ListField
else:
_check_field = EmbeddedDocumentListField
last_listField = len(
cleaned_fields) - field_classes.index(_check_field)
key = '.'.join(parts[:last_listField])
parts = parts[last_listField:]
parts.insert(0, key)
parts.reverse()
for key in parts:
value = {key: value}
elif op == 'addToSet' and isinstance(value, list):
value = {key: {'$each': value}}
elif op == 'push':
if parts[-1].isdigit():
key = parts[0]
position = int(parts[-1])
# $position expects an iterable. If pushing a single value,
# wrap it in a list.
if not isinstance(value, (set, tuple, list)):
value = [value]
value = {key: {'$each': value, '$position': position}}
else:
value = {key: value}
else:
value = {key: value}
key = '$' + op
if key not in mongo_update:
mongo_update[key] = value
elif key in mongo_update and isinstance(mongo_update[key], dict):
mongo_update[key].update(value)
return mongo_update
def purge_executions(logger,
timestamp,
action_ref=None,
purge_incomplete=False):
"""
:param timestamp: Exections older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
:param action_ref: Only delete executions for the provided actions.
:type action_ref: ``str``
:param purge_incomplete: True to also delete executions which are not in a done state.
:type purge_incomplete: ``bool``
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging executions older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
filters = {}
if purge_incomplete:
filters['start_timestamp__lt'] = timestamp
else:
filters['end_timestamp__lt'] = timestamp
filters['start_timestamp__lt'] = timestamp
filters['status'] = {'$in': DONE_STATES}
exec_filters = copy.copy(filters)
if action_ref:
exec_filters['action__ref'] = action_ref
liveaction_filters = copy.deepcopy(filters)
if action_ref:
liveaction_filters['action'] = action_ref
try:
deleted_count = ActionExecution.delete_by_query(**exec_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution instances: %s'
'Please contact support.' % (exec_filters, str(e)))
raise InvalidQueryError(msg)
except:
logger.exception(
'Deletion of execution models failed for query with filters: %s.',
exec_filters)
else:
logger.info('Deleted %s action execution objects' % (deleted_count))
try:
deleted_count = LiveAction.delete_by_query(**liveaction_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete liveaction instances: %s'
'Please contact support.' % (liveaction_filters, str(e)))
raise InvalidQueryError(msg)
except:
logger.exception(
'Deletion of liveaction models failed for query with filters: %s.',
liveaction_filters)
else:
logger.info('Deleted %s liveaction objects' % (deleted_count))
zombie_execution_instances = len(ActionExecution.query(**exec_filters))
zombie_liveaction_instances = len(LiveAction.query(**liveaction_filters))
if (zombie_execution_instances > 0) or (zombie_liveaction_instances > 0):
logger.error('Zombie execution instances left: %d.',
zombie_execution_instances)
logger.error('Zombie liveaction instances left: %s.',
zombie_liveaction_instances)
# Print stats
logger.info('All execution models older than timestamp %s were deleted.',
timestamp)
def purge_executions(logger, timestamp, action_ref=None, purge_incomplete=False):
"""
Purge action executions and corresponding live action, execution output objects.
:param timestamp: Exections older than this timestamp will be deleted.
:type timestamp: ``datetime.datetime
:param action_ref: Only delete executions for the provided actions.
:type action_ref: ``str``
:param purge_incomplete: True to also delete executions which are not in a done state.
:type purge_incomplete: ``bool``
"""
if not timestamp:
raise ValueError('Specify a valid timestamp to purge.')
logger.info('Purging executions older than timestamp: %s' %
timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ'))
filters = {}
if purge_incomplete:
filters['start_timestamp__lt'] = timestamp
else:
filters['end_timestamp__lt'] = timestamp
filters['start_timestamp__lt'] = timestamp
filters['status'] = {'$in': DONE_STATES}
exec_filters = copy.copy(filters)
if action_ref:
exec_filters['action__ref'] = action_ref
liveaction_filters = copy.deepcopy(filters)
if action_ref:
liveaction_filters['action'] = action_ref
to_delete_execution_dbs = []
# 1. Delete ActionExecutionDB objects
try:
# Note: We call list() on the query set object because it's lazyily evaluated otherwise
to_delete_execution_dbs = list(ActionExecution.query(only_fields=['id'],
no_dereference=True,
**exec_filters))
deleted_count = ActionExecution.delete_by_query(**exec_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution instances: %s'
'Please contact support.' % (exec_filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of execution models failed for query with filters: %s.',
exec_filters)
else:
logger.info('Deleted %s action execution objects' % (deleted_count))
# 2. Delete LiveActionDB objects
try:
deleted_count = LiveAction.delete_by_query(**liveaction_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete liveaction instances: %s'
'Please contact support.' % (liveaction_filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of liveaction models failed for query with filters: %s.',
liveaction_filters)
else:
logger.info('Deleted %s liveaction objects' % (deleted_count))
# 3. Delete ActionExecutionOutputDB objects
to_delete_exection_ids = [str(execution_db.id) for execution_db in to_delete_execution_dbs]
output_dbs_filters = {}
output_dbs_filters['execution_id'] = {'$in': to_delete_exection_ids}
try:
deleted_count = ActionExecutionOutput.delete_by_query(**output_dbs_filters)
except InvalidQueryError as e:
msg = ('Bad query (%s) used to delete execution output instances: %s'
'Please contact support.' % (output_dbs_filters, six.text_type(e)))
raise InvalidQueryError(msg)
except:
logger.exception('Deletion of execution output models failed for query with filters: %s.',
output_dbs_filters)
else:
logger.info('Deleted %s execution output objects' % (deleted_count))
zombie_execution_instances = len(ActionExecution.query(only_fields=['id'],
no_dereference=True,
**exec_filters))
zombie_liveaction_instances = len(LiveAction.query(only_fields=['id'],
no_dereference=True,
**liveaction_filters))
if (zombie_execution_instances > 0) or (zombie_liveaction_instances > 0):
logger.error('Zombie execution instances left: %d.', zombie_execution_instances)
logger.error('Zombie liveaction instances left: %s.', zombie_liveaction_instances)
# Print stats
logger.info('All execution models older than timestamp %s were deleted.', timestamp)
def query(_doc_cls=None, _field_operation=False, **query):
"""Transform a query from Django-style format to Mongo format.
"""
mongo_query = {}
merge_query = defaultdict(list)
for key, value in sorted(query.items()):
if key == "__raw__":
mongo_query.update(value)
continue
parts = key.rsplit('__')
indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()]
parts = [part for part in parts if not part.isdigit()]
# Check for an operator and transform to mongo-style if there is
op = None
if len(parts) > 1 and parts[-1] in MATCH_OPERATORS:
op = parts.pop()
negate = False
if len(parts) > 1 and parts[-1] == 'not':
parts.pop()
negate = True
if _doc_cls:
# Switch field names to proper names [set in Field(name='foo')]
try:
fields = _doc_cls._lookup_field(parts)
except Exception as e:
raise InvalidQueryError(e)
parts = []
CachedReferenceField = _import_class('CachedReferenceField')
cleaned_fields = []
for field in fields:
append_field = True
if isinstance(field, str):
parts.append(field)
append_field = False
# is last and CachedReferenceField
elif isinstance(field,
CachedReferenceField) and fields[-1] == field:
parts.append('%s._id' % field.db_field)
else:
parts.append(field.db_field)
if append_field:
cleaned_fields.append(field)
# Convert value to proper value
field = cleaned_fields[-1]
singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not']
singular_ops += STRING_OPERATORS
if op in singular_ops:
if isinstance(field, str):
if (op in STRING_OPERATORS and isinstance(value, str)):
StringField = _import_class('StringField')
value = StringField.prepare_query_value(op, value)
else:
value = field
else:
value = field.prepare_query_value(op, value)
if isinstance(field, CachedReferenceField) and value:
value = value['_id']
elif op in ('in', 'nin', 'all',
'near') and not isinstance(value, dict):
# 'in', 'nin' and 'all' require a list of values
value = [field.prepare_query_value(op, v) for v in value]
# if op and op not in COMPARISON_OPERATORS:
if op:
if op in GEO_OPERATORS:
value = _geo_operator(field, op, value)
elif op in CUSTOM_OPERATORS:
if op in ('elem_match', 'match'):
value = field.prepare_query_value(op, value)
value = {"$elemMatch": value}
else:
NotImplementedError("Custom method '%s' has not "
"been implemented" % op)
elif op not in STRING_OPERATORS:
value = {'$' + op: value}
if negate:
value = {'$not': value}
for i, part in indices:
parts.insert(i, part)
key = '.'.join(parts)
if op is None or key not in mongo_query:
mongo_query[key] = value
elif key in mongo_query:
if key in mongo_query and isinstance(mongo_query[key], dict):
mongo_query[key].update(value)
# $maxDistance needs to come last - convert to SON
value_dict = mongo_query[key]
if ('$maxDistance' in value_dict and '$near' in value_dict):
value_son = SON()
if isinstance(value_dict['$near'], dict):
for k, v in value_dict.items():
if k == '$maxDistance':
continue
value_son[k] = v
if (get_connection().max_wire_version <= 1):
value_son['$maxDistance'] = value_dict[
'$maxDistance']
else:
value_son['$near'] = SON(value_son['$near'])
value_son['$near']['$maxDistance'] = value_dict[
'$maxDistance']
else:
for k, v in value_dict.items():
if k == '$maxDistance':
continue
value_son[k] = v
value_son['$maxDistance'] = value_dict['$maxDistance']
mongo_query[key] = value_son
else:
# Store for manually merging later
merge_query[key].append(value)
# The queryset has been filter in such a way we must manually merge
for k, v in list(merge_query.items()):
merge_query[k].append(mongo_query[k])
del mongo_query[k]
if isinstance(v, list):
value = [{k: val} for val in v]
if '$and' in list(mongo_query.keys()):
mongo_query['$and'].extend(value)
else:
mongo_query['$and'] = value
return mongo_query
def query(_doc_cls=None, **kwargs):
"""Transform a query from Django-style format to Mongo format."""
mongo_query = {}
merge_query = defaultdict(list)
for key, value in sorted(kwargs.items()):
if key == "__raw__":
mongo_query.update(value)
continue
parts = key.rsplit("__")
indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()]
parts = [part for part in parts if not part.isdigit()]
# Check for an operator and transform to mongo-style if there is
op = None
if len(parts) > 1 and parts[-1] in MATCH_OPERATORS:
op = parts.pop()
# Allow to escape operator-like field name by __
if len(parts) > 1 and parts[-1] == "":
parts.pop()
negate = False
if len(parts) > 1 and parts[-1] == "not":
parts.pop()
negate = True
if _doc_cls:
# Switch field names to proper names [set in Field(name='foo')]
try:
fields = _doc_cls._lookup_field(parts)
except Exception as e:
raise InvalidQueryError(e)
parts = []
CachedReferenceField = _import_class("CachedReferenceField")
GenericReferenceField = _import_class("GenericReferenceField")
cleaned_fields = []
for field in fields:
append_field = True
if isinstance(field, str):
parts.append(field)
append_field = False
# is last and CachedReferenceField
elif isinstance(field,
CachedReferenceField) and fields[-1] == field:
parts.append("%s._id" % field.db_field)
else:
parts.append(field.db_field)
if append_field:
cleaned_fields.append(field)
# Convert value to proper value
field = cleaned_fields[-1]
singular_ops = [None, "ne", "gt", "gte", "lt", "lte", "not"]
singular_ops += STRING_OPERATORS
if op in singular_ops:
value = field.prepare_query_value(op, value)
if isinstance(field, CachedReferenceField) and value:
value = value["_id"]
elif op in ("in", "nin", "all",
"near") and not isinstance(value, dict):
# Raise an error if the in/nin/all/near param is not iterable.
value = _prepare_query_for_iterable(field, op, value)
# If we're querying a GenericReferenceField, we need to alter the
# key depending on the value:
# * If the value is a DBRef, the key should be "field_name._ref".
# * If the value is an ObjectId, the key should be "field_name._ref.$id".
if isinstance(field, GenericReferenceField):
if isinstance(value, DBRef):
parts[-1] += "._ref"
elif isinstance(value, ObjectId):
parts[-1] += "._ref.$id"
# if op and op not in COMPARISON_OPERATORS:
if op:
if op in GEO_OPERATORS:
value = _geo_operator(field, op, value)
elif op in ("match", "elemMatch"):
ListField = _import_class("ListField")
EmbeddedDocumentField = _import_class("EmbeddedDocumentField")
if (isinstance(value, dict) and isinstance(field, ListField)
and isinstance(field.field, EmbeddedDocumentField)):
value = query(field.field.document_type, **value)
else:
value = field.prepare_query_value(op, value)
value = {"$elemMatch": value}
elif op in CUSTOM_OPERATORS:
NotImplementedError('Custom method "%s" has not '
"been implemented" % op)
elif op not in STRING_OPERATORS:
value = {"$" + op: value}
if negate:
value = {"$not": value}
for i, part in indices:
parts.insert(i, part)
key = ".".join(parts)
if key not in mongo_query:
mongo_query[key] = value
else:
if isinstance(mongo_query[key], dict) and isinstance(value, dict):
mongo_query[key].update(value)
# $max/minDistance needs to come last - convert to SON
value_dict = mongo_query[key]
if ("$maxDistance" in value_dict or "$minDistance"
in value_dict) and ("$near" in value_dict
or "$nearSphere" in value_dict):
value_son = SON()
for k, v in value_dict.items():
if k == "$maxDistance" or k == "$minDistance":
continue
value_son[k] = v
# Required for MongoDB >= 2.6, may fail when combining
# PyMongo 3+ and MongoDB < 2.6
near_embedded = False
for near_op in ("$near", "$nearSphere"):
if isinstance(value_dict.get(near_op), dict):
value_son[near_op] = SON(value_son[near_op])
if "$maxDistance" in value_dict:
value_son[near_op][
"$maxDistance"] = value_dict[
"$maxDistance"]
if "$minDistance" in value_dict:
value_son[near_op][
"$minDistance"] = value_dict[
"$minDistance"]
near_embedded = True
if not near_embedded:
if "$maxDistance" in value_dict:
value_son["$maxDistance"] = value_dict[
"$maxDistance"]
if "$minDistance" in value_dict:
value_son["$minDistance"] = value_dict[
"$minDistance"]
mongo_query[key] = value_son
else:
# Store for manually merging later
merge_query[key].append(value)
# The queryset has been filter in such a way we must manually merge
for k, v in merge_query.items():
merge_query[k].append(mongo_query[k])
del mongo_query[k]
if isinstance(v, list):
value = [{k: val} for val in v]
if "$and" in mongo_query.keys():
mongo_query["$and"].extend(value)
else:
mongo_query["$and"] = value
return mongo_query
def update(_doc_cls=None, **update):
"""Transform an update spec from Django-style format to Mongo
format.
"""
mongo_update = {}
for key, value in update.items():
if key == "__raw__":
mongo_update.update(value)
continue
parts = key.split("__")
# if there is no operator, default to 'set'
if len(parts) < 3 and parts[0] not in UPDATE_OPERATORS:
parts.insert(0, "set")
# Check for an operator and transform to mongo-style if there is
op = None
if parts[0] in UPDATE_OPERATORS:
op = parts.pop(0)
# Convert Pythonic names to Mongo equivalents
operator_map = {
"push_all": "pushAll",
"pull_all": "pullAll",
"dec": "inc",
"add_to_set": "addToSet",
"set_on_insert": "setOnInsert",
}
if op == "dec":
# Support decrement by flipping a positive value's sign
# and using 'inc'
value = -value
# If the operator doesn't found from operator map, the op value
# will stay unchanged
op = operator_map.get(op, op)
match = None
if parts[-1] in COMPARISON_OPERATORS:
match = parts.pop()
# Allow to escape operator-like field name by __
if len(parts) > 1 and parts[-1] == "":
parts.pop()
if _doc_cls:
# Switch field names to proper names [set in Field(name='foo')]
try:
fields = _doc_cls._lookup_field(parts)
except Exception as e:
raise InvalidQueryError(e)
parts = []
cleaned_fields = []
appended_sub_field = False
for field in fields:
append_field = True
if isinstance(field, str):
# Convert the S operator to $
if field == "S":
field = "$"
parts.append(field)
append_field = False
else:
parts.append(field.db_field)
if append_field:
appended_sub_field = False
cleaned_fields.append(field)
if hasattr(field, "field"):
cleaned_fields.append(field.field)
appended_sub_field = True
# Convert value to proper value
if appended_sub_field:
field = cleaned_fields[-2]
else:
field = cleaned_fields[-1]
GeoJsonBaseField = _import_class("GeoJsonBaseField")
if isinstance(field, GeoJsonBaseField):
value = field.to_mongo(value)
if op == "pull":
if field.required or value is not None:
if match in ("in", "nin") and not isinstance(value, dict):
value = _prepare_query_for_iterable(field, op, value)
else:
value = field.prepare_query_value(op, value)
elif op == "push" and isinstance(value, (list, tuple, set)):
value = [field.prepare_query_value(op, v) for v in value]
elif op in (None, "set", "push"):
if field.required or value is not None:
value = field.prepare_query_value(op, value)
elif op in ("pushAll", "pullAll"):
value = [field.prepare_query_value(op, v) for v in value]
elif op in ("addToSet", "setOnInsert"):
if isinstance(value, (list, tuple, set)):
value = [field.prepare_query_value(op, v) for v in value]
elif field.required or value is not None:
value = field.prepare_query_value(op, value)
elif op == "unset":
value = 1
elif op == "inc":
value = field.prepare_query_value(op, value)
if match:
match = "$" + match
value = {match: value}
key = ".".join(parts)
if "pull" in op and "." in key:
# Dot operators don't work on pull operations
# unless they point to a list field
# Otherwise it uses nested dict syntax
if op == "pullAll":
raise InvalidQueryError(
"pullAll operations only support a single field depth")
# Look for the last list field and use dot notation until there
field_classes = [c.__class__ for c in cleaned_fields]
field_classes.reverse()
ListField = _import_class("ListField")
EmbeddedDocumentListField = _import_class(
"EmbeddedDocumentListField")
if ListField in field_classes or EmbeddedDocumentListField in field_classes:
# Join all fields via dot notation to the last ListField or EmbeddedDocumentListField
# Then process as normal
if ListField in field_classes:
_check_field = ListField
else:
_check_field = EmbeddedDocumentListField
last_listField = len(cleaned_fields) - field_classes.index(
_check_field)
key = ".".join(parts[:last_listField])
parts = parts[last_listField:]
parts.insert(0, key)
parts.reverse()
for key in parts:
value = {key: value}
elif op == "addToSet" and isinstance(value, list):
value = {key: {"$each": value}}
elif op in ("push", "pushAll"):
if parts[-1].isdigit():
key = ".".join(parts[0:-1])
position = int(parts[-1])
# $position expects an iterable. If pushing a single value,
# wrap it in a list.
if not isinstance(value, (set, tuple, list)):
value = [value]
value = {key: {"$each": value, "$position": position}}
else:
if op == "pushAll":
op = "push" # convert to non-deprecated keyword
if not isinstance(value, (set, tuple, list)):
value = [value]
value = {key: {"$each": value}}
else:
value = {key: value}
else:
value = {key: value}
key = "$" + op
if key not in mongo_update:
mongo_update[key] = value
elif key in mongo_update and isinstance(mongo_update[key], dict):
mongo_update[key].update(value)
return mongo_update
