def _infer_geometry(value): """Helper method that tries to infer the $geometry shape for a given value. """ if isinstance(value, dict): if '$geometry' in value: return value elif 'coordinates' in value and 'type' in value: return {'$geometry': value} raise InvalidQueryError('Invalid $geometry dictionary should have ' 'type and coordinates keys') elif isinstance(value, (list, set)): # TODO: shouldn't we test value[0][0][0][0] to see if it is MultiPolygon? # TODO: should both TypeError and IndexError be alike interpreted? try: value[0][0][0] return {'$geometry': {'type': 'Polygon', 'coordinates': value}} except (TypeError, IndexError): pass try: value[0][0] return {'$geometry': {'type': 'LineString', 'coordinates': value}} except (TypeError, IndexError): pass try: value[0] return {'$geometry': {'type': 'Point', 'coordinates': value}} except (TypeError, IndexError): pass raise InvalidQueryError('Invalid $geometry data. Can be either a ' 'dictionary or (nested) lists of coordinate(s)')
def _infer_geometry(value): """Helper method that tries to infer the $geometry shape for a given value""" if isinstance(value, dict): if "$geometry" in value: return value elif 'coordinates' in value and 'type' in value: return {"$geometry": value} raise InvalidQueryError("Invalid $geometry dictionary should have " "type and coordinates keys") elif isinstance(value, (list, set)): try: value[0][0][0] return {"$geometry": {"type": "Polygon", "coordinates": value}} except: pass try: value[0][0] return {"$geometry": {"type": "LineString", "coordinates": value}} except: pass try: value[0] return {"$geometry": {"type": "Point", "coordinates": value}} except: pass raise InvalidQueryError( "Invalid $geometry data. Can be either a dictionary " "or (nested) lists of coordinate(s)")
def _infer_geometry(value): """Helper method that tries to infer the $geometry shape for a given value. """ if isinstance(value, dict): if "$geometry" in value: return value elif "coordinates" in value and "type" in value: return {"$geometry": value} raise InvalidQueryError( "Invalid $geometry dictionary should have type and coordinates keys" ) elif isinstance(value, (list, set)): # TODO: shouldn't we test value[0][0][0][0] to see if it is MultiPolygon? try: value[0][0][0] return {"$geometry": {"type": "Polygon", "coordinates": value}} except (TypeError, IndexError): pass try: value[0][0] return {"$geometry": {"type": "LineString", "coordinates": value}} except (TypeError, IndexError): pass try: value[0] return {"$geometry": {"type": "Point", "coordinates": value}} except (TypeError, IndexError): pass raise InvalidQueryError("Invalid $geometry data. Can be either a " "dictionary or (nested) lists of coordinate(s)")
def purge_execution_output_objects(logger, timestamp, action_ref=None): """ Purge action executions output objects. :param timestamp: Objects older than this timestamp will be deleted. :type timestamp: ``datetime.datetime :param action_ref: Only delete objects for the provided actions. :type action_ref: ``str`` """ if not timestamp: raise ValueError('Specify a valid timestamp to purge.') logger.info('Purging action execution output objects older than timestamp: %s' % timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) filters = {} filters['timestamp__lt'] = timestamp if action_ref: filters['action_ref'] = action_ref try: deleted_count = ActionExecutionOutput.delete_by_query(**filters) except InvalidQueryError as e: msg = ('Bad query (%s) used to delete execution output instances: %s' 'Please contact support.' % (filters, six.text_type(e))) raise InvalidQueryError(msg) except: logger.exception('Deletion of execution output models failed for query with filters: %s.', filters) else: logger.info('Deleted %s execution output objects' % (deleted_count))
def purge_trigger_instances(logger, timestamp): """ :param timestamp: Trigger instances older than this timestamp will be deleted. :type timestamp: ``datetime.datetime """ if not timestamp: raise ValueError('Specify a valid timestamp to purge.') logger.info('Purging trigger instances older than timestamp: %s' % timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) query_filters = {'occurrence_time__lt': isotime.parse(timestamp)} try: deleted_count = TriggerInstance.delete_by_query(**query_filters) except InvalidQueryError as e: msg = ('Bad query (%s) used to delete trigger instances: %s' 'Please contact support.' % (query_filters, str(e))) raise InvalidQueryError(msg) except: logger.exception('Deleting instances using query_filters %s failed.', query_filters) else: logger.info('Deleted %s trigger instance objects' % (deleted_count)) # Print stats logger.info('All trigger instance models older than timestamp %s were deleted.', timestamp)
def purge_workflow_executions(logger, timestamp, purge_incomplete=False): """ Purge workflow execution output objects. :param timestamp: Exections older than this timestamp will be deleted. :type timestamp: ``datetime.datetime :param purge_incomplete: True to also delete executions which are not in a done state. :type purge_incomplete: ``bool`` """ if not timestamp: raise ValueError("Specify a valid timestamp to purge.") logger.info("Purging workflow executions older than timestamp: %s" % timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ")) filters = {} if purge_incomplete: filters["start_timestamp__lt"] = timestamp else: filters["end_timestamp__lt"] = timestamp filters["start_timestamp__lt"] = timestamp filters["status"] = {"$in": DONE_STATES} exec_filters = copy.copy(filters) # 1. Delete Workflow Execution objects try: # Note: We call list() on the query set object because it's lazyily evaluated otherwise # to_delete_execution_dbs = list(WorkflowExecution.query(only_fields=['id'], # no_dereference=True, # **exec_filters)) deleted_count = WorkflowExecution.delete_by_query(**exec_filters) except InvalidQueryError as e: msg = ("Bad query (%s) used to delete workflow execution instances: %s" "Please contact support." % (exec_filters, six.text_type(e))) raise InvalidQueryError(msg) except: logger.exception( "Deletion of workflow execution models failed for query with filters: %s.", exec_filters, ) else: logger.info("Deleted %s workflow execution objects" % deleted_count) zombie_execution_instances = len( WorkflowExecution.query(only_fields=["id"], no_dereference=True, **exec_filters)) if zombie_execution_instances > 0: logger.error("Zombie workflow execution instances left: %d.", zombie_execution_instances) # Print stats logger.info( "All workflow execution models older than timestamp %s were deleted.", timestamp, )
def purge_rule_enforcements(logger, timestamp): """ :param timestamp: Rule enforcement instances older than this timestamp will be deleted. :type timestamp: ``datetime.datetime """ if not timestamp: raise ValueError("Specify a valid timestamp to purge.") logger.info("Purging rule enforcements older than timestamp: %s" % timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ")) query_filters = {"enforced_at__lt": isotime.parse(timestamp)} try: deleted_count = RuleEnforcement.delete_by_query(**query_filters) except InvalidQueryError as e: msg = ("Bad query (%s) used to delete rule enforcements: %s" "Please contact support." % ( query_filters, six.text_type(e), )) raise InvalidQueryError(msg) except: logger.exception( "Deleting rule enforcements using query_filters %s failed.", query_filters) else: logger.info("Deleted %s rule enforcement objects" % (deleted_count)) # Print stats logger.info( "All rule enforcement models older than timestamp %s were deleted.", timestamp)
def purge_trigger_instances(logger, timestamp): """ :param timestamp: Trigger instances older than this timestamp will be deleted. :type timestamp: ``datetime.datetime """ if not timestamp: raise ValueError('Specify a valid timestamp to purge.') logger.info('Purging trigger instances older than timestamp: %s' % timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) query_filters = {'occurrence_time__lt': isotime.parse(timestamp)} # TODO: Update this code to return statistics on deleted objects once we # upgrade to newer version of MongoDB where delete_by_query actually returns # some data try: TriggerInstance.delete_by_query(**query_filters) except InvalidQueryError as e: msg = ('Bad query (%s) used to delete trigger instances: %s' 'Please contact support.' % (query_filters, str(e))) raise InvalidQueryError(msg) except: logger.exception('Deleting instances using query_filters %s failed.', query_filters) # Print stats logger.info( 'All trigger instance models older than timestamp %s were deleted.', timestamp)
async def delete(request): try: car = Car.objects(VIN=request.match_info['vin']) if not car.delete(): raise InvalidQueryError('Unknown car with this VIN') return web.json_response({'msg': 'Car deleted'}, status=204) except Exception as e: return web.json_response({'error': str(e)}, status=400)
async def update_car(request): car_data = await request.json() try: if car_data.get('VIN'): raise InvalidQueryError('Field VIN is Unchanged') Car.objects(VIN=request.match_info['vin']).update(**car_data) return web.json_response({'msg': 'Car updated'}, status=201) except Exception as e: return web.json_response({'msg': str(e)}, status=400)
def modify(self, query={}, **update): """Perform an atomic update of the document in the database and reload the document object using updated version. Returns True if the document has been updated or False if the document in the database doesn't match the query. .. note:: All unsaved changes that have been made to the document are rejected if the method returns True. :param query: the update will be performed only if the document in the database matches the query :param update: Django-style update keyword arguments """ if self.pk is None: raise InvalidDocumentError( "The document does not have a primary key.") id_field = self._meta["id_field"] query = query.copy() if isinstance(query, dict) else query.to_query(self) if id_field not in query: query[id_field] = self.pk elif query[id_field] != self.pk: msg = "Invalid document modify query: " msg += "it must modify only this document." raise InvalidQueryError(msg) updated_future = self._qs(**query).modify(new=True, **update) ret_future = get_future(self) def updated_cb(updated_future): try: updated = updated_future.result() if updated is None: ret_future.set_result(False) return for field in self._fields_ordered: setattr(self, field, self._reload(field, updated[field])) self._changed_fields = updated._changed_fields self._created = False ret_future.set_result(True) return except Exception as e: ret_future.set_exception(e) updated_future.add_done_callback(updated_cb) return ret_future
def _query_conjunction(self, queries): """Merges query dicts - effectively &ing them together. """ query_ops = set() combined_query = {} for query in queries: ops = set(query.keys()) # Make sure that the same operation isn't applied more than once # to a single field intersection = ops.intersection(query_ops) if intersection: msg = 'Duplicate query conditions: ' raise InvalidQueryError(msg + ', '.join(intersection)) query_ops.update(ops) combined_query.update(copy.deepcopy(query)) return combined_query
def modify(self, query=None, **update): """Perform an atomic update of the document in the database and reload the document object using updated version. Returns True if the document has been updated or False if the document in the database doesn't match the query. .. note:: All unsaved changes that have been made to the document are rejected if the method returns True. :param query: the update will be performed only if the document in the database matches the query :param update: Django-style update keyword arguments """ if query is None: query = {} if self.pk is None: raise InvalidDocumentError( 'The document does not have a primary key.') id_field = self._meta['id_field'] query = query.copy() if isinstance(query, dict) else query.to_query(self) if id_field not in query: query[id_field] = self.pk elif query[id_field] != self.pk: raise InvalidQueryError( 'Invalid document modify query: it must modify only this document.' ) # Need to add shard key to query, or you get an error query.update(self._object_key) updated = self._qs(**query).modify(new=True, **update) if updated is None: return False for field in self._fields_ordered: setattr(self, field, self._reload(field, updated[field])) self._changed_fields = updated._changed_fields self._created = False return True
def update(_doc_cls=None, **update): """Transform an update spec from Django-style format to Mongo format. """ mongo_update = {} for key, value in update.items(): if key == "__raw__": mongo_update.update(value) continue parts = key.split('__') # Check for an operator and transform to mongo-style if there is op = None if parts[0] in UPDATE_OPERATORS: op = parts.pop(0) # Convert Pythonic names to Mongo equivalents if op in ('push_all', 'pull_all'): op = op.replace('_all', 'All') elif op == 'dec': # Support decrement by flipping a positive value's sign # and using 'inc' op = 'inc' if value > 0: value = -value elif op == 'add_to_set': op = 'addToSet' elif op == 'set_on_insert': op = "setOnInsert" match = None if parts[-1] in COMPARISON_OPERATORS: match = parts.pop() if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] try: fields = _doc_cls._lookup_field(parts) except Exception, e: raise InvalidQueryError(e) parts = [] cleaned_fields = [] for field in fields: append_field = True if isinstance(field, basestring): # Convert the S operator to $ if field == 'S': field = '$' parts.append(field) append_field = False else: parts.append(field.db_field) if append_field: cleaned_fields.append(field) # Convert value to proper value field = cleaned_fields[-1] if op in (None, 'set', 'push', 'pull'): if field.required or value is not None: value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] elif op == 'addToSet': if isinstance(value, (list, tuple, set)): value = [field.prepare_query_value(op, v) for v in value] elif field.required or value is not None: value = field.prepare_query_value(op, value) if match: match = '$' + match value = {match: value} key = '.'.join(parts) if not op: raise InvalidQueryError("Updates must supply an operation " "eg: set__FIELD=value") if 'pull' in op and '.' in key: # Dot operators don't work on pull operations # it uses nested dict syntax if op == 'pullAll': raise InvalidQueryError("pullAll operations only support " "a single field depth") parts.reverse() for key in parts: value = {key: value} elif op == 'addToSet' and isinstance(value, list): value = {key: {"$each": value}} else: value = {key: value} key = '$' + op if key not in mongo_update: mongo_update[key] = value elif key in mongo_update and isinstance(mongo_update[key], dict): mongo_update[key].update(value)
def query(_doc_cls=None, **kwargs): """Transform a query from Django-style format to Mongo format.""" mongo_query = {} merge_query = defaultdict(list) for key, value in sorted(kwargs.items()): if key == '__raw__': mongo_query.update(value) continue parts = key.rsplit('__') indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] parts = [part for part in parts if not part.isdigit()] # Check for an operator and transform to mongo-style if there is op = None if len(parts) > 1 and parts[-1] in MATCH_OPERATORS: op = parts.pop() # Allow to escape operator-like field name by __ if len(parts) > 1 and parts[-1] == '': parts.pop() negate = False if len(parts) > 1 and parts[-1] == 'not': parts.pop() negate = True if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] try: fields = _doc_cls._lookup_field(parts) except Exception as e: raise InvalidQueryError(e) parts = [] CachedReferenceField = _import_class('CachedReferenceField') GenericReferenceField = _import_class('GenericReferenceField') cleaned_fields = [] for field in fields: append_field = True if isinstance(field, six.string_types): parts.append(field) append_field = False # is last and CachedReferenceField elif isinstance(field, CachedReferenceField) and fields[-1] == field: parts.append('%s._id' % field.db_field) else: parts.append(field.db_field) if append_field: cleaned_fields.append(field) # Convert value to proper value field = cleaned_fields[-1] singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not'] singular_ops += STRING_OPERATORS if op in singular_ops: if isinstance(field, six.string_types): if (op in STRING_OPERATORS and isinstance(value, six.string_types)): StringField = _import_class('StringField') value = StringField.prepare_query_value(op, value) else: value = field else: value = field.prepare_query_value(op, value) if isinstance(field, CachedReferenceField) and value: value = value['_id'] elif op in ('in', 'nin', 'all', 'near') and not isinstance(value, dict): # Raise an error if the in/nin/all/near param is not iterable. value = _prepare_query_for_iterable(field, op, value) # If we're querying a GenericReferenceField, we need to alter the # key depending on the value: # * If the value is a DBRef, the key should be "field_name._ref". # * If the value is an ObjectId, the key should be "field_name._ref.$id". if isinstance(field, GenericReferenceField): if isinstance(value, DBRef): parts[-1] += '._ref' elif isinstance(value, ObjectId): parts[-1] += '._ref.$id' # if op and op not in COMPARISON_OPERATORS: if op: if op in GEO_OPERATORS: value = _geo_operator(field, op, value) elif op in ('match', 'elemMatch'): ListField = _import_class('ListField') EmbeddedDocumentField = _import_class('EmbeddedDocumentField') if ( isinstance(value, dict) and isinstance(field, ListField) and isinstance(field.field, EmbeddedDocumentField) ): value = query(field.field.document_type, **value) else: value = field.prepare_query_value(op, value) value = {'$elemMatch': value} elif op in CUSTOM_OPERATORS: NotImplementedError('Custom method "%s" has not ' 'been implemented' % op) elif op not in STRING_OPERATORS: value = {'$' + op: value} if negate: value = {'$not': value} for i, part in indices: parts.insert(i, part) key = '.'.join(parts) if op is None or key not in mongo_query: mongo_query[key] = value elif key in mongo_query: if isinstance(mongo_query[key], dict): mongo_query[key].update(value) # $max/minDistance needs to come last - convert to SON value_dict = mongo_query[key] if ('$maxDistance' in value_dict or '$minDistance' in value_dict) and \ ('$near' in value_dict or '$nearSphere' in value_dict): value_son = SON() for k, v in value_dict.iteritems(): if k == '$maxDistance' or k == '$minDistance': continue value_son[k] = v # Required for MongoDB >= 2.6, may fail when combining # PyMongo 3+ and MongoDB < 2.6 near_embedded = False for near_op in ('$near', '$nearSphere'): if isinstance(value_dict.get(near_op), dict) and ( IS_PYMONGO_3 or get_connection().max_wire_version > 1): value_son[near_op] = SON(value_son[near_op]) if '$maxDistance' in value_dict: value_son[near_op][ '$maxDistance'] = value_dict['$maxDistance'] if '$minDistance' in value_dict: value_son[near_op][ '$minDistance'] = value_dict['$minDistance'] near_embedded = True if not near_embedded: if '$maxDistance' in value_dict: value_son['$maxDistance'] = value_dict['$maxDistance'] if '$minDistance' in value_dict: value_son['$minDistance'] = value_dict['$minDistance'] mongo_query[key] = value_son else: # Store for manually merging later merge_query[key].append(value) # The queryset has been filter in such a way we must manually merge for k, v in merge_query.items(): merge_query[k].append(mongo_query[k]) del mongo_query[k] if isinstance(v, list): value = [{k: val} for val in v] if '$and' in mongo_query.keys(): mongo_query['$and'].extend(value) else: mongo_query['$and'] = value return mongo_query
def update(_doc_cls=None, **update): """Transform an update spec from Django-style format to Mongo format. """ mongo_update = {} for key, value in update.items(): if key == "__raw__": mongo_update.update(value) continue parts = key.split('__') # Check for an operator and transform to mongo-style if there is op = None if parts[0] in UPDATE_OPERATORS: op = parts.pop(0) # Convert Pythonic names to Mongo equivalents if op in ('push_all', 'pull_all'): op = op.replace('_all', 'All') elif op == 'dec': # Support decrement by flipping a positive value's sign # and using 'inc' op = 'inc' if value > 0: value = -value elif op == 'add_to_set': op = 'addToSet' elif op == 'set_on_insert': op = "setOnInsert" match = None if parts[-1] in COMPARISON_OPERATORS: match = parts.pop() if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] try: fields = _doc_cls._lookup_field(parts) except Exception, e: raise InvalidQueryError(e) parts = [] cleaned_fields = [] appended_sub_field = False for field in fields: append_field = True if isinstance(field, basestring): # Convert the S operator to $ if field == 'S': field = '$' parts.append(field) append_field = False else: parts.append(field.db_field) if append_field: appended_sub_field = False cleaned_fields.append(field) if hasattr(field, 'field'): cleaned_fields.append(field.field) appended_sub_field = True # Convert value to proper value if appended_sub_field: field = cleaned_fields[-2] else: field = cleaned_fields[-1] GeoJsonBaseField = _import_class("GeoJsonBaseField") if isinstance(field, GeoJsonBaseField): value = field.to_mongo(value) if op in (None, 'set', 'push', 'pull'): if field.required or value is not None: value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] elif op in ('addToSet', 'setOnInsert'): if isinstance(value, (list, tuple, set)): value = [field.prepare_query_value(op, v) for v in value] elif field.required or value is not None: value = field.prepare_query_value(op, value) elif op == "unset": value = 1 if match: match = '$' + match value = {match: value} key = '.'.join(parts) if not op: raise InvalidQueryError("Updates must supply an operation " "eg: set__FIELD=value") if 'pull' in op and '.' in key: # Dot operators don't work on pull operations # unless they point to a list field # Otherwise it uses nested dict syntax if op == 'pullAll': raise InvalidQueryError("pullAll operations only support " "a single field depth") # Look for the last list field and use dot notation until there field_classes = [c.__class__ for c in cleaned_fields] field_classes.reverse() ListField = _import_class('ListField') if ListField in field_classes: # Join all fields via dot notation to the last ListField # Then process as normal last_listField = len(cleaned_fields) - field_classes.index( ListField) key = ".".join(parts[:last_listField]) parts = parts[last_listField:] parts.insert(0, key) parts.reverse() for key in parts: value = {key: value} elif op == 'addToSet' and isinstance(value, list): value = {key: {"$each": value}} else: value = {key: value} key = '$' + op if key not in mongo_update: mongo_update[key] = value elif key in mongo_update and isinstance(mongo_update[key], dict): mongo_update[key].update(value)
def query(_doc_cls=None, **query): """Transform a query from Django-style format to Mongo format. """ mongo_query = {} merge_query = defaultdict(list) for key, value in sorted(query.items()): if key == "__raw__": mongo_query.update(value) continue parts = key.rsplit('__') indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] parts = [part for part in parts if not part.isdigit()] # Check for an operator and transform to mongo-style if there is op = None if len(parts) > 1 and parts[-1] in MATCH_OPERATORS: op = parts.pop() # Allw to escape operator-like field name by __ if len(parts) > 1 and parts[-1] == "": parts.pop() negate = False if len(parts) > 1 and parts[-1] == 'not': parts.pop() negate = True if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] try: fields = _doc_cls._lookup_field(parts) except Exception, e: raise InvalidQueryError(e) parts = [] CachedReferenceField = _import_class('CachedReferenceField') cleaned_fields = [] for field in fields: append_field = True if isinstance(field, basestring): parts.append(field) append_field = False # is last and CachedReferenceField elif isinstance(field, CachedReferenceField) and fields[-1] == field: parts.append('%s._id' % field.db_field) else: parts.append(field.db_field) if append_field: cleaned_fields.append(field) # Convert value to proper value field = cleaned_fields[-1] singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not'] singular_ops += STRING_OPERATORS if op in singular_ops: if isinstance(field, basestring): if (op in STRING_OPERATORS and isinstance(value, basestring)): StringField = _import_class('StringField') value = StringField.prepare_query_value(op, value) else: value = field else: value = field.prepare_query_value(op, value) if isinstance(field, CachedReferenceField) and value: value = value['_id'] elif op in ('in', 'nin', 'all', 'near') and not isinstance(value, dict): # 'in', 'nin' and 'all' require a list of values value = [field.prepare_query_value(op, v) for v in value] # if op and op not in COMPARISON_OPERATORS: if op: if op in GEO_OPERATORS: value = _geo_operator(field, op, value) elif op in CUSTOM_OPERATORS: if op in ('elem_match', 'match'): value = field.prepare_query_value(op, value) value = {"$elemMatch": value} else: NotImplementedError("Custom method '%s' has not " "been implemented" % op) elif op not in STRING_OPERATORS: value = {'$' + op: value} if negate: value = {'$not': value} for i, part in indices: parts.insert(i, part) key = '.'.join(parts) if op is None or key not in mongo_query: mongo_query[key] = value elif key in mongo_query: if key in mongo_query and isinstance(mongo_query[key], dict): mongo_query[key].update(value) # $max/minDistance needs to come last - convert to SON value_dict = mongo_query[key] if ('$maxDistance' in value_dict or '$minDistance' in value_dict) and \ ('$near' in value_dict or '$nearSphere' in value_dict): value_son = SON() for k, v in value_dict.iteritems(): if k == '$maxDistance' or k == '$minDistance': continue value_son[k] = v # Required for MongoDB >= 2.6, may fail when combining # PyMongo 3+ and MongoDB < 2.6 near_embedded = False for near_op in ('$near', '$nearSphere'): if isinstance(value_dict.get(near_op), dict) and ( IS_PYMONGO_3 or get_connection().max_wire_version > 1): value_son[near_op] = SON(value_son[near_op]) if '$maxDistance' in value_dict: value_son[near_op][ '$maxDistance'] = value_dict['$maxDistance'] if '$minDistance' in value_dict: value_son[near_op][ '$minDistance'] = value_dict['$minDistance'] near_embedded = True if not near_embedded: if '$maxDistance' in value_dict: value_son['$maxDistance'] = value_dict['$maxDistance'] if '$minDistance' in value_dict: value_son['$minDistance'] = value_dict['$minDistance'] mongo_query[key] = value_son else: # Store for manually merging later merge_query[key].append(value)
def update(_doc_cls=None, **update): """Transform an update spec from Django-style format to Mongo format. """ mongo_update = {} for key, value in update.items(): if key == '__raw__': mongo_update.update(value) continue parts = key.split('__') # if there is no operator, default to 'set' if len(parts) < 3 and parts[0] not in UPDATE_OPERATORS: parts.insert(0, 'set') # Check for an operator and transform to mongo-style if there is op = None if parts[0] in UPDATE_OPERATORS: op = parts.pop(0) # Convert Pythonic names to Mongo equivalents if op in ('push_all', 'pull_all'): op = op.replace('_all', 'All') elif op == 'dec': # Support decrement by flipping a positive value's sign # and using 'inc' op = 'inc' value = -value elif op == 'add_to_set': op = 'addToSet' elif op == 'set_on_insert': op = 'setOnInsert' match = None if parts[-1] in COMPARISON_OPERATORS: match = parts.pop() # Allow to escape operator-like field name by __ if len(parts) > 1 and parts[-1] == '': parts.pop() if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] try: fields = _doc_cls._lookup_field(parts) except Exception as e: raise InvalidQueryError(e) parts = [] cleaned_fields = [] appended_sub_field = False for field in fields: append_field = True if isinstance(field, six.string_types): # Convert the S operator to $ if field == 'S': field = '$' parts.append(field) append_field = False else: parts.append(field.db_field) if append_field: appended_sub_field = False cleaned_fields.append(field) if hasattr(field, 'field'): cleaned_fields.append(field.field) appended_sub_field = True # Convert value to proper value if appended_sub_field: field = cleaned_fields[-2] else: field = cleaned_fields[-1] GeoJsonBaseField = _import_class('GeoJsonBaseField') if isinstance(field, GeoJsonBaseField): value = field.to_mongo(value) if op == 'pull': if field.required or value is not None: if match == 'in' and not isinstance(value, dict): value = _prepare_query_for_iterable(field, op, value) else: value = field.prepare_query_value(op, value) elif op == 'push' and isinstance(value, (list, tuple, set)): value = [field.prepare_query_value(op, v) for v in value] elif op in (None, 'set', 'push'): if field.required or value is not None: value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] elif op in ('addToSet', 'setOnInsert'): if isinstance(value, (list, tuple, set)): value = [field.prepare_query_value(op, v) for v in value] elif field.required or value is not None: value = field.prepare_query_value(op, value) elif op == 'unset': value = 1 if match: match = '$' + match value = {match: value} key = '.'.join(parts) if not op: raise InvalidQueryError('Updates must supply an operation ' 'eg: set__FIELD=value') if 'pull' in op and '.' in key: # Dot operators don't work on pull operations # unless they point to a list field # Otherwise it uses nested dict syntax if op == 'pullAll': raise InvalidQueryError('pullAll operations only support ' 'a single field depth') # Look for the last list field and use dot notation until there field_classes = [c.__class__ for c in cleaned_fields] field_classes.reverse() ListField = _import_class('ListField') EmbeddedDocumentListField = _import_class('EmbeddedDocumentListField') if ListField in field_classes or EmbeddedDocumentListField in field_classes: # Join all fields via dot notation to the last ListField or EmbeddedDocumentListField # Then process as normal if ListField in field_classes: _check_field = ListField else: _check_field = EmbeddedDocumentListField last_listField = len( cleaned_fields) - field_classes.index(_check_field) key = '.'.join(parts[:last_listField]) parts = parts[last_listField:] parts.insert(0, key) parts.reverse() for key in parts: value = {key: value} elif op == 'addToSet' and isinstance(value, list): value = {key: {'$each': value}} elif op == 'push': if parts[-1].isdigit(): key = parts[0] position = int(parts[-1]) # $position expects an iterable. If pushing a single value, # wrap it in a list. if not isinstance(value, (set, tuple, list)): value = [value] value = {key: {'$each': value, '$position': position}} else: value = {key: value} else: value = {key: value} key = '$' + op if key not in mongo_update: mongo_update[key] = value elif key in mongo_update and isinstance(mongo_update[key], dict): mongo_update[key].update(value) return mongo_update
def purge_executions(logger, timestamp, action_ref=None, purge_incomplete=False): """ :param timestamp: Exections older than this timestamp will be deleted. :type timestamp: ``datetime.datetime :param action_ref: Only delete executions for the provided actions. :type action_ref: ``str`` :param purge_incomplete: True to also delete executions which are not in a done state. :type purge_incomplete: ``bool`` """ if not timestamp: raise ValueError('Specify a valid timestamp to purge.') logger.info('Purging executions older than timestamp: %s' % timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) filters = {} if purge_incomplete: filters['start_timestamp__lt'] = timestamp else: filters['end_timestamp__lt'] = timestamp filters['start_timestamp__lt'] = timestamp filters['status'] = {'$in': DONE_STATES} exec_filters = copy.copy(filters) if action_ref: exec_filters['action__ref'] = action_ref liveaction_filters = copy.deepcopy(filters) if action_ref: liveaction_filters['action'] = action_ref try: deleted_count = ActionExecution.delete_by_query(**exec_filters) except InvalidQueryError as e: msg = ('Bad query (%s) used to delete execution instances: %s' 'Please contact support.' % (exec_filters, str(e))) raise InvalidQueryError(msg) except: logger.exception( 'Deletion of execution models failed for query with filters: %s.', exec_filters) else: logger.info('Deleted %s action execution objects' % (deleted_count)) try: deleted_count = LiveAction.delete_by_query(**liveaction_filters) except InvalidQueryError as e: msg = ('Bad query (%s) used to delete liveaction instances: %s' 'Please contact support.' % (liveaction_filters, str(e))) raise InvalidQueryError(msg) except: logger.exception( 'Deletion of liveaction models failed for query with filters: %s.', liveaction_filters) else: logger.info('Deleted %s liveaction objects' % (deleted_count)) zombie_execution_instances = len(ActionExecution.query(**exec_filters)) zombie_liveaction_instances = len(LiveAction.query(**liveaction_filters)) if (zombie_execution_instances > 0) or (zombie_liveaction_instances > 0): logger.error('Zombie execution instances left: %d.', zombie_execution_instances) logger.error('Zombie liveaction instances left: %s.', zombie_liveaction_instances) # Print stats logger.info('All execution models older than timestamp %s were deleted.', timestamp)
def purge_executions(logger, timestamp, action_ref=None, purge_incomplete=False): """ Purge action executions and corresponding live action, execution output objects. :param timestamp: Exections older than this timestamp will be deleted. :type timestamp: ``datetime.datetime :param action_ref: Only delete executions for the provided actions. :type action_ref: ``str`` :param purge_incomplete: True to also delete executions which are not in a done state. :type purge_incomplete: ``bool`` """ if not timestamp: raise ValueError('Specify a valid timestamp to purge.') logger.info('Purging executions older than timestamp: %s' % timestamp.strftime('%Y-%m-%dT%H:%M:%S.%fZ')) filters = {} if purge_incomplete: filters['start_timestamp__lt'] = timestamp else: filters['end_timestamp__lt'] = timestamp filters['start_timestamp__lt'] = timestamp filters['status'] = {'$in': DONE_STATES} exec_filters = copy.copy(filters) if action_ref: exec_filters['action__ref'] = action_ref liveaction_filters = copy.deepcopy(filters) if action_ref: liveaction_filters['action'] = action_ref to_delete_execution_dbs = [] # 1. Delete ActionExecutionDB objects try: # Note: We call list() on the query set object because it's lazyily evaluated otherwise to_delete_execution_dbs = list(ActionExecution.query(only_fields=['id'], no_dereference=True, **exec_filters)) deleted_count = ActionExecution.delete_by_query(**exec_filters) except InvalidQueryError as e: msg = ('Bad query (%s) used to delete execution instances: %s' 'Please contact support.' % (exec_filters, six.text_type(e))) raise InvalidQueryError(msg) except: logger.exception('Deletion of execution models failed for query with filters: %s.', exec_filters) else: logger.info('Deleted %s action execution objects' % (deleted_count)) # 2. Delete LiveActionDB objects try: deleted_count = LiveAction.delete_by_query(**liveaction_filters) except InvalidQueryError as e: msg = ('Bad query (%s) used to delete liveaction instances: %s' 'Please contact support.' % (liveaction_filters, six.text_type(e))) raise InvalidQueryError(msg) except: logger.exception('Deletion of liveaction models failed for query with filters: %s.', liveaction_filters) else: logger.info('Deleted %s liveaction objects' % (deleted_count)) # 3. Delete ActionExecutionOutputDB objects to_delete_exection_ids = [str(execution_db.id) for execution_db in to_delete_execution_dbs] output_dbs_filters = {} output_dbs_filters['execution_id'] = {'$in': to_delete_exection_ids} try: deleted_count = ActionExecutionOutput.delete_by_query(**output_dbs_filters) except InvalidQueryError as e: msg = ('Bad query (%s) used to delete execution output instances: %s' 'Please contact support.' % (output_dbs_filters, six.text_type(e))) raise InvalidQueryError(msg) except: logger.exception('Deletion of execution output models failed for query with filters: %s.', output_dbs_filters) else: logger.info('Deleted %s execution output objects' % (deleted_count)) zombie_execution_instances = len(ActionExecution.query(only_fields=['id'], no_dereference=True, **exec_filters)) zombie_liveaction_instances = len(LiveAction.query(only_fields=['id'], no_dereference=True, **liveaction_filters)) if (zombie_execution_instances > 0) or (zombie_liveaction_instances > 0): logger.error('Zombie execution instances left: %d.', zombie_execution_instances) logger.error('Zombie liveaction instances left: %s.', zombie_liveaction_instances) # Print stats logger.info('All execution models older than timestamp %s were deleted.', timestamp)
def query(_doc_cls=None, _field_operation=False, **query): """Transform a query from Django-style format to Mongo format. """ mongo_query = {} merge_query = defaultdict(list) for key, value in sorted(query.items()): if key == "__raw__": mongo_query.update(value) continue parts = key.rsplit('__') indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] parts = [part for part in parts if not part.isdigit()] # Check for an operator and transform to mongo-style if there is op = None if len(parts) > 1 and parts[-1] in MATCH_OPERATORS: op = parts.pop() negate = False if len(parts) > 1 and parts[-1] == 'not': parts.pop() negate = True if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] try: fields = _doc_cls._lookup_field(parts) except Exception as e: raise InvalidQueryError(e) parts = [] CachedReferenceField = _import_class('CachedReferenceField') cleaned_fields = [] for field in fields: append_field = True if isinstance(field, str): parts.append(field) append_field = False # is last and CachedReferenceField elif isinstance(field, CachedReferenceField) and fields[-1] == field: parts.append('%s._id' % field.db_field) else: parts.append(field.db_field) if append_field: cleaned_fields.append(field) # Convert value to proper value field = cleaned_fields[-1] singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not'] singular_ops += STRING_OPERATORS if op in singular_ops: if isinstance(field, str): if (op in STRING_OPERATORS and isinstance(value, str)): StringField = _import_class('StringField') value = StringField.prepare_query_value(op, value) else: value = field else: value = field.prepare_query_value(op, value) if isinstance(field, CachedReferenceField) and value: value = value['_id'] elif op in ('in', 'nin', 'all', 'near') and not isinstance(value, dict): # 'in', 'nin' and 'all' require a list of values value = [field.prepare_query_value(op, v) for v in value] # if op and op not in COMPARISON_OPERATORS: if op: if op in GEO_OPERATORS: value = _geo_operator(field, op, value) elif op in CUSTOM_OPERATORS: if op in ('elem_match', 'match'): value = field.prepare_query_value(op, value) value = {"$elemMatch": value} else: NotImplementedError("Custom method '%s' has not " "been implemented" % op) elif op not in STRING_OPERATORS: value = {'$' + op: value} if negate: value = {'$not': value} for i, part in indices: parts.insert(i, part) key = '.'.join(parts) if op is None or key not in mongo_query: mongo_query[key] = value elif key in mongo_query: if key in mongo_query and isinstance(mongo_query[key], dict): mongo_query[key].update(value) # $maxDistance needs to come last - convert to SON value_dict = mongo_query[key] if ('$maxDistance' in value_dict and '$near' in value_dict): value_son = SON() if isinstance(value_dict['$near'], dict): for k, v in value_dict.items(): if k == '$maxDistance': continue value_son[k] = v if (get_connection().max_wire_version <= 1): value_son['$maxDistance'] = value_dict[ '$maxDistance'] else: value_son['$near'] = SON(value_son['$near']) value_son['$near']['$maxDistance'] = value_dict[ '$maxDistance'] else: for k, v in value_dict.items(): if k == '$maxDistance': continue value_son[k] = v value_son['$maxDistance'] = value_dict['$maxDistance'] mongo_query[key] = value_son else: # Store for manually merging later merge_query[key].append(value) # The queryset has been filter in such a way we must manually merge for k, v in list(merge_query.items()): merge_query[k].append(mongo_query[k]) del mongo_query[k] if isinstance(v, list): value = [{k: val} for val in v] if '$and' in list(mongo_query.keys()): mongo_query['$and'].extend(value) else: mongo_query['$and'] = value return mongo_query
def query(_doc_cls=None, **kwargs): """Transform a query from Django-style format to Mongo format.""" mongo_query = {} merge_query = defaultdict(list) for key, value in sorted(kwargs.items()): if key == "__raw__": mongo_query.update(value) continue parts = key.rsplit("__") indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] parts = [part for part in parts if not part.isdigit()] # Check for an operator and transform to mongo-style if there is op = None if len(parts) > 1 and parts[-1] in MATCH_OPERATORS: op = parts.pop() # Allow to escape operator-like field name by __ if len(parts) > 1 and parts[-1] == "": parts.pop() negate = False if len(parts) > 1 and parts[-1] == "not": parts.pop() negate = True if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] try: fields = _doc_cls._lookup_field(parts) except Exception as e: raise InvalidQueryError(e) parts = [] CachedReferenceField = _import_class("CachedReferenceField") GenericReferenceField = _import_class("GenericReferenceField") cleaned_fields = [] for field in fields: append_field = True if isinstance(field, str): parts.append(field) append_field = False # is last and CachedReferenceField elif isinstance(field, CachedReferenceField) and fields[-1] == field: parts.append("%s._id" % field.db_field) else: parts.append(field.db_field) if append_field: cleaned_fields.append(field) # Convert value to proper value field = cleaned_fields[-1] singular_ops = [None, "ne", "gt", "gte", "lt", "lte", "not"] singular_ops += STRING_OPERATORS if op in singular_ops: value = field.prepare_query_value(op, value) if isinstance(field, CachedReferenceField) and value: value = value["_id"] elif op in ("in", "nin", "all", "near") and not isinstance(value, dict): # Raise an error if the in/nin/all/near param is not iterable. value = _prepare_query_for_iterable(field, op, value) # If we're querying a GenericReferenceField, we need to alter the # key depending on the value: # * If the value is a DBRef, the key should be "field_name._ref". # * If the value is an ObjectId, the key should be "field_name._ref.$id". if isinstance(field, GenericReferenceField): if isinstance(value, DBRef): parts[-1] += "._ref" elif isinstance(value, ObjectId): parts[-1] += "._ref.$id" # if op and op not in COMPARISON_OPERATORS: if op: if op in GEO_OPERATORS: value = _geo_operator(field, op, value) elif op in ("match", "elemMatch"): ListField = _import_class("ListField") EmbeddedDocumentField = _import_class("EmbeddedDocumentField") if (isinstance(value, dict) and isinstance(field, ListField) and isinstance(field.field, EmbeddedDocumentField)): value = query(field.field.document_type, **value) else: value = field.prepare_query_value(op, value) value = {"$elemMatch": value} elif op in CUSTOM_OPERATORS: NotImplementedError('Custom method "%s" has not ' "been implemented" % op) elif op not in STRING_OPERATORS: value = {"$" + op: value} if negate: value = {"$not": value} for i, part in indices: parts.insert(i, part) key = ".".join(parts) if key not in mongo_query: mongo_query[key] = value else: if isinstance(mongo_query[key], dict) and isinstance(value, dict): mongo_query[key].update(value) # $max/minDistance needs to come last - convert to SON value_dict = mongo_query[key] if ("$maxDistance" in value_dict or "$minDistance" in value_dict) and ("$near" in value_dict or "$nearSphere" in value_dict): value_son = SON() for k, v in value_dict.items(): if k == "$maxDistance" or k == "$minDistance": continue value_son[k] = v # Required for MongoDB >= 2.6, may fail when combining # PyMongo 3+ and MongoDB < 2.6 near_embedded = False for near_op in ("$near", "$nearSphere"): if isinstance(value_dict.get(near_op), dict): value_son[near_op] = SON(value_son[near_op]) if "$maxDistance" in value_dict: value_son[near_op][ "$maxDistance"] = value_dict[ "$maxDistance"] if "$minDistance" in value_dict: value_son[near_op][ "$minDistance"] = value_dict[ "$minDistance"] near_embedded = True if not near_embedded: if "$maxDistance" in value_dict: value_son["$maxDistance"] = value_dict[ "$maxDistance"] if "$minDistance" in value_dict: value_son["$minDistance"] = value_dict[ "$minDistance"] mongo_query[key] = value_son else: # Store for manually merging later merge_query[key].append(value) # The queryset has been filter in such a way we must manually merge for k, v in merge_query.items(): merge_query[k].append(mongo_query[k]) del mongo_query[k] if isinstance(v, list): value = [{k: val} for val in v] if "$and" in mongo_query.keys(): mongo_query["$and"].extend(value) else: mongo_query["$and"] = value return mongo_query
def update(_doc_cls=None, **update): """Transform an update spec from Django-style format to Mongo format. """ mongo_update = {} for key, value in update.items(): if key == "__raw__": mongo_update.update(value) continue parts = key.split("__") # if there is no operator, default to 'set' if len(parts) < 3 and parts[0] not in UPDATE_OPERATORS: parts.insert(0, "set") # Check for an operator and transform to mongo-style if there is op = None if parts[0] in UPDATE_OPERATORS: op = parts.pop(0) # Convert Pythonic names to Mongo equivalents operator_map = { "push_all": "pushAll", "pull_all": "pullAll", "dec": "inc", "add_to_set": "addToSet", "set_on_insert": "setOnInsert", } if op == "dec": # Support decrement by flipping a positive value's sign # and using 'inc' value = -value # If the operator doesn't found from operator map, the op value # will stay unchanged op = operator_map.get(op, op) match = None if parts[-1] in COMPARISON_OPERATORS: match = parts.pop() # Allow to escape operator-like field name by __ if len(parts) > 1 and parts[-1] == "": parts.pop() if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] try: fields = _doc_cls._lookup_field(parts) except Exception as e: raise InvalidQueryError(e) parts = [] cleaned_fields = [] appended_sub_field = False for field in fields: append_field = True if isinstance(field, str): # Convert the S operator to $ if field == "S": field = "$" parts.append(field) append_field = False else: parts.append(field.db_field) if append_field: appended_sub_field = False cleaned_fields.append(field) if hasattr(field, "field"): cleaned_fields.append(field.field) appended_sub_field = True # Convert value to proper value if appended_sub_field: field = cleaned_fields[-2] else: field = cleaned_fields[-1] GeoJsonBaseField = _import_class("GeoJsonBaseField") if isinstance(field, GeoJsonBaseField): value = field.to_mongo(value) if op == "pull": if field.required or value is not None: if match in ("in", "nin") and not isinstance(value, dict): value = _prepare_query_for_iterable(field, op, value) else: value = field.prepare_query_value(op, value) elif op == "push" and isinstance(value, (list, tuple, set)): value = [field.prepare_query_value(op, v) for v in value] elif op in (None, "set", "push"): if field.required or value is not None: value = field.prepare_query_value(op, value) elif op in ("pushAll", "pullAll"): value = [field.prepare_query_value(op, v) for v in value] elif op in ("addToSet", "setOnInsert"): if isinstance(value, (list, tuple, set)): value = [field.prepare_query_value(op, v) for v in value] elif field.required or value is not None: value = field.prepare_query_value(op, value) elif op == "unset": value = 1 elif op == "inc": value = field.prepare_query_value(op, value) if match: match = "$" + match value = {match: value} key = ".".join(parts) if "pull" in op and "." in key: # Dot operators don't work on pull operations # unless they point to a list field # Otherwise it uses nested dict syntax if op == "pullAll": raise InvalidQueryError( "pullAll operations only support a single field depth") # Look for the last list field and use dot notation until there field_classes = [c.__class__ for c in cleaned_fields] field_classes.reverse() ListField = _import_class("ListField") EmbeddedDocumentListField = _import_class( "EmbeddedDocumentListField") if ListField in field_classes or EmbeddedDocumentListField in field_classes: # Join all fields via dot notation to the last ListField or EmbeddedDocumentListField # Then process as normal if ListField in field_classes: _check_field = ListField else: _check_field = EmbeddedDocumentListField last_listField = len(cleaned_fields) - field_classes.index( _check_field) key = ".".join(parts[:last_listField]) parts = parts[last_listField:] parts.insert(0, key) parts.reverse() for key in parts: value = {key: value} elif op == "addToSet" and isinstance(value, list): value = {key: {"$each": value}} elif op in ("push", "pushAll"): if parts[-1].isdigit(): key = ".".join(parts[0:-1]) position = int(parts[-1]) # $position expects an iterable. If pushing a single value, # wrap it in a list. if not isinstance(value, (set, tuple, list)): value = [value] value = {key: {"$each": value, "$position": position}} else: if op == "pushAll": op = "push" # convert to non-deprecated keyword if not isinstance(value, (set, tuple, list)): value = [value] value = {key: {"$each": value}} else: value = {key: value} else: value = {key: value} key = "$" + op if key not in mongo_update: mongo_update[key] = value elif key in mongo_update and isinstance(mongo_update[key], dict): mongo_update[key].update(value) return mongo_update