def similar(exp, object_class, target_class, query): """Filter by relationships similarity. Note: only the first id from the list of ids is used. Args: object_name: the name of the class of the objects to which similarity will be computed. ids: the ids of similar objects of type `object_name`. Returns: sqlalchemy.sql.elements.BinaryExpression if an object of `object_class` is similar to one the given objects. """ similar_class = inflector.get_model(exp['object_name']) if not hasattr(similar_class, "get_similar_objects_query"): raise BadQueryException(u"{} does not define weights to count " u"relationships similarity".format( similar_class.__name__)) similar_objects_query = similar_class.get_similar_objects_query( id_=exp['ids'][0], types=[object_class.__name__], ) flask.g.similar_objects_query = similar_objects_query similar_objects_ids = [obj.id for obj in similar_objects_query] if similar_objects_ids: return object_class.id.in_(similar_objects_ids) return sqlalchemy.sql.false()
def unknown(exp, object_class, target_class, query): """A fake operator for invalid operator names.""" name = exp.get("op", {}).get("name") if name is None: msg = u"No operator name sent" else: msg = u"Unknown operator \"{}\"".format(name) raise BadQueryException(msg)
def build_expression(exp, object_class, target_class, query): """Make an SQLAlchemy filtering expression from exp expression tree.""" if OPS.get(exp.get("op", {}).get("name")) is None: return exp = autocast(exp, target_class) if not exp: raise BadQueryException("Invalid filter data") operation = OPS.get(exp.get("op", {}).get("name")) or unknown return operation(exp, object_class, target_class, query)
def expand_task_dates(exp): """Parse task dates from the specified expression.""" if not isinstance(exp, dict) or "op" not in exp: return operator_name = exp["op"]["name"] if operator_name in ["AND", "OR"]: expand_task_dates(exp["left"]) expand_task_dates(exp["right"]) elif isinstance(exp["left"], (str, unicode)): key = exp["left"] if key in ["start", "end"]: parts = exp["right"].split("/") if len(parts) == 3: try: month, day, year = [int(part) for part in parts] except Exception: raise BadQueryException( "Date must consist of numbers") exp["left"] = key + "_date" exp["right"] = datetime.date(year, month, day) elif len(parts) == 2: month, day = parts exp["op"] = {"name": u"AND"} exp["left"] = { "op": { "name": operator_name }, "left": "relative_" + key + "_month", "right": month, } exp["right"] = { "op": { "name": operator_name }, "left": "relative_" + key + "_day", "right": day, } elif len(parts) == 1: exp["left"] = "relative_" + key + "_day" else: raise BadQueryException( u"Field {} should be a date of one of the" u" following forms: DD, MM/DD, MM/DD/YYYY".format( key))
def decorated_operator(exp, *args, **kwargs): """Decorated operator """ operation_name = exp["op"]["name"] error_fields = required_fields_set - set(exp.keys()) if error_fields: raise BadQueryException("\n".join([ required_tmpl.format(field=field, operation=operation_name) for field in error_fields ])) return operation(exp, *args, **kwargs)
def _clean_query(self, query): """ sanitize the query object """ for object_query in query: if "object_name" not in object_query: raise BadQueryException( "`object_name` required for each object block") filters = object_query.get("filters", {}).get("expression") self._clean_filters(filters) self._macro_expand_object_query(object_query) return query
def build_expression(exp, object_class, target_class, query): """Make an SQLAlchemy filtering expression from exp expression tree.""" if not exp: # empty expression doesn't required filter return if autocast.is_autocast_required_for(exp): exp = validate("left", "right")(autocast.autocast)(exp, target_class) if not exp: # empty expression after autocast is invalid and should raise an exception raise BadQueryException("Invalid filter data") operation = OPS.get(exp.get("op", {}).get("name")) or unknown return operation(exp, object_class, target_class, query)
def _apply_limit(query, limit): """Apply limits for pagination. Args: query: filter query; limit: a tuple of indexes in format (from, to); objects is sliced to objects[from, to]. Returns: matched objects ids and total count. """ try: first, last = limit first, last = int(first), int(last) except (ValueError, TypeError): raise BadQueryException( "Invalid limit operator. Integers expected.") if first < 0 or last < 0: raise BadQueryException("Limit cannot contain negative numbers.") elif first >= last: raise BadQueryException("Limit start should be smaller than end.") else: page_size = last - first with benchmark("Apply limit: _apply_limit > query_limit"): # Note: limit request syntax is limit:[0,10]. We are counting # offset from 0 as the offset of the initial row for sql is 0 (not 1). ids = [obj.id for obj in query.limit(page_size).offset(first)] with benchmark("Apply limit: _apply_limit > query_count"): if len(ids) < page_size: total = len(ids) + first else: # Note: using func.count() as query.count() is generating additional # subquery count_q = query.statement.with_only_columns( [sa.func.count()]) total = db.session.execute(count_q).scalar() return ids, total
def is_filter(exp, object_class, target_class, query): """Handle 'is' operator. As in 'CA is empty' expression """ if exp['right'] != u"empty": raise BadQueryException(u"Invalid operator near 'is': {}".format( exp['right'])) left = exp['left'].lower() left, _ = target_class.attributes_map().get(left, (left, None)) subquery = db.session.query(Record.key).filter( Record.type == object_class.__name__, Record.property == left, sqlalchemy.not_( sqlalchemy.or_(Record.content == u"", Record.content.is_(None))), ) return object_class.id.notin_(subquery)
def _clean_filters(self, expression): """Prepare the filter expression for building the query.""" if not expression or not isinstance(expression, dict): return slugs = expression.get("slugs") if slugs: ids = expression.get("ids", []) ids.extend(self._slugs_to_ids(expression["object_name"], slugs)) expression["ids"] = ids try: expression["ids"] = [int(id_) for id_ in expression.get("ids", [])] except ValueError as error: # catch missing relevant filter (undefined id) if expression.get("op", {}).get("name", "") == "relevant": raise BadQueryException( u"Invalid relevant filter for {}".format( expression.get("object_name", ""))) raise error self._clean_filters(expression.get("left")) self._clean_filters(expression.get("right"))
def unknown(exp, object_class, target_class, query): """A fake operator for invalid operator names.""" raise BadQueryException( u"Unknown operator \"{}\"".format(exp["op"]["name"]))
def joins_and_order(clause): """Get join operations and ordering field from item of order_by list. Args: clause: {"name": the name of model's field, "desc": reverse sort on this field if True} Returns: ([joins], order) - a tuple of joins required for this ordering to work and ordering clause itself; join is None if no join required or [(aliased entity, relationship field)] if joins required. """ def by_similarity(): """Join similar_objects subquery, order by weight from it.""" join_target = flask.g.similar_objects_query.subquery() join_condition = model.id == join_target.c.id joins = [(join_target, join_condition)] order = join_target.c.weight return joins, order def by_fulltext(): """Join fulltext index table, order by indexed CA value.""" alias = sa.orm.aliased(Record, name=u"fulltext_{}".format(self._count)) joins = [(alias, sa.and_(alias.key == model.id, alias.type == model.__name__, alias.property == key, alias.subproperty.in_(["", "__sort__"])))] order = alias.content return joins, order def by_foreign_key(): """Join the related model, order by title or name/email.""" related_model = attr.property.mapper.class_ if issubclass(related_model, models.mixins.Titled): joins = [(alias, _)] = [(sa.orm.aliased(attr), attr)] order = alias.title else: raise NotImplementedError( u"Sorting by {model.__name__} is " u"not implemented yet.".format(model=related_model)) return joins, order # transform clause["name"] into a model's field name key = clause["name"].lower() if key == "__similarity__": # special case if hasattr(flask.g, "similar_objects_query"): joins, order = by_similarity() else: raise BadQueryException( "Can't order by '__similarity__' when no " "'similar' filter was applied.") else: key, _ = tgt_class.attributes_map().get(key, (key, None)) if key in custom_operators.GETATTR_WHITELIST: attr = getattr(model, key.encode('utf-8'), None) if (isinstance(attr, sa.orm.attributes.InstrumentedAttribute) and isinstance( attr.property, sa.orm.properties.RelationshipProperty)): joins, order = by_foreign_key() else: # a simple attribute joins, order = None, attr else: # Snapshot or non object attributes are treated as custom attributes self._count += 1 joins, order = by_fulltext() if clause.get("desc", False): order = order.desc() return joins, order