def similar(exp, object_class, target_class, query):
"""Filter by relationships similarity.
Note: only the first id from the list of ids is used.
Args:
object_name: the name of the class of the objects to which similarity
will be computed.
ids: the ids of similar objects of type `object_name`.
Returns:
sqlalchemy.sql.elements.BinaryExpression if an object of `object_class`
is similar to one the given objects.
"""
similar_class = inflector.get_model(exp['object_name'])
if not hasattr(similar_class, "get_similar_objects_query"):
raise BadQueryException(u"{} does not define weights to count "
u"relationships similarity".format(
similar_class.__name__))
similar_objects_query = similar_class.get_similar_objects_query(
id_=exp['ids'][0],
types=[object_class.__name__],
)
flask.g.similar_objects_query = similar_objects_query
similar_objects_ids = [obj.id for obj in similar_objects_query]
if similar_objects_ids:
return object_class.id.in_(similar_objects_ids)
return sqlalchemy.sql.false()
def unknown(exp, object_class, target_class, query):
"""A fake operator for invalid operator names."""
name = exp.get("op", {}).get("name")
if name is None:
msg = u"No operator name sent"
else:
msg = u"Unknown operator \"{}\"".format(name)
raise BadQueryException(msg)
def build_expression(exp, object_class, target_class, query):
"""Make an SQLAlchemy filtering expression from exp expression tree."""
if OPS.get(exp.get("op", {}).get("name")) is None:
return
exp = autocast(exp, target_class)
if not exp:
raise BadQueryException("Invalid filter data")
operation = OPS.get(exp.get("op", {}).get("name")) or unknown
return operation(exp, object_class, target_class, query)
def expand_task_dates(exp):
"""Parse task dates from the specified expression."""
if not isinstance(exp, dict) or "op" not in exp:
return
operator_name = exp["op"]["name"]
if operator_name in ["AND", "OR"]:
expand_task_dates(exp["left"])
expand_task_dates(exp["right"])
elif isinstance(exp["left"], (str, unicode)):
key = exp["left"]
if key in ["start", "end"]:
parts = exp["right"].split("/")
if len(parts) == 3:
try:
month, day, year = [int(part) for part in parts]
except Exception:
raise BadQueryException(
"Date must consist of numbers")
exp["left"] = key + "_date"
exp["right"] = datetime.date(year, month, day)
elif len(parts) == 2:
month, day = parts
exp["op"] = {"name": u"AND"}
exp["left"] = {
"op": {
"name": operator_name
},
"left": "relative_" + key + "_month",
"right": month,
}
exp["right"] = {
"op": {
"name": operator_name
},
"left": "relative_" + key + "_day",
"right": day,
}
elif len(parts) == 1:
exp["left"] = "relative_" + key + "_day"
else:
raise BadQueryException(
u"Field {} should be a date of one of the"
u" following forms: DD, MM/DD, MM/DD/YYYY".format(
key))
def decorated_operator(exp, *args, **kwargs):
"""Decorated operator """
operation_name = exp["op"]["name"]
error_fields = required_fields_set - set(exp.keys())
if error_fields:
raise BadQueryException("\n".join([
required_tmpl.format(field=field, operation=operation_name)
for field in error_fields
]))
return operation(exp, *args, **kwargs)
def _clean_query(self, query):
""" sanitize the query object """
for object_query in query:
if "object_name" not in object_query:
raise BadQueryException(
"`object_name` required for each object block")
filters = object_query.get("filters", {}).get("expression")
self._clean_filters(filters)
self._macro_expand_object_query(object_query)
return query
def build_expression(exp, object_class, target_class, query):
"""Make an SQLAlchemy filtering expression from exp expression tree."""
if not exp:
# empty expression doesn't required filter
return
if autocast.is_autocast_required_for(exp):
exp = validate("left", "right")(autocast.autocast)(exp, target_class)
if not exp:
# empty expression after autocast is invalid and should raise an exception
raise BadQueryException("Invalid filter data")
operation = OPS.get(exp.get("op", {}).get("name")) or unknown
return operation(exp, object_class, target_class, query)
def _apply_limit(query, limit):
"""Apply limits for pagination.
Args:
query: filter query;
limit: a tuple of indexes in format (from, to); objects is sliced to
objects[from, to].
Returns:
matched objects ids and total count.
"""
try:
first, last = limit
first, last = int(first), int(last)
except (ValueError, TypeError):
raise BadQueryException(
"Invalid limit operator. Integers expected.")
if first < 0 or last < 0:
raise BadQueryException("Limit cannot contain negative numbers.")
elif first >= last:
raise BadQueryException("Limit start should be smaller than end.")
else:
page_size = last - first
with benchmark("Apply limit: _apply_limit > query_limit"):
# Note: limit request syntax is limit:[0,10]. We are counting
# offset from 0 as the offset of the initial row for sql is 0 (not 1).
ids = [obj.id for obj in query.limit(page_size).offset(first)]
with benchmark("Apply limit: _apply_limit > query_count"):
if len(ids) < page_size:
total = len(ids) + first
else:
# Note: using func.count() as query.count() is generating additional
# subquery
count_q = query.statement.with_only_columns(
[sa.func.count()])
total = db.session.execute(count_q).scalar()
return ids, total
def is_filter(exp, object_class, target_class, query):
"""Handle 'is' operator.
As in 'CA is empty' expression
"""
if exp['right'] != u"empty":
raise BadQueryException(u"Invalid operator near 'is': {}".format(
exp['right']))
left = exp['left'].lower()
left, _ = target_class.attributes_map().get(left, (left, None))
subquery = db.session.query(Record.key).filter(
Record.type == object_class.__name__,
Record.property == left,
sqlalchemy.not_(
sqlalchemy.or_(Record.content == u"", Record.content.is_(None))),
)
return object_class.id.notin_(subquery)
def _clean_filters(self, expression):
"""Prepare the filter expression for building the query."""
if not expression or not isinstance(expression, dict):
return
slugs = expression.get("slugs")
if slugs:
ids = expression.get("ids", [])
ids.extend(self._slugs_to_ids(expression["object_name"], slugs))
expression["ids"] = ids
try:
expression["ids"] = [int(id_) for id_ in expression.get("ids", [])]
except ValueError as error:
# catch missing relevant filter (undefined id)
if expression.get("op", {}).get("name", "") == "relevant":
raise BadQueryException(
u"Invalid relevant filter for {}".format(
expression.get("object_name", "")))
raise error
self._clean_filters(expression.get("left"))
self._clean_filters(expression.get("right"))
def unknown(exp, object_class, target_class, query):
"""A fake operator for invalid operator names."""
raise BadQueryException(
u"Unknown operator \"{}\"".format(exp["op"]["name"]))
def joins_and_order(clause):
"""Get join operations and ordering field from item of order_by list.
Args:
clause: {"name": the name of model's field,
"desc": reverse sort on this field if True}
Returns:
([joins], order) - a tuple of joins required for this ordering to work
and ordering clause itself; join is None if no join
required or [(aliased entity, relationship field)]
if joins required.
"""
def by_similarity():
"""Join similar_objects subquery, order by weight from it."""
join_target = flask.g.similar_objects_query.subquery()
join_condition = model.id == join_target.c.id
joins = [(join_target, join_condition)]
order = join_target.c.weight
return joins, order
def by_fulltext():
"""Join fulltext index table, order by indexed CA value."""
alias = sa.orm.aliased(Record,
name=u"fulltext_{}".format(self._count))
joins = [(alias,
sa.and_(alias.key == model.id,
alias.type == model.__name__,
alias.property == key,
alias.subproperty.in_(["", "__sort__"])))]
order = alias.content
return joins, order
def by_foreign_key():
"""Join the related model, order by title or name/email."""
related_model = attr.property.mapper.class_
if issubclass(related_model, models.mixins.Titled):
joins = [(alias, _)] = [(sa.orm.aliased(attr), attr)]
order = alias.title
else:
raise NotImplementedError(
u"Sorting by {model.__name__} is "
u"not implemented yet.".format(model=related_model))
return joins, order
# transform clause["name"] into a model's field name
key = clause["name"].lower()
if key == "__similarity__":
# special case
if hasattr(flask.g, "similar_objects_query"):
joins, order = by_similarity()
else:
raise BadQueryException(
"Can't order by '__similarity__' when no "
"'similar' filter was applied.")
else:
key, _ = tgt_class.attributes_map().get(key, (key, None))
if key in custom_operators.GETATTR_WHITELIST:
attr = getattr(model, key.encode('utf-8'), None)
if (isinstance(attr,
sa.orm.attributes.InstrumentedAttribute)
and isinstance(
attr.property,
sa.orm.properties.RelationshipProperty)):
joins, order = by_foreign_key()
else:
# a simple attribute
joins, order = None, attr
else:
# Snapshot or non object attributes are treated as custom attributes
self._count += 1
joins, order = by_fulltext()
if clause.get("desc", False):
order = order.desc()
return joins, order
