def _get_lookup():
"""Get the where clause lookup just like Eve does.
Unfortunately, Eve only parses the `where` at a very low level and does not
provide any methods to elegantly access it, so we have use the same
internal functions as Eve does.
(Recently, Eve has at least somewhat exposed the parsing, but this
code is not part of an official release yet [1])
As soon as there is some 'official' support, this can be removed,
as it is basically copied, with the abort removed for simplicity
(as Eve itself will already abort if there's an error).
[1]: https://github.com/pyeve/eve/blob/master/eve/io/mongo/mongo.py
"""
req = parse_request('studydocuments')
if req and req.where:
try:
# Mongo Syntax
return current_app.data._sanitize(json.loads(req.where))
except (HTTPException, json.JSONDecodeError):
# Python Syntax
return parse(req.where)
return {} # No where clause
def _get_lookup():
"""Get the where clause lookup just like Eve does.
Unfortunately, Eve only parses the `where` at a very low level and does not
provide any methods to elegantly access it, so we have use the same
internal functions as Eve does.
(Recently, Eve has at least somewhat exposed the parsing, but this
code is not part of an official release yet [1])
As soon as there is some 'official' support, this can be removed,
as it is basically copied, with the abort removed for simplicity
(as Eve itself will already abort if there's an error).
[1]: https://github.com/pyeve/eve/blob/master/eve/io/mongo/mongo.py
"""
req = parse_request('studydocuments')
if req and req.where:
try:
# Mongo Syntax
return current_app.data._sanitize('studydocuments',
json.loads(req.where))
except (HTTPException, json.JSONDecodeError):
# Python Syntax
return parse(req.where)
return {} # No where clause
def test_nested_BoolOp(self):
r = parse('a == 1 or (b == 2 and c == 3)')
self.assertEqual(type(r), dict)
self.assertEqual(r,
{'$or': [{
'a': 1
}, {
'$and': [{
'b': 2
}, {
'c': 3
}]
}]})
def test_nested_BoolOp(self):
r = parse("a == 1 or (b == 2 and c == 3)")
self.assertEqual(type(r), dict)
self.assertEqual(r,
{"$or": [{
"a": 1
}, {
"$and": [{
"b": 2
}, {
"c": 3
}]
}]})
def aggregate(self, resource, req):
client_projection = {}
spec = {}
if req.where:
try:
spec = self._sanitize(
self._jsondatetime(json.loads(req.where, object_hook=json_util.object_hook)))
except:
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if req.projection:
try:
client_projection = json.loads(req.projection)
except:
abort(400, description=debug_error_message(
'Unable to parse `projection` clause'
))
datasource, spec, projection = self._datasource_ex(resource, spec,
client_projection)
groupers = config.DOMAIN[resource]["default_groupers"]
groupees = config.DOMAIN[resource]["default_groupees"]
group_val = {}
group_val["_id"] = {g: "$%s" % g for g in groupers}
for group_info in groupees:
name = group_info["name"]
group_type = group_info["type"]
group_val[name] = {"$%s" % group_type: "$%s" % name}
pipeline = []
pipeline.append({"$match": spec})
pipeline.append({"$project": projection})
pipeline.append({"$group": group_val})
pipeline.append({"$limit": 1000})
docs = self.driver.db[datasource].aggregate(pipeline)["result"]
cursor = Cursor(docs) #gives required functions to returned result
return cursor
def _convert_where_request_to_dict(self, req):
""" Converts the contents of a `ParsedRequest`'s `where` property to
a dict
"""
query = {}
if req and req.where:
try:
query = self._sanitize(json.loads(req.where))
except HTTPException:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
# couldn't parse as mongo query; give the python parser a shot.
try:
query = parse(req.where)
except ParseError:
abort(
400,
description=debug_error_message(
"Unable to parse `where` clause"),
)
return query
def _convert_where_request_to_dict(self, req):
""" Converts the contents of a `ParsedRequest`'s `where` property to
a dict
"""
query = {}
if req and req.where:
try:
query = self._sanitize(json.loads(req.where))
except HTTPException:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
# couldn't parse as mongo query; give the python parser a shot.
try:
query = parse(req.where)
except ParseError:
abort(
400,
description=debug_error_message(
"Unable to parse `where` clause"
),
)
return query
def test_Eq(self):
r = parse('a == "whatever"')
self.assertEqual(type(r), dict)
self.assertEqual(r, {"a": "whatever"})
def find(self, resource, req, sub_resource_lookup):
""" Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe"}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
"""
args = dict()
if req and req.max_results:
args['limit'] = req.max_results
if req and req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = {}
spec = {}
if req and req.sort:
try:
# assume it's mongo syntax (ie. ?sort=[("name", 1)])
client_sort = ast.literal_eval(req.sort)
except ValueError:
# it's not mongo so let's see if it's a comma delimited string
# instead (ie. "?sort=-age, name").
sort = []
for sort_arg in [s.strip() for s in req.sort.split(",")]:
if sort_arg[0] == "-":
sort.append((sort_arg[1:], -1))
else:
sort.append((sort_arg, 1))
if len(sort) > 0:
client_sort = sort
except Exception as e:
self.app.logger.exception(e)
abort(400, description=debug_error_message(str(e)))
if req and req.where:
try:
spec = self._sanitize(json.loads(req.where))
except HTTPException as e:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
# couldn't parse as mongo query; give the python parser a shot.
try:
spec = parse(req.where)
except ParseError:
abort(400,
description=debug_error_message(
'Unable to parse `where` clause'))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
if config.DOMAIN[resource]['soft_delete'] \
and not (req and req.show_deleted) \
and not self.query_contains_field(spec, config.DELETED):
# Soft delete filtering applied after validate_filters call as
# querying against the DELETED field must always be allowed when
# soft_delete is enabled
spec = self.combine_queries(spec, {config.DELETED: {"$ne": True}})
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource, spec, client_projection, client_sort)
if req and req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['filter'] = spec
if sort is not None:
args['sort'] = sort
if projection:
args['projection'] = projection
return self._find(resource, datasource, **args)
def test_Lt(self):
r = parse('a < 1')
self.assertIs(type(r), dict)
self.assertEqual(r, {'a': {'$lt': 1}})
def test_nested_BoolOp(self):
r = parse("a == 1 or (b == 2 and c == 3)")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"$or": [{"a": 1}, {"$and": [{"b": 2}, {"c": 3}]}]})
def test_LtE(self):
r = parse("a <= 1")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"a": {"$lte": 1}})
def find(self, resource, req, sub_resource_lookup):
""" Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
.. versionchanged:: 0.4
'allowed_filters' is now checked before adding 'sub_resource_lookup'
to the query, as it is considered safe.
Refactored to use self._client_projection since projection is now
honored by getitem() as well.
.. versionchanged:: 0.3
Support for new _mongotize() signature.
.. versionchagend:: 0.2
Support for sub-resources.
Support for 'default_sort'.
.. versionchanged:: 0.1.1
Better query handling. We're now properly casting objectid-like
strings to ObjectIds. Also, we're casting both datetimes and
objectids even when the query was originally in python syntax.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req.max_results:
args['limit'] = req.max_results
if req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = {}
spec = {}
if req.sort:
client_sort = ast.literal_eval(req.sort)
if req.where:
try:
spec = self._sanitize(json.loads(req.where))
except:
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource,
spec,
client_projection,
client_sort)
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['spec'] = spec
if sort is not None:
args['sort'] = sort
if projection is not None:
args['fields'] = projection
return self.driver.db[datasource].find(**args)
def test_And_BoolOp(self):
r = parse("a == 1 and b == 2")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"$and": [{"a": 1}, {"b": 2}]})
def test_Or_BoolOp(self):
r = parse("a == 1 or b == 2")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"$or": [{"a": 1}, {"b": 2}]})
def test_NotEq(self):
r = parse("a != 1")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"a": {"$ne": 1}})
def test_LtE(self):
r = parse("a <= 1")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"a": {"$lte": 1}})
def test_GtE(self):
r = parse("a >= 1")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"a": {"$gte": 1}})
def test_datetime_Call(self):
r = parse('born == datetime(2012, 11, 9)')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'born': datetime(2012, 11, 9)})
def test_datetime_Call(self):
r = parse("born == datetime(2012, 11, 9)")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"born": datetime(2012, 11, 9)})
def find(self, resource, req, sub_resource_lookup):
"""
Seach for results and return list of them.
:param resource: name of requested resource as string.
:param req: instance of :class:`eve.utils.ParsedRequest`.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
"""
qry = self.cls_map.objects(resource)
client_projection = {}
client_sort = {}
spec = {}
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
if req.sort:
try:
client_sort = ast.literal_eval(req.sort)
except Exception as e:
abort(400, description=debug_error_message(str(e)))
if req.where:
try:
spec = self._sanitize(json.loads(req.where))
except HTTPException as e:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
if sub_resource_lookup:
spec.update(sub_resource_lookup)
spec = self._mongotize(spec, resource)
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource,
spec,
client_projection,
client_sort)
# apply ordering
if sort:
for field, direction in _itemize(sort):
if direction < 0:
field = "-%s" % field
qry = qry.order_by(field)
# apply filters
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
qry = qry.filter(__raw__=spec)
# apply projection
qry = self._projection(resource, projection, qry)
# apply limits
if req.max_results:
qry = qry.limit(int(req.max_results))
if req.page > 1:
qry = qry.skip((req.page - 1) * req.max_results)
return PymongoQuerySet(qry)
def test_Attribute(self):
r = parse("Invoice.number == 1")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"Invoice.number": 1})
def test_Eq(self):
r = parse('a == "whatever"')
self.assertEqual(type(r), dict)
self.assertEqual(r, {"a": "whatever"})
def find(self, resource, req, sub_resource_lookup):
""" Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
.. versionchanged:: 0.5
Return the error if a blacklisted MongoDB operator is used in query.
Abort with 400 if unsupported query operator is used. #387.
Abort with 400 in case of invalid sort syntax. #387.
.. versionchanged:: 0.4
'allowed_filters' is now checked before adding 'sub_resource_lookup'
to the query, as it is considered safe.
Refactored to use self._client_projection since projection is now
honored by getitem() as well.
.. versionchanged:: 0.3
Support for new _mongotize() signature.
.. versionchagend:: 0.2
Support for sub-resources.
Support for 'default_sort'.
.. versionchanged:: 0.1.1
Better query handling. We're now properly casting objectid-like
strings to ObjectIds. Also, we're casting both datetimes and
objectids even when the query was originally in python syntax.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req.max_results:
args['limit'] = req.max_results
if req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = {}
spec = {}
if req.sort:
try:
client_sort = ast.literal_eval(req.sort)
except Exception as e:
abort(400, description=debug_error_message(str(e)))
if req.where:
try:
spec = self._sanitize(json.loads(req.where))
except HTTPException as e:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
# couldn't parse as mongo query; give the python parser a shot.
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource,
spec,
client_projection,
client_sort)
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['spec'] = spec
if sort is not None:
args['sort'] = sort
if projection is not None:
args['fields'] = projection
return self.driver.db[datasource].find(**args)
def test_And_BoolOp(self):
r = parse("a == 1 and b == 2")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"$and": [{"a": 1}, {"b": 2}]})
def test_Eq(self):
r = parse('a == "whatever"')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'a': 'whatever'})
def test_datetime_Call(self):
r = parse("born == datetime(2012, 11, 9)")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"born": datetime(2012, 11, 9)})
def test_LtE(self):
r = parse('a <= 1')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'a': {'$lte': 1}})
def find(self, resource, req):
"""Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req.max_results:
args['limit'] = req.max_results
if req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
if req.sort:
args['sort'] = ast.literal_eval(req.sort)
client_projection = {}
spec = {}
if req.where:
try:
spec = self._sanitize(
self._jsondatetime(json.loads(req.where)))
except:
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if req.projection:
try:
client_projection = json.loads(req.projection)
except:
abort(400, description=debug_error_message(
'Unable to parse `projection` clause'
))
datasource, spec, projection = self._datasource_ex(resource, spec,
client_projection)
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['spec'] = spec
if projection is not None:
args['fields'] = projection
return self.driver.db[datasource].find(**args)
def test_And_BoolOp(self):
r = parse('a == 1 and b == 2')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'$and': [{'a': 1}, {'b': 2}]})
def find(self, resource, req):
"""Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
.. versionchanged:: 0.1.1
Better query handling. We're now properly casting objectid-like
strings to ObjectIds. Also, we're casting both datetimes and
objectids even when the query was originally in python syntax.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req.max_results:
args['limit'] = req.max_results
if req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
if req.sort:
args['sort'] = ast.literal_eval(req.sort)
client_projection = {}
spec = {}
if req.where:
try:
spec = self._sanitize(json.loads(req.where))
except:
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
spec = self._mongotize(spec)
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if req.projection:
try:
client_projection = json.loads(req.projection)
except:
abort(400, description=debug_error_message(
'Unable to parse `projection` clause'
))
datasource, spec, projection = self._datasource_ex(resource, spec,
client_projection)
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['spec'] = spec
if projection is not None:
args['fields'] = projection
return self.driver.db[datasource].find(**args)
def find(self, resource, req, sub_resource_lookup):
"""Find documents for resource."""
args = getattr(req, 'args', request.args if request else {}) or {}
source_config = config.SOURCES[resource]
if args.get('source'):
query = json.loads(args.get('source'))
if 'filtered' not in query.get('query', {}):
_query = query.get('query')
query['query'] = {'filtered': {}}
if _query:
query['query']['filtered']['query'] = _query
else:
query = {'query': {'filtered': {}}}
if args.get('q', None):
query['query']['filtered']['query'] = _build_query_string(args.get('q'),
default_field=args.get('df', '_all'),
default_operator=args.get('default_operator', 'OR'))
if 'sort' not in query:
if req.sort:
sort = ast.literal_eval(req.sort)
set_sort(query, sort)
elif self._default_sort(resource) and 'query' not in query['query']['filtered']:
set_sort(query, self._default_sort(resource))
if req.max_results:
query.setdefault('size', req.max_results)
if req.page > 1:
query.setdefault('from', (req.page - 1) * req.max_results)
filters = []
filters.append(source_config.get('elastic_filter'))
filters.append(source_config.get('elastic_filter_callback', noop)())
filters.append({'and': _build_lookup_filter(sub_resource_lookup)} if sub_resource_lookup else None)
filters.append(json.loads(args.get('filter')) if 'filter' in args else None)
filters.extend(args.get('filters') if 'filters' in args else [])
if req.where:
try:
filters.append({'term': json.loads(req.where)})
except ValueError:
try:
filters.append({'term': parse(req.where)})
except ParseError:
abort(400)
set_filters(query, filters)
if 'facets' in source_config:
query['facets'] = source_config['facets']
if 'aggregations' in source_config and self.should_aggregate(req):
query['aggs'] = source_config['aggregations']
if 'es_highlight' in source_config and self.should_highlight(req):
query_string = query['query'].get('filtered', {}).get('query', {}).get('query_string')
highlights = source_config.get('es_highlight', noop)(query_string)
if highlights:
query['highlight'] = highlights
query['highlight'].setdefault('require_field_match', False)
source_projections = None
if self.should_project(req):
source_projections = self.get_projected_fields(req)
args = self._es_args(resource, source_projections=source_projections)
try:
hits = self.elastic(resource).search(body=query, **args)
except elasticsearch.exceptions.RequestError as e:
if e.status_code == 400 and "No mapping found for" in e.error:
hits = {}
elif e.status_code == 400 and 'SearchParseException' in e.error:
raise InvalidSearchString
else:
raise
return self._parse_hits(hits, resource)
def test_GtE(self):
r = parse('a >= 1')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'a': {'$gte': 1}})
def find(self, resource, req, sub_resource_lookup):
""" Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe"}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
.. versionchanged:: 0.6
Support for multiple databases.
Filter soft deleted documents by default
.. versionchanged:: 0.5
Support for comma delimited sort syntax. Addresses #443.
Return the error if a blacklisted MongoDB operator is used in query.
Abort with 400 if unsupported query operator is used. #387.
Abort with 400 in case of invalid sort syntax. #387.
.. versionchanged:: 0.4
'allowed_filters' is now checked before adding 'sub_resource_lookup'
to the query, as it is considered safe.
Refactored to use self._client_projection since projection is now
honored by getitem() as well.
.. versionchanged:: 0.3
Support for new _mongotize() signature.
.. versionchanged:: 0.2
Support for sub-resources.
Support for 'default_sort'.
.. versionchanged:: 0.1.1
Better query handling. We're now properly casting objectid-like
strings to ObjectIds. Also, we're casting both datetimes and
objectids even when the query was originally in python syntax.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req and req.max_results:
args['limit'] = req.max_results
if req and req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = {}
spec = {}
if req and req.sort:
try:
# assume it's mongo syntax (ie. ?sort=[("name", 1)])
client_sort = ast.literal_eval(req.sort)
except ValueError:
# it's not mongo so let's see if it's a comma delimited string
# instead (ie. "?sort=-age, name").
sort = []
for sort_arg in [s.strip() for s in req.sort.split(",")]:
if sort_arg[0] == "-":
sort.append((sort_arg[1:], -1))
else:
sort.append((sort_arg, 1))
if len(sort) > 0:
client_sort = sort
except Exception as e:
self.app.logger.exception(e)
abort(400, description=debug_error_message(str(e)))
if req and req.where:
try:
spec = self._sanitize(json.loads(req.where))
except HTTPException as e:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
# couldn't parse as mongo query; give the python parser a shot.
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
if config.DOMAIN[resource]['soft_delete'] \
and not (req and req.show_deleted) \
and not self.query_contains_field(spec, config.DELETED):
# Soft delete filtering applied after validate_filters call as
# querying against the DELETED field must always be allowed when
# soft_delete is enabled
spec = self.combine_queries(spec, {config.DELETED: {"$ne": True}})
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource,
spec,
client_projection,
client_sort)
if req and req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['filter'] = spec
if sort is not None:
args['sort'] = sort
if projection is not None:
args['projection'] = projection
return self.pymongo(resource).db[datasource].find(**args)
def test_NotEq(self):
r = parse('a != 1')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'a': {'$ne': 1}})
def find(self, resource, req, sub_resource_lookup):
"""Find documents for resource."""
args = getattr(req, 'args', request.args if request else {}) or {}
source_config = config.SOURCES[resource]
if args.get('source'):
query = json.loads(args.get('source'))
if 'filtered' not in query.get('query', {}):
_query = query.get('query')
query['query'] = {'filtered': {}}
if _query:
query['query']['filtered']['query'] = _query
else:
query = {'query': {'filtered': {}}}
if args.get('q', None):
query['query']['filtered']['query'] = _build_query_string(
args.get('q'),
default_field=args.get('df', '_all'),
default_operator=args.get('default_operator', 'OR'))
if 'sort' not in query:
if req.sort:
sort = ast.literal_eval(req.sort)
set_sort(query, sort)
elif self._default_sort(
resource) and 'query' not in query['query']['filtered']:
set_sort(query, self._default_sort(resource))
if req.max_results:
query.setdefault('size', req.max_results)
if req.page > 1:
query.setdefault('from', (req.page - 1) * req.max_results)
filters = []
filters.append(source_config.get('elastic_filter'))
filters.append(source_config.get('elastic_filter_callback', noop)())
filters.append({'and': _build_lookup_filter(sub_resource_lookup)}
if sub_resource_lookup else None)
filters.append(
json.loads(args.get('filter')) if 'filter' in args else None)
filters.extend(args.get('filters') if 'filters' in args else [])
if req.where:
try:
filters.append({'term': json.loads(req.where)})
except ValueError:
try:
filters.append({'term': parse(req.where)})
except ParseError:
abort(400)
set_filters(query, filters)
if 'facets' in source_config:
query['facets'] = source_config['facets']
if 'aggregations' in source_config and self.should_aggregate(req):
query['aggs'] = source_config['aggregations']
if 'es_highlight' in source_config and self.should_highlight(req):
query_string = query['query'].get('filtered',
{}).get('query',
{}).get('query_string')
highlights = source_config.get('es_highlight', noop)(query_string)
if highlights:
query['highlight'] = highlights
query['highlight'].setdefault('require_field_match', False)
source_projections = None
if self.should_project(req):
source_projections = self.get_projected_fields(req)
args = self._es_args(resource, source_projections=source_projections)
try:
hits = self.elastic(resource).search(body=query, **args)
except elasticsearch.exceptions.RequestError as e:
if e.status_code == 400 and "No mapping found for" in e.error:
hits = {}
elif e.status_code == 400 and 'SearchParseException' in e.error:
raise InvalidSearchString
else:
raise
return self._parse_hits(hits, resource)
def test_Or_BoolOp(self):
r = parse('a == 1 or b == 2')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'$or': [{'a': 1}, {'b': 2}]})
def test_Eq(self):
r = parse('a == "whatever"')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'a': 'whatever'})
def test_ObjectId_Call(self):
r = parse('_id == ObjectId("4f4644fbc88e20212c000000")')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'_id': ObjectId("4f4644fbc88e20212c000000")})
def test_GtE(self):
r = parse('a >= 1')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'a': {'$gte': 1}})
def test_Attribute(self):
r = parse('Invoice.number == 1')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'Invoice.number': 1})
def test_LtE(self):
r = parse('a <= 1')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'a': {'$lte': 1}})
def find(self, resource, req, sub_resource_lookup):
""" Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe"}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
.. versionchanged:: 0.6
Support for multiple databases.
Filter soft deleted documents by default
.. versionchanged:: 0.5
Support for comma delimited sort syntax. Addresses #443.
Return the error if a blacklisted MongoDB operator is used in query.
Abort with 400 if unsupported query operator is used. #387.
Abort with 400 in case of invalid sort syntax. #387.
.. versionchanged:: 0.4
'allowed_filters' is now checked before adding 'sub_resource_lookup'
to the query, as it is considered safe.
Refactored to use self._client_projection since projection is now
honored by getitem() as well.
.. versionchanged:: 0.3
Support for new _mongotize() signature.
.. versionchanged:: 0.2
Support for sub-resources.
Support for 'default_sort'.
.. versionchanged:: 0.1.1
Better query handling. We're now properly casting objectid-like
strings to ObjectIds. Also, we're casting both datetimes and
objectids even when the query was originally in python syntax.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req and req.max_results:
args['limit'] = req.max_results
if req and req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = {}
spec = {}
if req and req.sort:
try:
# assume it's mongo syntax (ie. ?sort=[("name", 1)])
client_sort = ast.literal_eval(req.sort)
except ValueError:
# it's not mongo so let's see if it's a comma delimited string
# instead (ie. "?sort=-age, name").
sort = []
for sort_arg in [s.strip() for s in req.sort.split(",")]:
if sort_arg[0] == "-":
sort.append((sort_arg[1:], -1))
else:
sort.append((sort_arg, 1))
if len(sort) > 0:
client_sort = sort
except Exception as e:
self.app.logger.exception(e)
abort(400, description=debug_error_message(str(e)))
if req and req.where:
try:
spec = self._sanitize(json.loads(req.where))
except HTTPException as e:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
# couldn't parse as mongo query; give the python parser a shot.
try:
spec = parse(req.where)
except ParseError:
abort(400,
description=debug_error_message(
'Unable to parse `where` clause'))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
if config.DOMAIN[resource]['soft_delete'] \
and not (req and req.show_deleted) \
and not self.query_contains_field(spec, config.DELETED):
# Soft delete filtering applied after validate_filters call as
# querying against the DELETED field must always be allowed when
# soft_delete is enabled
spec = self.combine_queries(spec, {config.DELETED: {"$ne": True}})
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource, spec, client_projection, client_sort)
if req and req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['filter'] = spec
if sort is not None:
args['sort'] = sort
if projection is not None:
args['projection'] = projection
return self.pymongo(resource).db[datasource].find(**args)
def test_NotEq(self):
r = parse('a != 1')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'a': {'$ne': 1}})
def test_And_BoolOp(self):
r = parse('a == 1 and b == 2')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'$and': [{'a': 1}, {'b': 2}]})
def test_Or_BoolOp(self):
r = parse('a == 1 or b == 2')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'$or': [{'a': 1}, {'b': 2}]})
def test_GtE(self):
r = parse("a >= 1")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"a": {"$gte": 1}})
def test_nested_BoolOp(self):
r = parse('a == 1 or (b == 2 and c == 3)')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'$or': [{'a': 1},
{'$and': [{'b': 2}, {'c': 3}]}]})
def test_NotEq(self):
r = parse("a != 1")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"a": {"$ne": 1}})
def test_datetime_Call(self):
r = parse('born == datetime(2012, 11, 9)')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'born': datetime(2012, 11, 9)})
def test_Or_BoolOp(self):
r = parse("a == 1 or b == 2")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"$or": [{"a": 1}, {"b": 2}]})
def test_Attribute(self):
r = parse('Invoice.number == 1')
self.assertEqual(type(r), dict)
self.assertEqual(r, {'Invoice.number': 1})
def test_ObjectId_Call(self):
r = parse('_id == ObjectId("4f4644fbc88e20212c000000")')
self.assertEqual(type(r), dict)
self.assertEqual(r, {"_id": ObjectId("4f4644fbc88e20212c000000")})
def find(self, resource, req, sub_resource_lookup):
"""
Seach for results and return list of them.
:param resource: name of requested resource as string.
:param req: instance of :class:`eve.utils.ParsedRequest`.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
"""
qry = self._objects(resource)
client_projection = {}
client_sort = {}
spec = {}
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
if req.sort:
try:
client_sort = ast.literal_eval(req.sort)
except Exception as e:
abort(400, description=debug_error_message(str(e)))
if req.where:
try:
spec = self._sanitize(json.loads(req.where))
except HTTPException as e:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
if sub_resource_lookup:
spec.update(sub_resource_lookup)
spec = self._mongotize(spec, resource)
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if req.projection:
try:
client_projection = json.loads(req.projection)
except Exception as e:
abort(400, description=debug_error_message(
'Unable to parse `projection` clause: '+str(e)
))
datasource, spec, projection, sort = self._datasource_ex(
resource,
spec,
client_projection,
client_sort)
# apply ordering
if sort:
for field, direction in _itemize(sort):
if direction < 0:
field = "-%s" % field
qry = qry.order_by(field)
# apply filters
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
qry = qry.filter(__raw__=spec)
# apply projection
qry = self._projection(resource, projection, qry)
# apply limits
if req.max_results:
qry = qry.limit(req.max_results)
if req.page > 1:
qry = qry.skip((req.page - 1) * req.max_results)
return PymongoQuerySet(qry)
def test_Attribute(self):
r = parse("Invoice.number == 1")
self.assertEqual(type(r), dict)
self.assertEqual(r, {"Invoice.number": 1})
def test_Gt(self):
r = parse('a > 1')
self.assertIs(type(r), dict)
self.assertEqual(r, {'a': {'$gt': 1}})
