def find(self, resource, req):
args = dict()
if req.max_results:
args['limit'] = req.max_results
if req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
if req.sortMethod:
sortMethod = req.sortMethod
else:
sortMethod = 1
if req.sort:
args['sort'] = ast.literal_eval(str([(req.sort,sortMethod)]))
client_projection = {}
spec = {}
if req.latLng:
spec['latLng'] = {'$within': {'$center': [[float(req.latLng[0]),float(req.latLng[1])],.75]}}
if not req.latLng and req.city:
spec['city'] = req.city
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if req.projection:
try:
client_projection = json.loads(req.projection)
except:
abort(400, description=debug_error_message(
'Unable to parse `projection` clause'
))
datasource, spec, projection = self._datasource_ex(resource, spec,
client_projection)
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['spec'] = spec
if projection is not None:
args['fields'] = projection
return self.driver.db[datasource].find(**args)
def aggregate(self, resource, req):
client_projection = {}
spec = {}
if req.where:
try:
spec = self._sanitize(
self._jsondatetime(json.loads(req.where, object_hook=json_util.object_hook)))
except:
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if req.projection:
try:
client_projection = json.loads(req.projection)
except:
abort(400, description=debug_error_message(
'Unable to parse `projection` clause'
))
datasource, spec, projection = self._datasource_ex(resource, spec,
client_projection)
groupers = config.DOMAIN[resource]["default_groupers"]
groupees = config.DOMAIN[resource]["default_groupees"]
group_val = {}
group_val["_id"] = {g: "$%s" % g for g in groupers}
for group_info in groupees:
name = group_info["name"]
group_type = group_info["type"]
group_val[name] = {"$%s" % group_type: "$%s" % name}
pipeline = []
pipeline.append({"$match": spec})
pipeline.append({"$project": projection})
pipeline.append({"$group": group_val})
pipeline.append({"$limit": 1000})
docs = self.driver.db[datasource].aggregate(pipeline)["result"]
cursor = Cursor(docs) #gives required functions to returned result
return cursor
def find(self, resource, req, sub_resource_lookup):
"""
Seach for results and return list of them.
:param resource: name of requested resource as string.
:param req: instance of :class:`eve.utils.ParsedRequest`.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
"""
qry = self.cls_map.objects(resource)
client_projection = {}
client_sort = {}
spec = {}
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
if req.sort:
try:
client_sort = ast.literal_eval(req.sort)
except Exception as e:
abort(400, description=debug_error_message(str(e)))
if req.where:
try:
spec = self._sanitize(json.loads(req.where))
except HTTPException as e:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
if sub_resource_lookup:
spec.update(sub_resource_lookup)
spec = self._mongotize(spec, resource)
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource,
spec,
client_projection,
client_sort)
# apply ordering
if sort:
for field, direction in _itemize(sort):
if direction < 0:
field = "-%s" % field
qry = qry.order_by(field)
# apply filters
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
qry = qry.filter(__raw__=spec)
# apply projection
qry = self._projection(resource, projection, qry)
# apply limits
if req.max_results:
qry = qry.limit(int(req.max_results))
if req.page > 1:
qry = qry.skip((req.page - 1) * req.max_results)
return PymongoQuerySet(qry)
def find(self, resource, req):
"""Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
.. versionchanged:: 0.1.1
Better query handling. We're now properly casting objectid-like
strings to ObjectIds. Also, we're casting both datetimes and
objectids even when the query was originally in python syntax.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req.max_results:
args['limit'] = req.max_results
if req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
if req.sort:
args['sort'] = ast.literal_eval(req.sort)
client_projection = {}
spec = {}
if req.where:
try:
spec = self._sanitize(json.loads(req.where))
except:
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
spec = self._mongotize(spec)
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if req.projection:
try:
client_projection = json.loads(req.projection)
except:
abort(400, description=debug_error_message(
'Unable to parse `projection` clause'
))
datasource, spec, projection = self._datasource_ex(resource, spec,
client_projection)
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['spec'] = spec
if projection is not None:
args['fields'] = projection
return self.driver.db[datasource].find(**args)
def find(self, resource, req, sub_resource_lookup):
""" Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe"}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
"""
args = dict()
if req and req.max_results:
args['limit'] = req.max_results
if req and req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = {}
spec = {}
if req and req.sort:
try:
# assume it's mongo syntax (ie. ?sort=[("name", 1)])
client_sort = ast.literal_eval(req.sort)
except ValueError:
# it's not mongo so let's see if it's a comma delimited string
# instead (ie. "?sort=-age, name").
sort = []
for sort_arg in [s.strip() for s in req.sort.split(",")]:
if sort_arg[0] == "-":
sort.append((sort_arg[1:], -1))
else:
sort.append((sort_arg, 1))
if len(sort) > 0:
client_sort = sort
except Exception as e:
self.app.logger.exception(e)
abort(400, description=debug_error_message(str(e)))
if req and req.where:
try:
spec = self._sanitize(json.loads(req.where))
except HTTPException as e:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
# couldn't parse as mongo query; give the python parser a shot.
try:
spec = parse(req.where)
except ParseError:
abort(400,
description=debug_error_message(
'Unable to parse `where` clause'))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
if config.DOMAIN[resource]['soft_delete'] \
and not (req and req.show_deleted) \
and not self.query_contains_field(spec, config.DELETED):
# Soft delete filtering applied after validate_filters call as
# querying against the DELETED field must always be allowed when
# soft_delete is enabled
spec = self.combine_queries(spec, {config.DELETED: {"$ne": True}})
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource, spec, client_projection, client_sort)
if req and req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['filter'] = spec
if sort is not None:
args['sort'] = sort
if projection:
args['projection'] = projection
return self._find(resource, datasource, **args)
def find(self, resource, req):
"""Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req.max_results:
args['limit'] = req.max_results
if req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
if req.sort:
args['sort'] = ast.literal_eval(req.sort)
client_projection = {}
spec = {}
if req.where:
try:
spec = self._sanitize(
self._jsondatetime(json.loads(req.where)))
except:
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if req.projection:
try:
client_projection = json.loads(req.projection)
except:
abort(400, description=debug_error_message(
'Unable to parse `projection` clause'
))
datasource, spec, projection = self._datasource_ex(resource, spec,
client_projection)
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['spec'] = spec
if projection is not None:
args['fields'] = projection
return self.driver.db[datasource].find(**args)
def find(self, resource, req, sub_resource_lookup):
"""
Seach for results and return list of them.
:param resource: name of requested resource as string.
:param req: instance of :class:`eve.utils.ParsedRequest`.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
"""
qry = self._objects(resource)
client_projection = {}
client_sort = {}
spec = {}
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
if req.sort:
try:
client_sort = ast.literal_eval(req.sort)
except Exception as e:
abort(400, description=debug_error_message(str(e)))
if req.where:
try:
spec = self._sanitize(json.loads(req.where))
except HTTPException as e:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
if sub_resource_lookup:
spec.update(sub_resource_lookup)
spec = self._mongotize(spec, resource)
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if req.projection:
try:
client_projection = json.loads(req.projection)
except Exception as e:
abort(400, description=debug_error_message(
'Unable to parse `projection` clause: '+str(e)
))
datasource, spec, projection, sort = self._datasource_ex(
resource,
spec,
client_projection,
client_sort)
# apply ordering
if sort:
for field, direction in _itemize(sort):
if direction < 0:
field = "-%s" % field
qry = qry.order_by(field)
# apply filters
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
qry = qry.filter(__raw__=spec)
# apply projection
qry = self._projection(resource, projection, qry)
# apply limits
if req.max_results:
qry = qry.limit(req.max_results)
if req.page > 1:
qry = qry.skip((req.page - 1) * req.max_results)
return PymongoQuerySet(qry)
def find(self, resource, req, sub_resource_lookup):
""" Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe"}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
.. versionchanged:: 0.6
Support for multiple databases.
Filter soft deleted documents by default
.. versionchanged:: 0.5
Support for comma delimited sort syntax. Addresses #443.
Return the error if a blacklisted MongoDB operator is used in query.
Abort with 400 if unsupported query operator is used. #387.
Abort with 400 in case of invalid sort syntax. #387.
.. versionchanged:: 0.4
'allowed_filters' is now checked before adding 'sub_resource_lookup'
to the query, as it is considered safe.
Refactored to use self._client_projection since projection is now
honored by getitem() as well.
.. versionchanged:: 0.3
Support for new _mongotize() signature.
.. versionchanged:: 0.2
Support for sub-resources.
Support for 'default_sort'.
.. versionchanged:: 0.1.1
Better query handling. We're now properly casting objectid-like
strings to ObjectIds. Also, we're casting both datetimes and
objectids even when the query was originally in python syntax.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req and req.max_results:
args['limit'] = req.max_results
if req and req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = {}
spec = {}
if req and req.sort:
try:
# assume it's mongo syntax (ie. ?sort=[("name", 1)])
client_sort = ast.literal_eval(req.sort)
except ValueError:
# it's not mongo so let's see if it's a comma delimited string
# instead (ie. "?sort=-age, name").
sort = []
for sort_arg in [s.strip() for s in req.sort.split(",")]:
if sort_arg[0] == "-":
sort.append((sort_arg[1:], -1))
else:
sort.append((sort_arg, 1))
if len(sort) > 0:
client_sort = sort
except Exception as e:
self.app.logger.exception(e)
abort(400, description=debug_error_message(str(e)))
if req and req.where:
try:
spec = self._sanitize(json.loads(req.where))
except HTTPException as e:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
# couldn't parse as mongo query; give the python parser a shot.
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
if config.DOMAIN[resource]['soft_delete'] \
and not (req and req.show_deleted) \
and not self.query_contains_field(spec, config.DELETED):
# Soft delete filtering applied after validate_filters call as
# querying against the DELETED field must always be allowed when
# soft_delete is enabled
spec = self.combine_queries(spec, {config.DELETED: {"$ne": True}})
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource,
spec,
client_projection,
client_sort)
if req and req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['filter'] = spec
if sort is not None:
args['sort'] = sort
if projection is not None:
args['projection'] = projection
return self.pymongo(resource).db[datasource].find(**args)
def find(self, resource, req, sub_resource_lookup, perform_count=True):
"""
Seach for results and return list of them.
:param resource: name of requested resource as string.
:param req: instance of :class:`eve.utils.ParsedRequest`.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
"""
args = dict()
if req and req.max_results:
args["limit"] = req.max_results
if req and req.page > 1:
args["skip"] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = self._convert_sort_request_to_dict(req)
spec = self._convert_where_request_to_dict(resource, req)
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
if (config.DOMAIN[resource]["soft_delete"]
and not (req and req.show_deleted)
and not self.query_contains_field(spec, config.DELETED)):
# Soft delete filtering applied after validate_filters call as
# querying against the DELETED field must always be allowed when
# soft_delete is enabled
spec = self.combine_queries(spec, {config.DELETED: {"$ne": True}})
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource, spec, client_projection, client_sort)
if len(spec) > 0:
args["filter"] = spec
if sort is not None:
args["sort"] = sort
if projection:
args["projection"] = projection
qry = self.cls_map.objects(resource)
# apply ordering
if sort:
sort_fields = []
for field, direction in _itemize(sort):
if direction < 0:
field = "-%s" % field
sort_fields.append(field)
qry = qry.order_by(*sort_fields)
if len(spec) > 0:
qry = qry.filter(__raw__=spec)
# apply projection
qry = self._projection(resource, projection, qry)
# apply limits
if args.get("limit"):
qry = qry.limit(int(args["limit"]))
if args.get("skip"):
qry = qry.skip(args["skip"])
count = None
if perform_count:
count = qry.count()
return PymongoQuerySet(qry), count
def test_validate_filters(self):
self.app.config["DOMAIN"][self.known_resource]["allowed_filters"] = []
with self.app.test_request_context():
self.assertTrue(
"key" in validate_filters({"key": "val"}, self.known_resource))
self.assertTrue("key" in validate_filters(
{"key": ["val1", "val2"]}, self.known_resource))
self.assertTrue(
"key" in validate_filters({"key": {
"$in": ["val1", "val2"]
}}, self.known_resource))
self.assertTrue("key" in validate_filters(
{"$or": [{
"key": "val1"
}, {
"key": "val2"
}]}, self.known_resource))
self.assertTrue(
"$or" in validate_filters({"$or": "val"}, self.known_resource))
self.assertTrue("$or" in validate_filters({"$or": {
"key": "val1"
}}, self.known_resource))
self.assertTrue("$or" in validate_filters({"$or": ["val"]},
self.known_resource))
self.app.config["DOMAIN"][self.known_resource]["allowed_filters"] = [
"key"
]
with self.app.test_request_context():
self.assertTrue(
validate_filters({"key": "val"}, self.known_resource) is None)
self.assertTrue(
validate_filters({"key": ["val1", "val2"]},
self.known_resource) is None)
self.assertTrue(
validate_filters({"key": {
"$in": ["val1", "val2"]
}}, self.known_resource) is None)
self.assertTrue(
validate_filters({"$or": [{
"key": "val1"
}, {
"key": "val2"
}]}, self.known_resource) is None)
def find(self, resource, req, sub_resource_lookup):
""" Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
.. versionchanged:: 0.5
Return the error if a blacklisted MongoDB operator is used in query.
Abort with 400 if unsupported query operator is used. #387.
Abort with 400 in case of invalid sort syntax. #387.
.. versionchanged:: 0.4
'allowed_filters' is now checked before adding 'sub_resource_lookup'
to the query, as it is considered safe.
Refactored to use self._client_projection since projection is now
honored by getitem() as well.
.. versionchanged:: 0.3
Support for new _mongotize() signature.
.. versionchagend:: 0.2
Support for sub-resources.
Support for 'default_sort'.
.. versionchanged:: 0.1.1
Better query handling. We're now properly casting objectid-like
strings to ObjectIds. Also, we're casting both datetimes and
objectids even when the query was originally in python syntax.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req.max_results:
args['limit'] = req.max_results
if req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = {}
spec = {}
if req.sort:
try:
client_sort = ast.literal_eval(req.sort)
except Exception as e:
abort(400, description=debug_error_message(str(e)))
if req.where:
try:
spec = self._sanitize(json.loads(req.where))
except HTTPException as e:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
# couldn't parse as mongo query; give the python parser a shot.
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource,
spec,
client_projection,
client_sort)
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['spec'] = spec
if sort is not None:
args['sort'] = sort
if projection is not None:
args['fields'] = projection
return self.driver.db[datasource].find(**args)
def test_validate_filters(self):
self.app.config['DOMAIN'][self.known_resource]['allowed_filters'] = []
with self.app.test_request_context():
self.assertTrue('key' in validate_filters(
{'key': 'val'},
self.known_resource))
self.assertTrue('key' in validate_filters(
{'key': ['val1', 'val2']},
self.known_resource))
self.assertTrue('key' in validate_filters(
{'key': {'$in': ['val1', 'val2']}},
self.known_resource))
self.assertTrue('key' in validate_filters(
{'$or': [{'key': 'val1'}, {'key': 'val2'}]},
self.known_resource))
self.assertTrue('$or' in validate_filters(
{'$or': 'val'},
self.known_resource))
self.assertTrue('$or' in validate_filters(
{'$or': {'key': 'val1'}},
self.known_resource))
self.assertTrue('$or' in validate_filters(
{'$or': ['val']},
self.known_resource))
self.app.config['DOMAIN'][self.known_resource]['allowed_filters'] = \
['key']
with self.app.test_request_context():
self.assertTrue(validate_filters(
{'key': 'val'},
self.known_resource) is None)
self.assertTrue(validate_filters(
{'key': ['val1', 'val2']},
self.known_resource) is None)
self.assertTrue(validate_filters(
{'key': {'$in': ['val1', 'val2']}},
self.known_resource) is None)
self.assertTrue(validate_filters(
{'$or': [{'key': 'val1'}, {'key': 'val2'}]},
self.known_resource) is None)
def test_validate_filters(self):
self.app.config['DOMAIN'][self.known_resource]['allowed_filters'] = []
with self.app.test_request_context():
self.assertTrue(
'key' in validate_filters({'key': 'val'}, self.known_resource))
self.assertTrue('key' in validate_filters(
{'key': ['val1', 'val2']}, self.known_resource))
self.assertTrue(
'key' in validate_filters({'key': {
'$in': ['val1', 'val2']
}}, self.known_resource))
self.assertTrue('key' in validate_filters(
{'$or': [{
'key': 'val1'
}, {
'key': 'val2'
}]}, self.known_resource))
self.assertTrue(
'$or' in validate_filters({'$or': 'val'}, self.known_resource))
self.assertTrue('$or' in validate_filters({'$or': {
'key': 'val1'
}}, self.known_resource))
self.assertTrue('$or' in validate_filters({'$or': ['val']},
self.known_resource))
self.app.config['DOMAIN'][self.known_resource]['allowed_filters'] = \
['key']
with self.app.test_request_context():
self.assertTrue(
validate_filters({'key': 'val'}, self.known_resource) is None)
self.assertTrue(
validate_filters({'key': ['val1', 'val2']},
self.known_resource) is None)
self.assertTrue(
validate_filters({'key': {
'$in': ['val1', 'val2']
}}, self.known_resource) is None)
self.assertTrue(
validate_filters({'$or': [{
'key': 'val1'
}, {
'key': 'val2'
}]}, self.known_resource) is None)
def test_validate_filters(self):
self.app.config["DOMAIN"][self.known_resource]["allowed_filters"] = []
with self.app.test_request_context():
self.assertTrue(
"key" in validate_filters({"key": "val"}, self.known_resource)
)
self.assertTrue(
"key"
in validate_filters({"key": ["val1", "val2"]}, self.known_resource)
)
self.assertTrue(
"key"
in validate_filters(
{"key": {"$in": ["val1", "val2"]}}, self.known_resource
)
)
self.assertTrue(
"key"
in validate_filters(
{"$or": [{"key": "val1"}, {"key": "val2"}]}, self.known_resource
)
)
self.assertTrue(
"$or" in validate_filters({"$or": "val"}, self.known_resource)
)
self.assertTrue(
"$or" in validate_filters({"$or": {"key": "val1"}}, self.known_resource)
)
self.assertTrue(
"$or" in validate_filters({"$or": ["val"]}, self.known_resource)
)
self.app.config["DOMAIN"][self.known_resource]["allowed_filters"] = ["key"]
with self.app.test_request_context():
self.assertTrue(
validate_filters({"key": "val"}, self.known_resource) is None
)
self.assertTrue(
validate_filters({"key": ["val1", "val2"]}, self.known_resource) is None
)
self.assertTrue(
validate_filters(
{"key": {"$in": ["val1", "val2"]}}, self.known_resource
)
is None
)
self.assertTrue(
validate_filters(
{"$or": [{"key": "val1"}, {"key": "val2"}]}, self.known_resource
)
is None
)
def find(self, resource, req, sub_resource_lookup):
""" Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
.. versionchanged:: 0.4
'allowed_filters' is now checked before adding 'sub_resource_lookup'
to the query, as it is considered safe.
Refactored to use self._client_projection since projection is now
honored by getitem() as well.
.. versionchanged:: 0.3
Support for new _mongotize() signature.
.. versionchagend:: 0.2
Support for sub-resources.
Support for 'default_sort'.
.. versionchanged:: 0.1.1
Better query handling. We're now properly casting objectid-like
strings to ObjectIds. Also, we're casting both datetimes and
objectids even when the query was originally in python syntax.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req.max_results:
args['limit'] = req.max_results
if req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = {}
spec = {}
if req.sort:
client_sort = ast.literal_eval(req.sort)
if req.where:
try:
spec = self._sanitize(json.loads(req.where))
except:
try:
spec = parse(req.where)
except ParseError:
abort(400, description=debug_error_message(
'Unable to parse `where` clause'
))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource,
spec,
client_projection,
client_sort)
if req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['spec'] = spec
if sort is not None:
args['sort'] = sort
if projection is not None:
args['fields'] = projection
return self.driver.db[datasource].find(**args)
def find(self, resource, req, sub_resource_lookup):
""" Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe"}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
.. versionchanged:: 0.6
Support for multiple databases.
Filter soft deleted documents by default
.. versionchanged:: 0.5
Support for comma delimited sort syntax. Addresses #443.
Return the error if a blacklisted MongoDB operator is used in query.
Abort with 400 if unsupported query operator is used. #387.
Abort with 400 in case of invalid sort syntax. #387.
.. versionchanged:: 0.4
'allowed_filters' is now checked before adding 'sub_resource_lookup'
to the query, as it is considered safe.
Refactored to use self._client_projection since projection is now
honored by getitem() as well.
.. versionchanged:: 0.3
Support for new _mongotize() signature.
.. versionchanged:: 0.2
Support for sub-resources.
Support for 'default_sort'.
.. versionchanged:: 0.1.1
Better query handling. We're now properly casting objectid-like
strings to ObjectIds. Also, we're casting both datetimes and
objectids even when the query was originally in python syntax.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req and req.max_results:
args['limit'] = req.max_results
if req and req.page > 1:
args['skip'] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = {}
spec = {}
if req and req.sort:
try:
# assume it's mongo syntax (ie. ?sort=[("name", 1)])
client_sort = ast.literal_eval(req.sort)
except ValueError:
# it's not mongo so let's see if it's a comma delimited string
# instead (ie. "?sort=-age, name").
sort = []
for sort_arg in [s.strip() for s in req.sort.split(",")]:
if sort_arg[0] == "-":
sort.append((sort_arg[1:], -1))
else:
sort.append((sort_arg, 1))
if len(sort) > 0:
client_sort = sort
except Exception as e:
self.app.logger.exception(e)
abort(400, description=debug_error_message(str(e)))
if req and req.where:
try:
spec = self._sanitize(json.loads(req.where))
except HTTPException as e:
# _sanitize() is raising an HTTP exception; let it fire.
raise
except:
# couldn't parse as mongo query; give the python parser a shot.
try:
spec = parse(req.where)
except ParseError:
abort(400,
description=debug_error_message(
'Unable to parse `where` clause'))
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
if config.DOMAIN[resource]['soft_delete'] \
and not (req and req.show_deleted) \
and not self.query_contains_field(spec, config.DELETED):
# Soft delete filtering applied after validate_filters call as
# querying against the DELETED field must always be allowed when
# soft_delete is enabled
spec = self.combine_queries(spec, {config.DELETED: {"$ne": True}})
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource, spec, client_projection, client_sort)
if req and req.if_modified_since:
spec[config.LAST_UPDATED] = \
{'$gt': req.if_modified_since}
if len(spec) > 0:
args['filter'] = spec
if sort is not None:
args['sort'] = sort
if projection is not None:
args['projection'] = projection
return self.pymongo(resource).db[datasource].find(**args)
def find(self, resource, req, sub_resource_lookup):
""" Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe"}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
.. versionchanged:: 0.6
Support for multiple databases.
Filter soft deleted documents by default
.. versionchanged:: 0.5
Support for comma delimited sort syntax. Addresses #443.
Return the error if a blacklisted MongoDB operator is used in query.
Abort with 400 if unsupported query operator is used. #387.
Abort with 400 in case of invalid sort syntax. #387.
.. versionchanged:: 0.4
'allowed_filters' is now checked before adding 'sub_resource_lookup'
to the query, as it is considered safe.
Refactored to use self._client_projection since projection is now
honored by getitem() as well.
.. versionchanged:: 0.3
Support for new _mongotize() signature.
.. versionchanged:: 0.2
Support for sub-resources.
Support for 'default_sort'.
.. versionchanged:: 0.1.1
Better query handling. We're now properly casting objectid-like
strings to ObjectIds. Also, we're casting both datetimes and
objectids even when the query was originally in python syntax.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req and req.max_results:
args["limit"] = req.max_results
if req and req.page > 1:
args["skip"] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = self._convert_sort_request_to_dict(req)
spec = self._convert_where_request_to_dict(req)
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
if (config.DOMAIN[resource]["soft_delete"]
and not (req and req.show_deleted)
and not self.query_contains_field(spec, config.DELETED)):
# Soft delete filtering applied after validate_filters call as
# querying against the DELETED field must always be allowed when
# soft_delete is enabled
spec = self.combine_queries(spec, {config.DELETED: {"$ne": True}})
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource, spec, client_projection, client_sort)
if req and req.if_modified_since:
spec[config.LAST_UPDATED] = {"$gt": req.if_modified_since}
if len(spec) > 0:
args["filter"] = spec
if sort is not None:
args["sort"] = sort
if projection:
args["projection"] = projection
self.__last_target = self.pymongo(resource).db[datasource], spec
try:
self.__last_cursor = self.pymongo(resource).db[datasource].find(
**args)
except TypeError as e:
# pymongo raises ValueError when invalid query paramenters are
# included. We do our best to catch them beforehand but, especially
# with key/value sort syntax, invalid ones might still slip in.
self.app.logger.exception(e)
abort(400, description=debug_error_message(str(e)))
return self.__last_cursor
def find(self, resource, req, sub_resource_lookup):
""" Retrieves a set of documents matching a given request. Queries can
be expressed in two different formats: the mongo query syntax, and the
python syntax. The first kind of query would look like: ::
?where={"name": "john doe"}
while the second would look like: ::
?where=name=="john doe"
The resultset if paginated.
:param resource: resource name.
:param req: a :class:`ParsedRequest`instance.
:param sub_resource_lookup: sub-resource lookup from the endpoint url.
.. versionchanged:: 0.6
Support for multiple databases.
Filter soft deleted documents by default
.. versionchanged:: 0.5
Support for comma delimited sort syntax. Addresses #443.
Return the error if a blacklisted MongoDB operator is used in query.
Abort with 400 if unsupported query operator is used. #387.
Abort with 400 in case of invalid sort syntax. #387.
.. versionchanged:: 0.4
'allowed_filters' is now checked before adding 'sub_resource_lookup'
to the query, as it is considered safe.
Refactored to use self._client_projection since projection is now
honored by getitem() as well.
.. versionchanged:: 0.3
Support for new _mongotize() signature.
.. versionchanged:: 0.2
Support for sub-resources.
Support for 'default_sort'.
.. versionchanged:: 0.1.1
Better query handling. We're now properly casting objectid-like
strings to ObjectIds. Also, we're casting both datetimes and
objectids even when the query was originally in python syntax.
.. versionchanged:: 0.0.9
More informative error messages.
.. versionchanged:: 0.0.7
Abort with a 400 if the query includes blacklisted operators.
.. versionchanged:: 0.0.6
Only retrieve fields in the resource schema
Support for projection queries ('?projection={"name": 1}')
.. versionchanged:: 0.0.5
handles the case where req.max_results is None because pagination
has been disabled.
.. versionchanged:: 0.0.4
retrieves the target collection via the new config.SOURCES helper.
"""
args = dict()
if req and req.max_results:
args["limit"] = req.max_results
if req and req.page > 1:
args["skip"] = (req.page - 1) * req.max_results
# TODO sort syntax should probably be coherent with 'where': either
# mongo-like # or python-like. Currently accepts only mongo-like sort
# syntax.
# TODO should validate on unknown sort fields (mongo driver doesn't
# return an error)
client_sort = self._convert_sort_request_to_dict(req)
spec = self._convert_where_request_to_dict(req)
bad_filter = validate_filters(spec, resource)
if bad_filter:
abort(400, bad_filter)
if sub_resource_lookup:
spec = self.combine_queries(spec, sub_resource_lookup)
if (
config.DOMAIN[resource]["soft_delete"]
and not (req and req.show_deleted)
and not self.query_contains_field(spec, config.DELETED)
):
# Soft delete filtering applied after validate_filters call as
# querying against the DELETED field must always be allowed when
# soft_delete is enabled
spec = self.combine_queries(spec, {config.DELETED: {"$ne": True}})
spec = self._mongotize(spec, resource)
client_projection = self._client_projection(req)
datasource, spec, projection, sort = self._datasource_ex(
resource, spec, client_projection, client_sort
)
if req and req.if_modified_since:
spec[config.LAST_UPDATED] = {"$gt": req.if_modified_since}
if len(spec) > 0:
args["filter"] = spec
if sort is not None:
args["sort"] = sort
if projection:
args["projection"] = projection
self.__last_target = self.pymongo(resource).db[datasource], spec
try:
self.__last_cursor = self.pymongo(resource).db[datasource].find(**args)
except TypeError as e:
# pymongo raises ValueError when invalid query paramenters are
# included. We do our best to catch them beforehand but, especially
# with key/value sort syntax, invalid ones might still slip in.
self.app.logger.exception(e)
abort(400, description=debug_error_message(str(e)))
return self.__last_cursor
