def _get_lookup(): """Get the where clause lookup just like Eve does. Unfortunately, Eve only parses the `where` at a very low level and does not provide any methods to elegantly access it, so we have use the same internal functions as Eve does. (Recently, Eve has at least somewhat exposed the parsing, but this code is not part of an official release yet [1]) As soon as there is some 'official' support, this can be removed, as it is basically copied, with the abort removed for simplicity (as Eve itself will already abort if there's an error). [1]: https://github.com/pyeve/eve/blob/master/eve/io/mongo/mongo.py """ req = parse_request('studydocuments') if req and req.where: try: # Mongo Syntax return current_app.data._sanitize(json.loads(req.where)) except (HTTPException, json.JSONDecodeError): # Python Syntax return parse(req.where) return {} # No where clause
def _get_lookup(): """Get the where clause lookup just like Eve does. Unfortunately, Eve only parses the `where` at a very low level and does not provide any methods to elegantly access it, so we have use the same internal functions as Eve does. (Recently, Eve has at least somewhat exposed the parsing, but this code is not part of an official release yet [1]) As soon as there is some 'official' support, this can be removed, as it is basically copied, with the abort removed for simplicity (as Eve itself will already abort if there's an error). [1]: https://github.com/pyeve/eve/blob/master/eve/io/mongo/mongo.py """ req = parse_request('studydocuments') if req and req.where: try: # Mongo Syntax return current_app.data._sanitize('studydocuments', json.loads(req.where)) except (HTTPException, json.JSONDecodeError): # Python Syntax return parse(req.where) return {} # No where clause
def test_nested_BoolOp(self): r = parse('a == 1 or (b == 2 and c == 3)') self.assertEqual(type(r), dict) self.assertEqual(r, {'$or': [{ 'a': 1 }, { '$and': [{ 'b': 2 }, { 'c': 3 }] }]})
def test_nested_BoolOp(self): r = parse("a == 1 or (b == 2 and c == 3)") self.assertEqual(type(r), dict) self.assertEqual(r, {"$or": [{ "a": 1 }, { "$and": [{ "b": 2 }, { "c": 3 }] }]})
def aggregate(self, resource, req): client_projection = {} spec = {} if req.where: try: spec = self._sanitize( self._jsondatetime(json.loads(req.where, object_hook=json_util.object_hook))) except: try: spec = parse(req.where) except ParseError: abort(400, description=debug_error_message( 'Unable to parse `where` clause' )) bad_filter = validate_filters(spec, resource) if bad_filter: abort(400, bad_filter) if req.projection: try: client_projection = json.loads(req.projection) except: abort(400, description=debug_error_message( 'Unable to parse `projection` clause' )) datasource, spec, projection = self._datasource_ex(resource, spec, client_projection) groupers = config.DOMAIN[resource]["default_groupers"] groupees = config.DOMAIN[resource]["default_groupees"] group_val = {} group_val["_id"] = {g: "$%s" % g for g in groupers} for group_info in groupees: name = group_info["name"] group_type = group_info["type"] group_val[name] = {"$%s" % group_type: "$%s" % name} pipeline = [] pipeline.append({"$match": spec}) pipeline.append({"$project": projection}) pipeline.append({"$group": group_val}) pipeline.append({"$limit": 1000}) docs = self.driver.db[datasource].aggregate(pipeline)["result"] cursor = Cursor(docs) #gives required functions to returned result return cursor
def _convert_where_request_to_dict(self, req): """ Converts the contents of a `ParsedRequest`'s `where` property to a dict """ query = {} if req and req.where: try: query = self._sanitize(json.loads(req.where)) except HTTPException: # _sanitize() is raising an HTTP exception; let it fire. raise except: # couldn't parse as mongo query; give the python parser a shot. try: query = parse(req.where) except ParseError: abort( 400, description=debug_error_message( "Unable to parse `where` clause"), ) return query
def _convert_where_request_to_dict(self, req): """ Converts the contents of a `ParsedRequest`'s `where` property to a dict """ query = {} if req and req.where: try: query = self._sanitize(json.loads(req.where)) except HTTPException: # _sanitize() is raising an HTTP exception; let it fire. raise except: # couldn't parse as mongo query; give the python parser a shot. try: query = parse(req.where) except ParseError: abort( 400, description=debug_error_message( "Unable to parse `where` clause" ), ) return query
def test_Eq(self): r = parse('a == "whatever"') self.assertEqual(type(r), dict) self.assertEqual(r, {"a": "whatever"})
def find(self, resource, req, sub_resource_lookup): """ Retrieves a set of documents matching a given request. Queries can be expressed in two different formats: the mongo query syntax, and the python syntax. The first kind of query would look like: :: ?where={"name": "john doe"} while the second would look like: :: ?where=name=="john doe" The resultset if paginated. :param resource: resource name. :param req: a :class:`ParsedRequest`instance. :param sub_resource_lookup: sub-resource lookup from the endpoint url. """ args = dict() if req and req.max_results: args['limit'] = req.max_results if req and req.page > 1: args['skip'] = (req.page - 1) * req.max_results # TODO sort syntax should probably be coherent with 'where': either # mongo-like # or python-like. Currently accepts only mongo-like sort # syntax. # TODO should validate on unknown sort fields (mongo driver doesn't # return an error) client_sort = {} spec = {} if req and req.sort: try: # assume it's mongo syntax (ie. ?sort=[("name", 1)]) client_sort = ast.literal_eval(req.sort) except ValueError: # it's not mongo so let's see if it's a comma delimited string # instead (ie. "?sort=-age, name"). sort = [] for sort_arg in [s.strip() for s in req.sort.split(",")]: if sort_arg[0] == "-": sort.append((sort_arg[1:], -1)) else: sort.append((sort_arg, 1)) if len(sort) > 0: client_sort = sort except Exception as e: self.app.logger.exception(e) abort(400, description=debug_error_message(str(e))) if req and req.where: try: spec = self._sanitize(json.loads(req.where)) except HTTPException as e: # _sanitize() is raising an HTTP exception; let it fire. raise except: # couldn't parse as mongo query; give the python parser a shot. try: spec = parse(req.where) except ParseError: abort(400, description=debug_error_message( 'Unable to parse `where` clause')) bad_filter = validate_filters(spec, resource) if bad_filter: abort(400, bad_filter) if sub_resource_lookup: spec = self.combine_queries(spec, sub_resource_lookup) if config.DOMAIN[resource]['soft_delete'] \ and not (req and req.show_deleted) \ and not self.query_contains_field(spec, config.DELETED): # Soft delete filtering applied after validate_filters call as # querying against the DELETED field must always be allowed when # soft_delete is enabled spec = self.combine_queries(spec, {config.DELETED: {"$ne": True}}) spec = self._mongotize(spec, resource) client_projection = self._client_projection(req) datasource, spec, projection, sort = self._datasource_ex( resource, spec, client_projection, client_sort) if req and req.if_modified_since: spec[config.LAST_UPDATED] = \ {'$gt': req.if_modified_since} if len(spec) > 0: args['filter'] = spec if sort is not None: args['sort'] = sort if projection: args['projection'] = projection return self._find(resource, datasource, **args)
def test_Lt(self): r = parse('a < 1') self.assertIs(type(r), dict) self.assertEqual(r, {'a': {'$lt': 1}})
def test_nested_BoolOp(self): r = parse("a == 1 or (b == 2 and c == 3)") self.assertEqual(type(r), dict) self.assertEqual(r, {"$or": [{"a": 1}, {"$and": [{"b": 2}, {"c": 3}]}]})
def test_LtE(self): r = parse("a <= 1") self.assertEqual(type(r), dict) self.assertEqual(r, {"a": {"$lte": 1}})
def find(self, resource, req, sub_resource_lookup): """ Retrieves a set of documents matching a given request. Queries can be expressed in two different formats: the mongo query syntax, and the python syntax. The first kind of query would look like: :: ?where={"name": "john doe} while the second would look like: :: ?where=name=="john doe" The resultset if paginated. :param resource: resource name. :param req: a :class:`ParsedRequest`instance. :param sub_resource_lookup: sub-resource lookup from the endpoint url. .. versionchanged:: 0.4 'allowed_filters' is now checked before adding 'sub_resource_lookup' to the query, as it is considered safe. Refactored to use self._client_projection since projection is now honored by getitem() as well. .. versionchanged:: 0.3 Support for new _mongotize() signature. .. versionchagend:: 0.2 Support for sub-resources. Support for 'default_sort'. .. versionchanged:: 0.1.1 Better query handling. We're now properly casting objectid-like strings to ObjectIds. Also, we're casting both datetimes and objectids even when the query was originally in python syntax. .. versionchanged:: 0.0.9 More informative error messages. .. versionchanged:: 0.0.7 Abort with a 400 if the query includes blacklisted operators. .. versionchanged:: 0.0.6 Only retrieve fields in the resource schema Support for projection queries ('?projection={"name": 1}') .. versionchanged:: 0.0.5 handles the case where req.max_results is None because pagination has been disabled. .. versionchanged:: 0.0.4 retrieves the target collection via the new config.SOURCES helper. """ args = dict() if req.max_results: args['limit'] = req.max_results if req.page > 1: args['skip'] = (req.page - 1) * req.max_results # TODO sort syntax should probably be coherent with 'where': either # mongo-like # or python-like. Currently accepts only mongo-like sort # syntax. # TODO should validate on unknown sort fields (mongo driver doesn't # return an error) client_sort = {} spec = {} if req.sort: client_sort = ast.literal_eval(req.sort) if req.where: try: spec = self._sanitize(json.loads(req.where)) except: try: spec = parse(req.where) except ParseError: abort(400, description=debug_error_message( 'Unable to parse `where` clause' )) bad_filter = validate_filters(spec, resource) if bad_filter: abort(400, bad_filter) if sub_resource_lookup: spec = self.combine_queries(spec, sub_resource_lookup) spec = self._mongotize(spec, resource) client_projection = self._client_projection(req) datasource, spec, projection, sort = self._datasource_ex( resource, spec, client_projection, client_sort) if req.if_modified_since: spec[config.LAST_UPDATED] = \ {'$gt': req.if_modified_since} if len(spec) > 0: args['spec'] = spec if sort is not None: args['sort'] = sort if projection is not None: args['fields'] = projection return self.driver.db[datasource].find(**args)
def test_And_BoolOp(self): r = parse("a == 1 and b == 2") self.assertEqual(type(r), dict) self.assertEqual(r, {"$and": [{"a": 1}, {"b": 2}]})
def test_Or_BoolOp(self): r = parse("a == 1 or b == 2") self.assertEqual(type(r), dict) self.assertEqual(r, {"$or": [{"a": 1}, {"b": 2}]})
def test_NotEq(self): r = parse("a != 1") self.assertEqual(type(r), dict) self.assertEqual(r, {"a": {"$ne": 1}})
def test_GtE(self): r = parse("a >= 1") self.assertEqual(type(r), dict) self.assertEqual(r, {"a": {"$gte": 1}})
def test_datetime_Call(self): r = parse('born == datetime(2012, 11, 9)') self.assertEqual(type(r), dict) self.assertEqual(r, {'born': datetime(2012, 11, 9)})
def test_datetime_Call(self): r = parse("born == datetime(2012, 11, 9)") self.assertEqual(type(r), dict) self.assertEqual(r, {"born": datetime(2012, 11, 9)})
def find(self, resource, req, sub_resource_lookup): """ Seach for results and return list of them. :param resource: name of requested resource as string. :param req: instance of :class:`eve.utils.ParsedRequest`. :param sub_resource_lookup: sub-resource lookup from the endpoint url. """ qry = self.cls_map.objects(resource) client_projection = {} client_sort = {} spec = {} # TODO sort syntax should probably be coherent with 'where': either # mongo-like # or python-like. Currently accepts only mongo-like sort # syntax. # TODO should validate on unknown sort fields (mongo driver doesn't # return an error) if req.sort: try: client_sort = ast.literal_eval(req.sort) except Exception as e: abort(400, description=debug_error_message(str(e))) if req.where: try: spec = self._sanitize(json.loads(req.where)) except HTTPException as e: # _sanitize() is raising an HTTP exception; let it fire. raise except: try: spec = parse(req.where) except ParseError: abort(400, description=debug_error_message( 'Unable to parse `where` clause' )) if sub_resource_lookup: spec.update(sub_resource_lookup) spec = self._mongotize(spec, resource) bad_filter = validate_filters(spec, resource) if bad_filter: abort(400, bad_filter) client_projection = self._client_projection(req) datasource, spec, projection, sort = self._datasource_ex( resource, spec, client_projection, client_sort) # apply ordering if sort: for field, direction in _itemize(sort): if direction < 0: field = "-%s" % field qry = qry.order_by(field) # apply filters if req.if_modified_since: spec[config.LAST_UPDATED] = \ {'$gt': req.if_modified_since} if len(spec) > 0: qry = qry.filter(__raw__=spec) # apply projection qry = self._projection(resource, projection, qry) # apply limits if req.max_results: qry = qry.limit(int(req.max_results)) if req.page > 1: qry = qry.skip((req.page - 1) * req.max_results) return PymongoQuerySet(qry)
def test_Attribute(self): r = parse("Invoice.number == 1") self.assertEqual(type(r), dict) self.assertEqual(r, {"Invoice.number": 1})
def find(self, resource, req, sub_resource_lookup): """ Retrieves a set of documents matching a given request. Queries can be expressed in two different formats: the mongo query syntax, and the python syntax. The first kind of query would look like: :: ?where={"name": "john doe} while the second would look like: :: ?where=name=="john doe" The resultset if paginated. :param resource: resource name. :param req: a :class:`ParsedRequest`instance. :param sub_resource_lookup: sub-resource lookup from the endpoint url. .. versionchanged:: 0.5 Return the error if a blacklisted MongoDB operator is used in query. Abort with 400 if unsupported query operator is used. #387. Abort with 400 in case of invalid sort syntax. #387. .. versionchanged:: 0.4 'allowed_filters' is now checked before adding 'sub_resource_lookup' to the query, as it is considered safe. Refactored to use self._client_projection since projection is now honored by getitem() as well. .. versionchanged:: 0.3 Support for new _mongotize() signature. .. versionchagend:: 0.2 Support for sub-resources. Support for 'default_sort'. .. versionchanged:: 0.1.1 Better query handling. We're now properly casting objectid-like strings to ObjectIds. Also, we're casting both datetimes and objectids even when the query was originally in python syntax. .. versionchanged:: 0.0.9 More informative error messages. .. versionchanged:: 0.0.7 Abort with a 400 if the query includes blacklisted operators. .. versionchanged:: 0.0.6 Only retrieve fields in the resource schema Support for projection queries ('?projection={"name": 1}') .. versionchanged:: 0.0.5 handles the case where req.max_results is None because pagination has been disabled. .. versionchanged:: 0.0.4 retrieves the target collection via the new config.SOURCES helper. """ args = dict() if req.max_results: args['limit'] = req.max_results if req.page > 1: args['skip'] = (req.page - 1) * req.max_results # TODO sort syntax should probably be coherent with 'where': either # mongo-like # or python-like. Currently accepts only mongo-like sort # syntax. # TODO should validate on unknown sort fields (mongo driver doesn't # return an error) client_sort = {} spec = {} if req.sort: try: client_sort = ast.literal_eval(req.sort) except Exception as e: abort(400, description=debug_error_message(str(e))) if req.where: try: spec = self._sanitize(json.loads(req.where)) except HTTPException as e: # _sanitize() is raising an HTTP exception; let it fire. raise except: # couldn't parse as mongo query; give the python parser a shot. try: spec = parse(req.where) except ParseError: abort(400, description=debug_error_message( 'Unable to parse `where` clause' )) bad_filter = validate_filters(spec, resource) if bad_filter: abort(400, bad_filter) if sub_resource_lookup: spec = self.combine_queries(spec, sub_resource_lookup) spec = self._mongotize(spec, resource) client_projection = self._client_projection(req) datasource, spec, projection, sort = self._datasource_ex( resource, spec, client_projection, client_sort) if req.if_modified_since: spec[config.LAST_UPDATED] = \ {'$gt': req.if_modified_since} if len(spec) > 0: args['spec'] = spec if sort is not None: args['sort'] = sort if projection is not None: args['fields'] = projection return self.driver.db[datasource].find(**args)
def test_Eq(self): r = parse('a == "whatever"') self.assertEqual(type(r), dict) self.assertEqual(r, {'a': 'whatever'})
def test_LtE(self): r = parse('a <= 1') self.assertEqual(type(r), dict) self.assertEqual(r, {'a': {'$lte': 1}})
def find(self, resource, req): """Retrieves a set of documents matching a given request. Queries can be expressed in two different formats: the mongo query syntax, and the python syntax. The first kind of query would look like: :: ?where={"name": "john doe} while the second would look like: :: ?where=name=="john doe" The resultset if paginated. :param resource: resource name. :param req: a :class:`ParsedRequest`instance. .. versionchanged:: 0.0.9 More informative error messages. .. versionchanged:: 0.0.7 Abort with a 400 if the query includes blacklisted operators. .. versionchanged:: 0.0.6 Only retrieve fields in the resource schema Support for projection queries ('?projection={"name": 1}') .. versionchanged:: 0.0.5 handles the case where req.max_results is None because pagination has been disabled. .. versionchanged:: 0.0.4 retrieves the target collection via the new config.SOURCES helper. """ args = dict() if req.max_results: args['limit'] = req.max_results if req.page > 1: args['skip'] = (req.page - 1) * req.max_results # TODO sort syntax should probably be coherent with 'where': either # mongo-like # or python-like. Currently accepts only mongo-like sort # syntax. # TODO should validate on unknown sort fields (mongo driver doesn't # return an error) if req.sort: args['sort'] = ast.literal_eval(req.sort) client_projection = {} spec = {} if req.where: try: spec = self._sanitize( self._jsondatetime(json.loads(req.where))) except: try: spec = parse(req.where) except ParseError: abort(400, description=debug_error_message( 'Unable to parse `where` clause' )) bad_filter = validate_filters(spec, resource) if bad_filter: abort(400, bad_filter) if req.projection: try: client_projection = json.loads(req.projection) except: abort(400, description=debug_error_message( 'Unable to parse `projection` clause' )) datasource, spec, projection = self._datasource_ex(resource, spec, client_projection) if req.if_modified_since: spec[config.LAST_UPDATED] = \ {'$gt': req.if_modified_since} if len(spec) > 0: args['spec'] = spec if projection is not None: args['fields'] = projection return self.driver.db[datasource].find(**args)
def test_And_BoolOp(self): r = parse('a == 1 and b == 2') self.assertEqual(type(r), dict) self.assertEqual(r, {'$and': [{'a': 1}, {'b': 2}]})
def find(self, resource, req): """Retrieves a set of documents matching a given request. Queries can be expressed in two different formats: the mongo query syntax, and the python syntax. The first kind of query would look like: :: ?where={"name": "john doe} while the second would look like: :: ?where=name=="john doe" The resultset if paginated. :param resource: resource name. :param req: a :class:`ParsedRequest`instance. .. versionchanged:: 0.1.1 Better query handling. We're now properly casting objectid-like strings to ObjectIds. Also, we're casting both datetimes and objectids even when the query was originally in python syntax. .. versionchanged:: 0.0.9 More informative error messages. .. versionchanged:: 0.0.7 Abort with a 400 if the query includes blacklisted operators. .. versionchanged:: 0.0.6 Only retrieve fields in the resource schema Support for projection queries ('?projection={"name": 1}') .. versionchanged:: 0.0.5 handles the case where req.max_results is None because pagination has been disabled. .. versionchanged:: 0.0.4 retrieves the target collection via the new config.SOURCES helper. """ args = dict() if req.max_results: args['limit'] = req.max_results if req.page > 1: args['skip'] = (req.page - 1) * req.max_results # TODO sort syntax should probably be coherent with 'where': either # mongo-like # or python-like. Currently accepts only mongo-like sort # syntax. # TODO should validate on unknown sort fields (mongo driver doesn't # return an error) if req.sort: args['sort'] = ast.literal_eval(req.sort) client_projection = {} spec = {} if req.where: try: spec = self._sanitize(json.loads(req.where)) except: try: spec = parse(req.where) except ParseError: abort(400, description=debug_error_message( 'Unable to parse `where` clause' )) spec = self._mongotize(spec) bad_filter = validate_filters(spec, resource) if bad_filter: abort(400, bad_filter) if req.projection: try: client_projection = json.loads(req.projection) except: abort(400, description=debug_error_message( 'Unable to parse `projection` clause' )) datasource, spec, projection = self._datasource_ex(resource, spec, client_projection) if req.if_modified_since: spec[config.LAST_UPDATED] = \ {'$gt': req.if_modified_since} if len(spec) > 0: args['spec'] = spec if projection is not None: args['fields'] = projection return self.driver.db[datasource].find(**args)
def find(self, resource, req, sub_resource_lookup): """Find documents for resource.""" args = getattr(req, 'args', request.args if request else {}) or {} source_config = config.SOURCES[resource] if args.get('source'): query = json.loads(args.get('source')) if 'filtered' not in query.get('query', {}): _query = query.get('query') query['query'] = {'filtered': {}} if _query: query['query']['filtered']['query'] = _query else: query = {'query': {'filtered': {}}} if args.get('q', None): query['query']['filtered']['query'] = _build_query_string(args.get('q'), default_field=args.get('df', '_all'), default_operator=args.get('default_operator', 'OR')) if 'sort' not in query: if req.sort: sort = ast.literal_eval(req.sort) set_sort(query, sort) elif self._default_sort(resource) and 'query' not in query['query']['filtered']: set_sort(query, self._default_sort(resource)) if req.max_results: query.setdefault('size', req.max_results) if req.page > 1: query.setdefault('from', (req.page - 1) * req.max_results) filters = [] filters.append(source_config.get('elastic_filter')) filters.append(source_config.get('elastic_filter_callback', noop)()) filters.append({'and': _build_lookup_filter(sub_resource_lookup)} if sub_resource_lookup else None) filters.append(json.loads(args.get('filter')) if 'filter' in args else None) filters.extend(args.get('filters') if 'filters' in args else []) if req.where: try: filters.append({'term': json.loads(req.where)}) except ValueError: try: filters.append({'term': parse(req.where)}) except ParseError: abort(400) set_filters(query, filters) if 'facets' in source_config: query['facets'] = source_config['facets'] if 'aggregations' in source_config and self.should_aggregate(req): query['aggs'] = source_config['aggregations'] if 'es_highlight' in source_config and self.should_highlight(req): query_string = query['query'].get('filtered', {}).get('query', {}).get('query_string') highlights = source_config.get('es_highlight', noop)(query_string) if highlights: query['highlight'] = highlights query['highlight'].setdefault('require_field_match', False) source_projections = None if self.should_project(req): source_projections = self.get_projected_fields(req) args = self._es_args(resource, source_projections=source_projections) try: hits = self.elastic(resource).search(body=query, **args) except elasticsearch.exceptions.RequestError as e: if e.status_code == 400 and "No mapping found for" in e.error: hits = {} elif e.status_code == 400 and 'SearchParseException' in e.error: raise InvalidSearchString else: raise return self._parse_hits(hits, resource)
def test_GtE(self): r = parse('a >= 1') self.assertEqual(type(r), dict) self.assertEqual(r, {'a': {'$gte': 1}})
def find(self, resource, req, sub_resource_lookup): """ Retrieves a set of documents matching a given request. Queries can be expressed in two different formats: the mongo query syntax, and the python syntax. The first kind of query would look like: :: ?where={"name": "john doe"} while the second would look like: :: ?where=name=="john doe" The resultset if paginated. :param resource: resource name. :param req: a :class:`ParsedRequest`instance. :param sub_resource_lookup: sub-resource lookup from the endpoint url. .. versionchanged:: 0.6 Support for multiple databases. Filter soft deleted documents by default .. versionchanged:: 0.5 Support for comma delimited sort syntax. Addresses #443. Return the error if a blacklisted MongoDB operator is used in query. Abort with 400 if unsupported query operator is used. #387. Abort with 400 in case of invalid sort syntax. #387. .. versionchanged:: 0.4 'allowed_filters' is now checked before adding 'sub_resource_lookup' to the query, as it is considered safe. Refactored to use self._client_projection since projection is now honored by getitem() as well. .. versionchanged:: 0.3 Support for new _mongotize() signature. .. versionchanged:: 0.2 Support for sub-resources. Support for 'default_sort'. .. versionchanged:: 0.1.1 Better query handling. We're now properly casting objectid-like strings to ObjectIds. Also, we're casting both datetimes and objectids even when the query was originally in python syntax. .. versionchanged:: 0.0.9 More informative error messages. .. versionchanged:: 0.0.7 Abort with a 400 if the query includes blacklisted operators. .. versionchanged:: 0.0.6 Only retrieve fields in the resource schema Support for projection queries ('?projection={"name": 1}') .. versionchanged:: 0.0.5 handles the case where req.max_results is None because pagination has been disabled. .. versionchanged:: 0.0.4 retrieves the target collection via the new config.SOURCES helper. """ args = dict() if req and req.max_results: args['limit'] = req.max_results if req and req.page > 1: args['skip'] = (req.page - 1) * req.max_results # TODO sort syntax should probably be coherent with 'where': either # mongo-like # or python-like. Currently accepts only mongo-like sort # syntax. # TODO should validate on unknown sort fields (mongo driver doesn't # return an error) client_sort = {} spec = {} if req and req.sort: try: # assume it's mongo syntax (ie. ?sort=[("name", 1)]) client_sort = ast.literal_eval(req.sort) except ValueError: # it's not mongo so let's see if it's a comma delimited string # instead (ie. "?sort=-age, name"). sort = [] for sort_arg in [s.strip() for s in req.sort.split(",")]: if sort_arg[0] == "-": sort.append((sort_arg[1:], -1)) else: sort.append((sort_arg, 1)) if len(sort) > 0: client_sort = sort except Exception as e: self.app.logger.exception(e) abort(400, description=debug_error_message(str(e))) if req and req.where: try: spec = self._sanitize(json.loads(req.where)) except HTTPException as e: # _sanitize() is raising an HTTP exception; let it fire. raise except: # couldn't parse as mongo query; give the python parser a shot. try: spec = parse(req.where) except ParseError: abort(400, description=debug_error_message( 'Unable to parse `where` clause' )) bad_filter = validate_filters(spec, resource) if bad_filter: abort(400, bad_filter) if sub_resource_lookup: spec = self.combine_queries(spec, sub_resource_lookup) if config.DOMAIN[resource]['soft_delete'] \ and not (req and req.show_deleted) \ and not self.query_contains_field(spec, config.DELETED): # Soft delete filtering applied after validate_filters call as # querying against the DELETED field must always be allowed when # soft_delete is enabled spec = self.combine_queries(spec, {config.DELETED: {"$ne": True}}) spec = self._mongotize(spec, resource) client_projection = self._client_projection(req) datasource, spec, projection, sort = self._datasource_ex( resource, spec, client_projection, client_sort) if req and req.if_modified_since: spec[config.LAST_UPDATED] = \ {'$gt': req.if_modified_since} if len(spec) > 0: args['filter'] = spec if sort is not None: args['sort'] = sort if projection is not None: args['projection'] = projection return self.pymongo(resource).db[datasource].find(**args)
def test_NotEq(self): r = parse('a != 1') self.assertEqual(type(r), dict) self.assertEqual(r, {'a': {'$ne': 1}})
def find(self, resource, req, sub_resource_lookup): """Find documents for resource.""" args = getattr(req, 'args', request.args if request else {}) or {} source_config = config.SOURCES[resource] if args.get('source'): query = json.loads(args.get('source')) if 'filtered' not in query.get('query', {}): _query = query.get('query') query['query'] = {'filtered': {}} if _query: query['query']['filtered']['query'] = _query else: query = {'query': {'filtered': {}}} if args.get('q', None): query['query']['filtered']['query'] = _build_query_string( args.get('q'), default_field=args.get('df', '_all'), default_operator=args.get('default_operator', 'OR')) if 'sort' not in query: if req.sort: sort = ast.literal_eval(req.sort) set_sort(query, sort) elif self._default_sort( resource) and 'query' not in query['query']['filtered']: set_sort(query, self._default_sort(resource)) if req.max_results: query.setdefault('size', req.max_results) if req.page > 1: query.setdefault('from', (req.page - 1) * req.max_results) filters = [] filters.append(source_config.get('elastic_filter')) filters.append(source_config.get('elastic_filter_callback', noop)()) filters.append({'and': _build_lookup_filter(sub_resource_lookup)} if sub_resource_lookup else None) filters.append( json.loads(args.get('filter')) if 'filter' in args else None) filters.extend(args.get('filters') if 'filters' in args else []) if req.where: try: filters.append({'term': json.loads(req.where)}) except ValueError: try: filters.append({'term': parse(req.where)}) except ParseError: abort(400) set_filters(query, filters) if 'facets' in source_config: query['facets'] = source_config['facets'] if 'aggregations' in source_config and self.should_aggregate(req): query['aggs'] = source_config['aggregations'] if 'es_highlight' in source_config and self.should_highlight(req): query_string = query['query'].get('filtered', {}).get('query', {}).get('query_string') highlights = source_config.get('es_highlight', noop)(query_string) if highlights: query['highlight'] = highlights query['highlight'].setdefault('require_field_match', False) source_projections = None if self.should_project(req): source_projections = self.get_projected_fields(req) args = self._es_args(resource, source_projections=source_projections) try: hits = self.elastic(resource).search(body=query, **args) except elasticsearch.exceptions.RequestError as e: if e.status_code == 400 and "No mapping found for" in e.error: hits = {} elif e.status_code == 400 and 'SearchParseException' in e.error: raise InvalidSearchString else: raise return self._parse_hits(hits, resource)
def test_Or_BoolOp(self): r = parse('a == 1 or b == 2') self.assertEqual(type(r), dict) self.assertEqual(r, {'$or': [{'a': 1}, {'b': 2}]})
def test_ObjectId_Call(self): r = parse('_id == ObjectId("4f4644fbc88e20212c000000")') self.assertEqual(type(r), dict) self.assertEqual(r, {'_id': ObjectId("4f4644fbc88e20212c000000")})
def test_Attribute(self): r = parse('Invoice.number == 1') self.assertEqual(type(r), dict) self.assertEqual(r, {'Invoice.number': 1})
def find(self, resource, req, sub_resource_lookup): """ Retrieves a set of documents matching a given request. Queries can be expressed in two different formats: the mongo query syntax, and the python syntax. The first kind of query would look like: :: ?where={"name": "john doe"} while the second would look like: :: ?where=name=="john doe" The resultset if paginated. :param resource: resource name. :param req: a :class:`ParsedRequest`instance. :param sub_resource_lookup: sub-resource lookup from the endpoint url. .. versionchanged:: 0.6 Support for multiple databases. Filter soft deleted documents by default .. versionchanged:: 0.5 Support for comma delimited sort syntax. Addresses #443. Return the error if a blacklisted MongoDB operator is used in query. Abort with 400 if unsupported query operator is used. #387. Abort with 400 in case of invalid sort syntax. #387. .. versionchanged:: 0.4 'allowed_filters' is now checked before adding 'sub_resource_lookup' to the query, as it is considered safe. Refactored to use self._client_projection since projection is now honored by getitem() as well. .. versionchanged:: 0.3 Support for new _mongotize() signature. .. versionchanged:: 0.2 Support for sub-resources. Support for 'default_sort'. .. versionchanged:: 0.1.1 Better query handling. We're now properly casting objectid-like strings to ObjectIds. Also, we're casting both datetimes and objectids even when the query was originally in python syntax. .. versionchanged:: 0.0.9 More informative error messages. .. versionchanged:: 0.0.7 Abort with a 400 if the query includes blacklisted operators. .. versionchanged:: 0.0.6 Only retrieve fields in the resource schema Support for projection queries ('?projection={"name": 1}') .. versionchanged:: 0.0.5 handles the case where req.max_results is None because pagination has been disabled. .. versionchanged:: 0.0.4 retrieves the target collection via the new config.SOURCES helper. """ args = dict() if req and req.max_results: args['limit'] = req.max_results if req and req.page > 1: args['skip'] = (req.page - 1) * req.max_results # TODO sort syntax should probably be coherent with 'where': either # mongo-like # or python-like. Currently accepts only mongo-like sort # syntax. # TODO should validate on unknown sort fields (mongo driver doesn't # return an error) client_sort = {} spec = {} if req and req.sort: try: # assume it's mongo syntax (ie. ?sort=[("name", 1)]) client_sort = ast.literal_eval(req.sort) except ValueError: # it's not mongo so let's see if it's a comma delimited string # instead (ie. "?sort=-age, name"). sort = [] for sort_arg in [s.strip() for s in req.sort.split(",")]: if sort_arg[0] == "-": sort.append((sort_arg[1:], -1)) else: sort.append((sort_arg, 1)) if len(sort) > 0: client_sort = sort except Exception as e: self.app.logger.exception(e) abort(400, description=debug_error_message(str(e))) if req and req.where: try: spec = self._sanitize(json.loads(req.where)) except HTTPException as e: # _sanitize() is raising an HTTP exception; let it fire. raise except: # couldn't parse as mongo query; give the python parser a shot. try: spec = parse(req.where) except ParseError: abort(400, description=debug_error_message( 'Unable to parse `where` clause')) bad_filter = validate_filters(spec, resource) if bad_filter: abort(400, bad_filter) if sub_resource_lookup: spec = self.combine_queries(spec, sub_resource_lookup) if config.DOMAIN[resource]['soft_delete'] \ and not (req and req.show_deleted) \ and not self.query_contains_field(spec, config.DELETED): # Soft delete filtering applied after validate_filters call as # querying against the DELETED field must always be allowed when # soft_delete is enabled spec = self.combine_queries(spec, {config.DELETED: {"$ne": True}}) spec = self._mongotize(spec, resource) client_projection = self._client_projection(req) datasource, spec, projection, sort = self._datasource_ex( resource, spec, client_projection, client_sort) if req and req.if_modified_since: spec[config.LAST_UPDATED] = \ {'$gt': req.if_modified_since} if len(spec) > 0: args['filter'] = spec if sort is not None: args['sort'] = sort if projection is not None: args['projection'] = projection return self.pymongo(resource).db[datasource].find(**args)
def test_nested_BoolOp(self): r = parse('a == 1 or (b == 2 and c == 3)') self.assertEqual(type(r), dict) self.assertEqual(r, {'$or': [{'a': 1}, {'$and': [{'b': 2}, {'c': 3}]}]})
def test_ObjectId_Call(self): r = parse('_id == ObjectId("4f4644fbc88e20212c000000")') self.assertEqual(type(r), dict) self.assertEqual(r, {"_id": ObjectId("4f4644fbc88e20212c000000")})
def find(self, resource, req, sub_resource_lookup): """ Seach for results and return list of them. :param resource: name of requested resource as string. :param req: instance of :class:`eve.utils.ParsedRequest`. :param sub_resource_lookup: sub-resource lookup from the endpoint url. """ qry = self._objects(resource) client_projection = {} client_sort = {} spec = {} # TODO sort syntax should probably be coherent with 'where': either # mongo-like # or python-like. Currently accepts only mongo-like sort # syntax. # TODO should validate on unknown sort fields (mongo driver doesn't # return an error) if req.sort: try: client_sort = ast.literal_eval(req.sort) except Exception as e: abort(400, description=debug_error_message(str(e))) if req.where: try: spec = self._sanitize(json.loads(req.where)) except HTTPException as e: # _sanitize() is raising an HTTP exception; let it fire. raise except: try: spec = parse(req.where) except ParseError: abort(400, description=debug_error_message( 'Unable to parse `where` clause' )) if sub_resource_lookup: spec.update(sub_resource_lookup) spec = self._mongotize(spec, resource) bad_filter = validate_filters(spec, resource) if bad_filter: abort(400, bad_filter) if req.projection: try: client_projection = json.loads(req.projection) except Exception as e: abort(400, description=debug_error_message( 'Unable to parse `projection` clause: '+str(e) )) datasource, spec, projection, sort = self._datasource_ex( resource, spec, client_projection, client_sort) # apply ordering if sort: for field, direction in _itemize(sort): if direction < 0: field = "-%s" % field qry = qry.order_by(field) # apply filters if req.if_modified_since: spec[config.LAST_UPDATED] = \ {'$gt': req.if_modified_since} if len(spec) > 0: qry = qry.filter(__raw__=spec) # apply projection qry = self._projection(resource, projection, qry) # apply limits if req.max_results: qry = qry.limit(req.max_results) if req.page > 1: qry = qry.skip((req.page - 1) * req.max_results) return PymongoQuerySet(qry)
def test_Gt(self): r = parse('a > 1') self.assertIs(type(r), dict) self.assertEqual(r, {'a': {'$gt': 1}})