def prefill_search_page(self, search):
""" Prefill the search page parameters
:param newsroom.search.SearchQuery search: The search query instance
:param ParsedRequest req: The parsed in request instance from the endpoint
"""
try:
search.args['page'] = int(search.args.get('page') or 1)
except ValueError:
raise BadParameterValueError('Page must be a number')
try:
search.args['page_size'] = int(search.args.get('page_size') or 25)
except ValueError:
raise BadParameterValueError('Page Size must be a number')
search.args['size'] = search.args['page_size']
search.args['from'] = (search.args['page'] -
1) * search.args['page_size']
super().prefill_search_page(search)
if isinstance(search.args['sort'], str):
if search.args['sort'] == 'versioncreated:desc':
search.args['sort'] = [{'versioncreated': 'desc'}]
elif search.args['sort'] == 'versioncreated:asc':
search.args['sort'] = [{'versioncreated': 'asc'}]
elif search.args['sort'] == 'score':
search.args['sort'] = [{'_score': 'desc'}]
else:
raise BadParameterValueError(
'Unknown sort option ({name})'.format(
name=search.args['sort']))
def _set_filter_for_arguments(self, req, orig_request_params):
"""Based on arguments creates elastic search filters and
assign to `filter` argument of the `req` object.
:param req: object representing the HTTP request
:type req: `eve.utils.ParsedRequest`
:param dict orig_request_params: request parameter names and their
corresponding values
"""
# request argments and elasticsearch fields
argument_fields = {
"service": "service.code",
"subject": "subject.code",
"urgency": "urgency",
"priority": "priority",
"genre": "genre.code",
"item_source": "source",
}
try:
filters = (json.loads(orig_request_params.get("filter"))
if orig_request_params
and orig_request_params.get("filter") else [])
except Exception:
raise BadParameterValueError(
"Bad parameter value for Parameter (filter)")
for argument_name, field_name in argument_fields.items():
if argument_name not in orig_request_params or orig_request_params.get(
argument_name) is None:
continue
filter_value = orig_request_params.get(argument_name)
try:
filter_value = json.loads(
orig_request_params.get(argument_name))
except Exception:
pass
if not filter_value:
raise BadParameterValueError(
"Bad parameter value for Parameter ({})".format(
argument_name))
if not isinstance(filter_value, list):
filter_value = [filter_value]
filters.append({"terms": {field_name: filter_value}})
# set the date range filter
start_date, end_date = self._get_date_range(orig_request_params)
date_filter = self._create_date_range_filter(start_date, end_date)
if date_filter:
filters.append(date_filter)
if filters:
req.args["filter"] = json.dumps({"bool": {"must": filters}})
def _set_filter_for_arguments(self, req, orig_request_params):
"""Based on arguments creates elastic search filters and
assign to `filter` argument of the `req` object.
:param req: object representing the HTTP request
:type req: `eve.utils.ParsedRequest`
:param dict orig_request_params: request parameter names and their
corresponding values
"""
# request argments and elasticsearch fields
argument_fields = {
'service': 'service.code',
'subject': 'subject.code',
'urgency': 'urgency',
'priority': 'priority',
'genre': 'genre.code',
'item_source': 'source'
}
try:
filters = json.loads(orig_request_params.get('filter')) \
if orig_request_params and orig_request_params.get('filter') else []
except:
raise BadParameterValueError(
"Bad parameter value for Parameter (filter)")
for argument_name, field_name in argument_fields.items():
if argument_name not in orig_request_params or orig_request_params.get(
argument_name) is None:
continue
filter_value = orig_request_params.get(argument_name)
try:
filter_value = json.loads(
orig_request_params.get(argument_name))
except:
pass
if not filter_value:
raise BadParameterValueError(
"Bad parameter value for Parameter ({})".format(
argument_name))
if not isinstance(filter_value, list):
filter_value = [filter_value]
filters.append({'terms': {field_name: filter_value}})
# set the date range filter
start_date, end_date = self._get_date_range(orig_request_params)
date_filter = self._create_date_range_filter(start_date, end_date)
if date_filter:
filters.append(date_filter)
if filters:
req.args['filter'] = json.dumps({'bool': {'must': filters}})
def _get_date_range(self, request_params):
"""Extract the start and end date limits from request parameters.
If start and/or end date parameter is not present, a default value is
returned for the missing parameter(s).
:param dict request_params: request parameter names and their
corresponding values
:return: a (start_date, end_date) tuple with both values being
instances of Python's datetime.date
:raises BadParameterValueError:
* if any of the dates is not in the ISO 8601 format
* if any of the dates is set in the future
* if the start date is bigger than the end date
"""
# check date limits' format...
err_msg = "{} parameter must be a valid ISO 8601 date (YYYY-MM-DD) " "without the time part"
try:
start_date = self._parse_iso_date(request_params.get("start_date"))
except ValueError:
raise BadParameterValueError(desc=err_msg.format("start_date")) from None
try:
end_date = self._parse_iso_date(request_params.get("end_date"))
except ValueError:
raise BadParameterValueError(desc=err_msg.format("end_date")) from None
# disallow dates in the future...
err_msg = "{} date ({}) must not be set in the future " "(current server date (UTC): {})"
today = utcnow().date()
if (start_date is not None) and (start_date > today):
raise BadParameterValueError(desc=err_msg.format("Start", start_date.isoformat(), today.isoformat()))
if (end_date is not None) and (end_date > today):
raise BadParameterValueError(desc=err_msg.format("End", end_date.isoformat(), today.isoformat()))
# make sure that the date range limits make sense...
if (start_date is not None) and (end_date is not None) and (start_date > end_date):
# NOTE: we allow start_date == end_date (for specific date queries)
raise BadParameterValueError(desc="Start date must not be greater than end date")
# set default date range values if missing...
if self._is_internal_api():
if end_date is None:
end_date = today
if start_date is None:
start_date = end_date - timedelta(days=7) # get last 7 days by default
return start_date, end_date
def _set_filter_parameter(self, req, filter, original_filter_param):
elastic_filter = []
try:
elastic_filter = json.loads(original_filter_param) if original_filter_param else []
if not isinstance(elastic_filter, list):
raise BadParameterValueError(desc="Invalid Parameter value for filter")
except Exception:
raise BadParameterValueError(desc="Invalid Parameter value for filter")
elastic_filter.append(filter)
req.args["filter"] = json.dumps(elastic_filter)
def _validate(self, token):
"""
Ensure that a no other tokens for the company exist. Ensure that the expiry date is in the future.
:param token:
:return:
"""
company_tokens = self.find(where={'company': token.get('company')})
if company_tokens.count() >= 1:
raise BadParameterValueError(
'A token for the company exists already')
if token.get('expiry'):
if token.get('expiry') < utcnow():
raise BadParameterValueError('Token has expired')
def apply_fields_filter(self, search):
""" Generate the field filters
:param newsroom.search.SearchQuery search: The search query instance
"""
# request argments and elasticsearch fields
argument_fields = {
'service': 'service.code',
'subject': 'subject.code',
'urgency': 'urgency',
'priority': 'priority',
'genre': 'genre.code',
'item_source': 'source'
}
filters = []
if search.args.get('filter'):
try:
filters = json.loads(search.args['filter'])
except Exception:
raise BadParameterValueError(
'Bad parameter value for Parameter (filter)')
for argument_name, field_name in argument_fields.items():
if argument_name not in search.args or search.args[
argument_name] is None:
continue
filter_value = search.args[argument_name]
try:
filter_value = json.loads(filter_value)
except Exception:
pass
if not filter_value:
raise BadParameterValueError(
"Bad parameter value for Parameter ({})".format(
argument_name))
if not isinstance(filter_value, list):
filter_value = [filter_value]
filters.append({'terms': {field_name: filter_value}})
if filters:
search.query['bool']['must'].extend(filters)
def validate_page(self, search):
""" Validate the page params
:param newsroom.search.SearchQuery search: The search query instance
"""
if search.args['page'] < 1:
raise BadParameterValueError(
'Page number must be greater or equal to 1')
elif search.args['page_size'] > app.config['QUERY_MAX_PAGE_SIZE']:
raise BadParameterValueError(
'Requested maximum number of results exceeds {max}'.format(
max=app.config['QUERY_MAX_PAGE_SIZE']))
elif (search.args['page'] - 1) * search.args['page_size'] >= 1000:
# https://www.elastic.co/guide/en/elasticsearch/guide/current/pagination.html#pagination
raise BadParameterValueError('Page limit exceeded')
def prefill_search_products(self, search):
""" Prefill the search products
:param SearchQuery search: The search query instance
"""
if search.args.get('requested_products'):
products = search.args['requested_products']
if isinstance(products, list):
search.requested_products = products
elif getattr(products, 'split', None):
search.requested_products = list(products.split(','))
else:
raise BadParameterValueError(
'Invalid requested_products parameter')
if search.is_admin:
if len(search.navigation_ids):
search.products = get_products_by_navigation(
search.navigation_ids, product_type=search.section)
else:
# admin will see everything by default,
# regardless of company products
search.products = []
elif search.company:
search.products = get_products_by_company(
search.company.get('_id'),
search.navigation_ids,
product_type=search.section)
else:
search.products = []
def _make_one(self, *args, **kwargs):
"""Create and return a new instance of the class under test.
Make the test fail immediately if the class cannot be imported.
"""
try:
from content_api.errors import BadParameterValueError
except ImportError:
self.fail("Could not import class under test")
else:
return BadParameterValueError(*args, **kwargs)
def prefill_search_navigation(self, search):
""" Prefill the search navigation
:param SearchQuery search: The search query instance
"""
if search.args.get('navigation'):
navigation = search.args['navigation']
if isinstance(navigation, list):
search.navigation_ids = navigation
elif getattr(navigation, 'split', None):
search.navigation_ids = list(navigation.split(','))
else:
raise BadParameterValueError('Invalid navigation parameter')
else:
search.navigation_ids = []
def get(self, req, lookup):
"""Retrieve a list of items that match the filter criteria (if any)
pssed along the HTTP request.
:param req: object representing the HTTP request
:type req: `eve.utils.ParsedRequest`
:param dict lookup: sub-resource lookup from the endpoint URL
:return: database results cursor object
:rtype: `pymongo.cursor.Cursor`
"""
internal_req = ParsedRequest() if req is None else deepcopy(req)
internal_req.args = MultiDict()
orig_request_params = getattr(req, "args", MultiDict())
self._check_for_unknown_params(req, whitelist=self.allowed_params)
self._set_search_field(internal_req.args, orig_request_params)
# combine elastic search filter for args as args.get('filter')
self._set_filter_for_arguments(internal_req, orig_request_params)
# projections
internal_req.args["exclude_fields"] = orig_request_params.get(
"exclude_fields")
internal_req.args["include_fields"] = orig_request_params.get(
"include_fields")
self._set_fields_filter(internal_req) # Eve's "projection"
# if subscribers is not allowed it is an external API request that should be filtered by the user
if self._is_internal_api():
# in case there is no subscriber set by auth return nothing
lookup["subscribers"] = g.get("user")
# apply default sorting if it was not provided explicitly in query.
# eve-elastic applies default sorting only if filtering was not provided in query
# https://github.com/petrjasek/eve-elastic/blob/master/eve_elastic/elastic.py#L455
self._set_default_sort(internal_req)
if "aggregations" in self.allowed_params:
internal_req.args["aggregations"] = orig_request_params.get(
"aggregations", 0)
try:
res = super().get(internal_req, lookup)
return res
except InvalidSearchString:
raise BadParameterValueError("invalid search text")
def apply_request_filter(self, search):
if search.args.get('q'):
search.query['bool']['must'].append(
query_string(search.args['q'],
search.args.get('default_operator') or 'AND'))
filters = None
if search.args.get('filter'):
if isinstance(search.args['filter'], dict):
filters = search.args['filter']
else:
try:
filters = json.loads(search.args['filter'])
except TypeError:
raise BadParameterValueError(
'Incorrect type supplied for filter parameter')
if not app.config.get('FILTER_BY_POST_FILTER', False):
if filters:
if app.config.get('FILTER_AGGREGATIONS', True):
search.query['bool']['must'] += self._filter_terms(filters)
else:
search.query['bool']['must'].append(filters)
if search.args.get('created_from') or search.args.get(
'created_to'):
search.query['bool']['must'].append(
self.versioncreated_range(search.args))
elif filters or search.args.get('created_from') or search.args.get(
'created_to'):
search.source['post_filter'] = {'bool': {'must': []}}
if filters:
if app.config.get('FILTER_AGGREGATIONS', True):
search.source['post_filter']['bool'][
'must'] += self._filter_terms(filters)
else:
search.source['post_filter']['bool']['must'].append(
filters)
if search.args.get('created_from') or search.args.get(
'created_to'):
search.source['post_filter']['bool']['must'].append(
self.versioncreated_range(search.args))
def prefill_search_query(self, search, req=None, lookup=None):
""" Generate the search query instance
:param newsroom.search.SearchQuery search: The search query instance
:param eve.utils.ParsedRequest req: The parsed in request instance from the endpoint
:param dict lookup: The parsed in lookup dictionary from the endpoint
"""
super().prefill_search_query(search, req, lookup)
if search.args.get('exclude_ids'):
search.args['exclude_ids'] = search.args['exclude_ids'].split(',')
try:
search.args['max_results'] = int(search.args.get('max_results') or 25)
except ValueError:
raise BadParameterValueError('Max Results must be a number')
search.args['size'] = search.args['max_results']
def get(self, req, lookup):
"""Retrieve a list of items that match the filter criteria (if any)
pssed along the HTTP request.
:param req: object representing the HTTP request
:type req: `eve.utils.ParsedRequest`
:param dict lookup: sub-resource lookup from the endpoint URL
:return: database results cursor object
:rtype: `pymongo.cursor.Cursor`
"""
internal_req = ParsedRequest() if req is None else deepcopy(req)
internal_req.args = MultiDict()
orig_request_params = getattr(req, 'args', MultiDict())
self._check_for_unknown_params(req, whitelist=self.allowed_params)
self._set_search_field(internal_req.args, orig_request_params)
# combine elastic search filter for args as args.get('filter')
self._set_filter_for_arguments(internal_req, orig_request_params)
# projections
internal_req.args['exclude_fields'] = orig_request_params.get(
'exclude_fields')
internal_req.args['include_fields'] = orig_request_params.get(
'include_fields')
self._set_fields_filter(internal_req) # Eve's "projection"
# if subscribers is not allowed it is an external API request that should be filtered by the user
if self._is_internal_api():
# in case there is no subscriber set by auth return nothing
lookup['subscribers'] = g.get('user')
if 'aggregations' in self.allowed_params:
internal_req.args['aggregations'] = orig_request_params.get(
'aggregations', 0)
try:
res = super().get(internal_req, lookup)
return res
except InvalidSearchString:
raise BadParameterValueError('invalid search text')
def get(self, req, lookup):
"""Retrieve a list of items that match the filter criteria (if any)
pssed along the HTTP request.
:param req: object representing the HTTP request
:type req: `eve.utils.ParsedRequest`
:param dict lookup: sub-resource lookup from the endpoint URL
:return: database results cursor object
:rtype: `pymongo.cursor.Cursor`
"""
internal_req = ParsedRequest() if req is None else deepcopy(req)
internal_req.args = MultiDict()
orig_request_params = getattr(req, 'args', MultiDict())
allowed_params = {
'start_date', 'end_date',
'include_fields', 'exclude_fields',
'max_results', 'page',
'where',
'version'
}
self._check_for_unknown_params(req, whitelist=allowed_params)
# set the date range filter
start_date, end_date = self._get_date_range(orig_request_params)
date_filter = self._create_date_range_filter(start_date, end_date)
internal_req.args['filter'] = json.dumps(date_filter)
self._set_fields_filter(internal_req) # Eve's "projection"
# in case there is no subscriber set by auth return nothing
lookup['subscribers'] = g.get('user')
try:
res = super().get(internal_req, lookup)
return res
except InvalidSearchString:
raise BadParameterValueError('invalid search text')
def get(self, req, lookup):
"""Retrieve a list of items that match the filter criteria (if any)
pssed along the HTTP request.
:param req: object representing the HTTP request
:type req: `eve.utils.ParsedRequest`
:param dict lookup: sub-resource lookup from the endpoint URL
:return: database results cursor object
:rtype: `pymongo.cursor.Cursor`
"""
if req is None:
req = ParsedRequest()
allowed_params = {
'q', 'start_date', 'end_date', 'include_fields', 'exclude_fields'
}
self._check_for_unknown_params(req, whitelist=allowed_params)
request_params = req.args or {}
req.args = {}
# set the search query
if 'q' in request_params:
req.args['q'] = request_params['q']
req.args['default_operator'] = 'OR'
# set the date range filter
start_date, end_date = self._get_date_range(request_params)
date_filter = self._create_date_range_filter(start_date, end_date)
req.args['filter'] = json.dumps(date_filter)
self._set_fields_filter(req) # Eve's "projection"
try:
return super().get(req, lookup)
except InvalidSearchString:
raise BadParameterValueError('invalid search text')
def _parse_iso_date(date_str, timezone=None):
""" Create a date object from the given string in ISO 8601 format.
:param date_str:
:type date_str: str or None
:return: resulting date object or None if None is given
:rtype: datetime.date
:raises ValueError: if `date_str` is not in the ISO 8601 date format
"""
if date_str is None:
return None
else:
dt = parser.parse(date_str)
if dt.tzinfo is None:
if timezone:
if timezone not in pytz.all_timezones:
raise BadParameterValueError("Bad parameter value for Parameter (timezone)")
dt = local_to_utc(timezone, dt)
else:
dt = pytz.timezone('UTC').localize(dt)
return dt
def _get_field_filter_params(self, request_params):
"""Extract the list of content fields to keep in or remove from
the response.
The parameter names are `include_fields` and `exclude_fields`. Both are
simple comma-separated lists, for example::
exclude_fields= field_1, field_2,field_3,, ,field_4,
NOTE: any redundant spaces, empty field values and duplicate values are
gracefully ignored and do not cause an error.
:param dict request_params: request parameter names and their
corresponding values
:return: a (include_fields, exclude_fields) tuple with each item being
either a `set` of field names (as strings) or None if the request
does not contain the corresponding parameter
:raises BadParameterValueError:
* if the request contains both parameters at the same time
* if any of the parameters contain an unknown field name (i.e. not
defined in the resource schema)
* if `exclude_params` parameter contains a field name that is
required to be present in the response according to the NINJS
standard
"""
include_fields = request_params.get("include_fields")
exclude_fields = request_params.get("exclude_fields")
# parse field filter parameters...
strip_items = functools.partial(map, lambda s: s.strip())
remove_empty = functools.partial(filter, None)
if include_fields is not None:
include_fields = include_fields.split(",")
include_fields = set(remove_empty(strip_items(include_fields)))
if exclude_fields is not None:
exclude_fields = exclude_fields.split(",")
exclude_fields = set(remove_empty(strip_items(exclude_fields)))
# check for semantically incorrect field filter values...
if (include_fields is not None) and (exclude_fields is not None):
err_msg = "Cannot both include and exclude content fields " "at the same time."
raise UnexpectedParameterError(desc=err_msg)
if include_fields is not None:
err_msg = "Unknown content field to include ({})."
for field in include_fields:
if field not in ItemsResource.schema:
raise BadParameterValueError(desc=err_msg.format(field))
if exclude_fields is not None:
if "uri" in exclude_fields:
err_msg = "Cannot exclude a content field required by the " "NINJS format (uri)."
raise BadParameterValueError(desc=err_msg)
err_msg = "Unknown content field to exclude ({})."
for field in exclude_fields:
if field not in ItemsResource.schema:
raise BadParameterValueError(desc=err_msg.format(field))
return include_fields, exclude_fields
def _get_date_range(self, request_params):
""" Extract the start and end date limits from request parameters.
If start and/or end date parameter is not present, a default value is
returned for the missing parameter(s).
:param dict request_params: request parameter names and their
corresponding values
:return: a (start_date, end_date) tuple with both values being
instances of Python's datetime.date
:raises BadParameterValueError:
* if any of the dates is not in the ISO 8601 format
* if any of the dates is set in the future
* if the start date is bigger than the end date
"""
# regex that should match likely relative elastic searcg date math
regex = r"now([-+][0-9]*([YMwdHhms]*)$|/d$)"
# check date limits' format...
err_msg = ("{} parameter must be a valid ISO 8601 date (YYYY-MM-DD) "
"with optional the time part")
relative_start = False
relative_end = False
try:
# check for a relative date
if re.match(regex, request_params.get('start_date', '')):
start_date = request_params.get('start_date')
relative_start = True
else:
start_date = self._parse_iso_date(
request_params.get('start_date'),
request_params.get('timezone'))
except BadParameterValueError:
raise
except ValueError:
raise BadParameterValueError(
desc=err_msg.format('start_date')) from None
try:
if request_params.get('end_date'):
if re.match(regex, request_params.get('end_date', '')):
end_date = request_params.get('end_date')
relative_end = True
else:
end_date = self._parse_iso_date(
request_params.get('end_date'),
request_params.get('timezone'))
else:
end_date = None
except ValueError:
raise BadParameterValueError(
desc=err_msg.format('end_date')) from None
# disallow dates in the future...
err_msg = ("{} date ({}) must not be set in the future "
"(current server date (UTC): {})")
today = utcnow()
if (start_date
is not None) and not relative_start and (start_date > today):
raise BadParameterValueError(desc=err_msg.format(
'Start', start_date.isoformat(), today.isoformat()))
if (end_date is not None) and not relative_end and (end_date > today):
raise BadParameterValueError(desc=err_msg.format(
'End', end_date.isoformat(), today.isoformat()))
# make sure that the date range limits make sense...
if ((not relative_start or not relative_end)
and (start_date is not None) and (end_date is not None)
and (start_date > end_date)):
# NOTE: we allow start_date == end_date (for specific date queries)
raise BadParameterValueError(
desc="Start date must not be greater than end date")
return start_date, end_date