def get_item(self, **kw):
_raise_on_empty = kw.pop('_raise_on_empty', True)
params = dict(index=self.index_name, doc_type=self.doc_type)
params.update(kw)
not_found_msg = "'{}({})' resource not found".format(
self.doc_type, params)
try:
data = self.api.get_source(**params)
except IndexNotFoundException:
if _raise_on_empty:
raise JHTTPNotFound("{} (Index does not exist)".format(
not_found_msg, self.doc_type, params))
data = {}
except JHTTPNotFound:
data = {}
if not data:
if _raise_on_empty:
raise JHTTPNotFound(not_found_msg)
else:
log.debug(not_found_msg)
if '_type' not in data:
data['_type'] = self.doc_type
return dict2obj(data)
def get_by_ids(self, ids, **params):
if not ids:
return _ESDocs()
_raise_on_empty = params.pop('_raise_on_empty', False)
fields = params.pop('_fields', [])
_limit = params.pop('_limit', len(ids))
_page = params.pop('_page', None)
_start = params.pop('_start', None)
_start, _limit = process_limit(_start, _page, _limit)
docs = []
for _id in ids:
docs.append(
dict(_index=self.index_name,
_type=self.src2type(_id['_type']),
_id=_id['_id']))
params = dict(body=dict(docs=docs))
if fields:
fields_params = process_fields_param(fields)
params.update(fields_params)
documents = _ESDocs()
documents._nefertari_meta = dict(
start=_start,
fields=fields,
)
try:
data = self.api.mget(**params)
except IndexNotFoundException:
if _raise_on_empty:
raise JHTTPNotFound(
'{}({}) resource not found (Index does not exist)'.format(
self.doc_type, params))
documents._nefertari_meta.update(total=0)
return documents
for found_doc in data['docs']:
try:
output_doc = found_doc['_source']
output_doc['_type'] = found_doc['_type']
except KeyError:
msg = "ES: '%s(%s)' resource not found" % (found_doc['_type'],
found_doc['_id'])
if _raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.error(msg)
continue
documents.append(dict2obj(dictset(output_doc)))
documents._nefertari_meta.update(total=len(documents), )
return documents
def get_collection(self, **params):
_raise_on_empty = params.pop('_raise_on_empty', False)
if 'body' in params:
_params = params
else:
_params = self.build_search_params(params)
if '_count' in params:
return self.do_count(_params)
fields = _params.pop('fields', '')
if fields:
fields_params = process_fields_param(fields)
_params.update(fields_params)
documents = _ESDocs()
documents._nefertari_meta = dict(start=_params.get('from_', 0),
fields=fields)
try:
data = self.api.search(**_params)
except IndexNotFoundException:
if _raise_on_empty:
raise JHTTPNotFound(
'{}({}) resource not found (Index does not exist)'.format(
self.doc_type, params))
documents._nefertari_meta.update(total=0, took=0)
return documents
for found_doc in data['hits']['hits']:
output_doc = found_doc['_source']
output_doc['_score'] = found_doc['_score']
output_doc['_type'] = found_doc['_type']
documents.append(dict2obj(output_doc))
documents._nefertari_meta.update(
total=data['hits']['total'],
took=data['took'],
)
if not documents:
msg = "%s(%s) resource not found" % (self.doc_type, params)
if _raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.debug(msg)
return documents
def view_mapper_wrapper(context, request):
matchdict = request.matchdict.copy()
matchdict.pop('action', None)
matchdict.pop('traverse', None)
# instance of BaseView (or child of)
view_obj = view(context, request)
action = getattr(view_obj, action_name)
request.action = action_name
# Tunneled collection PATCH/PUT doesn't support query params
tunneled = getattr(request, '_tunneled_get', False)
if tunneled and action_name in ('update_many', ):
view_obj._query_params = dictset()
# we should not run "after_calls" here, so lets save them in
# request as filters they will be ran in the renderer factory
request.filters = view_obj._after_calls
try:
# run before_calls (validators) before running the action
for call in view_obj._before_calls.get(action_name, []):
call(request=request)
except wrappers.ValidationError as e:
log.error('validation error: %s', e)
raise JHTTPBadRequest(e.args)
except wrappers.ResourceNotFound as e:
log.error('resource not found: %s', e)
raise JHTTPNotFound()
trigger_before_events(view_obj)
return action(**matchdict)
def filter_objects(cls, objects, first=False, **params):
""" Perform query with :params: on instances sequence :objects:
:param object: Sequence of :cls: instances on which query should be run.
:param params: Query parameters to filter :objects:.
"""
id_name = cls.pk_field()
ids = [getattr(obj, id_name, None) for obj in objects]
ids = [str(id_) for id_ in ids if id_ is not None]
field_obj = getattr(cls, id_name)
query_set = Session().query(cls).filter(field_obj.in_(ids))
if params:
params['query_set'] = query_set.from_self()
query_set = cls.get_collection(**params)
if first:
first_obj = query_set.first()
if not first_obj:
msg = "'{}({})' resource not found".format(
cls.__name__, params)
raise JHTTPNotFound(msg)
return first_obj
return query_set
def aggregate(self, **params):
""" Perform aggreration
Arguments:
:_aggregations_params: Dict of aggregation params. Root key is an
aggregation name. Required.
:_raise_on_empty: Boolean indicating whether to raise exception
when IndexNotFoundException exception happens. Optional,
defaults to False.
:_search_type: Type of search to use. Optional, defaults to
'count'. You might want to provide this argument explicitly
when performing nested aggregations on buckets.
"""
_aggregations_params = params.pop('_aggregations_params', None)
_raise_on_empty = params.pop('_raise_on_empty', False)
_search_type = params.pop('_search_type', 'count')
if not _aggregations_params:
raise Exception('Missing _aggregations_params')
# Set limit so ES won't complain. It is ignored in the end
params['_limit'] = 0
search_params = self.build_search_params(params)
search_params.pop('size', None)
search_params.pop('from_', None)
search_params.pop('sort', None)
search_params['search_type'] = _search_type
search_params['body']['aggregations'] = _aggregations_params
log.debug('Performing aggregation: {}'.format(_aggregations_params))
try:
response = self.api.search(**search_params)
except IndexNotFoundException:
if _raise_on_empty:
raise JHTTPNotFound('Aggregation failed: Index does not exist')
return {}
try:
return response['aggregations']
except KeyError:
raise JHTTPNotFound('No aggregations returned from ES')
def claim_token(self, **params):
"""Claim current token by POSTing 'login' and 'password'.
User's `Authorization` header value is returned in `WWW-Authenticate`
header.
"""
self._json_params.update(params)
success, self.user = self.Model.authenticate_by_password(
self._json_params)
if success:
headers = remember(self.request, self.user.username)
return JHTTPOk('Token claimed', headers=headers)
if self.user:
raise JHTTPUnauthorized('Wrong login or password')
else:
raise JHTTPNotFound('User not found')
def get_item(self, **kwargs):
""" Get collection item taking into account generated queryset
of parent view.
This method allows working with nested resources properly. Thus an item
returned by this method will belong to its parent view's queryset, thus
filtering out objects that don't belong to the parent object.
Returns an object from the applicable ACL. If ACL wasn't applied, it is
applied explicitly.
"""
if six.callable(self.context):
self.reload_context(es_based=False, **kwargs)
objects = self._parent_queryset()
if objects is not None and self.context not in objects:
raise JHTTPNotFound('{}({}) not found'.format(
self.Model.__name__, self._get_context_key(**kwargs)))
return self.context
def get_item_es(self, **kwargs):
""" Get ES collection item taking into account generated queryset
of parent view.
This method allows working with nested resources properly. Thus an item
returned by this method will belong to its parent view's queryset, thus
filtering out objects that don't belong to the parent object.
Returns an object retrieved from the applicable ACL. If an ACL wasn't
applied, it is applied explicitly.
"""
item_id = self._get_context_key(**kwargs)
objects_ids = self._parent_queryset_es()
if objects_ids is not None:
objects_ids = self.get_es_object_ids(objects_ids)
if six.callable(self.context):
self.reload_context(es_based=True, **kwargs)
if (objects_ids is not None) and (item_id not in objects_ids):
raise JHTTPNotFound('{}(id={}) resource not found'.format(
self.Model.__name__, item_id))
return self.context
def login(self, **params):
self._json_params.update(params)
next = self._query_params.get('next', '')
if self.request.path in next:
next = '' # never use the login form itself as next
unauthorized_url = self._query_params.get('unauthorized', None)
success, user = self.Model.authenticate_by_password(self._json_params)
if success:
pk_field = user.pk_field()
headers = remember(self.request, getattr(user, pk_field))
if next:
raise JHTTPFound(location=next, headers=headers)
else:
return JHTTPOk('Logged in', headers=headers)
if user:
if unauthorized_url:
return JHTTPUnauthorized(location=unauthorized_url +
'?error=1')
raise JHTTPUnauthorized('Failed to Login.')
else:
raise JHTTPNotFound('User not found')
def get_collection(cls, **params):
""" Query collection and return results.
Notes:
* Before validating that only model fields are present in params,
reserved params, query params and all params starting with
double underscore are dropped.
* Params which have value "_all" are dropped.
* When ``_count`` param is used, objects count is returned
before applying offset and limit.
:param bool _strict: If True ``params`` are validated to contain
only fields defined on model, exception is raised if invalid
fields are present. When False - invalid fields are dropped.
Defaults to ``True``.
:param bool _item_request: Indicates whether it is a single item
request or not. When True and DataError happens on DB request,
JHTTPNotFound is raised. JHTTPBadRequest is raised when False.
Defaults to ``False``.
:param list _sort: Field names to sort results by. If field name
is prefixed with "-" it is used for "descending" sorting.
Otherwise "ascending" sorting is performed by that field.
Defaults to an empty list in which case sorting is not
performed.
:param list _fields: Names of fields which should be included
or excluded from results. Fields to excluded should be
prefixed with "-". Defaults to an empty list in which
case all fields are returned.
:param int _limit: Number of results per page. Defaults
to None in which case all results are returned.
:param int _page: Number of page. In conjunction with
``_limit`` is used to calculate results offset. Defaults to
None in which case it is ignored. Params ``_page`` and
``_start` are mutually exclusive.
:param int _start: Results offset. If provided ``_limit`` and
``_page`` params are ignored when calculating offset. Defaults
to None. Params ``_page`` and ``_start`` are mutually
exclusive. If not offset-related params are provided, offset
equals to 0.
:param Query query_set: Existing queryset. If provided, all queries
are applied to it instead of creating new queryset. Defaults
to None.
:param _count: When provided, only results number is returned as
integer.
:param _explain: When provided, query performed(SQL) is returned
as a string instead of query results.
:param bool _raise_on_empty: When True JHTTPNotFound is raised
if query returned no results. Defaults to False in which case
error is just logged and empty query results are returned.
:returns: Query results as ``sqlalchemy.orm.query.Query`` instance.
May be sorted, offset, limited.
:returns: Dict of {'field_name': fieldval}, when ``_fields`` param
is provided.
:returns: Number of query results as an int when ``_count`` param
is provided.
:returns: String representing query ran when ``_explain`` param
is provided.
:raises JHTTPNotFound: When ``_raise_on_empty=True`` and no
results found.
:raises JHTTPNotFound: When ``_item_request=True`` and
``sqlalchemy.exc.DataError`` exception is raised during DB
query. Latter exception is raised when querying DB with
an identifier of a wrong type. E.g. when querying Int field
with a string.
:raises JHTTPBadRequest: When ``_item_request=False`` and
``sqlalchemy.exc.DataError`` exception is raised during DB
query.
:raises JHTTPBadRequest: When ``sqlalchemy.exc.InvalidRequestError``
or ``sqlalchemy.exc.IntegrityError`` errors happen during DB
query.
"""
log.debug('Get collection: {}, {}'.format(cls.__name__, params))
params.pop('__confirmation', False)
_strict = params.pop('_strict', True)
_item_request = params.pop('_item_request', False)
_sort = _split(params.pop('_sort', []))
_fields = _split(params.pop('_fields', []))
_limit = params.pop('_limit', None)
_page = params.pop('_page', None)
_start = params.pop('_start', None)
query_set = params.pop('query_set', None)
_count = '_count' in params
params.pop('_count', None)
_explain = '_explain' in params
params.pop('_explain', None)
_raise_on_empty = params.pop('_raise_on_empty', False)
if query_set is None:
query_set = Session().query(cls)
# Remove any __ legacy instructions from this point on
params = dictset({
key: val
for key, val in params.items() if not key.startswith('__')
})
iterables_exprs, params = cls._pop_iterables(params)
params = drop_reserved_params(params)
if _strict:
_check_fields = [
f.strip('-+') for f in list(params.keys()) + _fields + _sort
]
cls.check_fields_allowed(_check_fields)
else:
params = cls.filter_fields(params)
process_lists(params)
process_bools(params)
# If param is _all then remove it
params.pop_by_values('_all')
try:
query_set = query_set.filter_by(**params)
# Apply filtering by iterable expressions
for expr in iterables_exprs:
query_set = query_set.from_self().filter(expr)
_total = query_set.count()
if _count:
return _total
# Filtering by fields has to be the first thing to do on
# the query_set!
query_set = cls.apply_fields(query_set, _fields)
query_set = cls.apply_sort(query_set, _sort)
if _limit is not None:
_start, _limit = process_limit(_start, _page, _limit)
query_set = query_set.offset(_start).limit(_limit)
if not query_set.count():
msg = "'%s(%s)' resource not found" % (cls.__name__, params)
if _raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.debug(msg)
except DataError as ex:
if _item_request:
msg = "'{}({})' resource not found".format(
cls.__name__, params)
raise JHTTPNotFound(msg, explanation=ex.message)
else:
raise JHTTPBadRequest(str(ex), extra={'data': ex})
except (InvalidRequestError, ) as ex:
raise JHTTPBadRequest(str(ex), extra={'data': ex})
query_sql = str(query_set).replace('\n', '')
if _explain:
return query_sql
log.debug('get_collection.query_set: %s (%s)', cls.__name__, query_sql)
if _fields:
query_set = cls.add_field_names(query_set, _fields)
query_set._nefertari_meta = dict(total=_total,
start=_start,
fields=_fields)
return query_set
def get_collection(cls, **params):
"""
params may include '_limit', '_page', '_sort', '_fields'
returns paginated and sorted query set
raises JHTTPBadRequest for bad values in params
"""
params.pop('__confirmation', False)
__strict = params.pop('__strict', True)
_sort = _split(params.pop('_sort', []))
_fields = _split(params.pop('_fields', []))
_limit = params.pop('_limit', None)
_page = params.pop('_page', None)
_start = params.pop('_start', None)
_count = '_count' in params
params.pop('_count', None)
_explain = '_explain' in params
params.pop('_explain', None)
__raise_on_empty = params.pop('__raise_on_empty', False)
query_set = cls.objects
# Remove any __ legacy instructions from this point on
params = dictset(
filter(lambda item: not item[0].startswith('__'), params.items()))
if __strict:
_check_fields = [
f.strip('-+') for f in params.keys() + _fields + _sort
]
cls.check_fields_allowed(_check_fields)
else:
params = cls.filter_fields(params)
process_lists(params)
process_bools(params)
#if param is _all then remove it
params.pop_by_values('_all')
try:
query_set = query_set(**params)
_total = query_set.count()
if _count:
return _total
if _limit is None:
raise JHTTPBadRequest('Missing _limit')
_start, _limit = process_limit(_start, _page, _limit)
# Filtering by fields has to be the first thing to do on the query_set!
query_set = cls.apply_fields(query_set, _fields)
query_set = cls.apply_sort(query_set, _sort)
query_set = query_set[_start:_start + _limit]
if not query_set.count():
msg = "'%s(%s)' resource not found" % (cls.__name__, params)
if __raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.debug(msg)
except (mongo.ValidationError, mongo.InvalidQueryError) as e:
raise JHTTPBadRequest(str(e), extra={'data': e})
if _explain:
return query_set.explain()
log.debug('get_collection.query_set: %s(%s)', cls.__name__,
query_set._query)
query_set._nefertari_meta = dict(total=_total,
start=_start,
fields=_fields)
return query_set