def get_by_ids(self, ids, **params):
if not ids:
return _ESDocs()
_raise_on_empty = params.pop('_raise_on_empty', False)
fields = params.pop('_fields', [])
_limit = params.pop('_limit', len(ids))
_page = params.pop('_page', None)
_start = params.pop('_start', None)
_start, _limit = process_limit(_start, _page, _limit)
docs = []
for _id in ids:
docs.append(
dict(_index=self.index_name,
_type=self.src2type(_id['_type']),
_id=_id['_id']))
params = dict(body=dict(docs=docs))
if fields:
fields_params = process_fields_param(fields)
params.update(fields_params)
documents = _ESDocs()
documents._nefertari_meta = dict(
start=_start,
fields=fields,
)
try:
data = self.api.mget(**params)
except IndexNotFoundException:
if _raise_on_empty:
raise JHTTPNotFound(
'{}({}) resource not found (Index does not exist)'.format(
self.doc_type, params))
documents._nefertari_meta.update(total=0)
return documents
for found_doc in data['docs']:
try:
output_doc = found_doc['_source']
output_doc['_type'] = found_doc['_type']
except KeyError:
msg = "ES: '%s(%s)' resource not found" % (found_doc['_type'],
found_doc['_id'])
if _raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.error(msg)
continue
documents.append(dict2obj(dictset(output_doc)))
documents._nefertari_meta.update(total=len(documents), )
return documents
def get_by_ids(self, ids, **params):
if not ids:
return _ESDocs()
__raise_on_empty = params.pop('__raise_on_empty', False)
fields = params.pop('_fields', [])
_limit = params.pop('_limit', len(ids))
_page = params.pop('_page', None)
_start = params.pop('_start', None)
_start, _limit = process_limit(_start, _page, _limit)
docs = []
for _id in ids:
docs.append(
dict(
_index=self.index_name,
_type=self.src2type(_id['_type']),
_id=_id['_id']
)
)
params = dict(
body=dict(docs=docs)
)
if fields:
params['fields'] = fields
data = ES.api.mget(**params)
documents = _ESDocs()
for _d in data['docs']:
try:
_d = _d['fields'] if fields else _d['_source']
except KeyError:
msg = "ES: '%s(%s)' resource not found" % (
_d['_type'], _d['_id'])
if __raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.error(msg)
continue
documents.append(dict2obj(dictset(_d)))
documents._nefertari_meta = dict(
total=len(documents),
start=_start,
fields=fields,
)
return documents
def get_by_ids(self, ids, **params):
if not ids:
return _ESDocs()
__raise_on_empty = params.pop("__raise_on_empty", False)
fields = params.pop("_fields", [])
_limit = params.pop("_limit", len(ids))
_page = params.pop("_page", None)
_start = params.pop("_start", None)
_start, _limit = process_limit(_start, _page, _limit)
docs = []
for _id in ids:
docs.append(dict(_index=self.index_name, _type=self.src2type(_id["_type"]), _id=_id["_id"]))
params = dict(body=dict(docs=docs))
if fields:
fields_params = process_fields_param(fields)
params.update(fields_params)
documents = _ESDocs()
documents._nefertari_meta = dict(start=_start, fields=fields)
try:
data = self.api.mget(**params)
except IndexNotFoundException:
if __raise_on_empty:
raise JHTTPNotFound("{}({}) resource not found (Index does not exist)".format(self.doc_type, params))
documents._nefertari_meta.update(total=0)
return documents
for found_doc in data["docs"]:
try:
output_doc = found_doc["_source"]
output_doc["_type"] = found_doc["_type"]
except KeyError:
msg = "ES: '%s(%s)' resource not found" % (found_doc["_type"], found_doc["_id"])
if __raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.error(msg)
continue
documents.append(dict2obj(dictset(output_doc)))
documents._nefertari_meta.update(total=len(documents))
return documents
def build_search_params(self, params):
params = dictset(params)
_params = dict(
index=self.index_name,
doc_type=self.doc_type
)
_raw_terms = params.pop('q', '')
if 'body' not in params:
query_string = build_qs(params.remove(RESERVED_PARAMS), _raw_terms)
if query_string:
_params['body'] = {
'query': {
'query_string': {
'query': query_string
}
}
}
else:
_params['body'] = {"query": {"match_all": {}}}
else:
_params['body'] = params['body']
if '_limit' not in params:
params['_limit'] = self.api.count()['count']
_params['from_'], _params['size'] = process_limit(
params.get('_start', None),
params.get('_page', None),
params['_limit'])
if '_sort' in params:
_params['sort'] = apply_sort(params['_sort'])
if '_fields' in params:
_params['fields'] = params['_fields']
if '_search_fields' in params:
search_fields = params['_search_fields'].split(',')
search_fields.reverse()
search_fields = [s + '^' + str(i) for i, s in
enumerate(search_fields, 1)]
current_qs = _params['body']['query']['query_string']
if isinstance(current_qs, str):
_params['body']['query']['query_string'] = {'query': current_qs}
_params['body']['query']['query_string']['fields'] = search_fields
return _params
def build_search_params(self, params):
params = dictset(params)
_params = dict(index=self.index_name, doc_type=self.doc_type)
_raw_terms = params.pop('q', '')
if 'body' not in params:
query_string = build_qs(params.remove(RESERVED_PARAMS), _raw_terms)
if query_string:
_params['body'] = {
'query': {
'query_string': {
'query': query_string
}
}
}
else:
_params['body'] = {"query": {"match_all": {}}}
else:
_params['body'] = params['body']
if '_limit' not in params:
params['_limit'] = self.api.count()['count']
_params['from_'], _params['size'] = process_limit(
params.get('_start', None), params.get('_page', None),
params['_limit'])
if '_sort' in params:
_params['sort'] = apply_sort(params['_sort'])
if '_fields' in params:
_params['fields'] = params['_fields']
if '_search_fields' in params:
search_fields = params['_search_fields'].split(',')
search_fields.reverse()
search_fields = [
s + '^' + str(i) for i, s in enumerate(search_fields, 1)
]
current_qs = _params['body']['query']['query_string']
if isinstance(current_qs, str):
_params['body']['query']['query_string'] = {
'query': current_qs
}
_params['body']['query']['query_string']['fields'] = search_fields
return _params
def get_by_ids(self, ids, **params):
if not ids:
return _ESDocs()
__raise_on_empty = params.pop('__raise_on_empty', False)
fields = params.pop('_fields', [])
_limit = params.pop('_limit', len(ids))
_page = params.pop('_page', None)
_start = params.pop('_start', None)
_start, _limit = process_limit(_start, _page, _limit)
docs = []
for _id in ids:
docs.append(
dict(_index=self.index_name,
_type=self.src2type(_id['_type']),
_id=_id['_id']))
params = dict(body=dict(docs=docs))
if fields:
params['fields'] = fields
data = ES.api.mget(**params)
documents = _ESDocs()
for _d in data['docs']:
try:
_d = _d['fields'] if fields else _d['_source']
except KeyError:
msg = "ES: '%s(%s)' resource not found" % (_d['_type'],
_d['_id'])
if __raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.error(msg)
continue
documents.append(dict2obj(dictset(_d)))
documents._nefertari_meta = dict(
total=len(documents),
start=_start,
fields=fields,
)
return documents
def build_search_params(self, params):
params = dictset(params)
_params = dict(
index=self.index_name,
doc_type=self.doc_type
)
if 'body' not in params:
query_string = build_qs(
params.remove(RESERVED),
params.get('_raw_terms', ''))
if query_string:
_params['body'] = {
'query': {
'query_string': {
'query': query_string
}
}
}
else:
_params['body'] = {"query": {"match_all": {}}}
if '_limit' not in params:
raise JHTTPBadRequest('Missing _limit')
_params['from_'], _params['size'] = process_limit(
params.get('_start', None),
params.get('_page', None),
params['_limit'])
if '_sort' in params:
_params['sort'] = apply_sort(params['_sort'])
if '_fields' in params:
_params['fields'] = params['_fields']
if '_search_fields' in params:
search_fields = params['_search_fields'].split(',')
search_fields.reverse()
search_fields = [s + '^' + str(i) for i, s in
enumerate(search_fields, 1)]
_params['body']['query']['query_string']['fields'] = search_fields
return _params
def build_search_params(self, params):
params = dictset(params)
_params = dict(index=self.index_name, doc_type=self.doc_type)
if 'body' not in params:
query_string = build_qs(params.remove(RESERVED),
params.get('_raw_terms', ''))
if query_string:
_params['body'] = {
'query': {
'query_string': {
'query': query_string
}
}
}
print query_string
else:
_params['body'] = {"query": {"match_all": {}}}
if '_limit' in params:
_params['from_'], _params['size'] = process_limit(
params.get('_start', None), params.get('_page', None),
params['_limit'])
if '_sort' in params:
_params['sort'] = apply_sort(params['_sort'])
if '_fields' in params:
_params['fields'] = params['_fields']
if '_search_fields' in params:
search_fields = params['_search_fields'].split(',')
search_fields.reverse()
search_fields = [
s + '^' + str(i) for i, s in enumerate(search_fields, 1)
]
_params['body']['query']['query_string']['fields'] = search_fields
return _params
def build_search_params(self, params):
params = dictset(params)
_params = dict(index=self.index_name, doc_type=self.doc_type)
_raw_terms = params.pop("q", "")
if "body" not in params:
query_string = build_qs(params.remove(RESERVED_PARAMS), _raw_terms)
if query_string:
_params["body"] = {"query": {"query_string": {"query": query_string}}}
else:
_params["body"] = {"query": {"match_all": {}}}
else:
_params["body"] = params["body"]
if "_limit" not in params:
params["_limit"] = self.api.count()["count"]
_params["from_"], _params["size"] = process_limit(
params.get("_start", None), params.get("_page", None), params["_limit"]
)
if "_sort" in params:
_params["sort"] = apply_sort(params["_sort"])
if "_fields" in params:
_params["fields"] = params["_fields"]
if "_search_fields" in params:
search_fields = params["_search_fields"].split(",")
search_fields.reverse()
search_fields = [s + "^" + str(i) for i, s in enumerate(search_fields, 1)]
current_qs = _params["body"]["query"]["query_string"]
if isinstance(current_qs, str):
_params["body"]["query"]["query_string"] = {"query": current_qs}
_params["body"]["query"]["query_string"]["fields"] = search_fields
return _params
def get_by_ids(self, ids, **params):
if not ids:
return _ESDocs()
_raise_on_empty = params.pop('_raise_on_empty', False)
fields = params.pop('_fields', [])
_limit = params.pop('_limit', len(ids))
_page = params.pop('_page', None)
_start = params.pop('_start', None)
_start, _limit = process_limit(_start, _page, _limit)
docs = []
for _id in ids:
docs.append(
dict(
_index=self.index_name,
_type=self.src2type(_id['_type']),
_id=_id['_id']
)
)
params = dict(
body=dict(docs=docs)
)
if fields:
fields_params = process_fields_param(fields)
params.update(fields_params)
documents = _ESDocs()
documents._nefertari_meta = dict(
start=_start,
fields=fields,
)
try:
data = self.api.mget(**params)
except IndexNotFoundException:
if _raise_on_empty:
raise JHTTPNotFound(
'{}({}) resource not found (Index does not exist)'.format(
self.doc_type, params))
documents._nefertari_meta.update(total=0)
return documents
for found_doc in data['docs']:
try:
output_doc = found_doc['_source']
output_doc['_type'] = found_doc['_type']
except KeyError:
msg = "ES: '%s(%s)' resource not found" % (
found_doc['_type'], found_doc['_id'])
if _raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.error(msg)
continue
documents.append(dict2proxy(dictset(output_doc), ES.document_proxy.get_document_proxies_by_type(found_doc['_type'])))
documents._nefertari_meta.update(
total=len(documents),
)
return documents
def get_collection(cls, **params):
"""
params may include '_limit', '_page', '_sort', '_fields'
returns paginated and sorted query set
raises JHTTPBadRequest for bad values in params
"""
params.pop('__confirmation', False)
__strict = params.pop('__strict', True)
_sort = _split(params.pop('_sort', []))
_fields = _split(params.pop('_fields', []))
_limit = params.pop('_limit', None)
_page = params.pop('_page', None)
_start = params.pop('_start', None)
_count = '_count' in params; params.pop('_count', None)
_explain = '_explain' in params; params.pop('_explain', None)
__raise_on_empty = params.pop('__raise_on_empty', False)
query_set = cls.objects
# Remove any __ legacy instructions from this point on
params = dictset(filter(lambda item: not item[0].startswith('__'), params.items()))
if __strict:
_check_fields = [f.strip('-+') for f in params.keys() + _fields + _sort]
cls.check_fields_allowed(_check_fields)
else:
params = cls.filter_fields(params)
process_lists(params)
process_bools(params)
#if param is _all then remove it
params.pop_by_values('_all')
try:
query_set = query_set(**params)
_total = query_set.count()
if _count:
return _total
if _limit is None:
raise JHTTPBadRequest('Missing _limit')
_start, _limit = process_limit(_start, _page, _limit)
# Filtering by fields has to be the first thing to do on the query_set!
query_set = cls.apply_fields(query_set, _fields)
query_set = cls.apply_sort(query_set, _sort)
query_set = query_set[_start:_start+_limit]
if not query_set.count():
msg = "'%s(%s)' resource not found" % (cls.__name__, params)
if __raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.debug(msg)
except (mongo.ValidationError, mongo.InvalidQueryError) as e:
raise JHTTPBadRequest(str(e), extra={'data': e})
if _explain:
return query_set.explain()
log.debug('get_collection.query_set: %s(%s)',
cls.__name__, query_set._query)
query_set._nefertari_meta = dict(
total=_total,
start=_start,
fields=_fields)
return query_set
def get_collection(cls, **params):
""" Query collection and return results.
Notes:
* Before validating that only model fields are present in params,
reserved params, query params and all params starting with
double underscore are dropped.
* Params which have value "_all" are dropped.
* When ``_count`` param is used, objects count is returned
before applying offset and limit.
:param bool _strict: If True ``params`` are validated to contain
only fields defined on model, exception is raised if invalid
fields are present. When False - invalid fields are dropped.
Defaults to ``True``.
:param bool _item_request: Indicates whether it is a single item
request or not. When True and DataError happens on DB request,
JHTTPNotFound is raised. JHTTPBadRequest is raised when False.
Defaults to ``False``.
:param list _sort: Field names to sort results by. If field name
is prefixed with "-" it is used for "descending" sorting.
Otherwise "ascending" sorting is performed by that field.
Defaults to an empty list in which case sorting is not
performed.
:param list _fields: Names of fields which should be included
or excluded from results. Fields to excluded should be
prefixed with "-". Defaults to an empty list in which
case all fields are returned.
:param int _limit: Number of results per page. Defaults
to None in which case all results are returned.
:param int _page: Number of page. In conjunction with
``_limit`` is used to calculate results offset. Defaults to
None in which case it is ignored. Params ``_page`` and
``_start` are mutually exclusive.
:param int _start: Results offset. If provided ``_limit`` and
``_page`` params are ignored when calculating offset. Defaults
to None. Params ``_page`` and ``_start`` are mutually
exclusive. If not offset-related params are provided, offset
equals to 0.
:param Query query_set: Existing queryset. If provided, all queries
are applied to it instead of creating new queryset. Defaults
to None.
:param _count: When provided, only results number is returned as
integer.
:param _explain: When provided, query performed(SQL) is returned
as a string instead of query results.
:param bool _raise_on_empty: When True JHTTPNotFound is raised
if query returned no results. Defaults to False in which case
error is just logged and empty query results are returned.
:returns: Query results as ``sqlalchemy.orm.query.Query`` instance.
May be sorted, offset, limited.
:returns: Dict of {'field_name': fieldval}, when ``_fields`` param
is provided.
:returns: Number of query results as an int when ``_count`` param
is provided.
:returns: String representing query ran when ``_explain`` param
is provided.
:raises JHTTPNotFound: When ``_raise_on_empty=True`` and no
results found.
:raises JHTTPNotFound: When ``_item_request=True`` and
``sqlalchemy.exc.DataError`` exception is raised during DB
query. Latter exception is raised when querying DB with
an identifier of a wrong type. E.g. when querying Int field
with a string.
:raises JHTTPBadRequest: When ``_item_request=False`` and
``sqlalchemy.exc.DataError`` exception is raised during DB
query.
:raises JHTTPBadRequest: When ``sqlalchemy.exc.InvalidRequestError``
or ``sqlalchemy.exc.IntegrityError`` errors happen during DB
query.
"""
log.debug('Get collection: {}, {}'.format(cls.__name__, params))
params.pop('__confirmation', False)
_strict = params.pop('_strict', True)
_item_request = params.pop('_item_request', False)
_sort = _split(params.pop('_sort', []))
_fields = _split(params.pop('_fields', []))
_limit = params.pop('_limit', None)
_page = params.pop('_page', None)
_start = params.pop('_start', None)
query_set = params.pop('query_set', None)
_count = '_count' in params
params.pop('_count', None)
_explain = '_explain' in params
params.pop('_explain', None)
_raise_on_empty = params.pop('_raise_on_empty', False)
if query_set is None:
query_set = Session().query(cls)
# Remove any __ legacy instructions from this point on
params = dictset({
key: val
for key, val in params.items() if not key.startswith('__')
})
iterables_exprs, params = cls._pop_iterables(params)
params = drop_reserved_params(params)
if _strict:
_check_fields = [
f.strip('-+') for f in list(params.keys()) + _fields + _sort
]
cls.check_fields_allowed(_check_fields)
else:
params = cls.filter_fields(params)
process_lists(params)
process_bools(params)
# If param is _all then remove it
params.pop_by_values('_all')
try:
query_set = query_set.filter_by(**params)
# Apply filtering by iterable expressions
for expr in iterables_exprs:
query_set = query_set.from_self().filter(expr)
_total = query_set.count()
if _count:
return _total
# Filtering by fields has to be the first thing to do on
# the query_set!
query_set = cls.apply_fields(query_set, _fields)
query_set = cls.apply_sort(query_set, _sort)
if _limit is not None:
_start, _limit = process_limit(_start, _page, _limit)
query_set = query_set.offset(_start).limit(_limit)
if not query_set.count():
msg = "'%s(%s)' resource not found" % (cls.__name__, params)
if _raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.debug(msg)
except DataError as ex:
if _item_request:
msg = "'{}({})' resource not found".format(
cls.__name__, params)
raise JHTTPNotFound(msg, explanation=ex.message)
else:
raise JHTTPBadRequest(str(ex), extra={'data': ex})
except (InvalidRequestError, ) as ex:
raise JHTTPBadRequest(str(ex), extra={'data': ex})
query_sql = str(query_set).replace('\n', '')
if _explain:
return query_sql
log.debug('get_collection.query_set: %s (%s)', cls.__name__, query_sql)
if _fields:
query_set = cls.add_field_names(query_set, _fields)
query_set._nefertari_meta = dict(total=_total,
start=_start,
fields=_fields)
return query_set
def build_search_params(self, params):
params = dictset(params)
_params = dict(
index=self.index_name,
doc_type=self.doc_type
)
_raw_terms = params.pop('q', '')
if 'body' not in params and 'es_q' not in params:
analyzed_terms = apply_analyzer(params, self.doc_type, engine.get_document_cls)
query_string = self.build_qs(params.remove(RESERVED_PARAMS), _raw_terms)
query = {'must': []}
if query_string:
query['must'].append({'query_string': {'query': query_string}})
if analyzed_terms:
query['must'].append(analyzed_terms)
if query['must']:
_params['body'] = {'query': {'bool': query}}
else:
_params['body'] = {'query': {'match_all': {}}}
if 'body' in params:
raise JHTTPUnprocessableEntity('Illegal parameter "body"')
if '_limit' not in params:
params['_limit'] = self.api.count(index=self.index_name)['count']
_params['from_'], _params['size'] = process_limit(
params.get('_start', None),
params.get('_page', None),
params['_limit'])
if 'es_q' in params:
_params['body'] = {}
try:
_params['body']['query'] = compile_es_query(params)
except Exception as exc:
log.exception('es_q parsing error: {exc}'.format(exc=exc))
raise JHTTPBadRequest('Bad query string for {params}'
.format(
params=_params['body']['query']['query_string']['query']))
log.debug('Parsed ES request body {body}'.format(body=_params['body']['query']))
if '_sort' in params and self.proxy:
params['_sort'] = substitute_nested_terms(params['_sort'], self.proxy.substitutions)
if '_sort' in params:
_params['sort'] = apply_sort(params['_sort'])
if '_fields' in params:
params['_fields'] = self.add_nested_fields(params['_fields'], ',')
_params['fields'] = params['_fields']
if '_search_fields' in params:
search_fields = params['_search_fields'].split(',')
search_fields.reverse()
# Substitute search fields and add ^index
for index, search_field in enumerate(search_fields):
sf_terms = search_field.split('.')
if self.proxy is not None:
if len(sf_terms) > 0 and sf_terms[0] in self.proxy.substitutions:
sf_terms[0] += "_nested"
search_field = '.'.join(sf_terms)
search_fields[index] = search_field + '^' + str(index + 1)
must_query = _params['body']['query']['bool']['must']
query_string = {}
for query_item in must_query:
if 'query_string' in query_item:
query_string = query_item
break
current_qs = query_string.get('query_string', None)
if current_qs:
query_string['query_string']['fields'] = search_fields
return _params
def get_collection(cls, **params):
""" Query collection and return results.
Notes:
* Before validating that only model fields are present in params,
reserved params, query params and all params starting with
double underscore are dropped.
* Params which have value "_all" are dropped.
* When ``_count`` param is used, objects count is returned
before applying offset and limit.
:param bool __strict: If True ``params`` are validated to contain
only fields defined on model, exception is raised if invalid
fields are present. When False - invalid fields are dropped.
Defaults to ``True``.
:param bool _item_request: Indicates whether it is a single item
request or not. When True and DataError happens on DB request,
JHTTPNotFound is raised. JHTTPBadRequest is raised when False.
Defaults to ``False``.
:param list _sort: Field names to sort results by. If field name
is prefixed with "-" it is used for "descending" sorting.
Otherwise "ascending" sorting is performed by that field.
Defaults to an empty list in which case sorting is not
performed.
:param list _fields: Names of fields which should be included
or excluded from results. Fields to excluded should be
prefixed with "-". Defaults to an empty list in which
case all fields are returned.
:param int _limit: Number of results per page. Defaults
to None in which case all results are returned.
:param int _page: Number of page. In conjunction with
``_limit`` is used to calculate results offset. Defaults to
None in which case it is ignored. Params ``_page`` and
``_start` are mutually exclusive.
:param int _start: Results offset. If provided ``_limit`` and
``_page`` params are ignored when calculating offset. Defaults
to None. Params ``_page`` and ``_start`` are mutually
exclusive. If not offset-related params are provided, offset
equals to 0.
:param Query query_set: Existing queryset. If provided, all queries
are applied to it instead of creating new queryset. Defaults
to None.
:param _count: When provided, only results number is returned as
integer.
:param _explain: When provided, query performed(SQL) is returned
as a string instead of query results.
:param bool __raise_on_empty: When True JHTTPNotFound is raised
if query returned no results. Defaults to False in which case
error is just logged and empty query results are returned.
:returns: Query results as ``sqlalchemy.orm.query.Query`` instance.
May be sorted, offset, limited.
:returns: Dict of {'field_name': fieldval}, when ``_fields`` param
is provided.
:returns: Number of query results as an int when ``_count`` param
is provided.
:returns: String representing query ran when ``_explain`` param
is provided.
:raises JHTTPNotFound: When ``__raise_on_empty=True`` and no
results found.
:raises JHTTPNotFound: When ``_item_request=True`` and
``sqlalchemy.exc.DataError`` exception is raised during DB
query. Latter exception is raised when querying DB with
an identifier of a wrong type. E.g. when querying Int field
with a string.
:raises JHTTPBadRequest: When ``_item_request=False`` and
``sqlalchemy.exc.DataError`` exception is raised during DB
query.
:raises JHTTPBadRequest: When ``sqlalchemy.exc.InvalidRequestError``
or ``sqlalchemy.exc.IntegrityError`` errors happen during DB
query.
"""
log.debug('Get collection: {}, {}'.format(cls.__name__, params))
params.pop('__confirmation', False)
__strict = params.pop('__strict', True)
_item_request = params.pop('_item_request', False)
_sort = _split(params.pop('_sort', []))
_fields = _split(params.pop('_fields', []))
_limit = params.pop('_limit', None)
_page = params.pop('_page', None)
_start = params.pop('_start', None)
query_set = params.pop('query_set', None)
_count = '_count' in params
params.pop('_count', None)
_explain = '_explain' in params
params.pop('_explain', None)
__raise_on_empty = params.pop('__raise_on_empty', False)
if query_set is None:
query_set = Session().query(cls)
# Remove any __ legacy instructions from this point on
params = dictset({
key: val for key, val in params.items()
if not key.startswith('__')
})
iterables_exprs, params = cls._pop_iterables(params)
params = drop_reserved_params(params)
if __strict:
_check_fields = [
f.strip('-+') for f in list(params.keys()) + _fields + _sort]
cls.check_fields_allowed(_check_fields)
else:
params = cls.filter_fields(params)
process_lists(params)
process_bools(params)
# If param is _all then remove it
params.pop_by_values('_all')
try:
query_set = query_set.filter_by(**params)
# Apply filtering by iterable expressions
for expr in iterables_exprs:
query_set = query_set.from_self().filter(expr)
_total = query_set.count()
if _count:
return _total
# Filtering by fields has to be the first thing to do on
# the query_set!
query_set = cls.apply_fields(query_set, _fields)
query_set = cls.apply_sort(query_set, _sort)
if _limit is not None:
_start, _limit = process_limit(_start, _page, _limit)
query_set = query_set.offset(_start).limit(_limit)
if not query_set.count():
msg = "'%s(%s)' resource not found" % (cls.__name__, params)
if __raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.debug(msg)
except DataError as ex:
if _item_request:
msg = "'{}({})' resource not found".format(
cls.__name__, params)
raise JHTTPNotFound(msg, explanation=ex.message)
else:
raise JHTTPBadRequest(str(ex), extra={'data': ex})
except (InvalidRequestError,) as ex:
raise JHTTPBadRequest(str(ex), extra={'data': ex})
query_sql = str(query_set).replace('\n', '')
if _explain:
return query_sql
log.debug('get_collection.query_set: %s (%s)', cls.__name__, query_sql)
if _fields:
query_set = cls.add_field_names(query_set, _fields)
query_set._nefertari_meta = dict(
total=_total,
start=_start,
fields=_fields)
return query_set
def get_collection(cls, **params):
"""
Params may include '_limit', '_page', '_sort', '_fields'.
Returns paginated and sorted query set.
Raises JHTTPBadRequest for bad values in params.
"""
log.debug('Get collection: {}, {}'.format(cls.__name__, params))
params.pop('__confirmation', False)
_strict = params.pop('_strict', True)
_item_request = params.pop('_item_request', False)
_sort = _split(params.pop('_sort', []))
_fields = _split(params.pop('_fields', []))
_limit = params.pop('_limit', None)
_page = params.pop('_page', None)
_start = params.pop('_start', None)
query_set = params.pop('query_set', None)
_count = '_count' in params
params.pop('_count', None)
_explain = '_explain' in params
params.pop('_explain', None)
_raise_on_empty = params.pop('_raise_on_empty', False)
if query_set is None:
query_set = cls.objects
# Remove any __ legacy instructions from this point on
params = dictset({
key: val for key, val in params.items()
if not key.startswith('__')
})
params = drop_reserved_params(params)
if _strict:
_check_fields = [
f.strip('-+') for f in list(params.keys()) + _fields + _sort]
cls.check_fields_allowed(_check_fields)
else:
params = cls.filter_fields(params)
process_lists(params)
process_bools(params)
# If param is _all then remove it
params.pop_by_values('_all')
try:
query_set = query_set(**params)
_total = query_set.count()
if _count:
return _total
# Filtering by fields has to be the first thing to do on the
# query_set!
query_set = cls.apply_fields(query_set, _fields)
query_set = cls.apply_sort(query_set, _sort)
if _limit is not None:
_start, _limit = process_limit(_start, _page, _limit)
query_set = query_set[_start:_start+_limit]
if not query_set.count():
msg = "'%s(%s)' resource not found" % (cls.__name__, params)
if _raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.debug(msg)
except mongo.ValidationError as ex:
if _item_request:
msg = "'%s(%s)' resource not found" % (cls.__name__, params)
raise JHTTPNotFound(msg, explanation=ex.message)
else:
raise JHTTPBadRequest(str(ex), extra={'data': ex})
except mongo.InvalidQueryError as ex:
raise JHTTPBadRequest(str(ex), extra={'data': ex})
if _explain:
return query_set.explain()
log.debug('get_collection.query_set: %s(%s)',
cls.__name__, query_set._query)
query_set._nefertari_meta = dict(
total=_total,
start=_start,
fields=_fields)
return query_set
def build_search_params(self, params):
params = dictset(params)
_params = dict(
index=self.index_name,
doc_type=self.doc_type
)
_raw_terms = params.pop('q', '')
if 'body' not in params:
query_string = self.build_qs(params.remove(RESERVED_PARAMS), _raw_terms)
if query_string:
_params['body'] = {
'query': {
'query_string': {
'query': query_string
}
}
}
else:
_params['body'] = {"query": {"match_all": {}}}
else:
_params['body'] = params['body']
if '_limit' not in params:
params['_limit'] = self.api.count(index=self.index_name)['count']
_params['from_'], _params['size'] = process_limit(
params.get('_start', None),
params.get('_page', None),
params['_limit'])
if '_nested' in params:
if 'query' in _params['body']:
old_query = _params['body']['query']
_params['body']['query'] = {'bool': {'must': []}}
nested = dict()
nested['nested'] = self.build_nested_query(params)
_params['body']['query']['bool']['must'].append(nested)
_params['body']['query']['bool']['must'].append(old_query)
if '_sort' in params and self.proxy:
params['_sort'] = substitute_nested_terms(params['_sort'], self.proxy.substitutions)
if '_sort' in params:
_params['sort'] = apply_sort(params['_sort'])
if '_fields' in params:
params['_fields'] = self.add_nested_fields(params['_fields'], ',')
_params['fields'] = params['_fields']
if '_search_fields' in params:
search_fields = params['_search_fields'].split(',')
search_fields.reverse()
# Substitute search fields and add ^index
for index, search_field in enumerate(search_fields):
sf_terms = search_field.split('.')
if self.proxy is not None:
if len(sf_terms) > 0 and sf_terms[0] in self.proxy.substitutions:
sf_terms[0] += "_nested"
search_field = '.'.join(sf_terms)
search_fields[index] = search_field + '^' + str(index + 1)
current_qs = _params['body']['query']['query_string']
if isinstance(current_qs, str):
_params['body']['query']['query_string'] = {'query': current_qs}
_params['body']['query']['query_string']['fields'] = search_fields
return _params
def get_collection(cls, _count=False, _strict=True, _sort=None,
_fields=None, _limit=None, _page=None, _start=None,
_query_set=None, _item_request=False, _explain=None,
_search_fields=None, q=None, **params):
""" Query collection and return results.
Notes:
* Before validating that only model fields are present in params,
reserved params, query params and all params starting with
double underscore are dropped.
* Params which have value "_all" are dropped.
* When ``_count`` param is used, objects count is returned
before applying offset and limit.
:param bool _strict: If True ``params`` are validated to contain
only fields defined on model, exception is raised if invalid
fields are present. When False - invalid fields are dropped.
Defaults to ``True``.
:param list _sort: Field names to sort results by. If field name
is prefixed with "-" it is used for "descending" sorting.
Otherwise "ascending" sorting is performed by that field.
Defaults to an empty list in which case sorting is not
performed.
:param list _fields: Names of fields which should be included
or excluded from results. Fields to excluded should be
prefixed with "-". Defaults to an empty list in which
case all fields are returned.
:param int _limit: Number of results per page. Defaults
to None in which case all results are returned.
:param int _page: Number of page. In conjunction with
``_limit`` is used to calculate results offset. Defaults to
None in which case it is ignored. Params ``_page`` and
``_start` are mutually exclusive.
:param int _start: Results offset. If provided ``_limit`` and
``_page`` params are ignored when calculating offset. Defaults
to None. Params ``_page`` and ``_start`` are mutually
exclusive. If not offset-related params are provided, offset
equals to 0.
:param Query _query_set: Existing queryset. If provided, all queries
are applied to it instead of creating new queryset. Defaults
to None.
:param bool _item_request: Indicates whether it is a single item
request or not. When True and DataError happens on DB request,
JHTTPNotFound is raised. JHTTPBadRequest is raised when False.
Defaults to ``False``.
:param _count: When provided, only results number is returned as
integer.
:param _explain: When provided, query performed(SQL) is returned
as a string instead of query results.
:param bool _raise_on_empty: When True JHTTPNotFound is raised
if query returned no results. Defaults to False in which case
error is just logged and empty query results are returned.
:param q: Query string to perform full-text search with.
:param _search_fields: Coma-separated list of field names to use
with full-text search(q param) to limit fields which are
searched.
:returns: Query results as ``elasticsearch_dsl.XXX`` instance.
May be sorted, offset, limited.
:returns: Dict of {'field_name': fieldval}, when ``_fields`` param
is provided.
:returns: Number of query results as an int when ``_count`` param
is provided.
:raises JHTTPNotFound: When ``_raise_on_empty=True`` and no
results found.
:raises JHTTPNotFound: When ``_item_request=True`` and
``sqlalchemy.exc.DataError`` exception is raised during DB
query. Latter exception is raised when querying DB with
an identifier of a wrong type. E.g. when querying Int field
with a string.
:raises JHTTPBadRequest: When ``_item_request=False`` and
``sqlalchemy.exc.DataError`` exception is raised during DB
query.
:raises JHTTPBadRequest: When ``sqlalchemy.exc.InvalidRequestError``
or ``sqlalchemy.exc.IntegrityError`` errors happen during DB
query.
"""
# see if the items are cached
pk_field = cls.pk_field()
if (list(params.keys()) == [pk_field] and _count==False
and _strict==True and _sort==None and _fields==None
and _limit==None and _page==None and _start==None
and _query_set==None and _item_request==False and _explain==None
and _search_fields==None and q==None):
ids = params[pk_field]
if not isinstance(ids, (list, tuple)):
ids = [ids]
results = []
for id in ids:
if not id in cls._cache:
break
results.append(cls._cache[id])
else:
return results
search_obj = cls.search()
if _limit is not None:
_start, limit = process_limit(_start, _page, _limit)
search_obj = search_obj.extra(from_=_start, size=limit)
if _fields:
include, exclude = process_fields(_fields)
if _strict:
_validate_fields(cls, include + exclude)
# XXX partial fields support isn't yet released. for now
# we just use fields, later we'll add support for excluded fields
search_obj = search_obj.fields(include)
if params:
params = _cleaned_query_params(cls, params, _strict)
params = _restructure_params(cls, params)
if params:
search_obj = search_obj.filter('terms', **params)
if q is not None:
query_kw = {'query': q}
if _search_fields is not None:
query_kw['fields'] = _search_fields.split(',')
search_obj = search_obj.query('query_string', **query_kw)
if _count:
return search_obj.count()
if _explain:
return search_obj.to_dict()
if _sort:
sort_fields = split_strip(_sort)
if _strict:
_validate_fields(
cls,
[f[1:] if f.startswith('-') else f for f in sort_fields]
)
search_obj = search_obj.sort(*sort_fields)
hits = search_obj.execute().hits
hits._nefertari_meta = dict(
total=hits.total,
start=_start,
fields=_fields
)
return hits
def get_collection(cls, **params):
"""
params may include '_limit', '_page', '_sort', '_fields'
returns paginated and sorted query set
raises JHTTPBadRequest for bad values in params
"""
params.pop('__confirmation', False)
__strict = params.pop('__strict', True)
_sort = _split(params.pop('_sort', []))
_fields = _split(params.pop('_fields', []))
_limit = params.pop('_limit', None)
_page = params.pop('_page', None)
_start = params.pop('_start', None)
_count = '_count' in params
params.pop('_count', None)
_explain = '_explain' in params
params.pop('_explain', None)
__raise_on_empty = params.pop('__raise_on_empty', False)
query_set = cls.objects
# Remove any __ legacy instructions from this point on
params = dictset(
filter(lambda item: not item[0].startswith('__'), params.items()))
if __strict:
_check_fields = [
f.strip('-+') for f in params.keys() + _fields + _sort
]
cls.check_fields_allowed(_check_fields)
else:
params = cls.filter_fields(params)
process_lists(params)
process_bools(params)
#if param is _all then remove it
params.pop_by_values('_all')
try:
query_set = query_set(**params)
_total = query_set.count()
if _count:
return _total
if _limit is None:
raise JHTTPBadRequest('Missing _limit')
_start, _limit = process_limit(_start, _page, _limit)
# Filtering by fields has to be the first thing to do on the query_set!
query_set = cls.apply_fields(query_set, _fields)
query_set = cls.apply_sort(query_set, _sort)
query_set = query_set[_start:_start + _limit]
if not query_set.count():
msg = "'%s(%s)' resource not found" % (cls.__name__, params)
if __raise_on_empty:
raise JHTTPNotFound(msg)
else:
log.debug(msg)
except (mongo.ValidationError, mongo.InvalidQueryError) as e:
raise JHTTPBadRequest(str(e), extra={'data': e})
if _explain:
return query_set.explain()
log.debug('get_collection.query_set: %s(%s)', cls.__name__,
query_set._query)
query_set._nefertari_meta = dict(total=_total,
start=_start,
fields=_fields)
return query_set
