def test_search_is_selective_about_sort_field(self, app, config):
sort_fields = [
v['fields'] for (k, v) in
config['RECORDS_REST_SORT_OPTIONS']['records'].items()
]
assert ['-_score'] in sort_fields
assert ['_score'] in sort_fields
assert ['_created'] in sort_fields
assert ['-_created'] in sort_fields
assert ['_updated'] in sort_fields
assert ['-_updated'] in sort_fields
assert ['title.raw'] in sort_fields
assert ['-title.raw'] in sort_fields
with app.test_request_context('?sort=title_asc'):
search = RecordsSearch().query()
search, args = default_sorter_factory(search, "records")
search_dict = search.to_dict()
assert 'sort' in search_dict
with app.test_request_context('?sort=invalid'):
search = RecordsSearch().query()
search, args = default_sorter_factory(search, "records")
search_dict = search.to_dict()
assert 'sort' not in search_dict
def inspire_search_factory(self, search):
"""Parse query using Inspire-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
query_string = request.values.get('q', '')
urlkwargs = MultiDict()
try:
search = search.query_from_iq(query_string)
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = inspire_filter_factory(search, urlkwargs, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
search = select_source(search)
urlkwargs.add('q', query_string)
current_app.logger.debug(json.dumps(search.to_dict(), indent=4))
return search, urlkwargs
def cernopendata_search_factory(self, search):
"""Customized parse query using invenio query parser.
:param self: REST view
:param search: Elastic search DSL search instance
:return: Tuple with search instance and URL arguments
"""
query_string = request.values.get("q")
try:
search = search.query(cernopendata_query_parser(query_string))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get("q", "")),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, url_kwargs = default_facets_factory(search, search_index)
search, sort_kwargs = default_sorter_factory(search, search_index)
for key, value in sort_kwargs.items():
url_kwargs.add(key, value)
url_kwargs.add("q", query_string)
return search, url_kwargs
def search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
# Apply 'identity' grouping by default
if 'group_by' not in request.values:
search = search.filter(Q('term', Grouping='identity'))
urlkwargs['group_by'] = 'identity'
try:
query_string = request.values.get('q')
if query_string:
search = search.query(
Q('query_string',
query=query_string,
default_field='_search_all'))
urlkwargs['q'] = query_string
except SyntaxError:
raise InvalidQueryRESTError()
# Exclude the identifiers by which the search was made (large aggregate)
search = search.source(exclude=['*.SearchIdentifier'])
return search, urlkwargs
def inspire_search_factory(self, search):
"""Parse query using Invenio-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q', '')
try:
search = search.query(IQ(query_string, search))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
finally:
if current_app.debug:
current_app.logger.debug(json.dumps(search.to_dict(), indent=4))
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def cap_search_factory(self, search, query_parser=None):
"""Customized Parse query using Invenio-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr)
return Q()
query_string = request.values.get('q')
query_parser = query_parser or _default_parser
try:
search = search.query(query_parser(query_string))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = cap_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def _ils_search_factory(self, search, qs_validator):
"""Search factory with Query String validator.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def query_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr)
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get("q")
query = query_parser(qs_validator(query_string))
try:
search = search.query(query)
except SyntaxError:
raise SearchQueryError(query_string)
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add("q", query_string)
return search, urlkwargs
def inspire_search_factory(self, search):
"""Parse query using Invenio-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q', '')
try:
search = search.query(IQ(query_string))
except SyntaxError:
current_app.logger.debug(
"Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def inspire_search_factory(self, search):
"""Parse query using Inspire-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
query_string = request.values.get('q', '')
urlkwargs = MultiDict()
try:
search = search.query_from_iq(query_string)
except SyntaxError:
current_app.logger.debug(
"Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = inspire_filter_factory(search, urlkwargs, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
search = select_source(search)
urlkwargs.add('q', query_string)
current_app.logger.debug(json.dumps(search.to_dict(), indent=4))
return search, urlkwargs
def cap_search_factory(self, search, query_parser=None):
"""Customize Parse query using Invenio-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Use of the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr)
return Q()
query_string = request.values.get('q')
query_parser = query_parser or _default_parser
try:
search = search.query(query_parser(query_string))
except SyntaxError:
current_app.logger.debug(
"Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = cap_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
search_index = search._index[0]
# TODO: make "scheme" optional?
for field in ('id', 'scheme', 'relation'):
if field not in request.values:
raise RESTValidationError(
errors=[FieldError(field, 'Required field.')])
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
# Apply 'identity' grouping by default
if 'groupBy' not in request.values:
search = search.filter(Q('term', Grouping='identity'))
urlkwargs['groupBy'] = 'identity'
# Exclude the identifiers by which the search was made (large aggregate)
search = search.source(exclude=['*.SearchIdentifier'])
return search, urlkwargs
def meta_search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
# for field in (['keyword']):
# if field not in request.values:
# raise RESTValidationError(
# errors=[FieldError(field, 'Required field.')])
search, urlkwargs = default_facets_factory(search, "metadata")
search, sortkwargs = default_sorter_factory(search, "metadata")
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
search = search.extra(size=0)
# Apply 'identity' grouping by default
search = search.filter(Q('term', RelationshipType='Cites'))
if 'group_by' not in request.values:
search = search.filter(Q('term', Grouping='identity'))
urlkwargs['group_by'] = 'identity'
try:
query_string = request.values.get('q')
if query_string:
search = search.query(
Q('query_string',
query=query_string,
default_field='_search_all'))
urlkwargs['q'] = query_string
except SyntaxError:
raise InvalidQueryRESTError()
size = 10
if 'size' in request.values:
size = request.values.get('size')
start = 0
if 'page' in request.values:
start = int(int(request.values.get('page')) - 1) * int(size)
search.aggs.bucket('Target', 'terms', field='Target.ID', size=1000)\
.metric("first", "top_hits", _source=dict(include=["Target.Identifier.*", "Target.Creator.Name", "Target.Title"]), size=1)
kwargs = {
'from': start,
'size': size,
'sort': [{
'_count': {
'order': 'desc'
}
}]
}
search.aggs['Target'].bucket('pagination', 'bucket_sort', **kwargs)
return search, urlkwargs
def circulation_search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Return default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr)
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q', '')
query = _default_parser(qstr=query_string)
# if the logged in user in not librarian or admin, validate the query
if not backoffice_permission().allows(g.identity):
# patron can find only his loans
try:
if not query_string:
# force query to be patron_pid:<logged in user>
only_patron_loans = 'patron_pid:{}'.format(g.identity.id)
query = _default_parser(qstr=only_patron_loans)
else:
# check for patron_pid query value
match = re.match(r"patron_pid:(?P<pid>\d)", query_string)
if match and match.group('pid') != str(g.identity.id):
raise UnauthorizedSearch()
except UnauthorizedSearch:
current_app.logger.debug(
"Search for `{0}` not allowed by `patron_pid:{1}`".format(
query_string, str(g.identity.id))
)
abort(403)
try:
search = search.query(query)
except SyntaxError:
current_app.logger.debug(
"Failed parsing query: {0}".format(query_string), exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def cap_search_factory(self, search, query_parser=None):
"""Customize Parse query using Invenio-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None, **kwargs):
"""Use of the Q() from elasticsearch_dsl."""
def _escape_qstr(qstr):
return ''.join((ESCAPE_CHAR_MAP.get(char, char) for char in qstr))
query = Q('query_string',
query=_escape_qstr(qstr),
analyzer="lowercase_whitespace_analyzer",
analyze_wildcard=True,
default_operator='AND') if qstr else Q()
# resolve keywords to queries
for k, v in kwargs.items():
if k in KEYWORD_TO_QUERY:
if v == 'True':
query = query & KEYWORD_TO_QUERY[k]()
elif v == 'False':
query = query & ~KEYWORD_TO_QUERY[k]()
return query
query_string = request.values.get('q')
# parse url params to search for keywords
query_keywords = {
k: request.values[k]
for k in KEYWORD_TO_QUERY.keys() if k in request.values
}
query_parser = query_parser or _default_parser
try:
search = search.query(query_parser(query_string, **query_keywords))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = cap_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def test_default_sorter_factory(app):
"""Test default sorter factory."""
app.config["RECORDS_REST_SORT_OPTIONS"] = dict(
myindex=dict(myfield=dict(fields=['field1', '-field2'], )), )
app.config["RECORDS_REST_DEFAULT_SORT"] = dict(myindex=dict(
query='-myfield',
noquery='myfield',
), )
# Sort
with app.test_request_context("?sort=myfield"):
query, urlargs = default_sorter_factory(Search(), 'myindex')
assert query.to_dict()['sort'] == \
[{'field1': {'order': 'asc'}}, {'field2': {'order': 'desc'}}]
assert urlargs['sort'] == 'myfield'
# Reverse sort
with app.test_request_context("?sort=-myfield"):
query, urlargs = default_sorter_factory(Search(), 'myindex')
assert query.to_dict()['sort'] == \
[{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}]
assert urlargs['sort'] == '-myfield'
# Invalid sort key
with app.test_request_context("?sort=invalid"):
query, urlargs = default_sorter_factory(Search(), 'myindex')
assert 'sort' not in query.to_dict()
assert urlargs == {}
# Default sort without query
with app.test_request_context("/?q="):
query, urlargs = default_sorter_factory(Search(), 'myindex')
assert query.to_dict()['sort'] == \
[{'field1': {'order': 'asc'}}, {'field2': {'order': 'desc'}}]
assert urlargs == dict(sort='myfield')
# Default sort with query
with app.test_request_context("/?q=test"):
query, urlargs = default_sorter_factory(Search(), 'myindex')
assert query.to_dict()['sort'] == \
[{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}]
assert urlargs == dict(sort='-myfield')
# Default sort with query that includes unicodes
with app.test_request_context("/?q=tést"):
query, urlargs = default_sorter_factory(Search(), 'myindex')
assert query.to_dict()['sort'] == \
[{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}]
assert urlargs == dict(sort='-myfield')
# Default sort another index
with app.test_request_context("/?q=test"):
query, urlargs = default_sorter_factory(Search(), 'aidx')
assert 'sort' not in query.to_dict()
def filter_by_patron_search_factory(self, search, query_parser=None):
"""Filter search queries to only show records for the logged in patron.
If the logged in user has backoffice permissions do not filter by patron.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Return default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr)
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q', '')
if not current_user.is_authenticated:
raise UnauthorizedSearchError(query_string)
parser = query_parser or _default_parser
query = parser(qstr=query_string)
# if the logged in user in not librarian or admin, validate the query
if has_request_context() and not backoffice_permission().allows(g.identity):
# patron can find only his document requests
if not query_string:
# force query to be patron_pid:<logged in user>
patron_pid_filter = 'patron_pid:{}'.format(g.identity.id)
query = _default_parser(qstr=patron_pid_filter)
else:
# check for patron_pid query value
match = re.match(r"patron_pid:(?P<pid>\d)", query_string)
if match and match.group('pid') != str(g.identity.id):
raise UnauthorizedSearchError(query_string, g.identity.id)
try:
search = search.query(query)
except SyntaxError:
raise SearchQueryError(query_string)
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def search_factory_without_aggs(self, search):
query_string, search = inspire_search_factory(self, search)
search_index = search.base_index
search, urlkwargs = inspire_filter_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
search = get_search_with_source(search)
urlkwargs.add("q", query_string)
return search, urlkwargs
def default_search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param search: Elastic search DSL search instance.
:param query_parser: Custom query parser.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl.
:param qstr: Query string.
:returns: Query object.
"""
if not qstr:
return Q()
operator, query_type = get_operator_and_query_type(qstr)
return Q(query_type, query=qstr, default_operator=operator)
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q')
# Use query parser given to function or the default one.
query_parser = query_parser or _default_parser
# Search query
search = search.query(query_parser(query_string))
# Get index corresponding to record type.
search_index = getattr(search, '_original_index', search._index)[0]
# Build facets
search, urlkwargs = default_facets_factory(search, search_index)
# Sort records
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
# Add explanation to hits
if request.args.get('debug'):
search = search.extra(explain=True)
return search, urlkwargs
def ils_search_factory(self, search, validator=None):
"""Search factory with Query String validator.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def query_parser(search, qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
boosted = getattr(search, "boosted_fields", [])
extra_params = {}
if boosted:
extra_params["fields"] = boosted + ["*"]
# add lenient parameter in order to fix
# parsing exception on data fields, see known issues
# https://www.elastic.co/guide/en/elasticsearch/reference/current/release-notes-7.1.1.html # noqa
extra_params["lenient"] = True
return Q("query_string", query=qstr, **extra_params)
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get("q")
if validator:
search, query_string = validator(search, query_string)
query = query_parser(search, qstr=query_string)
try:
search = search.query(query)
except SyntaxError:
current_app.logger.debug(
"Failed parsing query: {0}".format(request.values.get("q", "")),
exc_info=True,
)
raise SearchQueryError(query_string)
search_index = getattr(search, "_original_index", search._index)[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add("q", query_string)
return search, urlkwargs
def test_default_sorter_factory(app, user_factory):
"""Test default sorter factory."""
app.config["RECORDS_REST_SORT_OPTIONS"] = dict(
myindex=dict(
myfield=dict(
fields=['field1', '-field2'],
)
),
)
app.config["RECORDS_REST_DEFAULT_SORT"] = dict(
myindex=dict(
query='-myfield',
noquery='myfield',
),
)
# Sort
with app.test_request_context("?sort=myfield"):
query, urlargs = default_sorter_factory(Query("value"), 'myindex')
assert query.body['sort'] == \
[{'field1': {'order': 'asc'}}, {'field2': {'order': 'desc'}}]
assert urlargs['sort'] == 'myfield'
# Reverse sort
with app.test_request_context("?sort=-myfield"):
query, urlargs = default_sorter_factory(Query("value"), 'myindex')
assert query.body['sort'] == \
[{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}]
assert urlargs['sort'] == '-myfield'
# Invalid sort key
with app.test_request_context("?sort=invalid"):
query, urlargs = default_sorter_factory(Query("value"), 'myindex')
assert 'sort' not in query.body
assert urlargs == {}
# Default sort without query
with app.test_request_context("/?q="):
query, urlargs = default_sorter_factory(Query("value"), 'myindex')
assert query.body['sort'] == \
[{'field1': {'order': 'asc'}}, {'field2': {'order': 'desc'}}]
assert urlargs == dict(sort='myfield')
# Default sort with query
with app.test_request_context("/?q=test"):
query, urlargs = default_sorter_factory(Query("value"), 'myindex')
assert query.body['sort'] == \
[{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}]
assert urlargs == dict(sort='-myfield')
# Default sort another index
with app.test_request_context("/?q=test"):
query, urlargs = default_sorter_factory(Query("value"), 'aidx')
assert 'sort' not in query.body
def and_search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr, default_operator='AND')
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q')
query_parser = query_parser or _default_parser
try:
search = search.query(query_parser(query_string))
except SyntaxError:
current_app.logger.debug(
f'Failed parsing query: {request.values.get("q", "")}',
exc_info=True,
)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
# include deleted
deleted = request.args.get('deleted')
if not deleted:
search = search.filter('bool', must_not=[Q('exists', field='deleted')])
return search, urlkwargs
def cernopendata_search_factory(self, search, query_parser=None):
"""Customized Parse query using Invenio-Query-Parser.
By default we hide the results that have availability:ondemand.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None, ondemand=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
q = Q('query_string', query=qstr) if qstr else Q()
# by default hide ondemand ones
if not ondemand:
q = q & ~Q('match', **{'availability.keyword': 'ondemand'})
return q
query_string = request.values.get('q')
ondemand = request.values.get('ondemand', False) # this is a workaround
query_parser = query_parser or _default_parser
try:
search = search.query(query_parser(query_string, ondemand))
except SyntaxError:
current_app.logger.debug(
"Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = cernopendata_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def cernopendata_search_factory(self, search, query_parser=None):
"""Customized Parse query using Invenio-Query-Parser.
By default we hide the results that have availability:ondemand.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None, ondemand=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
q = Q('query_string', query=qstr) if qstr else Q()
# by default hide ondemand ones
if not ondemand:
q = q & ~Q('match', **{'availability.keyword': 'ondemand'})
return q
query_string = request.values.get('q')
ondemand = request.values.get('ondemand', False) # this is a workaround
query_parser = query_parser or _default_parser
try:
search = search.query(query_parser(query_string, ondemand))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = cernopendata_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def deposit_search_factory(self, search):
"""Replace default search factory to use custom facet factory."""
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q', '')
query_parser = Q('query_string',
query=query_string) if query_string else Q()
try:
search = search.query(query_parser)
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = deposit_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def _ils_search_factory(self, search, validator):
"""Search factory with Query String validator.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def query_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr)
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get("q")
search, query_string = validator(search, query_string)
query = query_parser(query_string)
try:
search = search.query(query)
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise SearchQueryError(query_string)
search_index = getattr(search, '_original_index', search._index)[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add("q", query_string)
return search, urlkwargs
def item_path_search_factory(self, search, index_id=None):
"""Parse query using Weko-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _get_index_earch_query():
query_q = {
"_source": {
"exclude": ['content']
},
"query": {
"match": {
"path.tree": "@index"
}
},
"aggs": {
"path": {
"terms": {
"field": "path.tree",
"include": "@index|@index/[^/]+"
},
"aggs": {
"date_range": {
"filter": {
"match": {
"publish_status": "0"
}
},
"aggs": {
"available": {
"range": {
"field":
"publish_date",
"ranges": [{
"from": "now+1d/d"
}, {
"to": "now+1d/d"
}]
},
}
}
},
"no_available": {
"filter": {
"bool": {
"must_not": [{
"match": {
"publish_status": "0"
}
}]
}
}
}
}
}
},
"post_filter": {
"term": {
"path": "@index"
}
}
}
# add item type aggs
query_q['aggs']['path']['aggs']. \
update(get_item_type_aggs(search._index[0]))
mut = get_permission_filter()
if mut:
mut = list(map(lambda x: x.to_dict(), mut))
post_filter = query_q['post_filter']
if mut[0].get('bool'):
post_filter['bool'] = {
'must': [{
'term': post_filter.pop('term')
}, mut[0]['bool']['must'][0]],
'should':
mut[0]['bool']['should']
}
# post_filter['bool'] = {'must': [{'term': post_filter.pop('term')}],
# 'should': mut[0]['bool']['should']}
else:
mut.append({'term': post_filter.pop('term')})
post_filter['bool'] = {'must': mut}
# create search query
q = request.values.get('q') if index_id is None else index_id
if q:
try:
fp = Indexes.get_self_path(q)
if fp:
query_q = json.dumps(query_q).replace("@index", fp.path)
query_q = json.loads(query_q)
except BaseException:
pass
return query_q
# create a index search query
query_q = _get_index_earch_query()
urlkwargs = MultiDict()
try:
# Aggregations.
extr = search._extra.copy()
search.update_from_dict(query_q)
search._extra.update(extr)
except SyntaxError:
q = request.values.get('q', '') if index_id is None else index_id
current_app.logger.debug("Failed parsing query: {0}".format(q),
exc_info=True)
raise InvalidQueryRESTError()
from invenio_records_rest.sorter import default_sorter_factory
search_index = search._index[0]
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_q)
return search, urlkwargs
def search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
Terms defined by: RERO_ILS_QUERY_BOOSTING will be boosted
at the query level.
:param self: REST view.
:param search: Elastic search DSL search instance.
:param query_parser: a specific query parser
:return: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None, query_boosting=[]):
"""Default parser that uses the Q() from elasticsearch_dsl."""
query_type = 'query_string'
default_operator = 'OR'
if request.args.get('simple'):
query_type = 'simple_query_string'
default_operator = 'AND'
if qstr:
# TODO: remove this bad hack
qstr = _PUNCTUATION_REGEX.sub(' ', qstr)
qstr = re.sub('\s+', ' ', qstr).rstrip()
if not query_boosting:
return Q(query_type,
query=qstr,
default_operator=default_operator)
else:
return Q('bool',
should=[
Q(query_type,
query=qstr,
boost=2,
fields=query_boosting,
default_operator=default_operator),
Q(query_type,
query=qstr,
default_operator=default_operator)
])
return Q()
def _boosting_parser(query_boosting, search_index):
"""Elasticsearch boosting fields parser."""
boosting = []
if search_index in query_boosting:
for field, boost in query_boosting[search_index].items():
boosting.append('{}^{}'.format(field, boost))
return boosting
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q')
display_score = request.values.get('display_score')
if display_score:
search = search.extra(explain=True)
query_parser = query_parser or _default_parser
search_index = search._index[0]
query_boosting = _boosting_parser(
current_app.config['RERO_ILS_QUERY_BOOSTING'], search_index)
try:
search = search.query(query_parser(query_string, query_boosting))
except SyntaxError:
current_app.logger.debug(
'Failed parsing query: {0}'.format(request.values.get('q', '')),
exc_info=True,
)
raise InvalidQueryRESTError()
search, urlkwargs = default_facets_factory(search, search_index)
# i18n translated facets
search = i18n_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def default_search_factory(self, search, query_parser=None, search_type=None):
"""Parse query using Weko-Query-Parser. MetaData Search.
:param self: REST view.
:param search: Elastic search DSL search instance.
:param query_parser: Query parser. (Default: ``None``)
:returns: Tuple with search instance and URL arguments.
"""
def _get_search_qs_query(qs=None):
"""Qs of search bar keywords for detail simple search.
:param qs: Query string.
:return: Query parser.
"""
q = Q('query_string',
query=qs,
default_operator='and',
fields=['search_*', 'search_*.ja']) if qs else None
return q
def _get_detail_keywords_query():
"""Get keywords query.
:return: Query parser.
"""
def _get_keywords_query(k, v):
qry = None
kv = request.values.get(k)
if not kv:
return
if isinstance(v, str):
name_dict = dict(operator="and")
name_dict.update(dict(query=kv))
qry = Q('match', **{v: name_dict})
elif isinstance(v, list):
qry = Q('multi_match',
query=kv,
type='most_fields',
minimum_should_match='75%',
operator='and',
fields=v)
elif isinstance(v, dict):
for key, vlst in v.items():
if isinstance(vlst, list):
shud = []
kvl = [
x for x in kv.split(',')
if x.isdecimal() and int(x) < len(vlst)
]
for j in map(partial(lambda x, y: x[int(y)], vlst),
kvl):
name_dict = dict(operator="and")
name_dict.update(dict(query=j))
shud.append(Q('match', **{key: name_dict}))
kvl = [
x for x in kv.split(',')
if not x.isdecimal() and x in vlst
]
for j in kvl:
name_dict = dict(operator="and")
name_dict.update(dict(query=j))
shud.append(Q('match', **{key: name_dict}))
if shud:
return Q('bool', should=shud)
elif isinstance(v, tuple) and len(v) >= 2:
shud = []
for i in map(lambda x: v[1](x), kv.split(',')):
shud.append(Q('term', **{v[0]: i}))
if shud:
qry = Q('bool', should=shud)
return qry
def _get_nested_query(k, v):
# text value
kv = request.values.get(k)
if not kv:
return
shuld = []
if isinstance(v, tuple) and len(v) > 1 and isinstance(v[1], dict):
# attr keyword in request url
for attr_key, attr_val_str in map(
lambda x: (x, request.values.get(x)),
list(v[1].keys())):
attr_obj = v[1].get(attr_key)
if isinstance(attr_obj, dict) and attr_val_str:
if isinstance(v[0], str) and not len(v[0]):
# For ID search
for key in attr_val_str.split(','):
attr = attr_obj.get(key)
if isinstance(attr, tuple):
attr = [attr]
if isinstance(attr, list):
for alst in attr:
if isinstance(alst, tuple):
val_attr_lst = alst[1].split('=')
name = alst[0] + ".value"
name_dict = dict(operator="and")
name_dict.update(dict(query=kv))
mut = [
Q('match', **{name: name_dict})
]
qt = None
if '=*' not in alst[1]:
name = alst[0] + \
"." + val_attr_lst[0]
qt = [
Q(
'term', **{
name:
val_attr_lst[1]
})
]
mut.extend(qt or [])
qry = Q('bool', must=mut)
shuld.append(
Q('nested',
path=alst[0],
query=qry))
else:
attr_key_hit = [
x for x in attr_obj.keys() if v[0] + "." in x
]
if attr_key_hit:
vlst = attr_obj.get(attr_key_hit[0])
if isinstance(vlst, list):
attr_val = [
x for x in attr_val_str.split(',')
if x.isdecimal() and int(x) < len(vlst)
]
if attr_val:
shud = []
name = v[0] + ".value"
name_dict = dict(operator="and")
name_dict.update(dict(query=kv))
qm = Q('match', **{name: name_dict})
for j in map(
partial(
lambda m, n: m[int(n)],
vlst), attr_val):
name = attr_key_hit[0]
qm = Q('term', **{name: j})
shud.append(qm)
shuld.append(
Q('nested',
path=v[0],
query=Q('bool',
should=shud,
must=[qm])))
return Q('bool', should=shuld) if shuld else None
def _get_date_query(k, v):
# text value
qry = None
if isinstance(v, list) and len(v) >= 2:
date_from = request.values.get(k + "_" + v[0][0])
date_to = request.values.get(k + "_" + v[0][1])
if not date_from or not date_to:
return
date_from = datetime.strptime(date_from,
'%Y%m%d').strftime('%Y-%m-%d')
date_to = datetime.strptime(date_to,
'%Y%m%d').strftime('%Y-%m-%d')
qv = {}
qv.update(dict(gte=date_from))
qv.update(dict(lte=date_to))
if isinstance(v[1], str):
qry = Q('range', **{v[1]: qv})
elif isinstance(v[1], tuple) and len(v[1]) >= 2:
path = v[1][0]
dt = v[1][1]
if isinstance(dt, dict):
for attr_key, attr_val_str in map(
lambda x: (x, request.values.get(x)),
list(dt.keys())):
attr_obj = dt.get(attr_key)
if isinstance(attr_obj, dict) and attr_val_str:
attr_key_hit = [
x for x in attr_obj.keys()
if path + "." in x
]
if attr_key_hit:
vlst = attr_obj.get(attr_key_hit[0])
if isinstance(vlst, list):
attr_val = [
x for x in attr_val_str.split(',')
]
shud = []
for j in map(
partial(
lambda m, n: m[int(n)],
vlst), attr_val):
qt = Q('term',
**{attr_key_hit[0]: j})
shud.append(qt)
qry = Q('range',
**{path + ".value": qv})
qry = Q('nested',
path=path,
query=Q('bool',
should=shud,
must=[qry]))
return qry
kwd = current_app.config['WEKO_SEARCH_KEYWORDS_DICT']
ks = kwd.get('string')
kd = kwd.get('date')
kn = kwd.get('nested')
mut = []
try:
for k, v in ks.items():
qy = _get_keywords_query(k, v)
if qy:
mut.append(qy)
for k, v in kn.items():
qy = _get_nested_query(k, v)
if qy:
mut.append(qy)
for k, v in kd.items():
qy = _get_date_query(k, v)
if qy:
mut.append(qy)
except Exception as e:
current_app.logger.exception(
'Detail search query parser failed. err:{0}'.format(e))
return mut
def _get_simple_search_query(qs=None):
"""Query parser for simple search.
:param qs: Query string.
:return: Query parser.
"""
# add Permission filter by publish date and status
mt = get_permission_filter()
q = _get_search_qs_query(qs)
if q:
mt.append(q)
mt.extend(_get_detail_keywords_query())
return Q('bool', must=mt) if mt else Q()
def _get_simple_search_community_query(community_id, qs=None):
"""Query parser for simple search.
:param qs: Query string.
:return: Query parser.
"""
# add Permission filter by publish date and status
comm = Community.get(community_id)
root_node_id = comm.root_node_id
mt = get_permission_filter(root_node_id)
q = _get_search_qs_query(qs)
if q:
mt.append(q)
mt.extend(_get_detail_keywords_query())
return Q('bool', must=mt) if mt else Q()
def _default_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl.
Full text Search.
Detail Search.
:param qstr: Query string.
:returns: Query parser.
"""
# add Permission filter by publish date and status
mt = get_permission_filter()
# multi keywords search filter
kmt = _get_detail_keywords_query()
# detail search
if kmt:
mt.extend(kmt)
q = _get_search_qs_query(qs)
if q:
mt.append(q)
else:
# Full Text Search
if qstr:
q_s = Q('multi_match',
query=qstr,
operator='and',
fields=[
'content.file.content^1.5',
'content.file.content.ja^1.2', '_all',
'search_string'
],
type='most_fields',
minimum_should_match='75%')
mt.append(q_s)
return Q('bool', must=mt) if mt else Q()
def _default_parser_community(community_id, qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl.
Full text Search.
Detail Search.
:param qstr: Query string.
:returns: Query parser.
"""
# add Permission filter by publish date and status
comm = Community.get(community_id)
root_node_id = comm.root_node_id
mt = get_permission_filter(root_node_id)
# multi keywords search filter
kmt = _get_detail_keywords_query()
# detail search
if kmt:
mt.extend(kmt)
q = _get_search_qs_query(qs)
if q:
mt.append(q)
else:
# Full Text Search
if qstr:
q_s = Q('multi_match',
query=qstr,
operator='and',
fields=[
'content.file.content^1.5',
'content.file.content.ja^1.2', '_all',
'search_string'
],
type='most_fields',
minimum_should_match='75%')
mt.append(q_s)
return Q('bool', must=mt) if mt else Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
# add by ryuu at 1004 start curate
comm_ide = request.values.get('provisional_communities')
# simple search
comm_id_simple = request.values.get('community')
# add by ryuu at 1004 end
if comm_id_simple is not None:
query_parser = query_parser or _default_parser_community
else:
query_parser = query_parser or _default_parser
if search_type is None:
search_type = request.values.get('search_type')
qs = request.values.get('q')
# full text search
if search_type and '0' in search_type:
if comm_id_simple is not None:
query_q = query_parser(comm_id_simple, qs)
else:
query_q = query_parser(qs)
else:
# simple search
if comm_ide is not None:
query_q = _get_simple_search_community_query(comm_ide, qs)
elif comm_id_simple is not None:
query_q = _get_simple_search_community_query(comm_id_simple, qs)
else:
query_q = _get_simple_search_query(qs)
src = {'_source': {'excludes': ['content']}}
search._extra.update(src)
try:
search = search.filter(query_q)
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
# defalult sort
if not sortkwargs:
sort_key, sort = SearchSetting.get_default_sort(
current_app.config['WEKO_SEARCH_TYPE_KEYWORD'])
sort_obj = dict()
key_fileds = SearchSetting.get_sort_key(sort_key)
if sort == 'desc':
sort_obj[key_fileds] = dict(order='desc')
sort_key = '-' + sort_key
else:
sort_obj[key_fileds] = dict(order='asc')
search._sort.append(sort_obj)
urlkwargs.add('sort', sort_key)
urlkwargs.add('q', query_q)
return search, urlkwargs
def item_path_search_factory(self, search, index_id=None):
"""Parse query using Weko-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _get_index_earch_query():
query_q = {
"_source": {
"excludes": ['content']
},
"query": {
"bool": {
"must": [{
"match": {
"path.tree": "@index"
}
}, {
"match": {
"relation_version_is_last": "true"
}
}]
}
},
"aggs": {
"path": {
"terms": {
"field": "path.tree",
"include": "@index|@index/[^/]+",
"size": "@count"
},
"aggs": {
"date_range": {
"filter": {
"match": {
"publish_status": "0"
}
},
"aggs": {
"available": {
"range": {
"field":
"publish_date",
"ranges": [{
"from": "now+1d/d"
}, {
"to": "now+1d/d"
}]
},
}
}
},
"no_available": {
"filter": {
"bool": {
"must_not": [{
"match": {
"publish_status": "0"
}
}]
}
}
}
}
}
},
"post_filter": {}
}
# add item type aggs
query_q['aggs']['path']['aggs']. \
update(get_item_type_aggs(search._index[0]))
q = request.values.get('q') if index_id is None else index_id
if q:
mut = get_permission_filter(q)
else:
mut = get_permission_filter()
if mut:
mut = list(map(lambda x: x.to_dict(), mut))
post_filter = query_q['post_filter']
if mut[0].get('bool'):
post_filter['bool'] = mut[0]['bool']
else:
post_filter['bool'] = {'must': mut}
# create search query
if q:
try:
fp = Indexes.get_self_path(q)
if fp:
query_q = json.dumps(query_q).replace("@index", fp.path)
query_q = json.loads(query_q)
except BaseException:
pass
query_q = json.dumps(query_q).replace("@count",
str(Indexes.get_index_count()))
query_q = json.loads(query_q)
return query_q
# create a index search query
query_q = _get_index_earch_query()
urlkwargs = MultiDict()
try:
# Aggregations.
extr = search._extra.copy()
search.update_from_dict(query_q)
search._extra.update(extr)
except SyntaxError:
q = request.values.get('q', '') if index_id is None else index_id
current_app.logger.debug("Failed parsing query: {0}".format(q),
exc_info=True)
raise InvalidQueryRESTError()
from invenio_records_rest.sorter import default_sorter_factory
search_index = search._index[0]
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
# set custom sort option
if 'custom_sort' in value:
ind_id = request.values.get('q', '')
search._sort = []
if value == 'custom_sort':
script_str, default_sort = SearchSetting.get_custom_sort(
ind_id, 'asc')
else:
script_str, default_sort = SearchSetting.get_custom_sort(
ind_id, 'desc')
search._sort.append(script_str)
search._sort.append(default_sort)
# set selectbox
urlkwargs.add(key, value)
# default sort
if not sortkwargs:
sort_key, sort = SearchSetting.get_default_sort(
current_app.config['WEKO_SEARCH_TYPE_INDEX'])
sort_obj = dict()
key_fileds = SearchSetting.get_sort_key(sort_key)
if 'custom_sort' not in sort_key:
if sort == 'desc':
sort_obj[key_fileds] = dict(order='desc')
sort_key = '-' + sort_key
else:
sort_obj[key_fileds] = dict(order='asc')
search._sort.append(sort_obj)
else:
if sort == 'desc':
ind_id = request.values.get('q', '')
script_str, default_sort = SearchSetting.get_custom_sort(
ind_id, 'desc')
sort_key = '-' + sort_key
else:
script_str, default_sort = SearchSetting.get_custom_sort(
ind_id, 'asc')
search._sort = []
search._sort.append(script_str)
search._sort.append(default_sort)
urlkwargs.add('sort', sort_key)
urlkwargs.add('q', query_q)
return search, urlkwargs
