def test_default_facets_factory(app):
"""Test aggregations."""
defs = dict(
aggs=dict(
type=dict(
terms=dict(field='upload_type'),
),
subtype=dict(
terms=dict(field='subtype'),
)
),
filters=dict(
subtype=terms_filter('subtype'),
),
post_filters=dict(
type=terms_filter('type'),
),
)
app.config['RECORDS_REST_FACETS']['testidx'] = defs
with app.test_request_context('?type=a&subtype=b'):
search = Search().query(Q(query='value'))
search, urlkwargs = default_facets_factory(search, 'testidx')
assert search.to_dict()['aggs'] == defs['aggs']
assert 'post_filter' in search.to_dict()
assert search.to_dict(
)['query']['bool']['filter'][0]['terms']['subtype']
search = Search().query(Q(query='value'))
search, urlkwargs = default_facets_factory(search, 'anotheridx')
assert 'aggs' not in search.to_dict()
assert 'post_filter' not in search.to_dict()
assert 'bool' not in search.to_dict()['query']
def test_default_facets_factory(app, user_factory):
"""Test aggregations."""
defs = dict(
aggs=dict(
type=dict(
terms=dict(field="upload_type"),
),
subtype=dict(
terms=dict(field="subtype"),
)
),
filters=dict(
subtype=terms_filter('subtype'),
),
post_filters=dict(
type=terms_filter('type'),
),
)
app.config['RECORDS_REST_FACETS']['testidx'] = defs
with app.test_request_context("?type=a&subtype=b"):
q = Query("value")
query, urlkwargs = default_facets_factory(q, 'testidx')
assert query.body['aggs'] == defs['aggs']
assert 'post_filter' in query.body
assert 'filtered' in query.body['query']
q = Query("value")
query, urlkwargs = default_facets_factory(q, 'anotheridx')
assert 'aggs' not in query.body
assert 'post_filter' not in query.body
assert 'filtered' not in query.body['query']
def and_search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr, default_operator='AND')
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q')
query_parser = query_parser or _default_parser
try:
search = search.query(query_parser(query_string))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def inspire_search_factory(self, search):
"""Parse query using Invenio-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q', '')
try:
search = search.query(IQ(query_string, search))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
finally:
if current_app.debug:
current_app.logger.debug(json.dumps(search.to_dict(), indent=4))
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def _ils_search_factory(self, search, qs_validator):
"""Search factory with Query String validator.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def query_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr)
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get("q")
query = query_parser(qs_validator(query_string))
try:
search = search.query(query)
except SyntaxError:
raise SearchQueryError(query_string)
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add("q", query_string)
return search, urlkwargs
def inspire_search_factory(self, search):
"""Parse query using Invenio-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q', '')
try:
search = search.query(IQ(query_string))
except SyntaxError:
current_app.logger.debug(
"Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def cernopendata_search_factory(self, search):
"""Customized parse query using invenio query parser.
:param self: REST view
:param search: Elastic search DSL search instance
:return: Tuple with search instance and URL arguments
"""
query_string = request.values.get("q")
try:
search = search.query(cernopendata_query_parser(query_string))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get("q", "")),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, url_kwargs = default_facets_factory(search, search_index)
search, sort_kwargs = default_sorter_factory(search, search_index)
for key, value in sort_kwargs.items():
url_kwargs.add(key, value)
url_kwargs.add("q", query_string)
return search, url_kwargs
def search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
# Apply 'identity' grouping by default
if 'group_by' not in request.values:
search = search.filter(Q('term', Grouping='identity'))
urlkwargs['group_by'] = 'identity'
try:
query_string = request.values.get('q')
if query_string:
search = search.query(
Q('query_string',
query=query_string,
default_field='_search_all'))
urlkwargs['q'] = query_string
except SyntaxError:
raise InvalidQueryRESTError()
# Exclude the identifiers by which the search was made (large aggregate)
search = search.source(exclude=['*.SearchIdentifier'])
return search, urlkwargs
def search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
search_index = search._index[0]
# TODO: make "scheme" optional?
for field in ('id', 'scheme', 'relation'):
if field not in request.values:
raise RESTValidationError(
errors=[FieldError(field, 'Required field.')])
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
# Apply 'identity' grouping by default
if 'groupBy' not in request.values:
search = search.filter(Q('term', Grouping='identity'))
urlkwargs['groupBy'] = 'identity'
# Exclude the identifiers by which the search was made (large aggregate)
search = search.source(exclude=['*.SearchIdentifier'])
return search, urlkwargs
def meta_search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
# for field in (['keyword']):
# if field not in request.values:
# raise RESTValidationError(
# errors=[FieldError(field, 'Required field.')])
search, urlkwargs = default_facets_factory(search, "metadata")
search, sortkwargs = default_sorter_factory(search, "metadata")
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
search = search.extra(size=0)
# Apply 'identity' grouping by default
search = search.filter(Q('term', RelationshipType='Cites'))
if 'group_by' not in request.values:
search = search.filter(Q('term', Grouping='identity'))
urlkwargs['group_by'] = 'identity'
try:
query_string = request.values.get('q')
if query_string:
search = search.query(
Q('query_string',
query=query_string,
default_field='_search_all'))
urlkwargs['q'] = query_string
except SyntaxError:
raise InvalidQueryRESTError()
size = 10
if 'size' in request.values:
size = request.values.get('size')
start = 0
if 'page' in request.values:
start = int(int(request.values.get('page')) - 1) * int(size)
search.aggs.bucket('Target', 'terms', field='Target.ID', size=1000)\
.metric("first", "top_hits", _source=dict(include=["Target.Identifier.*", "Target.Creator.Name", "Target.Title"]), size=1)
kwargs = {
'from': start,
'size': size,
'sort': [{
'_count': {
'order': 'desc'
}
}]
}
search.aggs['Target'].bucket('pagination', 'bucket_sort', **kwargs)
return search, urlkwargs
def circulation_search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Return default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr)
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q', '')
query = _default_parser(qstr=query_string)
# if the logged in user in not librarian or admin, validate the query
if not backoffice_permission().allows(g.identity):
# patron can find only his loans
try:
if not query_string:
# force query to be patron_pid:<logged in user>
only_patron_loans = 'patron_pid:{}'.format(g.identity.id)
query = _default_parser(qstr=only_patron_loans)
else:
# check for patron_pid query value
match = re.match(r"patron_pid:(?P<pid>\d)", query_string)
if match and match.group('pid') != str(g.identity.id):
raise UnauthorizedSearch()
except UnauthorizedSearch:
current_app.logger.debug(
"Search for `{0}` not allowed by `patron_pid:{1}`".format(
query_string, str(g.identity.id))
)
abort(403)
try:
search = search.query(query)
except SyntaxError:
current_app.logger.debug(
"Failed parsing query: {0}".format(query_string), exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def test_default_facets_factory(app, user_factory):
"""Test aggregations."""
defs = dict(
aggs=dict(type=dict(terms=dict(field="upload_type"), ),
subtype=dict(terms=dict(field="subtype"), )),
filters=dict(subtype=terms_filter('subtype'), ),
post_filters=dict(type=terms_filter('type'), ),
)
app.config['RECORDS_REST_FACETS']['testidx'] = defs
with app.test_request_context("?type=a&subtype=b"):
q = Query("value")
query, urlkwargs = default_facets_factory(q, 'testidx')
assert query.body['aggs'] == defs['aggs']
assert 'post_filter' in query.body
assert 'filtered' in query.body['query']
q = Query("value")
query, urlkwargs = default_facets_factory(q, 'anotheridx')
assert 'aggs' not in query.body
assert 'post_filter' not in query.body
assert 'filtered' not in query.body['query']
def filter_by_patron_search_factory(self, search, query_parser=None):
"""Filter search queries to only show records for the logged in patron.
If the logged in user has backoffice permissions do not filter by patron.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Return default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr)
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q', '')
if not current_user.is_authenticated:
raise UnauthorizedSearchError(query_string)
parser = query_parser or _default_parser
query = parser(qstr=query_string)
# if the logged in user in not librarian or admin, validate the query
if has_request_context() and not backoffice_permission().allows(g.identity):
# patron can find only his document requests
if not query_string:
# force query to be patron_pid:<logged in user>
patron_pid_filter = 'patron_pid:{}'.format(g.identity.id)
query = _default_parser(qstr=patron_pid_filter)
else:
# check for patron_pid query value
match = re.match(r"patron_pid:(?P<pid>\d)", query_string)
if match and match.group('pid') != str(g.identity.id):
raise UnauthorizedSearchError(query_string, g.identity.id)
try:
search = search.query(query)
except SyntaxError:
raise SearchQueryError(query_string)
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def test_default_facets_factory(app):
"""Test aggregations."""
defs = dict(
aggs=dict(type=dict(terms=dict(field='upload_type'), ),
subtype=dict(terms=dict(field='subtype'), )),
filters=dict(subtype=terms_filter('subtype'), ),
post_filters=dict(type=terms_filter('type'), ),
)
app.config['RECORDS_REST_FACETS']['testidx'] = defs
with app.test_request_context('?type=a&subtype=b'):
search = Search().query(Q(query='value'))
search, urlkwargs = default_facets_factory(search, 'testidx')
assert search.to_dict()['aggs'] == defs['aggs']
assert 'post_filter' in search.to_dict()
assert search.to_dict(
)['query']['bool']['filter'][0]['terms']['subtype']
search = Search().query(Q(query='value'))
search, urlkwargs = default_facets_factory(search, 'anotheridx')
assert 'aggs' not in search.to_dict()
assert 'post_filter' not in search.to_dict()
assert 'bool' not in search.to_dict()['query']
def default_search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param search: Elastic search DSL search instance.
:param query_parser: Custom query parser.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl.
:param qstr: Query string.
:returns: Query object.
"""
if not qstr:
return Q()
operator, query_type = get_operator_and_query_type(qstr)
return Q(query_type, query=qstr, default_operator=operator)
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q')
# Use query parser given to function or the default one.
query_parser = query_parser or _default_parser
# Search query
search = search.query(query_parser(query_string))
# Get index corresponding to record type.
search_index = getattr(search, '_original_index', search._index)[0]
# Build facets
search, urlkwargs = default_facets_factory(search, search_index)
# Sort records
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
# Add explanation to hits
if request.args.get('debug'):
search = search.extra(explain=True)
return search, urlkwargs
def ils_search_factory(self, search, validator=None):
"""Search factory with Query String validator.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def query_parser(search, qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
boosted = getattr(search, "boosted_fields", [])
extra_params = {}
if boosted:
extra_params["fields"] = boosted + ["*"]
# add lenient parameter in order to fix
# parsing exception on data fields, see known issues
# https://www.elastic.co/guide/en/elasticsearch/reference/current/release-notes-7.1.1.html # noqa
extra_params["lenient"] = True
return Q("query_string", query=qstr, **extra_params)
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get("q")
if validator:
search, query_string = validator(search, query_string)
query = query_parser(search, qstr=query_string)
try:
search = search.query(query)
except SyntaxError:
current_app.logger.debug(
"Failed parsing query: {0}".format(request.values.get("q", "")),
exc_info=True,
)
raise SearchQueryError(query_string)
search_index = getattr(search, "_original_index", search._index)[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add("q", query_string)
return search, urlkwargs
def and_search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr, default_operator='AND')
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q')
query_parser = query_parser or _default_parser
try:
search = search.query(query_parser(query_string))
except SyntaxError:
current_app.logger.debug(
f'Failed parsing query: {request.values.get("q", "")}',
exc_info=True,
)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
# include deleted
deleted = request.args.get('deleted')
if not deleted:
search = search.filter('bool', must_not=[Q('exists', field='deleted')])
return search, urlkwargs
def _ils_search_factory(self, search, validator):
"""Search factory with Query String validator.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def query_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr)
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get("q")
search, query_string = validator(search, query_string)
query = query_parser(query_string)
try:
search = search.query(query)
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise SearchQueryError(query_string)
search_index = getattr(search, '_original_index', search._index)[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add("q", query_string)
return search, urlkwargs
def default_search_factory(self, search, query_parser=None, search_type=None):
"""Parse query using Weko-Query-Parser. MetaData Search.
:param self: REST view.
:param search: Elastic search DSL search instance.
:param query_parser: Query parser. (Default: ``None``)
:returns: Tuple with search instance and URL arguments.
"""
def _get_search_qs_query(qs=None):
"""Qs of search bar keywords for detail simple search.
:param qs: Query string.
:return: Query parser.
"""
q = Q('query_string',
query=qs,
default_operator='and',
fields=['search_*', 'search_*.ja']) if qs else None
return q
def _get_detail_keywords_query():
"""Get keywords query.
:return: Query parser.
"""
def _get_keywords_query(k, v):
qry = None
kv = request.values.get(k)
if not kv:
return
if isinstance(v, str):
name_dict = dict(operator="and")
name_dict.update(dict(query=kv))
qry = Q('match', **{v: name_dict})
elif isinstance(v, list):
qry = Q('multi_match',
query=kv,
type='most_fields',
minimum_should_match='75%',
operator='and',
fields=v)
elif isinstance(v, dict):
for key, vlst in v.items():
if isinstance(vlst, list):
shud = []
kvl = [
x for x in kv.split(',')
if x.isdecimal() and int(x) < len(vlst)
]
for j in map(partial(lambda x, y: x[int(y)], vlst),
kvl):
name_dict = dict(operator="and")
name_dict.update(dict(query=j))
shud.append(Q('match', **{key: name_dict}))
kvl = [
x for x in kv.split(',')
if not x.isdecimal() and x in vlst
]
for j in kvl:
name_dict = dict(operator="and")
name_dict.update(dict(query=j))
shud.append(Q('match', **{key: name_dict}))
if shud:
return Q('bool', should=shud)
elif isinstance(v, tuple) and len(v) >= 2:
shud = []
for i in map(lambda x: v[1](x), kv.split(',')):
shud.append(Q('term', **{v[0]: i}))
if shud:
qry = Q('bool', should=shud)
return qry
def _get_nested_query(k, v):
# text value
kv = request.values.get(k)
if not kv:
return
shuld = []
if isinstance(v, tuple) and len(v) > 1 and isinstance(v[1], dict):
# attr keyword in request url
for attr_key, attr_val_str in map(
lambda x: (x, request.values.get(x)),
list(v[1].keys())):
attr_obj = v[1].get(attr_key)
if isinstance(attr_obj, dict) and attr_val_str:
if isinstance(v[0], str) and not len(v[0]):
# For ID search
for key in attr_val_str.split(','):
attr = attr_obj.get(key)
if isinstance(attr, tuple):
attr = [attr]
if isinstance(attr, list):
for alst in attr:
if isinstance(alst, tuple):
val_attr_lst = alst[1].split('=')
name = alst[0] + ".value"
name_dict = dict(operator="and")
name_dict.update(dict(query=kv))
mut = [
Q('match', **{name: name_dict})
]
qt = None
if '=*' not in alst[1]:
name = alst[0] + \
"." + val_attr_lst[0]
qt = [
Q(
'term', **{
name:
val_attr_lst[1]
})
]
mut.extend(qt or [])
qry = Q('bool', must=mut)
shuld.append(
Q('nested',
path=alst[0],
query=qry))
else:
attr_key_hit = [
x for x in attr_obj.keys() if v[0] + "." in x
]
if attr_key_hit:
vlst = attr_obj.get(attr_key_hit[0])
if isinstance(vlst, list):
attr_val = [
x for x in attr_val_str.split(',')
if x.isdecimal() and int(x) < len(vlst)
]
if attr_val:
shud = []
name = v[0] + ".value"
name_dict = dict(operator="and")
name_dict.update(dict(query=kv))
qm = Q('match', **{name: name_dict})
for j in map(
partial(
lambda m, n: m[int(n)],
vlst), attr_val):
name = attr_key_hit[0]
qm = Q('term', **{name: j})
shud.append(qm)
shuld.append(
Q('nested',
path=v[0],
query=Q('bool',
should=shud,
must=[qm])))
return Q('bool', should=shuld) if shuld else None
def _get_date_query(k, v):
# text value
qry = None
if isinstance(v, list) and len(v) >= 2:
date_from = request.values.get(k + "_" + v[0][0])
date_to = request.values.get(k + "_" + v[0][1])
if not date_from or not date_to:
return
date_from = datetime.strptime(date_from,
'%Y%m%d').strftime('%Y-%m-%d')
date_to = datetime.strptime(date_to,
'%Y%m%d').strftime('%Y-%m-%d')
qv = {}
qv.update(dict(gte=date_from))
qv.update(dict(lte=date_to))
if isinstance(v[1], str):
qry = Q('range', **{v[1]: qv})
elif isinstance(v[1], tuple) and len(v[1]) >= 2:
path = v[1][0]
dt = v[1][1]
if isinstance(dt, dict):
for attr_key, attr_val_str in map(
lambda x: (x, request.values.get(x)),
list(dt.keys())):
attr_obj = dt.get(attr_key)
if isinstance(attr_obj, dict) and attr_val_str:
attr_key_hit = [
x for x in attr_obj.keys()
if path + "." in x
]
if attr_key_hit:
vlst = attr_obj.get(attr_key_hit[0])
if isinstance(vlst, list):
attr_val = [
x for x in attr_val_str.split(',')
]
shud = []
for j in map(
partial(
lambda m, n: m[int(n)],
vlst), attr_val):
qt = Q('term',
**{attr_key_hit[0]: j})
shud.append(qt)
qry = Q('range',
**{path + ".value": qv})
qry = Q('nested',
path=path,
query=Q('bool',
should=shud,
must=[qry]))
return qry
kwd = current_app.config['WEKO_SEARCH_KEYWORDS_DICT']
ks = kwd.get('string')
kd = kwd.get('date')
kn = kwd.get('nested')
mut = []
try:
for k, v in ks.items():
qy = _get_keywords_query(k, v)
if qy:
mut.append(qy)
for k, v in kn.items():
qy = _get_nested_query(k, v)
if qy:
mut.append(qy)
for k, v in kd.items():
qy = _get_date_query(k, v)
if qy:
mut.append(qy)
except Exception as e:
current_app.logger.exception(
'Detail search query parser failed. err:{0}'.format(e))
return mut
def _get_simple_search_query(qs=None):
"""Query parser for simple search.
:param qs: Query string.
:return: Query parser.
"""
# add Permission filter by publish date and status
mt = get_permission_filter()
q = _get_search_qs_query(qs)
if q:
mt.append(q)
mt.extend(_get_detail_keywords_query())
return Q('bool', must=mt) if mt else Q()
def _get_simple_search_community_query(community_id, qs=None):
"""Query parser for simple search.
:param qs: Query string.
:return: Query parser.
"""
# add Permission filter by publish date and status
comm = Community.get(community_id)
root_node_id = comm.root_node_id
mt = get_permission_filter(root_node_id)
q = _get_search_qs_query(qs)
if q:
mt.append(q)
mt.extend(_get_detail_keywords_query())
return Q('bool', must=mt) if mt else Q()
def _default_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl.
Full text Search.
Detail Search.
:param qstr: Query string.
:returns: Query parser.
"""
# add Permission filter by publish date and status
mt = get_permission_filter()
# multi keywords search filter
kmt = _get_detail_keywords_query()
# detail search
if kmt:
mt.extend(kmt)
q = _get_search_qs_query(qs)
if q:
mt.append(q)
else:
# Full Text Search
if qstr:
q_s = Q('multi_match',
query=qstr,
operator='and',
fields=[
'content.file.content^1.5',
'content.file.content.ja^1.2', '_all',
'search_string'
],
type='most_fields',
minimum_should_match='75%')
mt.append(q_s)
return Q('bool', must=mt) if mt else Q()
def _default_parser_community(community_id, qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl.
Full text Search.
Detail Search.
:param qstr: Query string.
:returns: Query parser.
"""
# add Permission filter by publish date and status
comm = Community.get(community_id)
root_node_id = comm.root_node_id
mt = get_permission_filter(root_node_id)
# multi keywords search filter
kmt = _get_detail_keywords_query()
# detail search
if kmt:
mt.extend(kmt)
q = _get_search_qs_query(qs)
if q:
mt.append(q)
else:
# Full Text Search
if qstr:
q_s = Q('multi_match',
query=qstr,
operator='and',
fields=[
'content.file.content^1.5',
'content.file.content.ja^1.2', '_all',
'search_string'
],
type='most_fields',
minimum_should_match='75%')
mt.append(q_s)
return Q('bool', must=mt) if mt else Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
# add by ryuu at 1004 start curate
comm_ide = request.values.get('provisional_communities')
# simple search
comm_id_simple = request.values.get('community')
# add by ryuu at 1004 end
if comm_id_simple is not None:
query_parser = query_parser or _default_parser_community
else:
query_parser = query_parser or _default_parser
if search_type is None:
search_type = request.values.get('search_type')
qs = request.values.get('q')
# full text search
if search_type and '0' in search_type:
if comm_id_simple is not None:
query_q = query_parser(comm_id_simple, qs)
else:
query_q = query_parser(qs)
else:
# simple search
if comm_ide is not None:
query_q = _get_simple_search_community_query(comm_ide, qs)
elif comm_id_simple is not None:
query_q = _get_simple_search_community_query(comm_id_simple, qs)
else:
query_q = _get_simple_search_query(qs)
src = {'_source': {'excludes': ['content']}}
search._extra.update(src)
try:
search = search.filter(query_q)
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
# defalult sort
if not sortkwargs:
sort_key, sort = SearchSetting.get_default_sort(
current_app.config['WEKO_SEARCH_TYPE_KEYWORD'])
sort_obj = dict()
key_fileds = SearchSetting.get_sort_key(sort_key)
if sort == 'desc':
sort_obj[key_fileds] = dict(order='desc')
sort_key = '-' + sort_key
else:
sort_obj[key_fileds] = dict(order='asc')
search._sort.append(sort_obj)
urlkwargs.add('sort', sort_key)
urlkwargs.add('q', query_q)
return search, urlkwargs
def search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
Terms defined by: RERO_ILS_QUERY_BOOSTING will be boosted
at the query level.
:param self: REST view.
:param search: Elastic search DSL search instance.
:param query_parser: a specific query parser
:return: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None, query_boosting=[]):
"""Default parser that uses the Q() from elasticsearch_dsl."""
query_type = 'query_string'
default_operator = 'OR'
if request.args.get('simple'):
query_type = 'simple_query_string'
default_operator = 'AND'
if qstr:
# TODO: remove this bad hack
qstr = _PUNCTUATION_REGEX.sub(' ', qstr)
qstr = re.sub('\s+', ' ', qstr).rstrip()
if not query_boosting:
return Q(query_type,
query=qstr,
default_operator=default_operator)
else:
return Q('bool',
should=[
Q(query_type,
query=qstr,
boost=2,
fields=query_boosting,
default_operator=default_operator),
Q(query_type,
query=qstr,
default_operator=default_operator)
])
return Q()
def _boosting_parser(query_boosting, search_index):
"""Elasticsearch boosting fields parser."""
boosting = []
if search_index in query_boosting:
for field, boost in query_boosting[search_index].items():
boosting.append('{}^{}'.format(field, boost))
return boosting
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q')
display_score = request.values.get('display_score')
if display_score:
search = search.extra(explain=True)
query_parser = query_parser or _default_parser
search_index = search._index[0]
query_boosting = _boosting_parser(
current_app.config['RERO_ILS_QUERY_BOOSTING'], search_index)
try:
search = search.query(query_parser(query_string, query_boosting))
except SyntaxError:
current_app.logger.debug(
'Failed parsing query: {0}'.format(request.values.get('q', '')),
exc_info=True,
)
raise InvalidQueryRESTError()
search, urlkwargs = default_facets_factory(search, search_index)
# i18n translated facets
search = i18n_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
