def cernopendata_search_factory(self, search):
"""Customized parse query using invenio query parser.
:param self: REST view
:param search: Elastic search DSL search instance
:return: Tuple with search instance and URL arguments
"""
query_string = request.values.get("q")
try:
search = search.query(cernopendata_query_parser(query_string))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get("q", "")),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, url_kwargs = default_facets_factory(search, search_index)
search, sort_kwargs = default_sorter_factory(search, search_index)
for key, value in sort_kwargs.items():
url_kwargs.add(key, value)
url_kwargs.add("q", query_string)
return search, url_kwargs
def search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
# Apply 'identity' grouping by default
if 'group_by' not in request.values:
search = search.filter(Q('term', Grouping='identity'))
urlkwargs['group_by'] = 'identity'
try:
query_string = request.values.get('q')
if query_string:
search = search.query(
Q('query_string',
query=query_string,
default_field='_search_all'))
urlkwargs['q'] = query_string
except SyntaxError:
raise InvalidQueryRESTError()
# Exclude the identifiers by which the search was made (large aggregate)
search = search.source(exclude=['*.SearchIdentifier'])
return search, urlkwargs
def cap_search_factory(self, search, query_parser=None):
"""Customized Parse query using Invenio-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr)
return Q()
query_string = request.values.get('q')
query_parser = query_parser or _default_parser
try:
search = search.query(query_parser(query_string))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = cap_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def inspire_search_factory(self, search):
"""Parse query using Invenio-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q', '')
try:
search = search.query(IQ(query_string, search))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
finally:
if current_app.debug:
current_app.logger.debug(json.dumps(search.to_dict(), indent=4))
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def inspire_search_factory(self, search):
"""Parse query using Inspire-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
query_string = request.values.get('q', '')
urlkwargs = MultiDict()
try:
search = search.query_from_iq(query_string)
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = inspire_filter_factory(search, urlkwargs, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
search = select_source(search)
urlkwargs.add('q', query_string)
current_app.logger.debug(json.dumps(search.to_dict(), indent=4))
return search, urlkwargs
def meta_search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
# for field in (['keyword']):
# if field not in request.values:
# raise RESTValidationError(
# errors=[FieldError(field, 'Required field.')])
search, urlkwargs = default_facets_factory(search, "metadata")
search, sortkwargs = default_sorter_factory(search, "metadata")
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
search = search.extra(size=0)
# Apply 'identity' grouping by default
search = search.filter(Q('term', RelationshipType='Cites'))
if 'group_by' not in request.values:
search = search.filter(Q('term', Grouping='identity'))
urlkwargs['group_by'] = 'identity'
try:
query_string = request.values.get('q')
if query_string:
search = search.query(
Q('query_string',
query=query_string,
default_field='_search_all'))
urlkwargs['q'] = query_string
except SyntaxError:
raise InvalidQueryRESTError()
size = 10
if 'size' in request.values:
size = request.values.get('size')
start = 0
if 'page' in request.values:
start = int(int(request.values.get('page')) - 1) * int(size)
search.aggs.bucket('Target', 'terms', field='Target.ID', size=1000)\
.metric("first", "top_hits", _source=dict(include=["Target.Identifier.*", "Target.Creator.Name", "Target.Title"]), size=1)
kwargs = {
'from': start,
'size': size,
'sort': [{
'_count': {
'order': 'desc'
}
}]
}
search.aggs['Target'].bucket('pagination', 'bucket_sort', **kwargs)
return search, urlkwargs
def inspire_search_factory(self, search):
query_string = request.values.get("q", "")
try:
search = search.query_from_iq(query_string)
except SyntaxError as exc:
LOGGER.warning("Failed parsing query",
query=request.values.get("q", ""))
raise InvalidQueryRESTError() from exc
return query_string, search
def circulation_search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Return default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr)
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q', '')
query = _default_parser(qstr=query_string)
# if the logged in user in not librarian or admin, validate the query
if not backoffice_permission().allows(g.identity):
# patron can find only his loans
try:
if not query_string:
# force query to be patron_pid:<logged in user>
only_patron_loans = 'patron_pid:{}'.format(g.identity.id)
query = _default_parser(qstr=only_patron_loans)
else:
# check for patron_pid query value
match = re.match(r"patron_pid:(?P<pid>\d)", query_string)
if match and match.group('pid') != str(g.identity.id):
raise UnauthorizedSearch()
except UnauthorizedSearch:
current_app.logger.debug(
"Search for `{0}` not allowed by `patron_pid:{1}`".format(
query_string, str(g.identity.id))
)
abort(403)
try:
search = search.query(query)
except SyntaxError:
current_app.logger.debug(
"Failed parsing query: {0}".format(query_string), exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def cap_search_factory(self, search, query_parser=None):
"""Customize Parse query using Invenio-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None, **kwargs):
"""Use of the Q() from elasticsearch_dsl."""
def _escape_qstr(qstr):
return ''.join((ESCAPE_CHAR_MAP.get(char, char) for char in qstr))
query = Q('query_string',
query=_escape_qstr(qstr),
analyzer="lowercase_whitespace_analyzer",
analyze_wildcard=True,
default_operator='AND') if qstr else Q()
# resolve keywords to queries
for k, v in kwargs.items():
if k in KEYWORD_TO_QUERY:
if v == 'True':
query = query & KEYWORD_TO_QUERY[k]()
elif v == 'False':
query = query & ~KEYWORD_TO_QUERY[k]()
return query
query_string = request.values.get('q')
# parse url params to search for keywords
query_keywords = {
k: request.values[k]
for k in KEYWORD_TO_QUERY.keys() if k in request.values
}
query_parser = query_parser or _default_parser
try:
search = search.query(query_parser(query_string, **query_keywords))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = cap_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def perform_query(query_string, page, size):
"""
:param query_string:
:param page:
:param size:
:return:
"""
try:
query = InspireQuery(query_string)[(page - 1) * size:page * size]
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
return query, {'q': query_string}
def and_search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
if qstr:
return Q('query_string', query=qstr, default_operator='AND')
return Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q')
query_parser = query_parser or _default_parser
try:
search = search.query(query_parser(query_string))
except SyntaxError:
current_app.logger.debug(
f'Failed parsing query: {request.values.get("q", "")}',
exc_info=True,
)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
# include deleted
deleted = request.args.get('deleted')
if not deleted:
search = search.filter('bool', must_not=[Q('exists', field='deleted')])
return search, urlkwargs
def cernopendata_search_factory(self, search, query_parser=None):
"""Customized Parse query using Invenio-Query-Parser.
By default we hide the results that have availability:ondemand.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None, ondemand=None):
"""Default parser that uses the Q() from elasticsearch_dsl."""
q = Q('query_string', query=qstr) if qstr else Q()
# by default hide ondemand ones
if not ondemand:
q = q & ~Q('match', **{'availability.keyword': 'ondemand'})
return q
query_string = request.values.get('q')
ondemand = request.values.get('ondemand', False) # this is a workaround
query_parser = query_parser or _default_parser
try:
search = search.query(query_parser(query_string, ondemand))
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = cernopendata_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def deposit_search_factory(self, search):
"""Replace default search factory to use custom facet factory."""
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q', '')
query_parser = Q('query_string',
query=query_string) if query_string else Q()
try:
search = search.query(query_parser)
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = deposit_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def item_search_factory(self,
search,
start_date,
end_date,
list_index_id=None):
"""Factory for opensearch.
:param self:
:param search: Record Search's instance
:param start_date: Start date for search
:param end_date: End date for search
:param list_index_id: index tree list or None
:return:
"""
def _get_query(start_term, end_term, indexes):
query_string = "_type:{} AND " \
"relation_version_is_last:true AND " \
"publish_status:0 AND " \
"publish_date:[{} TO {}]".format(current_app.config[
"INDEXER_DEFAULT_DOC_TYPE"],
start_term,
end_term)
query_filter = []
if indexes:
for index in indexes:
q_wildcard = {"wildcard": {"path": "*{}*".format(index)}}
query_filter.append(q_wildcard)
query_q = {
"size": 10000,
"query": {
"bool": {
"must": [{
"query_string": {
"query": query_string
}
}, {
"bool": {
"should": query_filter
}
}]
}
},
"sort": [{
"publish_date": {
"order": "desc"
}
}]
}
return query_q
query_q = _get_query(start_date, end_date, list_index_id)
urlkwargs = MultiDict()
try:
extr = search._extra.copy()
search.update_from_dict(query_q)
search._extra.update(extr)
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(query_q),
exc_info=True)
raise InvalidQueryRESTError()
return search, urlkwargs
def item_path_search_factory(search, index_id="0"):
"""Parse query using Weko-Query-Parser.
:param search: Elastic search DSL search instance.
:param index_id: Index Identifier contains item's path
:returns: Tuple with search instance and URL arguments.
"""
def _get_index_search_query():
"""Get index search query."""
query_q = {
"from": 0,
"size": 10000,
"_source": {
"excludes": ["content", "_item_metadata"]
},
"query": {
"bool": {
"must": [{
"match": {
"path.tree": "@index"
}
}, {
"match": {
"relation_version_is_last": "true"
}
}]
}
},
"post_filter": {
"bool": {
"must": [{
"match": {
"publish_status": "0"
}
}, {
"range": {
"publish_date": {
"lte": "now/d"
}
}
}]
}
}
}
q = str(index_id)
if q != str(current_app.config.get("WEKO_ROOT_INDEX",
WEKO_ROOT_INDEX)):
post_filter = query_q['post_filter']
if post_filter:
list_path = Indexes.get_list_path_publish(index_id)
post_filter['bool']['must'].append(
{"terms": {
"path": list_path
}})
# create search query
try:
fp = Indexes.get_self_path(q)
query_q = json.dumps(query_q).replace("@index", fp.path)
query_q = json.loads(query_q)
except BaseException:
pass
else:
post_filter = query_q['post_filter']
if post_filter:
list_path = Indexes.get_list_path_publish(index_id)
post_filter['bool']['must'].append(
{"terms": {
"path": list_path
}})
wild_card = []
child_list = Indexes.get_child_list(q)
if child_list:
for item in child_list:
wc = {"wildcard": {"path.tree": item.cid}}
wild_card.append(wc)
query_q['query']['bool']['must'] = [{
"bool": {
"should": wild_card
}
}, {
"match": {
"relation_version_is_last": "true"
}
}]
return query_q
# create a index search query
query_q = _get_index_search_query()
try:
# Aggregations.
extr = search._extra.copy()
search.update_from_dict(query_q)
search._extra.update(extr)
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(query_q),
exc_info=True)
raise InvalidQueryRESTError()
return search
def item_path_search_factory(self, search, index_id=None):
"""Parse query using Weko-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _get_index_earch_query():
query_q = {
"_source": {
"exclude": ['content']
},
"query": {
"match": {
"path.tree": "@index"
}
},
"aggs": {
"path": {
"terms": {
"field": "path.tree",
"include": "@index|@index/[^/]+"
},
"aggs": {
"date_range": {
"filter": {
"match": {
"publish_status": "0"
}
},
"aggs": {
"available": {
"range": {
"field":
"publish_date",
"ranges": [{
"from": "now+1d/d"
}, {
"to": "now+1d/d"
}]
},
}
}
},
"no_available": {
"filter": {
"bool": {
"must_not": [{
"match": {
"publish_status": "0"
}
}]
}
}
}
}
}
},
"post_filter": {
"term": {
"path": "@index"
}
}
}
# add item type aggs
query_q['aggs']['path']['aggs']. \
update(get_item_type_aggs(search._index[0]))
mut = get_permission_filter()
if mut:
mut = list(map(lambda x: x.to_dict(), mut))
post_filter = query_q['post_filter']
if mut[0].get('bool'):
post_filter['bool'] = {
'must': [{
'term': post_filter.pop('term')
}, mut[0]['bool']['must'][0]],
'should':
mut[0]['bool']['should']
}
# post_filter['bool'] = {'must': [{'term': post_filter.pop('term')}],
# 'should': mut[0]['bool']['should']}
else:
mut.append({'term': post_filter.pop('term')})
post_filter['bool'] = {'must': mut}
# create search query
q = request.values.get('q') if index_id is None else index_id
if q:
try:
fp = Indexes.get_self_path(q)
if fp:
query_q = json.dumps(query_q).replace("@index", fp.path)
query_q = json.loads(query_q)
except BaseException:
pass
return query_q
# create a index search query
query_q = _get_index_earch_query()
urlkwargs = MultiDict()
try:
# Aggregations.
extr = search._extra.copy()
search.update_from_dict(query_q)
search._extra.update(extr)
except SyntaxError:
q = request.values.get('q', '') if index_id is None else index_id
current_app.logger.debug("Failed parsing query: {0}".format(q),
exc_info=True)
raise InvalidQueryRESTError()
from invenio_records_rest.sorter import default_sorter_factory
search_index = search._index[0]
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_q)
return search, urlkwargs
def search_factory(self, search, query_parser=None):
"""Parse query using elasticsearch DSL query.
Terms defined by: RERO_ILS_QUERY_BOOSTING will be boosted
at the query level.
:param self: REST view.
:param search: Elastic search DSL search instance.
:param query_parser: a specific query parser
:return: Tuple with search instance and URL arguments.
"""
def _default_parser(qstr=None, query_boosting=[]):
"""Default parser that uses the Q() from elasticsearch_dsl."""
query_type = 'query_string'
default_operator = 'OR'
if request.args.get('simple'):
query_type = 'simple_query_string'
default_operator = 'AND'
if qstr:
# TODO: remove this bad hack
qstr = _PUNCTUATION_REGEX.sub(' ', qstr)
qstr = re.sub('\s+', ' ', qstr).rstrip()
if not query_boosting:
return Q(query_type,
query=qstr,
default_operator=default_operator)
else:
return Q('bool',
should=[
Q(query_type,
query=qstr,
boost=2,
fields=query_boosting,
default_operator=default_operator),
Q(query_type,
query=qstr,
default_operator=default_operator)
])
return Q()
def _boosting_parser(query_boosting, search_index):
"""Elasticsearch boosting fields parser."""
boosting = []
if search_index in query_boosting:
for field, boost in query_boosting[search_index].items():
boosting.append('{}^{}'.format(field, boost))
return boosting
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
query_string = request.values.get('q')
display_score = request.values.get('display_score')
if display_score:
search = search.extra(explain=True)
query_parser = query_parser or _default_parser
search_index = search._index[0]
query_boosting = _boosting_parser(
current_app.config['RERO_ILS_QUERY_BOOSTING'], search_index)
try:
search = search.query(query_parser(query_string, query_boosting))
except SyntaxError:
current_app.logger.debug(
'Failed parsing query: {0}'.format(request.values.get('q', '')),
exc_info=True,
)
raise InvalidQueryRESTError()
search, urlkwargs = default_facets_factory(search, search_index)
# i18n translated facets
search = i18n_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
urlkwargs.add('q', query_string)
return search, urlkwargs
def default_search_factory(self, search, query_parser=None, search_type=None):
"""Parse query using Weko-Query-Parser. MetaData Search.
:param self: REST view.
:param search: Elastic search DSL search instance.
:param query_parser: Query parser. (Default: ``None``)
:returns: Tuple with search instance and URL arguments.
"""
def _get_search_qs_query(qs=None):
"""Qs of search bar keywords for detail simple search.
:param qs: Query string.
:return: Query parser.
"""
q = Q('query_string',
query=qs,
default_operator='and',
fields=['search_*', 'search_*.ja']) if qs else None
return q
def _get_detail_keywords_query():
"""Get keywords query.
:return: Query parser.
"""
def _get_keywords_query(k, v):
qry = None
kv = request.values.get(k)
if not kv:
return
if isinstance(v, str):
name_dict = dict(operator="and")
name_dict.update(dict(query=kv))
qry = Q('match', **{v: name_dict})
elif isinstance(v, list):
qry = Q('multi_match',
query=kv,
type='most_fields',
minimum_should_match='75%',
operator='and',
fields=v)
elif isinstance(v, dict):
for key, vlst in v.items():
if isinstance(vlst, list):
shud = []
kvl = [
x for x in kv.split(',')
if x.isdecimal() and int(x) < len(vlst)
]
for j in map(partial(lambda x, y: x[int(y)], vlst),
kvl):
name_dict = dict(operator="and")
name_dict.update(dict(query=j))
shud.append(Q('match', **{key: name_dict}))
kvl = [
x for x in kv.split(',')
if not x.isdecimal() and x in vlst
]
for j in kvl:
name_dict = dict(operator="and")
name_dict.update(dict(query=j))
shud.append(Q('match', **{key: name_dict}))
if shud:
return Q('bool', should=shud)
elif isinstance(v, tuple) and len(v) >= 2:
shud = []
for i in map(lambda x: v[1](x), kv.split(',')):
shud.append(Q('term', **{v[0]: i}))
if shud:
qry = Q('bool', should=shud)
return qry
def _get_nested_query(k, v):
# text value
kv = request.values.get(k)
if not kv:
return
shuld = []
if isinstance(v, tuple) and len(v) > 1 and isinstance(v[1], dict):
# attr keyword in request url
for attr_key, attr_val_str in map(
lambda x: (x, request.values.get(x)),
list(v[1].keys())):
attr_obj = v[1].get(attr_key)
if isinstance(attr_obj, dict) and attr_val_str:
if isinstance(v[0], str) and not len(v[0]):
# For ID search
for key in attr_val_str.split(','):
attr = attr_obj.get(key)
if isinstance(attr, tuple):
attr = [attr]
if isinstance(attr, list):
for alst in attr:
if isinstance(alst, tuple):
val_attr_lst = alst[1].split('=')
name = alst[0] + ".value"
name_dict = dict(operator="and")
name_dict.update(dict(query=kv))
mut = [
Q('match', **{name: name_dict})
]
qt = None
if '=*' not in alst[1]:
name = alst[0] + \
"." + val_attr_lst[0]
qt = [
Q(
'term', **{
name:
val_attr_lst[1]
})
]
mut.extend(qt or [])
qry = Q('bool', must=mut)
shuld.append(
Q('nested',
path=alst[0],
query=qry))
else:
attr_key_hit = [
x for x in attr_obj.keys() if v[0] + "." in x
]
if attr_key_hit:
vlst = attr_obj.get(attr_key_hit[0])
if isinstance(vlst, list):
attr_val = [
x for x in attr_val_str.split(',')
if x.isdecimal() and int(x) < len(vlst)
]
if attr_val:
shud = []
name = v[0] + ".value"
name_dict = dict(operator="and")
name_dict.update(dict(query=kv))
qm = Q('match', **{name: name_dict})
for j in map(
partial(
lambda m, n: m[int(n)],
vlst), attr_val):
name = attr_key_hit[0]
qm = Q('term', **{name: j})
shud.append(qm)
shuld.append(
Q('nested',
path=v[0],
query=Q('bool',
should=shud,
must=[qm])))
return Q('bool', should=shuld) if shuld else None
def _get_date_query(k, v):
# text value
qry = None
if isinstance(v, list) and len(v) >= 2:
date_from = request.values.get(k + "_" + v[0][0])
date_to = request.values.get(k + "_" + v[0][1])
if not date_from or not date_to:
return
date_from = datetime.strptime(date_from,
'%Y%m%d').strftime('%Y-%m-%d')
date_to = datetime.strptime(date_to,
'%Y%m%d').strftime('%Y-%m-%d')
qv = {}
qv.update(dict(gte=date_from))
qv.update(dict(lte=date_to))
if isinstance(v[1], str):
qry = Q('range', **{v[1]: qv})
elif isinstance(v[1], tuple) and len(v[1]) >= 2:
path = v[1][0]
dt = v[1][1]
if isinstance(dt, dict):
for attr_key, attr_val_str in map(
lambda x: (x, request.values.get(x)),
list(dt.keys())):
attr_obj = dt.get(attr_key)
if isinstance(attr_obj, dict) and attr_val_str:
attr_key_hit = [
x for x in attr_obj.keys()
if path + "." in x
]
if attr_key_hit:
vlst = attr_obj.get(attr_key_hit[0])
if isinstance(vlst, list):
attr_val = [
x for x in attr_val_str.split(',')
]
shud = []
for j in map(
partial(
lambda m, n: m[int(n)],
vlst), attr_val):
qt = Q('term',
**{attr_key_hit[0]: j})
shud.append(qt)
qry = Q('range',
**{path + ".value": qv})
qry = Q('nested',
path=path,
query=Q('bool',
should=shud,
must=[qry]))
return qry
kwd = current_app.config['WEKO_SEARCH_KEYWORDS_DICT']
ks = kwd.get('string')
kd = kwd.get('date')
kn = kwd.get('nested')
mut = []
try:
for k, v in ks.items():
qy = _get_keywords_query(k, v)
if qy:
mut.append(qy)
for k, v in kn.items():
qy = _get_nested_query(k, v)
if qy:
mut.append(qy)
for k, v in kd.items():
qy = _get_date_query(k, v)
if qy:
mut.append(qy)
except Exception as e:
current_app.logger.exception(
'Detail search query parser failed. err:{0}'.format(e))
return mut
def _get_simple_search_query(qs=None):
"""Query parser for simple search.
:param qs: Query string.
:return: Query parser.
"""
# add Permission filter by publish date and status
mt = get_permission_filter()
q = _get_search_qs_query(qs)
if q:
mt.append(q)
mt.extend(_get_detail_keywords_query())
return Q('bool', must=mt) if mt else Q()
def _get_simple_search_community_query(community_id, qs=None):
"""Query parser for simple search.
:param qs: Query string.
:return: Query parser.
"""
# add Permission filter by publish date and status
comm = Community.get(community_id)
root_node_id = comm.root_node_id
mt = get_permission_filter(root_node_id)
q = _get_search_qs_query(qs)
if q:
mt.append(q)
mt.extend(_get_detail_keywords_query())
return Q('bool', must=mt) if mt else Q()
def _default_parser(qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl.
Full text Search.
Detail Search.
:param qstr: Query string.
:returns: Query parser.
"""
# add Permission filter by publish date and status
mt = get_permission_filter()
# multi keywords search filter
kmt = _get_detail_keywords_query()
# detail search
if kmt:
mt.extend(kmt)
q = _get_search_qs_query(qs)
if q:
mt.append(q)
else:
# Full Text Search
if qstr:
q_s = Q('multi_match',
query=qstr,
operator='and',
fields=[
'content.file.content^1.5',
'content.file.content.ja^1.2', '_all',
'search_string'
],
type='most_fields',
minimum_should_match='75%')
mt.append(q_s)
return Q('bool', must=mt) if mt else Q()
def _default_parser_community(community_id, qstr=None):
"""Default parser that uses the Q() from elasticsearch_dsl.
Full text Search.
Detail Search.
:param qstr: Query string.
:returns: Query parser.
"""
# add Permission filter by publish date and status
comm = Community.get(community_id)
root_node_id = comm.root_node_id
mt = get_permission_filter(root_node_id)
# multi keywords search filter
kmt = _get_detail_keywords_query()
# detail search
if kmt:
mt.extend(kmt)
q = _get_search_qs_query(qs)
if q:
mt.append(q)
else:
# Full Text Search
if qstr:
q_s = Q('multi_match',
query=qstr,
operator='and',
fields=[
'content.file.content^1.5',
'content.file.content.ja^1.2', '_all',
'search_string'
],
type='most_fields',
minimum_should_match='75%')
mt.append(q_s)
return Q('bool', must=mt) if mt else Q()
from invenio_records_rest.facets import default_facets_factory
from invenio_records_rest.sorter import default_sorter_factory
# add by ryuu at 1004 start curate
comm_ide = request.values.get('provisional_communities')
# simple search
comm_id_simple = request.values.get('community')
# add by ryuu at 1004 end
if comm_id_simple is not None:
query_parser = query_parser or _default_parser_community
else:
query_parser = query_parser or _default_parser
if search_type is None:
search_type = request.values.get('search_type')
qs = request.values.get('q')
# full text search
if search_type and '0' in search_type:
if comm_id_simple is not None:
query_q = query_parser(comm_id_simple, qs)
else:
query_q = query_parser(qs)
else:
# simple search
if comm_ide is not None:
query_q = _get_simple_search_community_query(comm_ide, qs)
elif comm_id_simple is not None:
query_q = _get_simple_search_community_query(comm_id_simple, qs)
else:
query_q = _get_simple_search_query(qs)
src = {'_source': {'excludes': ['content']}}
search._extra.update(src)
try:
search = search.filter(query_q)
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = default_facets_factory(search, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
urlkwargs.add(key, value)
# defalult sort
if not sortkwargs:
sort_key, sort = SearchSetting.get_default_sort(
current_app.config['WEKO_SEARCH_TYPE_KEYWORD'])
sort_obj = dict()
key_fileds = SearchSetting.get_sort_key(sort_key)
if sort == 'desc':
sort_obj[key_fileds] = dict(order='desc')
sort_key = '-' + sort_key
else:
sort_obj[key_fileds] = dict(order='asc')
search._sort.append(sort_obj)
urlkwargs.add('sort', sort_key)
urlkwargs.add('q', query_q)
return search, urlkwargs
def item_path_search_factory(self, search, index_id=None):
"""Parse query using Weko-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _get_index_earch_query():
query_q = {
"_source": {
"excludes": ['content']
},
"query": {
"bool": {
"must": [{
"match": {
"path.tree": "@index"
}
}, {
"match": {
"relation_version_is_last": "true"
}
}]
}
},
"aggs": {
"path": {
"terms": {
"field": "path.tree",
"include": "@index|@index/[^/]+",
"size": "@count"
},
"aggs": {
"date_range": {
"filter": {
"match": {
"publish_status": "0"
}
},
"aggs": {
"available": {
"range": {
"field":
"publish_date",
"ranges": [{
"from": "now+1d/d"
}, {
"to": "now+1d/d"
}]
},
}
}
},
"no_available": {
"filter": {
"bool": {
"must_not": [{
"match": {
"publish_status": "0"
}
}]
}
}
}
}
}
},
"post_filter": {}
}
# add item type aggs
query_q['aggs']['path']['aggs']. \
update(get_item_type_aggs(search._index[0]))
q = request.values.get('q') if index_id is None else index_id
if q:
mut = get_permission_filter(q)
else:
mut = get_permission_filter()
if mut:
mut = list(map(lambda x: x.to_dict(), mut))
post_filter = query_q['post_filter']
if mut[0].get('bool'):
post_filter['bool'] = mut[0]['bool']
else:
post_filter['bool'] = {'must': mut}
# create search query
if q:
try:
fp = Indexes.get_self_path(q)
if fp:
query_q = json.dumps(query_q).replace("@index", fp.path)
query_q = json.loads(query_q)
except BaseException:
pass
query_q = json.dumps(query_q).replace("@count",
str(Indexes.get_index_count()))
query_q = json.loads(query_q)
return query_q
# create a index search query
query_q = _get_index_earch_query()
urlkwargs = MultiDict()
try:
# Aggregations.
extr = search._extra.copy()
search.update_from_dict(query_q)
search._extra.update(extr)
except SyntaxError:
q = request.values.get('q', '') if index_id is None else index_id
current_app.logger.debug("Failed parsing query: {0}".format(q),
exc_info=True)
raise InvalidQueryRESTError()
from invenio_records_rest.sorter import default_sorter_factory
search_index = search._index[0]
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
# set custom sort option
if 'custom_sort' in value:
ind_id = request.values.get('q', '')
search._sort = []
if value == 'custom_sort':
script_str, default_sort = SearchSetting.get_custom_sort(
ind_id, 'asc')
else:
script_str, default_sort = SearchSetting.get_custom_sort(
ind_id, 'desc')
search._sort.append(script_str)
search._sort.append(default_sort)
# set selectbox
urlkwargs.add(key, value)
# default sort
if not sortkwargs:
sort_key, sort = SearchSetting.get_default_sort(
current_app.config['WEKO_SEARCH_TYPE_INDEX'])
sort_obj = dict()
key_fileds = SearchSetting.get_sort_key(sort_key)
if 'custom_sort' not in sort_key:
if sort == 'desc':
sort_obj[key_fileds] = dict(order='desc')
sort_key = '-' + sort_key
else:
sort_obj[key_fileds] = dict(order='asc')
search._sort.append(sort_obj)
else:
if sort == 'desc':
ind_id = request.values.get('q', '')
script_str, default_sort = SearchSetting.get_custom_sort(
ind_id, 'desc')
sort_key = '-' + sort_key
else:
script_str, default_sort = SearchSetting.get_custom_sort(
ind_id, 'asc')
search._sort = []
search._sort.append(script_str)
search._sort.append(default_sort)
urlkwargs.add('sort', sort_key)
urlkwargs.add('q', query_q)
return search, urlkwargs
def feedback_email_search_factory(self, search):
"""Factory for search feedback email list.
:param self:
:param search:
:return:
"""
def _get_query():
query_string = "_type:{} AND " \
"relation_version_is_last:true " \
.format(current_app.config['INDEXER_DEFAULT_DOC_TYPE'])
query_q = {
"size": 0,
"query": {
"bool": {
"must": [{
"nested": {
"path": "feedback_mail_list",
"query": {
"bool": {
"must": [{
"exists": {
"field": "feedback_mail_list.email"
}
}]
}
}
}
}, {
"query_string": {
"query": query_string
}
}]
}
},
"aggs": {
"feedback_mail_list": {
"nested": {
"path": "feedback_mail_list"
},
"aggs": {
"email_list": {
"terms": {
"field": "feedback_mail_list.email",
"size": config.WEKO_SEARCH_MAX_FEEDBACK_MAIL
},
"aggs": {
"top_tag_hits": {
"top_hits": {}
}
}
}
}
}
}
}
return query_q
query_q = _get_query()
try:
# Aggregations.
extr = search._extra.copy()
search.update_from_dict(query_q)
search._extra.update(extr)
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(query_q),
exc_info=True)
raise InvalidQueryRESTError()
return search