def api():
"""Search API for search UI demo.
.. note::
WARNING! This search API is just for demo proposes only.
"""
page = request.values.get('page', 1, type=int)
size = request.values.get('size', 1, type=int)
query = Query(request.values.get('q', ''))[(page-1)*size:page*size]
# dummy facets
query.body["aggs"] = {
"by_body": {
"terms": {
"field": "summary.summary"
}
},
"by_title": {
"terms": {
"field": "title_statement.title"
}
}
}
response = current_search_client.search(
index=request.values.get('index', 'records'),
doc_type=request.values.get('type'),
body=query.body,
)
return jsonify(**response)
def es_query_factory(index, page, size):
"""Send query directly as query string query to Elasticsearch.
.. warning:
All fields in a record that a user can access are searchable! This means
that if a user can access a record, you cannot include confidential
information into the record (or you must remove it when indexing).
Otherwise a user is able to search for the information.
The reason is that the query string is passed directly to Elasticsearch,
which takes care of parsing the string.
:param index: Index to search in.
:param page: Requested page.
:param size: Request results size.
:returns: Tuple of (query, URL arguments).
"""
query_string = request.values.get('q', '')
query = Query()
if query_string.strip():
query.body['query'] = dict(query_string=dict(
query=query_string,
allow_leading_wildcard=False,
))
query = query[(page - 1) * size:page * size]
return (query, {'q': query_string})
def es_query_factory(index, page, size):
"""Send query directly as query string query to Elasticsearch.
.. warning:
All fields in a record that a user can access are searchable! This means
that if a user can access a record, you cannot include confidential
information into the record (or you must remove it when indexing).
Otherwise a user is able to search for the information.
The reason is that the query string is passed directly to Elasticsearch,
which takes care of parsing the string.
:param index: Index to search in.
:param page: Requested page.
:param size: Request results size.
:returns: Tuple of (query, URL arguments).
"""
query_string = request.values.get('q', '')
query = Query()
if query_string.strip():
query.body['query'] = dict(
query_string=dict(
query=query_string,
allow_leading_wildcard=False,
)
)
query = query[(page-1)*size:page*size]
return (query, {'q': query_string})
def index():
"""Frontpage blueprint."""
query = Query("")
query.body["size"] = 10
query.body["sort"] = [{"creation_date": "desc"}]
response = current_search_client.search(
index='records',
body=query.body,
)
return render_template('zenodo_frontpage/index.html',
records=(h['_source']
for h in response['hits']['hits']))
def index():
"""Frontpage blueprint."""
query = Query("communities:zenodo AND access_right:open")
query.body["size"] = 10
query.body["sort"] = [{"creation_date": "desc"}]
response = current_search_client.search(
index='records',
body=query.body,
)
return render_template(
'zenodo_frontpage/index.html',
records=(h['_source'] for h in response['hits']['hits'])
)
def default_query_factory(index, page, size, query_string=None):
"""Create default ES query based on query-string pattern."""
if not query_string:
query_string = request.values.get('q', '')
query = Query()
if query_string.strip():
query.body['query'] = dict(
query_string=dict(
query=query_string,
allow_leading_wildcard=False,
)
)
query = query[(page-1)*size:page*size]
return (query, {'q': query_string})
def submit_set():
"""Insert a new set."""
form = get_NewSetForm(request.form)
if request.method == 'POST' and form.validate():
new_set = OAISet(spec=form.spec.data,
name=form.name.data,
description=form.description.data,
search_pattern=form.search_pattern.data,
parent=form.parent.data)
db.session.add(new_set)
# this shoul be moved to UPDATER (celery task) and it sould always
# take care of adding records to sets.
##########
query = Query(form.query.data)
response = current_search_client.search(
index='records', # make configurable PER SET
doc_type='record', # make configurable PER SET
body=query.body,
fields='_id, oaiid' # path to oaiid as a configurable
)
ids = [(a['_id'], a['oaiid']) for a in response['hits']['hits']]
add_records_to_set(ids)
#########
db.session.commit()
flash(_('New set %(spec)s was added.', spec=new_set.spec))
return redirect(url_for('.manage_sets'))
return render_template('make_set.html', new_set_form=form)
def get(self, **kwargs):
"""Search records.
:returns: the search result containing hits and aggregations as
returned by invenio-search.
"""
page = request.values.get('page', 1, type=int)
size = request.values.get('size', 10, type=int)
if page*size >= self.max_result_window:
raise MaxResultWindowRESTError()
# Parse and slice query
try:
query = Query(request.values.get('q', ''))[(page-1)*size:page*size]
except SyntaxError:
raise InvalidQueryRESTError()
# Arguments that must be added in prev/next links
urlkwargs = dict()
# Facets
query, qs_kwargs = self.facets_factory(query, self.search_index)
urlkwargs.update(qs_kwargs)
# Sort
query, qs_kwargs = self.sorter_factory(query, self.search_index)
urlkwargs.update(qs_kwargs)
# Execute search
response = current_search_client.search(
index=self.search_index,
doc_type=self.search_type,
body=query.body,
version=True,
)
# Generate links for prev/next
urlkwargs.update(
size=size,
q=request.values.get('q', ''),
_external=True,
)
endpoint = 'invenio_records_rest.{0}_list'.format(self.pid_type)
links = dict(self=url_for(endpoint, page=page, **urlkwargs))
if page > 1:
links['prev'] = url_for(endpoint, page=page-1, **urlkwargs)
if size * page < int(response['hits']['total']) and \
size * page < self.max_result_window:
links['next'] = url_for(endpoint, page=page+1, **urlkwargs)
return self.make_response(
pid_fetcher=self.pid_fetcher,
search_result=response,
links=links,
)
def index():
"""Query Elasticsearch using Invenio query syntax."""
page = request.values.get('page', 1, type=int)
size = request.values.get('size', 1, type=int)
query = Query(request.values.get('q', ''))[(page - 1) * size:page * size]
response = current_search_client.search(
index=request.values.get('index', 'demo'),
doc_type=request.values.get('type'),
body=query.body,
)
return jsonify(**response)
def get(self, **kwargs):
"""Search records.
:returns: the search result containing hits and aggregations as
returned by invenio-search.
"""
page = request.values.get("page", 1, type=int)
size = request.values.get("size", 10, type=int)
sort = request.values.get("sort", "", type=str)
query = Query(request.values.get("q", ""))[(page - 1) * size : page * size]
for sort_key in sort.split(","):
if sort_key:
query = query.sort(sort_key)
response = current_search_client.search(
index=self.search_index, doc_type=self.search_type, body=query.body, version=True
)
links = {}
if page > 1:
links["prev"] = url_for(
"invenio_records_rest.{0}_list".format(self.pid_type),
page=page - 1,
size=size,
sort=sort,
q=request.values.get("q", ""),
_external=True,
)
if size * page < int(response["hits"]["total"]):
links["next"] = url_for(
"invenio_records_rest.{0}_list".format(self.pid_type),
page=page + 1,
size=size,
sort=sort,
q=request.values.get("q", ""),
_external=True,
)
return self.make_response(pid_fetcher=self.pid_fetcher, search_result=response, links=links)
def submit_edit_set(spec):
"""Insert a new set."""
form = get_NewSetForm(request.form)
if request.method == 'POST' and form.validate():
old_set = OAISet.query.filter(spec=spec)
query = Query(old_set.search_pattern)
old_recid = current_search_client.search(index='records',
doc_type='record',
body=query.body,
fields='_id, oaiid')
query = Query(form.search_pattern)
new_recid = current_search_client.search(index='records',
doc_type='record',
body=query.body,
fields='_id, oaiid')
recids_to_delete = set(old_recid) - set(new_recid)
# TODO: marks records as deleted from set
remove_recids_from_set(recids_to_delete)
add_records_to_set(new_recid)
flash('Set was changed')
return redirect(url_for('.manage_sets'))
return render_template('edit_set.html', edit_set_form=form, spec=spec)
def get_expired_embargos(cls):
"""Get records for which the embargo period have expired."""
query_str = 'access_right:{0} AND embargo_date:{{* TO {1}}}'.format(
cls.EMBARGOED,
datetime.utcnow().isoformat()
)
query = Query()
query.body['from'] = 0
query.body['size'] = 1000
query.body['query'] = {
'query_string': {
'query': query_str,
'allow_leading_wildcard': False,
},
}
endpoints = current_app.config['RECORDS_REST_ENDPOINTS']
index = endpoints['recid']['search_index']
response = current_search_client.search(
index=index, body=query.body
)
return [hit['_id'] for hit in response['hits']['hits']]
def default_query_factory(index, page, size):
"""Parse and slice query using Invenio-Query-Parser.
:param index: Index to search in.
:param page: Requested page.
:param size: Request results size.
:returns: Tuple of (query, URL arguments).
"""
query_string = request.values.get('q', '')
try:
query = Query(query_string)[(page - 1) * size:page * size]
except SyntaxError:
current_app.logger.debug("Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
return (query, {'q': query_string})
