def remove(self,
taxonomy_term: TaxonomyTerm = None,
taxonomy_code: str = None,
slug: str = None) -> None:
"""
Remove taxonomy term from elasticsearch index. It takes either TaxonomyTerm class or
taxonomy code with slug as strings.
:param taxonomy_term: Taxonomy term class from flask-taxonomies
:type taxonomy_term: TaxonomyTerm
:param taxonomy_code: Code of taxonomy.
:type taxonomy_code: str
:param slug: Taxonomy slug as string
:type slug: str
:return: None
:rtype: None
"""
if taxonomy_term:
id_ = taxonomy_term.id
elif taxonomy_code and slug:
id_ = self.get(taxonomy_code, slug)["id"]
else:
raise InvalidTermIdentification(
"TaxonomyTerm or Taxonomy Code with slug must be specified")
current_search_client.delete(index=self.index, id=id_)
def sample_record(app):
record = {
"$schema": "http://localhost:5000/schemas/records/hep.json",
"control_number": "123",
"titles": [
{
"title": "Supersymmetric gauge field theory and string theory"
}
],
"collections": [
{
"primary": "HEP"
},
{
"primary": "THESIS"
}
]
}
record = _create_and_index_record(record)
yield record
pid = PersistentIdentifier.get('literature', '123')
db.session.delete(pid)
record.delete(force=True)
es.delete(index='records-hep', doc_type='hep', id=pid.object_uuid)
db.session.commit()
def index_record_modification(sender, changes):
"""Example handler for indexing record metadata."""
for obj, change in changes:
if isinstance(obj, RecordMetadata):
if change in ("insert", "update"):
current_search_client.index(index="records", doc_type="record", id=obj.id, body=obj.json)
elif change in ("delete"):
current_search_client.delete(index="records", doc_type="record", id=obj.id)
def restricted_record(app):
restricted_collection = Collection(
name='Restricted Collection',
dbquery='collections.primary:"Restricted Collection"',
)
another_restricted_collection = Collection(
name='Another Restricted Collection',
dbquery='collections.primary:"Another Restricted Collection"',
)
db.session.add_all([restricted_collection, another_restricted_collection])
db.session.commit()
record = {
"$schema": "http://localhost:5000/schemas/records/hep.json",
"control_number": "222",
"titles": [
{
"title": "Supersymmetric gauge field theory and string theory"
}
],
"collections": [
{
"primary": "HEP"
},
{
"primary": "THESIS"
},
{
"primary": "Restricted Collection"
},
{
"primary": "Another Restricted Collection"
}
]
}
record = _create_and_index_record(record)
yield record
with app.app_context():
# app context needed to avoid API app from being used after using
# the api client. Needed for invenio-collections to be available.
collection = Collection.query.filter_by(
name='Restricted Collection'
).one()
another_collection = Collection.query.filter_by(
name='Another Restricted Collection'
).one()
pid = PersistentIdentifier.get('literature', '222')
es.delete(index='records-hep', doc_type='hep', id=pid.object_uuid)
db.session.delete(collection)
db.session.delete(another_collection)
db.session.delete(pid)
db.session.commit()
def _delete_percolator(spec, search_pattern):
"""Delete percolator associated with the new oaiset."""
if spec:
for index in current_search.mappings.keys():
# Create the percolator doc_type in the existing index for >= ES5
percolator_doc_type = _get_percolator_doc_type(index)
_create_percolator_mapping(index, percolator_doc_type)
current_search_client.delete(
index=index, doc_type=percolator_doc_type,
id='oaiset-{}'.format(spec), ignore=[404]
)
def _delete_percolator(spec, search_pattern):
"""Delete percolator associated with the new oaiset."""
if spec:
for index in current_search.mappings.keys():
# Create the percolator doc_type in the existing index for >= ES5
percolator_doc_type = _get_percolator_doc_type(index)
_create_percolator_mapping(index, percolator_doc_type)
current_search_client.delete(index=index,
doc_type=percolator_doc_type,
id='oaiset-{}'.format(spec),
ignore=[404])
def delete_item_from_index(id, index, doc_type, parent=None):
"""
Given an id, deletes an item from the index.
:param id:
:param index:
:param doc_type:
:return:
"""
if parent:
es.delete(index=index, doc_type=doc_type, id=id, parent=parent)
else:
es.delete(index=index, doc_type=doc_type, id=id)
def delete_from_index(self, index=None, doc_type=None):
"""Delete given record from index."""
config = current_app.config['WORKFLOWS_UI_DATA_TYPES'].get(
self["_workflow"]["data_type"]
)
if config or (index and doc_type):
current_search_client.delete(
index=index or config.get('search_index'),
doc_type=doc_type or config.get('search_type'),
id=str(self['id']),
ignore=404
)
def delete_item_from_index(id, index, doc_type, parent=None):
"""
Given an id, deletes an item from the index.
:param id:
:param index:
:param doc_type:
:return:
"""
if parent:
es.delete(index=index, doc_type=doc_type, id=id, parent=parent)
else:
es.delete(index=index, doc_type=doc_type, id=id)
def delete_item_from_index(id, index, doc_type, parent=None):
"""
Given an id, deletes an item from the index.
:param id:
:param index:
:param doc_type:
:param parent: the parent record id
:return:
"""
if parent:
es.delete(index=index, id=id, routing=parent)
else:
es.delete(index=index, id=id, routing=id)
def _delete_percolator(spec, search_pattern):
"""Delete percolator associated with the removed/updated oaiset."""
oai_records_index = current_app.config["OAISERVER_RECORD_INDEX"]
# Create the percolator doc_type in the existing index for >= ES5
for index, mapping_path in current_search.mappings.items():
# Skip indices/mappings not used by OAI-PMH
if not index.startswith(oai_records_index):
continue
percolator_doc_type = _get_percolator_doc_type(index)
current_search_client.delete(
index=_build_percolator_index_name(index),
doc_type=percolator_doc_type,
id="oaiset-{}".format(spec),
ignore=[404],
)
def index_record_modification(sender, changes):
"""Example handler for indexing record metadata."""
for obj, change in changes:
if isinstance(obj, RecordMetadata):
if change in ('insert', 'update'):
current_search_client.index(
index='records',
doc_type='record',
id=obj.id,
body=obj.json,
)
elif change in ('delete'):
current_search_client.delete(
index='records',
doc_type='record',
id=obj.id,
)
def delete_from_index(self, index=None, doc_type=None):
"""Delete given record from index."""
config = current_app.config['WORKFLOWS_UI_DATA_TYPES'].get(
self["_workflow"]["data_type"])
if config or (index and doc_type):
try:
current_search_client.delete(index=index
or config.get('search_index'),
doc_type=doc_type
or config.get('search_type'),
id=str(self['id']),
ignore=404)
except TransportError:
current_app.logger.exception()
current_app.logger.error(
"Problem while indexing workflow object {0}".format(
self.model.id))
def delete_from_index(self, index=None, doc_type=None):
"""Delete given record from index."""
config = current_app.config['WORKFLOWS_UI_DATA_TYPES'].get(
self["_workflow"]["data_type"]
)
if config or (index and doc_type):
try:
current_search_client.delete(
index=index or config.get('search_index'),
doc_type=doc_type or config.get('search_type'),
id=str(self['id']),
ignore=404
)
except TransportError:
current_app.logger.exception()
current_app.logger.error(
"Problem while indexing workflow object {0}".format(
self.model.id
)
)
def index_record_modification(sender, changes):
"""Reset the set of processed records for the next session."""
records_to_index = flask.g.get('invenio_search_records_to_index', dict())
records_to_delete = flask.g.get('invenio_search_records_to_delete', set())
for id in records_to_index:
if id not in records_to_delete:
current_search_client.index(
index='invenio_records_rest_test_index',
doc_type='record',
id=id,
body=records_to_index[id].body,
version=records_to_index[id].version,
version_type='external_gte',
)
for id in records_to_delete:
current_search_client.delete(
index='invenio_records_rest_test_index',
doc_type='record',
id=id,
)
flask.g.invenio_search_records_to_index = dict()
flask.g.invenio_search_records_to_delete = set()
def index_record_modification(sender, changes):
"""Reset the set of processed records for the next session."""
records_to_index = flask.g.get('invenio_search_records_to_index', dict())
records_to_delete = flask.g.get('invenio_search_records_to_delete', set())
for id in records_to_index:
if id not in records_to_delete:
current_search_client.index(
index='invenio_records_rest_test_index',
doc_type='record',
id=id,
body=records_to_index[id].body,
version=records_to_index[id].version,
version_type='external_gte',
)
for id in records_to_delete:
current_search_client.delete(
index='invenio_records_rest_test_index',
doc_type='record',
id=id,
)
flask.g.invenio_search_records_to_index = dict()
flask.g.invenio_search_records_to_delete = set()
def index_record_modification(sender, changes):
"""Reset the set of processed records for the next session."""
records_to_index = flask.g.get('invenio_search_records_to_index', dict())
records_to_delete = flask.g.get('invenio_search_records_to_delete', set())
es_index = current_app.config["RECORDS_REST_DEFAULT_SEARCH_INDEX"]
for id in records_to_index:
if id not in records_to_delete:
current_search_client.index(
index=es_index,
doc_type='testrecord-v1.0.0',
id=id,
body=records_to_index[id].body,
version=records_to_index[id].version,
version_type='external_gte',
)
for id in records_to_delete:
current_search_client.delete(
index=es_index,
doc_type='testrecord-v1.0.0',
id=id,
)
flask.g.invenio_search_records_to_index = dict()
flask.g.invenio_search_records_to_delete = set()
def delete(self):
"""Delete acl from any internal representation / index for the acl."""
for schema in self.schemas:
acl_index_name = self.get_acl_index_name(schema_to_index(schema)[0])
try:
return current_search_client.delete(
index=acl_index_name,
**add_doc_type(current_app.config['INVENIO_EXPLICIT_ACLS_DOCTYPE_NAME']),
id=self.id,
refresh='wait_for'
)
except: # pragma: no cover
logger.exception('Strange, the ACL has not been indexed: %s', repr(self))
finally:
current_search_client.indices.flush(index=acl_index_name)
def test_bookmark_removal(app, es_with_templates, mock_event_queue):
"""Remove aggregation bookmark and restart aggregation.
This simulates the scenario where aggregations have been created but the
the bookmarks have not been set due to an error.
"""
mock_event_queue.consume.return_value = [
_create_file_download_event(date) for date in [
(2017, 6, 2, 15), # second event on the same date
(2017, 7, 1)
]
]
indexer = EventsIndexer(mock_event_queue)
indexer.run()
current_search_client.indices.refresh(index='*')
def aggregate_and_check_version(expected_version):
# Aggregate events
StatAggregator(name='file-download-agg',
event='file-download',
aggregation_field='file_id',
aggregation_interval='day',
query_modifiers=[]).run()
current_search_client.indices.refresh(index='*')
res = current_search_client.search(
index='stats-file-download',
doc_type='file-download-day-aggregation',
version=True)
for hit in res['hits']['hits']:
assert hit['_version'] == expected_version
aggregate_and_check_version(1)
aggregate_and_check_version(1)
# Delete all bookmarks
bookmarks = Search(
using=current_search_client,
index='stats-file-download',
doc_type='file-download-agg-bookmark').query('match_all')
for bookmark in bookmarks:
res = current_search_client.delete(
index=bookmark.meta.index,
id=bookmark.meta.id,
doc_type='file-download-agg-bookmark')
current_search_client.indices.refresh(index='*')
# the aggregations should have been overwritten
aggregate_and_check_version(2)
def teardown(app):
with app.app_context():
es.delete(index='records-authors', doc_type='authors', id=10)
def teardown(app):
with app.app_context():
es.delete(index='records-authors', doc_type='authors', id=10)
def _delete_es_records(index, doc_type, merged_uuid, deleted_uuid):
es.delete(index=index, doc_type=doc_type, id=merged_uuid)
es.delete(index=index, doc_type=doc_type, id=deleted_uuid)
