def remove(self, taxonomy_term: TaxonomyTerm = None, taxonomy_code: str = None, slug: str = None) -> None: """ Remove taxonomy term from elasticsearch index. It takes either TaxonomyTerm class or taxonomy code with slug as strings. :param taxonomy_term: Taxonomy term class from flask-taxonomies :type taxonomy_term: TaxonomyTerm :param taxonomy_code: Code of taxonomy. :type taxonomy_code: str :param slug: Taxonomy slug as string :type slug: str :return: None :rtype: None """ if taxonomy_term: id_ = taxonomy_term.id elif taxonomy_code and slug: id_ = self.get(taxonomy_code, slug)["id"] else: raise InvalidTermIdentification( "TaxonomyTerm or Taxonomy Code with slug must be specified") current_search_client.delete(index=self.index, id=id_)
def sample_record(app): record = { "$schema": "http://localhost:5000/schemas/records/hep.json", "control_number": "123", "titles": [ { "title": "Supersymmetric gauge field theory and string theory" } ], "collections": [ { "primary": "HEP" }, { "primary": "THESIS" } ] } record = _create_and_index_record(record) yield record pid = PersistentIdentifier.get('literature', '123') db.session.delete(pid) record.delete(force=True) es.delete(index='records-hep', doc_type='hep', id=pid.object_uuid) db.session.commit()
def index_record_modification(sender, changes): """Example handler for indexing record metadata.""" for obj, change in changes: if isinstance(obj, RecordMetadata): if change in ("insert", "update"): current_search_client.index(index="records", doc_type="record", id=obj.id, body=obj.json) elif change in ("delete"): current_search_client.delete(index="records", doc_type="record", id=obj.id)
def restricted_record(app): restricted_collection = Collection( name='Restricted Collection', dbquery='collections.primary:"Restricted Collection"', ) another_restricted_collection = Collection( name='Another Restricted Collection', dbquery='collections.primary:"Another Restricted Collection"', ) db.session.add_all([restricted_collection, another_restricted_collection]) db.session.commit() record = { "$schema": "http://localhost:5000/schemas/records/hep.json", "control_number": "222", "titles": [ { "title": "Supersymmetric gauge field theory and string theory" } ], "collections": [ { "primary": "HEP" }, { "primary": "THESIS" }, { "primary": "Restricted Collection" }, { "primary": "Another Restricted Collection" } ] } record = _create_and_index_record(record) yield record with app.app_context(): # app context needed to avoid API app from being used after using # the api client. Needed for invenio-collections to be available. collection = Collection.query.filter_by( name='Restricted Collection' ).one() another_collection = Collection.query.filter_by( name='Another Restricted Collection' ).one() pid = PersistentIdentifier.get('literature', '222') es.delete(index='records-hep', doc_type='hep', id=pid.object_uuid) db.session.delete(collection) db.session.delete(another_collection) db.session.delete(pid) db.session.commit()
def _delete_percolator(spec, search_pattern): """Delete percolator associated with the new oaiset.""" if spec: for index in current_search.mappings.keys(): # Create the percolator doc_type in the existing index for >= ES5 percolator_doc_type = _get_percolator_doc_type(index) _create_percolator_mapping(index, percolator_doc_type) current_search_client.delete( index=index, doc_type=percolator_doc_type, id='oaiset-{}'.format(spec), ignore=[404] )
def _delete_percolator(spec, search_pattern): """Delete percolator associated with the new oaiset.""" if spec: for index in current_search.mappings.keys(): # Create the percolator doc_type in the existing index for >= ES5 percolator_doc_type = _get_percolator_doc_type(index) _create_percolator_mapping(index, percolator_doc_type) current_search_client.delete(index=index, doc_type=percolator_doc_type, id='oaiset-{}'.format(spec), ignore=[404])
def delete_item_from_index(id, index, doc_type, parent=None): """ Given an id, deletes an item from the index. :param id: :param index: :param doc_type: :return: """ if parent: es.delete(index=index, doc_type=doc_type, id=id, parent=parent) else: es.delete(index=index, doc_type=doc_type, id=id)
def delete_from_index(self, index=None, doc_type=None): """Delete given record from index.""" config = current_app.config['WORKFLOWS_UI_DATA_TYPES'].get( self["_workflow"]["data_type"] ) if config or (index and doc_type): current_search_client.delete( index=index or config.get('search_index'), doc_type=doc_type or config.get('search_type'), id=str(self['id']), ignore=404 )
def delete_item_from_index(id, index, doc_type, parent=None): """ Given an id, deletes an item from the index. :param id: :param index: :param doc_type: :param parent: the parent record id :return: """ if parent: es.delete(index=index, id=id, routing=parent) else: es.delete(index=index, id=id, routing=id)
def _delete_percolator(spec, search_pattern): """Delete percolator associated with the removed/updated oaiset.""" oai_records_index = current_app.config["OAISERVER_RECORD_INDEX"] # Create the percolator doc_type in the existing index for >= ES5 for index, mapping_path in current_search.mappings.items(): # Skip indices/mappings not used by OAI-PMH if not index.startswith(oai_records_index): continue percolator_doc_type = _get_percolator_doc_type(index) current_search_client.delete( index=_build_percolator_index_name(index), doc_type=percolator_doc_type, id="oaiset-{}".format(spec), ignore=[404], )
def index_record_modification(sender, changes): """Example handler for indexing record metadata.""" for obj, change in changes: if isinstance(obj, RecordMetadata): if change in ('insert', 'update'): current_search_client.index( index='records', doc_type='record', id=obj.id, body=obj.json, ) elif change in ('delete'): current_search_client.delete( index='records', doc_type='record', id=obj.id, )
def delete_from_index(self, index=None, doc_type=None): """Delete given record from index.""" config = current_app.config['WORKFLOWS_UI_DATA_TYPES'].get( self["_workflow"]["data_type"]) if config or (index and doc_type): try: current_search_client.delete(index=index or config.get('search_index'), doc_type=doc_type or config.get('search_type'), id=str(self['id']), ignore=404) except TransportError: current_app.logger.exception() current_app.logger.error( "Problem while indexing workflow object {0}".format( self.model.id))
def delete_from_index(self, index=None, doc_type=None): """Delete given record from index.""" config = current_app.config['WORKFLOWS_UI_DATA_TYPES'].get( self["_workflow"]["data_type"] ) if config or (index and doc_type): try: current_search_client.delete( index=index or config.get('search_index'), doc_type=doc_type or config.get('search_type'), id=str(self['id']), ignore=404 ) except TransportError: current_app.logger.exception() current_app.logger.error( "Problem while indexing workflow object {0}".format( self.model.id ) )
def index_record_modification(sender, changes): """Reset the set of processed records for the next session.""" records_to_index = flask.g.get('invenio_search_records_to_index', dict()) records_to_delete = flask.g.get('invenio_search_records_to_delete', set()) for id in records_to_index: if id not in records_to_delete: current_search_client.index( index='invenio_records_rest_test_index', doc_type='record', id=id, body=records_to_index[id].body, version=records_to_index[id].version, version_type='external_gte', ) for id in records_to_delete: current_search_client.delete( index='invenio_records_rest_test_index', doc_type='record', id=id, ) flask.g.invenio_search_records_to_index = dict() flask.g.invenio_search_records_to_delete = set()
def index_record_modification(sender, changes): """Reset the set of processed records for the next session.""" records_to_index = flask.g.get('invenio_search_records_to_index', dict()) records_to_delete = flask.g.get('invenio_search_records_to_delete', set()) es_index = current_app.config["RECORDS_REST_DEFAULT_SEARCH_INDEX"] for id in records_to_index: if id not in records_to_delete: current_search_client.index( index=es_index, doc_type='testrecord-v1.0.0', id=id, body=records_to_index[id].body, version=records_to_index[id].version, version_type='external_gte', ) for id in records_to_delete: current_search_client.delete( index=es_index, doc_type='testrecord-v1.0.0', id=id, ) flask.g.invenio_search_records_to_index = dict() flask.g.invenio_search_records_to_delete = set()
def delete(self): """Delete acl from any internal representation / index for the acl.""" for schema in self.schemas: acl_index_name = self.get_acl_index_name(schema_to_index(schema)[0]) try: return current_search_client.delete( index=acl_index_name, **add_doc_type(current_app.config['INVENIO_EXPLICIT_ACLS_DOCTYPE_NAME']), id=self.id, refresh='wait_for' ) except: # pragma: no cover logger.exception('Strange, the ACL has not been indexed: %s', repr(self)) finally: current_search_client.indices.flush(index=acl_index_name)
def test_bookmark_removal(app, es_with_templates, mock_event_queue): """Remove aggregation bookmark and restart aggregation. This simulates the scenario where aggregations have been created but the the bookmarks have not been set due to an error. """ mock_event_queue.consume.return_value = [ _create_file_download_event(date) for date in [ (2017, 6, 2, 15), # second event on the same date (2017, 7, 1) ] ] indexer = EventsIndexer(mock_event_queue) indexer.run() current_search_client.indices.refresh(index='*') def aggregate_and_check_version(expected_version): # Aggregate events StatAggregator(name='file-download-agg', event='file-download', aggregation_field='file_id', aggregation_interval='day', query_modifiers=[]).run() current_search_client.indices.refresh(index='*') res = current_search_client.search( index='stats-file-download', doc_type='file-download-day-aggregation', version=True) for hit in res['hits']['hits']: assert hit['_version'] == expected_version aggregate_and_check_version(1) aggregate_and_check_version(1) # Delete all bookmarks bookmarks = Search( using=current_search_client, index='stats-file-download', doc_type='file-download-agg-bookmark').query('match_all') for bookmark in bookmarks: res = current_search_client.delete( index=bookmark.meta.index, id=bookmark.meta.id, doc_type='file-download-agg-bookmark') current_search_client.indices.refresh(index='*') # the aggregations should have been overwritten aggregate_and_check_version(2)
def teardown(app): with app.app_context(): es.delete(index='records-authors', doc_type='authors', id=10)
def _delete_es_records(index, doc_type, merged_uuid, deleted_uuid): es.delete(index=index, doc_type=doc_type, id=merged_uuid) es.delete(index=index, doc_type=doc_type, id=deleted_uuid)