def import_populate_db(): es_host = app.config['ELASTICSEARCH'] es_idx = app.config['ES_IDX'] es = Elasticsearch(f'{es_host}:9200') idx = IndicesClient(es) if idx.exists(index=es_idx): existing_docs = es.count(index=es_idx).get('count') if existing_docs: return existing_docs, 0 app.logger.info("Drop and creating index and mapping definition") idx.delete(es_idx) else: idx.create(index=es_idx, body=mapping) app.logger.info("Populating elasticsearch with documents") errors = [] for pdv in pdvs: document_id = ''.join(filter(str.isdigit, pdv.get('document'))) pdv['document'] = document_id del pdv['id'] try: es.index(index=es_idx, body=pdv, id=document_id) except Exception as ex: app.logger.exception(ex) errors.append({'id': pdv.get('id'), 'description': ex.args}) inserted = len(pdvs) - len(errors) return errors, inserted
class ElasticIndiceDriver: def __init__(self, client: Elasticsearch): self.client = IndicesClient(client) def create_index(self, index: str, mapping: dict): self.client.create(index, json.dumps(mapping)) def clean_index(self, index: str): self.client.delete(index) self.client.delete(f'{index}-finished')
def create(cls, user, **kwargs): """Create user index.""" # Create index for user client = Elasticsearch(cls.__url__) indice = IndicesClient(client) if indice.exists(index=user.user_id): if 'delete_existing' in kwargs and kwargs['delete_existing']: log.warn('Deleting existing index for user %s' % user.user_id) indice.delete(index=user.user_id) else: log.warn('Index already exists for user %s' % user.user_id) return False log.info('Creating index for user %s' % user.user_id) indice.create(index=user.user_id) return True
def recreate_index_model(self, model: Union[type[Gallery], type[Archive]]): from elasticsearch.client.indices import IndicesClient indices_client = IndicesClient(client=self.es_client) index_name = model._meta.es_index_name # type: ignore if indices_client.exists(index_name): indices_client.delete(index=index_name) indices_client.create(index=index_name) indices_client.close(index=index_name) indices_client.put_settings( index=index_name, body={ "index": { "max_result_window": settings.MAX_RESULT_WINDOW }, "analysis": { "filter": { "edge_ngram_filter": { "type": "edge_ngram", "min_gram": 2, "max_gram": 20 } }, "analyzer": { "edge_ngram_analyzer": { "type": "custom", "tokenizer": "standard", "filter": ["lowercase", "edge_ngram_filter"] } } } }) indices_client.put_mapping( body=model._meta.es_mapping, # type: ignore index=index_name, ) indices_client.open(index=index_name)
async def shutdown_elastic_search(elastic_search: Elasticsearch, indices_client: IndicesClient) -> None: indices_client.delete(index=INDEX) elastic_search.close()