def import_populate_db():
es_host = app.config['ELASTICSEARCH']
es_idx = app.config['ES_IDX']
es = Elasticsearch(f'{es_host}:9200')
idx = IndicesClient(es)
if idx.exists(index=es_idx):
existing_docs = es.count(index=es_idx).get('count')
if existing_docs:
return existing_docs, 0
app.logger.info("Drop and creating index and mapping definition")
idx.delete(es_idx)
else:
idx.create(index=es_idx, body=mapping)
app.logger.info("Populating elasticsearch with documents")
errors = []
for pdv in pdvs:
document_id = ''.join(filter(str.isdigit, pdv.get('document')))
pdv['document'] = document_id
del pdv['id']
try:
es.index(index=es_idx, body=pdv, id=document_id)
except Exception as ex:
app.logger.exception(ex)
errors.append({'id': pdv.get('id'), 'description': ex.args})
inserted = len(pdvs) - len(errors)
return errors, inserted
class ElasticIndiceDriver:
def __init__(self, client: Elasticsearch):
self.client = IndicesClient(client)
def create_index(self, index: str, mapping: dict):
self.client.create(index, json.dumps(mapping))
def clean_index(self, index: str):
self.client.delete(index)
self.client.delete(f'{index}-finished')
def create(cls, user, **kwargs):
"""Create user index."""
# Create index for user
client = Elasticsearch(cls.__url__)
indice = IndicesClient(client)
if indice.exists(index=user.user_id):
if 'delete_existing' in kwargs and kwargs['delete_existing']:
log.warn('Deleting existing index for user %s' % user.user_id)
indice.delete(index=user.user_id)
else:
log.warn('Index already exists for user %s' % user.user_id)
return False
log.info('Creating index for user %s' % user.user_id)
indice.create(index=user.user_id)
return True
def create(cls, user, **kwargs):
"""Create user index."""
# Create index for user
client = Elasticsearch(cls.__url__)
indice = IndicesClient(client)
if indice.exists(index=user.user_id):
if 'delete_existing' in kwargs and kwargs['delete_existing']:
log.warn('Deleting existing index for user %s' % user.user_id)
indice.delete(index=user.user_id)
else:
log.warn('Index already exists for user %s' % user.user_id)
return False
log.info('Creating index for user %s' % user.user_id)
indice.create(index=user.user_id)
return True
def recreate_index_model(self, model: Union[type[Gallery], type[Archive]]):
from elasticsearch.client.indices import IndicesClient
indices_client = IndicesClient(client=self.es_client)
index_name = model._meta.es_index_name # type: ignore
if indices_client.exists(index_name):
indices_client.delete(index=index_name)
indices_client.create(index=index_name)
indices_client.close(index=index_name)
indices_client.put_settings(
index=index_name,
body={
"index": {
"max_result_window": settings.MAX_RESULT_WINDOW
},
"analysis": {
"filter": {
"edge_ngram_filter": {
"type": "edge_ngram",
"min_gram": 2,
"max_gram": 20
}
},
"analyzer": {
"edge_ngram_analyzer": {
"type": "custom",
"tokenizer": "standard",
"filter": ["lowercase", "edge_ngram_filter"]
}
}
}
})
indices_client.put_mapping(
body=model._meta.es_mapping, # type: ignore
index=index_name,
)
indices_client.open(index=index_name)
async def shutdown_elastic_search(elastic_search: Elasticsearch,
indices_client: IndicesClient) -> None:
indices_client.delete(index=INDEX)
elastic_search.close()