def load_test_data_into_es(dataset_amt):
log.info("Loading test data into Elasticsearch..")
es_client = ElasticSearchService.get_elasticsearch_service(es_config)
if es_client is None:
log.error("Unable to initialize Elastisearch client")
return False
metax_api = MetaxAPIService.get_metax_api_service(metax_api_config)
if metax_api is None:
log.error("Unable to initialize Metax API client")
return False
if not es_client.ensure_index_existence():
return False
cr_identifiers = metax_api.get_latest_catalog_record_identifiers()
if cr_identifiers:
identifiers_to_load = cr_identifiers[0:min(len(cr_identifiers), dataset_amt)]
identifiers_to_delete = []
es_data_models = convert_identifiers_to_es_data_models(metax_api, identifiers_to_load, identifiers_to_delete)
es_client.do_bulk_request_for_datasets(es_data_models, identifiers_to_delete)
log.info("Test data loaded into Elasticsearch")
return True
log.error("No catalog record identifiers to load")
return False
def delete_search_index():
es_client = ElasticSearchService.get_elasticsearch_service(es_config)
if es_client is None:
log.error("Unable to initialize Elasticsearch client")
return
es_client.delete_index()
def __init__(self):
self.log = get_logger(__name__)
self.event_processing_completed = True
self.init_ok = False
# Get configs
# If these raise errors, let consumer init fail
self.rabbit_settings = get_metax_rabbit_mq_config()
es_settings = get_elasticsearch_config()
self.is_local_dev = True if os.path.isdir("/etsin/ansible") else False
if not self.rabbit_settings or not es_settings:
self.log.error(
"Unable to load RabbitMQ configuration or Elasticsearch configuration"
)
return
self.credentials = pika.PlainCredentials(
self.rabbit_settings['USER'], self.rabbit_settings['PASSWORD'])
self.exchange = self.rabbit_settings['EXCHANGE']
self._set_queue_names(self.is_local_dev)
self.es_client = ElasticSearchService.get_elasticsearch_service(
es_settings)
if self.es_client is None:
return
if not self.es_client.ensure_index_existence():
return
self.init_ok = True
def create_search_index_and_doc_type_mapping_if_not_exist():
es_client = ElasticSearchService.get_elasticsearch_service(es_config)
if es_client is None:
log.error("Unable to initialize Elasticsearch client")
return False
if not es_client.ensure_index_existence():
return False
return True
def __init__(self):
self.metax_api = MetaxAPIService.get_metax_api_service(metax_api_config)
self.es_client = ElasticSearchService.get_elasticsearch_service(es_config)