def __init__(self):
self.log = get_logger(__name__)
self.event_processing_completed = True
self.init_ok = False
# Get configs
# If these raise errors, let consumer init fail
self.rabbit_settings = get_metax_rabbit_mq_config()
es_settings = get_elasticsearch_config()
self.is_local_dev = True if os.path.isdir("/etsin/ansible") else False
if not self.rabbit_settings or not es_settings:
self.log.error(
"Unable to load RabbitMQ configuration or Elasticsearch configuration"
)
return
self.credentials = pika.PlainCredentials(
self.rabbit_settings['USER'], self.rabbit_settings['PASSWORD'])
self.exchange = self.rabbit_settings['EXCHANGE']
self._set_queue_names(self.is_local_dev)
self.es_client = ElasticSearchService.get_elasticsearch_service(
es_settings)
if self.es_client is None:
return
if not self.es_client.ensure_index_existence():
return
self.init_ok = True
# This file is part of the Etsin service
#
# Copyright 2017-2018 Ministry of Education and Culture, Finland
#
# :author: CSC - IT Center for Science Ltd., Espoo Finland <*****@*****.**>
# :license: MIT
import requests
from requests import HTTPError, ConnectionError, Timeout
import json
from time import sleep
from etsin_finder_search.reindexing_log import get_logger
log = get_logger(__name__)
TIMEOUT = 1200
NUM_RETRIES = 3
class MetaxAPIService:
def __init__(self, metax_api_config):
self.METAX_CATALOG_RECORDS_BASE_URL = 'https://{0}/rest/datasets'.format(metax_api_config['HOST'])
self.METAX_GET_PIDS_URL = self.METAX_CATALOG_RECORDS_BASE_URL + '/identifiers?latest'
self.METAX_GET_ALL_LATEST_DATASETS = \
self.METAX_CATALOG_RECORDS_BASE_URL + '?no_pagination=true&latest&expand_relation=data_catalog'
self.METAX_GET_CATALOG_RECORD_URL = self.METAX_CATALOG_RECORDS_BASE_URL + '/{0}?expand_relation=data_catalog'
self.USER = metax_api_config['USER']
self.PW = metax_api_config['PASSWORD']
self.VERIFY_SSL = metax_api_config.get('VERIFY_SSL', True)
def __init__(self):
self.log = get_logger(__name__)
self.event_processing_completed = True
self.init_ok = False
# Get configs
# If these raise errors, let consumer init fail
rabbit_settings = get_metax_rabbit_mq_config()
es_settings = get_elasticsearch_config()
is_local_dev = True if os.path.isdir("/etsin/ansible") else False
if not rabbit_settings or not es_settings:
self.log.error("Unable to load RabbitMQ configuration or Elasticsearch configuration")
return
# Set up RabbitMQ connection, channel, exchange and queues
credentials = pika.PlainCredentials(
rabbit_settings['USER'], rabbit_settings['PASSWORD'])
# Try connecting every one minute 30 times
try:
connection = pika.BlockingConnection(
pika.ConnectionParameters(
rabbit_settings['HOST'],
rabbit_settings['PORT'],
rabbit_settings['VHOST'],
credentials,
connection_attempts=30,
retry_delay=60))
except Exception as e:
self.log.error(e)
self.log.error("Unable to open RabbitMQ connection")
return
# Set up ElasticSearch client. In case connection cannot be established, try every 2 seconds for 30 seconds
es_conn_ok = False
i = 0
while not es_conn_ok and i < 15:
self.es_client = ElasticSearchService(es_settings)
if self.es_client.client_ok():
es_conn_ok = True
else:
time.sleep(2)
i += 1
if not es_conn_ok or not self._ensure_index_existence():
return
self.channel = connection.channel()
self.exchange = rabbit_settings['EXCHANGE']
self._set_queue_names(is_local_dev)
self._create_and_bind_queues(is_local_dev)
def callback_create(ch, method, properties, body):
if not self._init_event_callback_ok("create", ch, method):
return
body_as_json = self._get_event_json_body(ch, method, body)
if not body_as_json:
return
self._convert_to_es_doc_and_reindex(ch, method, body_as_json)
def callback_update(ch, method, properties, body):
if not self._init_event_callback_ok("update", ch, method):
return
body_as_json = self._get_event_json_body(ch, method, body)
if not body_as_json:
return
self._convert_to_es_doc_and_reindex(ch, method, body_as_json)
def callback_delete(ch, method, properties, body):
if not self._init_event_callback_ok("delete", ch, method):
return
body_as_json = self._get_event_json_body(ch, method, body)
if not body_as_json:
return
self._delete_from_index(ch, method, body_as_json)
# Set up consumers so that acks are required
self.create_consumer_tag = self.channel.basic_consume(callback_create, queue=self.create_queue, no_ack=False)
self.update_consumer_tag = self.channel.basic_consume(callback_update, queue=self.update_queue, no_ack=False)
self.delete_consumer_tag = self.channel.basic_consume(callback_delete, queue=self.delete_queue, no_ack=False)
self.init_ok = True