def _replace_index(self, new_index, alias_list):
"""Replace an existing index based on an alias."""
update = dict([('actions', [])])
old_index = []
reference_alias = next(iter(alias_list or []), None)
prefix = "{0}_".format(reference_alias)
try:
alias_indexes = self._alias_index_list().get(reference_alias)
if alias_indexes:
old_index = [x for x in alias_indexes if x.startswith(prefix)]
for alias in alias_list:
update["actions"].append(
{"add": {
"index": new_index,
"alias": alias
}})
for elem in old_index:
update["actions"].append(
{"remove": {
"index": elem,
"alias": alias
}})
self.es.indices.update_aliases(body=json.dumps(update))
self._index_delete(old_index)
app_logger.info("Replace index finished.")
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
finally:
return json.dumps(update)
def __call__(self, message):
"""Process the RPC Payload.
:param Message message:
:return:
"""
try:
processed_message = self._handle_message(message)
except Exception as e:
app_logger.error('Something went wrong: {0}'.format(e))
properties = {'correlation_id': message.correlation_id}
error_message = "Error Type: {0}, with message: {1}".format(
e.__class__.__name__, e.message)
message_data = json.loads(message.body)
processed_message = response_message(message_data["provenance"],
status="error",
status_messsage=error_message)
response = Message.create(
message.channel,
str(
json.dumps(processed_message,
indent=4,
separators=(',', ': '))), properties)
response.publish(message.reply_to)
message.reject(requeue=False)
else:
properties = {'correlation_id': message.correlation_id}
response = Message.create(message.channel, processed_message,
properties)
response.publish(message.reply_to)
message.ack()
def _bulk_index(self, doc_type, data, target_index):
"""Index one or more documents via the bulk operation."""
try:
if doc_type is None:
doc_type = "General"
self.es.bulk(index=target_index,
doc_type=doc_type,
body=data,
refresh=True)
app_logger.info(
"Bulk Index in Elasticsearch at index: \"{0}\" with type: \"{1}\"."
.format(target_index, doc_type))
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
def _api_index(self, doc_id, doc_type, data, target_index):
"""Index single document from an ."""
try:
if doc_type is None:
doc_type = "General"
self.es.index(index=target_index,
id=doc_id,
doc_type=doc_type,
body=data,
refresh=True)
app_logger.info(
"Index document {0} in Elasticsearch at index: \"{1}\" with type: \"{2}\"."
.format(doc_id, target_index, doc_type))
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
def _add_alias(self, new_index, alias_list):
"""Add index based on an alias."""
update = dict([('actions', [])])
try:
for alias in alias_list:
update["actions"].append(
{"add": {
"index": new_index,
"alias": alias
}})
self.es.indices.update_aliases(body=json.dumps(update))
app_logger.info("Add alias finished.")
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
finally:
return json.dumps(update)
def add_message(message_data, replace_index, alias_list):
"""Index data and associate aliases to the indexes."""
startTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
PUBLISHER = Publisher(broker['host'], broker['user'], broker['pass'], broker['provqueue'])
elastic = ElasticIndex()
try:
elastic._add_alias(replace_index, alias_list)
endTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
if bool(message_data["provenance"]):
PUBLISHER.push(prov_message(message_data, "success", startTime, endTime, replace_index))
app_logger.info('Indexed data with: {0} aliases'.format(alias_list))
return json.dumps(response_message(message_data["provenance"], status="success"), indent=4, separators=(',', ': '))
except Exception as error:
endTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
PUBLISHER.push(prov_message(message_data, "error", startTime, endTime, replace_index))
app_logger.error('Something is wrong: {0}'.format(error))
raise
def _index_create(self, reference_alias, target_index=None):
"""Create a new index if it does not exist."""
try:
if target_index is not None and not (
self.es.indices.exists(target_index)):
self.es.indices.create(index=target_index,
ignore=400,
refresh=True)
else:
target_index = "{0}_service_{1}".format(
reference_alias,
datetime.now().strftime('%Y%m%d_%H%M%S'))
app_logger.info("New index created: \"{0}\".".format(target_index))
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
finally:
return target_index
def replace_message(message_data, replace_index, alias_list):
"""Replace an old index with a new index for a given alias list."""
startTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
PUBLISHER = Publisher(broker['host'], broker['user'], broker['pass'], broker['provqueue'])
elastic = ElasticIndex()
try:
# This is what makes it a replace operation.
elastic._replace_index(replace_index, alias_list)
endTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
if bool(message_data["provenance"]):
PUBLISHER.push(prov_message(message_data, "success", startTime, endTime, replace_index))
app_logger.info('Replaced Indexed data with: {0} aliases'.format(alias_list))
return json.dumps(response_message(message_data["provenance"], status="success"), indent=4, separators=(',', ': '))
except Exception as error:
endTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
PUBLISHER.push(prov_message(message_data, "error", startTime, endTime, replace_index))
app_logger.error('Something is wrong: {0}'.format(error))
raise
def retrieve_data(inputType, input_data):
"""Retrieve data from a specific URI."""
s = requests.Session()
allowed = ('http', 'https', 'ftp')
local = ('file')
if inputType == "Data":
return input_data
elif inputType == "URI":
try:
if urlparse(input_data).scheme in allowed:
request = s.get(input_data, timeout=1)
return request.text
elif urlparse(input_data).scheme in local:
s.mount('file://', FileAdapter())
request = s.get(input_data)
return handle_fileAdapter(request, input_data)
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
def start_server(self):
"""Start the RPC Server.
:return:
"""
self._stopped.clear()
if not self._connection or self._connection.is_closed:
self._create_connection()
while not self._stopped.is_set():
try:
# Check our connection for errors.
self._connection.check_for_errors()
self._update_consumers()
except amqpstorm.AMQPError as why:
# If an error occurs, re-connect and let update_consumers
# re-open the channels.
app_logger.error(why)
self._stop_consumers()
self._create_connection()
time.sleep(1)
def wrapper(message_data, *args, **kwargs):
"""Wrap it nicely."""
aliases = message_data["payload"]["indexingServiceInput"]["targetAlias"]
alias_list = [str(r) for r in aliases]
# If there are multiple aliases in the message we need the first one for reference.
# The reference alias will act as a unique identified for the index to be replaced.
reference_alias = next(iter(alias_list or []), None)
source_data = message_data["payload"]["indexingServiceInput"]["sourceData"]
elastic = ElasticIndex()
# We will be using only one index for all data sources.
replace_index = elastic._index_create(reference_alias)
try:
for source in iter(source_data or []):
data = retrieve_data(source["inputType"], source["input"])
if source["useBulk"] and source["useBulk"] is True:
elastic._bulk_index(source["docType"], data, replace_index)
elif source["documentID"]:
elastic._api_index(source["documentID"], source["docType"], data, replace_index)
else:
raise KeyError("Missing useBulk operation or documentID for single document indexing.")
return func(message_data, replace_index=replace_index, alias_list=alias_list)
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
def _create_connection(self):
"""Create a connection.
:return:
"""
attempts = 0
while True:
attempts += 1
if self._stopped.is_set():
break
try:
self._connection = Connection(self.hostname, self.username,
self.password)
app_logger.info(
'Established connection with AMQP server {0}'.format(
self._connection))
break
except amqpstorm.AMQPError as why:
app_logger.error(why)
if self.max_retries and attempts > self.max_retries:
raise Exception('max number of retries reached')
time.sleep(min(attempts * 2, 30))
except KeyboardInterrupt:
break
