def on_get(self, req, resp):
"""Respond on GET request to map endpoint."""
# if you manange to call this it means the API is running
resp.data = healthcheck_response("Running")
resp.content_type = 'application/json'
resp.status = falcon.HTTP_200
app_logger.info('Finished operations on /health GET Request.')
def _replace_index(self, new_index, alias_list):
"""Replace an existing index based on an alias."""
update = dict([('actions', [])])
old_index = []
reference_alias = next(iter(alias_list or []), None)
prefix = "{0}_".format(reference_alias)
try:
alias_indexes = self._alias_index_list().get(reference_alias)
if alias_indexes:
old_index = [x for x in alias_indexes if x.startswith(prefix)]
for alias in alias_list:
update["actions"].append(
{"add": {
"index": new_index,
"alias": alias
}})
for elem in old_index:
update["actions"].append(
{"remove": {
"index": elem,
"alias": alias
}})
self.es.indices.update_aliases(body=json.dumps(update))
self._index_delete(old_index)
app_logger.info("Replace index finished.")
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
finally:
return json.dumps(update)
def on_get(self, req, resp):
"""Execution of the GET aliases list request."""
elastic = ElasticIndex()
resp.data = json.dumps(list(elastic._alias_list()),
indent=1,
sort_keys=True)
resp.content_type = 'application/json'
resp.status = falcon.HTTP_200
app_logger.info('Finished operations on /alias/list GET Request.')
def push(self, message):
"""Start channel to provenance inbox."""
with Connection(self.hostname, self.username,
self.password) as connection:
with connection.channel() as channel:
channel.queue.declare(self.queue)
properties = {'content_type': 'application/json'}
message = Message.create(channel, message, properties)
message.publish(self.queue)
app_logger.info('Pushed message: {0}.'.format(message))
def prov_message(message_data, status, start_time, end_time, replace_index):
"""Construct Indexing related provenance message."""
message = dict()
message["provenance"] = dict()
message["provenance"]["agent"] = dict()
message["provenance"]["agent"]["ID"] = artifact_id
message["provenance"]["agent"]["role"] = agent_role
activity_id = message_data["provenance"]["context"]["activityID"]
workflow_id = message_data["provenance"]["context"]["workflowID"]
prov_message = message["provenance"]
prov_message["context"] = dict()
prov_message["context"]["activityID"] = str(activity_id)
prov_message["context"]["workflowID"] = str(workflow_id)
if message_data["provenance"]["context"].get('stepID'):
prov_message["context"]["stepID"] = message_data["provenance"]["context"]["stepID"]
prov_message["activity"] = dict()
prov_message["activity"]["type"] = "ServiceExecution"
prov_message["activity"]["title"] = "Indexing Service Operations."
prov_message["activity"]["status"] = status
prov_message["activity"]["startTime"] = start_time
prov_message["activity"]["endTime"] = end_time
message["provenance"]["input"] = []
message["provenance"]["output"] = []
message["payload"] = {}
alias_list = [str(r) for r in message_data["payload"]["indexingServiceInput"]["targetAlias"]]
for alias in alias_list:
key = "outputAlias_{0}".format(alias_list.index(alias))
output_data = {
"key": key,
"role": "PublishedDataset"
}
message["provenance"]["output"].append(output_data)
message["payload"][key] = alias
source_data = message_data["payload"]["indexingServiceInput"]["sourceData"]
for elem in source_data:
key = "inputData_{0}".format(source_data.index(elem))
input_data = {
"key": key,
"role": "Dataset"
}
if elem["inputType"] == "Data":
message["payload"][key] = "attx:tempDataset"
if elem["inputType"] == "URI":
message["payload"][key] = elem["input"]
message["provenance"]["input"].append(input_data)
app_logger.info('Construct provenance metadata for Indexing Service.')
return json.dumps(message)
def init_api():
"""Create the API endpoint."""
indexservice = falcon.API()
indexservice.add_route('/health', HealthCheck())
indexservice.add_route('/%s/alias/list' % (api_version), AliasesList())
indexservice.add_route('/%s/data/index' % (api_version), IndexClass())
app_logger.info('IndexService REST API is running.')
return indexservice
def _bulk_index(self, doc_type, data, target_index):
"""Index one or more documents via the bulk operation."""
try:
if doc_type is None:
doc_type = "General"
self.es.bulk(index=target_index,
doc_type=doc_type,
body=data,
refresh=True)
app_logger.info(
"Bulk Index in Elasticsearch at index: \"{0}\" with type: \"{1}\"."
.format(target_index, doc_type))
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
def _api_index(self, doc_id, doc_type, data, target_index):
"""Index single document from an ."""
try:
if doc_type is None:
doc_type = "General"
self.es.index(index=target_index,
id=doc_id,
doc_type=doc_type,
body=data,
refresh=True)
app_logger.info(
"Index document {0} in Elasticsearch at index: \"{1}\" with type: \"{2}\"."
.format(doc_id, target_index, doc_type))
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
def _add_alias(self, new_index, alias_list):
"""Add index based on an alias."""
update = dict([('actions', [])])
try:
for alias in alias_list:
update["actions"].append(
{"add": {
"index": new_index,
"alias": alias
}})
self.es.indices.update_aliases(body=json.dumps(update))
app_logger.info("Add alias finished.")
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
finally:
return json.dumps(update)
def add_message(message_data, replace_index, alias_list):
"""Index data and associate aliases to the indexes."""
startTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
PUBLISHER = Publisher(broker['host'], broker['user'], broker['pass'], broker['provqueue'])
elastic = ElasticIndex()
try:
elastic._add_alias(replace_index, alias_list)
endTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
if bool(message_data["provenance"]):
PUBLISHER.push(prov_message(message_data, "success", startTime, endTime, replace_index))
app_logger.info('Indexed data with: {0} aliases'.format(alias_list))
return json.dumps(response_message(message_data["provenance"], status="success"), indent=4, separators=(',', ': '))
except Exception as error:
endTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
PUBLISHER.push(prov_message(message_data, "error", startTime, endTime, replace_index))
app_logger.error('Something is wrong: {0}'.format(error))
raise
def _index_create(self, reference_alias, target_index=None):
"""Create a new index if it does not exist."""
try:
if target_index is not None and not (
self.es.indices.exists(target_index)):
self.es.indices.create(index=target_index,
ignore=400,
refresh=True)
else:
target_index = "{0}_service_{1}".format(
reference_alias,
datetime.now().strftime('%Y%m%d_%H%M%S'))
app_logger.info("New index created: \"{0}\".".format(target_index))
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
finally:
return target_index
def replace_message(message_data, replace_index, alias_list):
"""Replace an old index with a new index for a given alias list."""
startTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
PUBLISHER = Publisher(broker['host'], broker['user'], broker['pass'], broker['provqueue'])
elastic = ElasticIndex()
try:
# This is what makes it a replace operation.
elastic._replace_index(replace_index, alias_list)
endTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
if bool(message_data["provenance"]):
PUBLISHER.push(prov_message(message_data, "success", startTime, endTime, replace_index))
app_logger.info('Replaced Indexed data with: {0} aliases'.format(alias_list))
return json.dumps(response_message(message_data["provenance"], status="success"), indent=4, separators=(',', ': '))
except Exception as error:
endTime = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
PUBLISHER.push(prov_message(message_data, "error", startTime, endTime, replace_index))
app_logger.error('Something is wrong: {0}'.format(error))
raise
def start(self, connection):
"""Start the Consumers."""
self.channel = None
try:
self.active = True
self.channel = connection.channel(rpc_timeout=10)
self.channel.basic.qos(1)
self.channel.queue.declare(self.rpc_queue)
self.channel.basic.consume(self, self.rpc_queue, no_ack=False)
self.channel.start_consuming(to_tuple=False)
app_logger.info('Connected to queue {0}'.format(self.queue))
if not self.channel.consumer_tags:
# Only close the channel if there is nothing consuming.
# This is to allow messages that are still being processed
# in __call__ to finish processing.
self.channel.close()
except amqpstorm.AMQPError:
pass
finally:
self.active = False
def _create_connection(self):
"""Create a connection.
:return:
"""
attempts = 0
while True:
attempts += 1
if self._stopped.is_set():
break
try:
self._connection = Connection(self.hostname, self.username,
self.password)
app_logger.info(
'Established connection with AMQP server {0}'.format(
self._connection))
break
except amqpstorm.AMQPError as why:
app_logger.error(why)
if self.max_retries and attempts > self.max_retries:
raise Exception('max number of retries reached')
time.sleep(min(attempts * 2, 30))
except KeyboardInterrupt:
break
def _index_delete(self, index_list):
"""Delete an index and remove it aliases."""
for target_index in index_list:
if self.es.indices.exists(target_index):
self.es.indices.delete(index=target_index, ignore=[400, 404])
app_logger.info("Index deleted: \"{0}\".".format(target_index))
def on_post(self, req, resp, parsed):
"""Respond on GET request to index endpoint."""
resp.data = add_message(parsed)
resp.content_type = 'application/json'
resp.status = falcon.HTTP_202
app_logger.info('Index/POST data with specified aliases.')