def _construct_provenance(self):
"""Parse Provenance Object and construct Provenance Graph."""
bind_prefix(self.graph)
try:
activity_id = ''.join(
filter(None, ('activity',
str(self.prov_object['context']['activityID']))))
workflow_id = ''.join(
filter(None, ('workflow',
str(self.prov_object['context']['workflowID']))))
# if an activity does not include step ID it is an WorkflowExecution
if self.prov_object['context'].get('stepID'):
step_id = ''.join(
filter(
None,
('step', str(self.prov_object['context']['stepID']))))
else:
step_id = None
base_uri = "_".join(
filter(None, (workflow_id, activity_id, step_id)))
wf_base_uri = "{0}_{1}".format(workflow_id, activity_id)
app_logger.info('Constructed base ID: {0}'.format(base_uri))
if self.prov_object['activity']['type'] == "DescribeStepExecution":
self._prov_dataset(base_uri)
else:
self._prov_activity(base_uri, wf_base_uri)
except Exception as error:
app_logger.error(
'Something is wrong with parsing the prov_object: {0}'.format(
error))
raise error
else:
self._store_provenance(wf_base_uri)
self._store_provenance_graph()
return self.graph.serialize(format='turtle')
def _graph_statistics(self):
"""Graph Store statistics agregated."""
result = {}
try:
request = requests.get("{0}stats/{1}".format(
self.server_address, self.dataset),
auth=('admin', self.key))
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
else:
stats = request.json()
result['dataset'] = "/{0}".format(self.dataset)
result['requests'] = {}
result['requests']['totalRequests'] = stats['datasets'][
'/{0}'.format(self.dataset)]['Requests']
result['requests']['failedRequests'] = stats['datasets'][
'/{0}'.format(self.dataset)]['RequestsBad']
triples = 0
graphs = self._graph_list()
for e in graphs['graphs']:
triples += int(e['tripleCount'])
result['totalTriples'] = triples
app_logger.info(
'Constructed statistics list for dataset: "/{0}".'.format(
self.dataset))
return result
def __call__(self, message):
"""Process the message body."""
try:
self.handle_message(message)
except Exception as error:
app_logger.error('Something went wrong: {0}'.format(error))
message.reject(requeue=False)
else:
message.ack()
def _graph_health(self):
"""Do the Health check for Graph Store."""
status = None
try:
request = requests.get("{0}ping".format(self.server_address))
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
status = False
raise ConnectionError(
'Tried getting graph health, with error {}'.format(error))
else:
app_logger.info('Response from Graph Store is {0}'.format(request))
status = True
return status
def execute_indexing():
"""Index provenance data by applying ld frame."""
try:
frame_request = requests.get("http://{0}:{1}/health".format(
ldframe["host"], ldframe["port"]))
index_request = requests.get("http://{0}:{1}/health".format(
index["host"], index["port"]))
if frame_request.status_code == 200 and index_request.status_code == 200:
response = index_task.delay()
except Exception:
app_logger.error(
'Could not find Services for either ldFrame, esIndexing or both.')
pass
else:
return response
def _graph_add(self, named_graph, data):
"""Update named graph in Graph Store."""
headers = {'content-type': "text/turtle", 'cache-control': "no-cache"}
try:
request = requests.post("{0}/data?graph={1}".format(
self.request_address, named_graph),
data=data,
headers=headers)
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
else:
app_logger.info('Updated named graph: {0}.'.format(named_graph))
app_logger.info('Graph Store response: {0}.'.format(request.text))
return request.text
def _graph_sparql(self, named_graph, query):
"""Execute SPARQL query on the Graph Store."""
store_api = "{0}/query".format(self.request_address)
try:
sparql = SPARQLWrapper(store_api)
# add a default graph, though that can also be in the query string
sparql.addDefaultGraph(named_graph)
sparql.setQuery(query)
data = sparql.query().convert()
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
else:
app_logger.info(
'Execture SPARQL query on named graph: {0}.'.format(
named_graph))
return data.toxml()
def handle_message(self, message):
"""Handle provenance message."""
prov = json.loads(message.body)
try:
if isinstance(prov, dict):
response = prov_task.delay(prov["provenance"], prov["payload"])
result = {'task_id': response.id}
elif isinstance(prov, list):
tasks = []
for obj in prov:
response = prov_task.delay(obj["provenance"],
obj["payload"])
tasks.append(response.id)
result = {'task_id': tasks}
app_logger.info(
'Processed provenance message with result {0}.'.format(result))
except Exception as error:
app_logger.error('Something went wrong: {0}'.format(error))
def _drop_graph(self, named_graph):
"""Drop named graph from Graph Store."""
drop_query = quote(" DROP GRAPH <{0}>".format(named_graph))
payload = "update={0}".format(drop_query)
headers = {
'content-type': "application/x-www-form-urlencoded",
'cache-control': "no-cache"
}
try:
request = requests.post("{0}/update".format(self.request_address),
data=payload,
headers=headers)
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise error
else:
app_logger.info('Deleted named graph: {0}.'.format(named_graph))
return request.text
def _graph_retrieve(self, named_graph):
"""Retrieve named graph from Graph Store."""
try:
request = requests.get("{0}/data?graph={1}".format(
self.request_address, named_graph))
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
else:
if request.status_code == 200:
app_logger.info(
'Retrived named graph: {0}.'.format(named_graph))
return request.text
elif request.status_code == 404:
app_logger.info(
'Retrived named graph: {0} does not exist.'.format(
named_graph))
return None
def start(self):
"""Start the Consumers.
:return:
"""
if not self.connection:
self.create_connection()
while True:
try:
channel = self.connection.channel()
channel.queue.declare(self.queue)
channel.basic.consume(self, self.queue, no_ack=False)
app_logger.info('Connected to queue {0}'.format(self.queue))
channel.start_consuming(to_tuple=False)
if not channel.consumer_tags:
channel.close()
except amqpstorm.AMQPError as error:
app_logger.error('Something went wrong: {0}'.format(error))
self.create_connection()
except KeyboardInterrupt:
self.connection.close()
break
def create_connection(self):
"""Create a connection.
:return:
"""
attempts = 0
while True:
attempts += 1
try:
self.connection = Connection(self.hostname, self.username,
self.password)
app_logger.info(
'Established connection with AMQP server {0}'.format(
self.connection))
break
except amqpstorm.AMQPError as error:
app_logger.error('Something went wrong: {0}'.format(error))
if self.max_retries and attempts > self.max_retries:
break
time.sleep(min(attempts * 2, 30))
except KeyboardInterrupt:
break
def _prov_list(self):
"""List Graph Store Provenance Named Graphs."""
result = {}
temp_list = []
list_query = quote(
"select ?g {{graph ?g {{?s ?p ?o}} filter(regex(str(?g), '{0}'))}} group by ?g"
.format(ATTXPROVURL))
try:
request = requests.get("{0}/sparql?query={1}".format(
self.request_address, list_query))
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
else:
graphs = request.json()
result['graphsCount'] = len(graphs['results']['bindings'])
for g in graphs['results']['bindings']:
temp_list.append(g['g']['value'])
result['graphs'] = temp_list
app_logger.info(
'Constructed list of prov Named Graphs from "/{0}" dataset.'.
format(self.dataset))
return result
def _graph_list(self):
"""List Graph Store Named Graphs."""
result = {}
temp_list = []
list_query = quote(
"select ?g (count(*) as ?count) {graph ?g {?s ?p ?o}} group by ?g")
try:
request = requests.get("{0}/sparql?query={1}".format(
self.request_address, list_query))
except Exception as error:
app_logger.error('Something is wrong: {0}'.format(error))
raise
else:
graphs = request.json()
result['graphsCount'] = len(graphs['results']['bindings'])
for g in graphs['results']['bindings']:
temp_graph = dict([('graphURI', g['g']['value']),
('tripleCount', g['count']['value'])])
temp_list.append(temp_graph)
result['graphs'] = temp_list
app_logger.info(
'Constructed list of Named Graphs from "/{0}" dataset.'.format(
self.dataset))
return result