class RetrieveProvenance(object):
"""Update Provenance on request."""
@validate(load_schema('idtype'))
def on_get(self, req, resp, provID):
"""Respond on GET request to map endpoint."""
resp.status = falcon.HTTP_200
app_logger.info(
'Finished operations on /prov/{0} GET Request.'.format(provID))
class GraphSPARQL(object):
"""Execute SPARQL Query on Graph Store."""
@validate(load_schema('query'))
def on_post(self, req, resp, parsed):
"""Execution of the POST SPARQL query request."""
fuseki = GraphStore()
data = fuseki._graph_sparql(parsed['namedGraph'], parsed['query'])
resp.data = str(data)
resp.content_type = 'application/xml' # for now just this type
resp.status = falcon.HTTP_200
app_logger.info('Finished operations on /graph/query POST Request.')
class GraphUpdate(object):
"""Update Graph Store using a SPARQL Query."""
@validate(load_schema('update'))
def on_post(self, req, resp, parsed):
"""Execution of the POST update query request."""
fuseki = GraphStore()
resp.data = json.dumps(
fuseki._graph_add(parsed['namedGraph'], parsed['triples']))
resp.content_type = 'application/json'
resp.status = falcon.HTTP_200
app_logger.info('Finished operations on /graph/update POST Request.')
class ConstructProvenance(object):
"""Construct Provenance based on provided request."""
@validate(load_schema('provschema'), load_schema('altprovschema'))
def on_post(self, req, resp, parsed):
"""Respond on GET request to map endpoint."""
if isinstance(parsed, dict):
response = prov_task.delay(parsed["provenance"], parsed["payload"])
result = {'taskID': response.id}
resp.body = json.dumps(result)
resp.content_type = 'application/json'
elif isinstance(parsed, list):
tasks = []
for obj in parsed:
response = prov_task.delay(obj["provenance"], obj["payload"])
tasks.append(response.id)
result = {'taskID': tasks}
resp.body = json.dumps(result)
resp.content_type = 'application/json'
# result = construct_provenance(parsed["provenance"], parsed["payload"])
# resp.body = result
# resp.content_type = 'text/turtle'
resp.status = falcon.HTTP_200
app_logger.info('Accepted POST Request for /prov.')
class RetrieveQueueTask(object):
"""Update Provenance on request."""
@validate(load_schema('idtype'))
def on_get(self, req, resp, task_id):
"""Respond on GET request to queued task endpoint."""
task_output = AsyncResult(task_id)
result = {
'status': task_output.status.lower(),
'output': str(task_output.result)
}
resp.status = falcon.HTTP_200
resp.body = json.dumps(result)
app_logger.info(
'Finished operations on /status/task/{0} GET Request.'.format(
task_id))
class Consumer(object):
"""Provenance message consumer."""
def __init__(self,
hostname='127.0.0.1',
username='******',
password='******',
queue='base.queue',
max_retries=None):
"""Consumer init function."""
self.hostname = hostname
self.username = username
self.password = password
self.queue = queue
self.max_retries = max_retries
self.connection = None
def create_connection(self):
"""Create a connection.
:return:
"""
attempts = 0
while True:
attempts += 1
try:
self.connection = Connection(self.hostname, self.username,
self.password)
app_logger.info(
'Established connection with AMQP server {0}'.format(
self.connection))
break
except amqpstorm.AMQPError as error:
app_logger.error('Something went wrong: {0}'.format(error))
if self.max_retries and attempts > self.max_retries:
break
time.sleep(min(attempts * 2, 30))
except KeyboardInterrupt:
break
def start(self):
"""Start the Consumers.
:return:
"""
if not self.connection:
self.create_connection()
while True:
try:
channel = self.connection.channel()
channel.queue.declare(self.queue)
channel.basic.consume(self, self.queue, no_ack=False)
app_logger.info('Connected to queue {0}'.format(self.queue))
channel.start_consuming(to_tuple=False)
if not channel.consumer_tags:
channel.close()
except amqpstorm.AMQPError as error:
app_logger.error('Something went wrong: {0}'.format(error))
self.create_connection()
except KeyboardInterrupt:
self.connection.close()
break
@valid_message(load_schema('provschema'), load_schema('altprovschema'))
def handle_message(self, message):
"""Handle provenance message."""
prov = json.loads(message.body)
try:
if isinstance(prov, dict):
response = prov_task.delay(prov["provenance"], prov["payload"])
result = {'task_id': response.id}
elif isinstance(prov, list):
tasks = []
for obj in prov:
response = prov_task.delay(obj["provenance"],
obj["payload"])
tasks.append(response.id)
result = {'task_id': tasks}
app_logger.info(
'Processed provenance message with result {0}.'.format(result))
except Exception as error:
app_logger.error('Something went wrong: {0}'.format(error))
def __call__(self, message):
"""Process the message body."""
try:
self.handle_message(message)
except Exception as error:
app_logger.error('Something went wrong: {0}'.format(error))
message.reject(requeue=False)
else:
message.ack()
import json
# from json import JSONEncoder
from prov.utils.logs import app_logger
from prov.utils.queue import init_celery
from prov.utils.broker import broker
from prov.applib.messaging_client import RpcClient
from prov.applib.graph_store import GraphStore
from prov.schemas import load_schema
app = init_celery(broker['user'], broker['pass'], broker['host'])
prov_alias = "attx"
prov_ld_frame = json.dumps(load_schema('prov_frame'))
@app.task(name="construct.index", max_retries=5)
def index_task():
"""Parse Provenance Object and construct Provenance Graph."""
prov = ProvenanceIndex(broker["framequeue"], broker["indexqueue"])
prov._index_prov()
# return result
class ProvenanceIndex(object):
"""Indexing Provenance in Elasticsearch with an LD Frame."""
def __init__(self, frame_queue, index_queue):
"""Initialize Provenance index."""
self.frame_queue = frame_queue
self.index_queue = index_queue
def _index_prov(self):
"""Index provenance in Elasticsearch."""