class RetrieveProvenance(object): """Update Provenance on request.""" @validate(load_schema('idtype')) def on_get(self, req, resp, provID): """Respond on GET request to map endpoint.""" resp.status = falcon.HTTP_200 app_logger.info( 'Finished operations on /prov/{0} GET Request.'.format(provID))
class GraphSPARQL(object): """Execute SPARQL Query on Graph Store.""" @validate(load_schema('query')) def on_post(self, req, resp, parsed): """Execution of the POST SPARQL query request.""" fuseki = GraphStore() data = fuseki._graph_sparql(parsed['namedGraph'], parsed['query']) resp.data = str(data) resp.content_type = 'application/xml' # for now just this type resp.status = falcon.HTTP_200 app_logger.info('Finished operations on /graph/query POST Request.')
class GraphUpdate(object): """Update Graph Store using a SPARQL Query.""" @validate(load_schema('update')) def on_post(self, req, resp, parsed): """Execution of the POST update query request.""" fuseki = GraphStore() resp.data = json.dumps( fuseki._graph_add(parsed['namedGraph'], parsed['triples'])) resp.content_type = 'application/json' resp.status = falcon.HTTP_200 app_logger.info('Finished operations on /graph/update POST Request.')
class ConstructProvenance(object): """Construct Provenance based on provided request.""" @validate(load_schema('provschema'), load_schema('altprovschema')) def on_post(self, req, resp, parsed): """Respond on GET request to map endpoint.""" if isinstance(parsed, dict): response = prov_task.delay(parsed["provenance"], parsed["payload"]) result = {'taskID': response.id} resp.body = json.dumps(result) resp.content_type = 'application/json' elif isinstance(parsed, list): tasks = [] for obj in parsed: response = prov_task.delay(obj["provenance"], obj["payload"]) tasks.append(response.id) result = {'taskID': tasks} resp.body = json.dumps(result) resp.content_type = 'application/json' # result = construct_provenance(parsed["provenance"], parsed["payload"]) # resp.body = result # resp.content_type = 'text/turtle' resp.status = falcon.HTTP_200 app_logger.info('Accepted POST Request for /prov.')
class RetrieveQueueTask(object): """Update Provenance on request.""" @validate(load_schema('idtype')) def on_get(self, req, resp, task_id): """Respond on GET request to queued task endpoint.""" task_output = AsyncResult(task_id) result = { 'status': task_output.status.lower(), 'output': str(task_output.result) } resp.status = falcon.HTTP_200 resp.body = json.dumps(result) app_logger.info( 'Finished operations on /status/task/{0} GET Request.'.format( task_id))
class Consumer(object): """Provenance message consumer.""" def __init__(self, hostname='127.0.0.1', username='******', password='******', queue='base.queue', max_retries=None): """Consumer init function.""" self.hostname = hostname self.username = username self.password = password self.queue = queue self.max_retries = max_retries self.connection = None def create_connection(self): """Create a connection. :return: """ attempts = 0 while True: attempts += 1 try: self.connection = Connection(self.hostname, self.username, self.password) app_logger.info( 'Established connection with AMQP server {0}'.format( self.connection)) break except amqpstorm.AMQPError as error: app_logger.error('Something went wrong: {0}'.format(error)) if self.max_retries and attempts > self.max_retries: break time.sleep(min(attempts * 2, 30)) except KeyboardInterrupt: break def start(self): """Start the Consumers. :return: """ if not self.connection: self.create_connection() while True: try: channel = self.connection.channel() channel.queue.declare(self.queue) channel.basic.consume(self, self.queue, no_ack=False) app_logger.info('Connected to queue {0}'.format(self.queue)) channel.start_consuming(to_tuple=False) if not channel.consumer_tags: channel.close() except amqpstorm.AMQPError as error: app_logger.error('Something went wrong: {0}'.format(error)) self.create_connection() except KeyboardInterrupt: self.connection.close() break @valid_message(load_schema('provschema'), load_schema('altprovschema')) def handle_message(self, message): """Handle provenance message.""" prov = json.loads(message.body) try: if isinstance(prov, dict): response = prov_task.delay(prov["provenance"], prov["payload"]) result = {'task_id': response.id} elif isinstance(prov, list): tasks = [] for obj in prov: response = prov_task.delay(obj["provenance"], obj["payload"]) tasks.append(response.id) result = {'task_id': tasks} app_logger.info( 'Processed provenance message with result {0}.'.format(result)) except Exception as error: app_logger.error('Something went wrong: {0}'.format(error)) def __call__(self, message): """Process the message body.""" try: self.handle_message(message) except Exception as error: app_logger.error('Something went wrong: {0}'.format(error)) message.reject(requeue=False) else: message.ack()
import json # from json import JSONEncoder from prov.utils.logs import app_logger from prov.utils.queue import init_celery from prov.utils.broker import broker from prov.applib.messaging_client import RpcClient from prov.applib.graph_store import GraphStore from prov.schemas import load_schema app = init_celery(broker['user'], broker['pass'], broker['host']) prov_alias = "attx" prov_ld_frame = json.dumps(load_schema('prov_frame')) @app.task(name="construct.index", max_retries=5) def index_task(): """Parse Provenance Object and construct Provenance Graph.""" prov = ProvenanceIndex(broker["framequeue"], broker["indexqueue"]) prov._index_prov() # return result class ProvenanceIndex(object): """Indexing Provenance in Elasticsearch with an LD Frame.""" def __init__(self, frame_queue, index_queue): """Initialize Provenance index.""" self.frame_queue = frame_queue self.index_queue = index_queue def _index_prov(self): """Index provenance in Elasticsearch."""