def __init__(self, progID, ousUID, recipe): self._progID = progID self._ousUID = ousUID self._recipe = recipe #Initialization code for pipeline execution # Make sure we know where we are self.location = os.environ.get( 'DRAWS_LOCATION' ) if self.location == None: raise RuntimeError( "DRAWS_LOCATION env variable is not defined" ) # Make sure we know where the local replicated cache directory is self.replicatedCache = os.environ.get( 'DRAWS_LOCAL_CACHE' ) if self.replicatedCache == None: raise RuntimeError( "DRAWS_LOCAL_CACHE env variable is not defined" ) self.pipelineScript = "pipeline.py" self.thisDirectory = os.getcwd() # Assume the pipeline script is in this same directory self.pipelineExecutable = self.thisDirectory + "/" + self.pipelineScript self.pipelineRunDirectory = tempfile.mkdtemp( prefix="drw-" ) self.workingDirectory = tempfile.mkdtemp( prefix="drw-" ) print( ">>> PipelineDriver: pipelineRunDirectory:", self.pipelineRunDirectory ) print( ">>> PipelineDriver: workingDirectory:", self.workingDirectory ) self.xtss = ExecutorClient( 'localhost', 'msgq', 'xtss' ) self.mq = MqConnection( 'localhost', 'msgq' ) params = collections.OrderedDict() params['progID'] = self._progID params['ousUID'] = self._ousUID params['recipe'] = self._recipe params['location'] = self.location params['pipelineRunDir'] = self.pipelineRunDirectory params['replicatedCache'] = self.replicatedCache self._temp = Template(params)
def __init__(self, location): self._baseUrl = "http://localhost:5984" # CouchDB self._dbconn = DbConnection(baseUrl) self._xtss = ExecutorClient('localhost', 'msgq', 'xtss') self._mq = MqConnection('localhost', 'msgq') self._broker = RabbitMqMessageBroker() self.location = location
def __init__(self, localCache): self.baseUrl = "http://localhost:5984" # CouchDB self.ngascon = NgasConnection() self.xtss = ExecutorClient('localhost', 'msgq', 'xtss') self.select = "ingest.JAO" self.mq = MqConnection('localhost', 'msgq', self.select) self.localCache = localCache
def __init__(self): self.weblogsBaseUrl = "http://localhost:8000" self.baseUrl = "http://localhost:5984" # CouchDB self.dbconn = DbConnection(self.baseUrl) self.dbName = "pipeline-reports" self.xtss = ExecutorClient('localhost', 'msgq', 'xtss') self.select = "pipeline.report.JAO" self.mq = MqConnection('localhost', 'msgq', select)
if location == None: raise RuntimeError("DRAWS_LOCATION env variable is not defined") # Make sure we know where the local replicated cache directory is replicatedCache = os.environ.get('DRAWS_LOCAL_CACHE') if replicatedCache == None: raise RuntimeError("DRAWS_LOCAL_CACHE env variable is not defined") parser = argparse.ArgumentParser(description='Pipeline Driver mock-up') parser.add_argument(dest="progID", help="ID of the project containing the OUS") parser.add_argument(dest="ousUID", help="ID of the OUS that should be processed") parser.add_argument(dest="recipe", help="Pipeline recipe to run") args = parser.parse_args() print(">>> PipelineDriver: progID=%s, ousUID=%s, recipe=%s" % (args.progID, args.ousUID, args.recipe)) pipelineScript = "pipeline.py" thisDirectory = os.getcwd( ) # Assume the pipeline script is in this same directory pipelineExecutable = thisDirectory + "/" + pipelineScript pipelineRunDirectory = tempfile.mkdtemp(prefix="drw-") workingDirectory = tempfile.mkdtemp(prefix="drw-") print(">>> PipelineDriver: pipelineRunDirectory:", pipelineRunDirectory) print(">>> PipelineDriver: workingDirectory:", workingDirectory) xtss = ExecutorClient('localhost', 'msgq', 'xtss') mq = MqConnection('localhost', 'msgq') wrapPipeline(args.progID, args.ousUID, args.recipe)
def __init__(self): self._mq = MqConnection('localhost', 'msgq') self._xtss = ExecutorClient('localhost', 'msgq', 'xtss')
#!/usr/bin/env python3 import argparse import sys sys.path.insert(0, "../shared") from dbmsgq import ExecutorClient ################################################################### ## Send an RPC request to an Executor ## Example: ## ./send-rpc.py xtss \ ## '{ "operation":"get-ouss-by-state-substate", "state":"ReadyForProcessing", "substate":"^Pipeline" }' ## parser = argparse.ArgumentParser( description='Temporary component, sends an arbitrary RPC request') parser.add_argument(dest="selector", help="Selector to send the message to") parser.add_argument(dest="message", help="Message (JSON text)") args = parser.parse_args() executor = ExecutorClient('localhost', 'msgq', args.selector) response = executor.call(args.message) print("Response:", response)
#!/usr/bin/env python3 from dbmsgq import ExecutorClient import argparse parser = argparse.ArgumentParser( description='qqq' ) parser.add_argument( dest="n", help="TODO" ) args=parser.parse_args() queue = ExecutorClient( "localhost", "msgq", "fibonacci" ) msg = {} msg['n'] = args.n print( "Executor returned:", queue.call( msg ))