def __init__(self, progID, ousUID, recipe): self._progID = progID self._ousUID = ousUID self._recipe = recipe #Initialization code for pipeline execution # Make sure we know where we are self.location = os.environ.get( 'DRAWS_LOCATION' ) if self.location == None: raise RuntimeError( "DRAWS_LOCATION env variable is not defined" ) # Make sure we know where the local replicated cache directory is self.replicatedCache = os.environ.get( 'DRAWS_LOCAL_CACHE' ) if self.replicatedCache == None: raise RuntimeError( "DRAWS_LOCAL_CACHE env variable is not defined" ) self.pipelineScript = "pipeline.py" self.thisDirectory = os.getcwd() # Assume the pipeline script is in this same directory self.pipelineExecutable = self.thisDirectory + "/" + self.pipelineScript self.pipelineRunDirectory = tempfile.mkdtemp( prefix="drw-" ) self.workingDirectory = tempfile.mkdtemp( prefix="drw-" ) print( ">>> PipelineDriver: pipelineRunDirectory:", self.pipelineRunDirectory ) print( ">>> PipelineDriver: workingDirectory:", self.workingDirectory ) self._broker = RabbitMqMessageBroker() params = collections.OrderedDict() params['progID'] = self._progID params['ousUID'] = self._ousUID params['recipe'] = self._recipe params['location'] = self.location params['pipelineRunDir'] = self.pipelineRunDirectory params['replicatedCache'] = self.replicatedCache self._temp = Template(params)
class BasicSender(object): #private Logger logger = LoggerFactory.getLogger( BasicReceiver.class ); def __init__(self): self.__broker = RabbitMqMessageBroker("amqp://localhost:5672", "guest", "guest") def run(self, args): publisher = Publisher(self.__broker, args.qname) freddie = Person("Freddie Mercury", 45, False) print("Sending to " + args.qname) envelope = publisher.publish(freddie) print(">>> Sent to " + args.qname + ": " + str(envelope.getMessage())) self.__broker.closeConnection()
def setUp(self): self.envelopeRepository = PersistedEnvelopeRepository() self.groupRepository = RecipientGroupRepository() self.broker = RabbitMqMessageBroker("amqp://localhost:5672", "guest", "guest", TestExecutor.__EXCHANGE_NAME, self.envelopeRepository, self.groupRepository) self.publisher = Publisher(self.broker, TestExecutor.__QUEUE_NAME) self.subscriber = Subscriber(self.broker, TestExecutor.__QUEUE_NAME, "test") self.broker.drainLoggingQueue() self.broker.drainQueue(self.subscriber.getQueue())
class BasicReceiver(object): def __init__(self): self.__broker = RabbitMqMessageBroker("amqp://localhost:5672", "guest", "guest") def run(self, args): subscriber = Subscriber(self.__broker, args.qname, args.sname) print("Waiting for message") received = subscriber.receive(120 * 1000) msg = args.sname + "(" + str(os.getpid()) + ") received: " + str( received.getMessage()) self.__broker.closeConnection() print(msg)
def __init__(self, location): self._baseUrl = "http://localhost:5984" # CouchDB self._dbconn = DbConnection(baseUrl) self._xtss = ExecutorClient('localhost', 'msgq', 'xtss') self._mq = MqConnection('localhost', 'msgq') self._broker = RabbitMqMessageBroker() self.location = location
def setUp(self): self.envelopeRepository = PersistedEnvelopeRepository() self.groupRepository = RecipientGroupRepository() self.broker = RabbitMqMessageBroker( exchangeName=TestTokenSecurityRabbitMQ.__EXCHANGE_NAME, envelopeRepository=self.envelopeRepository, groupRepository=self.groupRepository) #self.queue = self.broker.messageQueue(TestTokenSecurityRabbitMQ.__QUEUE_NAME) self.publisher = Publisher(self.broker, TestTokenSecurityRabbitMQ.__QUEUE_NAME) self.subscriber = Subscriber(self.broker, TestTokenSecurityRabbitMQ.__QUEUE_NAME, TestTokenSecurityRabbitMQ.__SERVICE_NAME) #Drain any existing messages in the logging queue self.broker.drainLoggingQueue() self.envelopeRepository.deleteAll() self.groupRepository.deleteAll() self.brian = TestMessage("Brian May", 71, True) #self.tokenFactory = MockedTokenFactory.getFactory(); self.tokenFactory = JWTFactory() self.broker.setTokenFactory(self.tokenFactory)
class TestExecutor(unittest.TestCase): __QUEUE_NAME = "test.executor.queue" __EXCHANGE_NAME = "unit-test-exchange" #Doubles its input class Doubler(RequestProcessor): def process(self, message): request = message print(">>> Received request with number: " + str(request.number)) response = DoubleResponse() response.doubled = request.number + request.number return response class TestMessageConsumer(MessageConsumer): def __init__(self): self.doubled = None def consume(self, message): if not isinstance(message, DoubleResponse): msg = "Not a " + DoubleResponse.__name__ + ": " + str(message) print(">>>>> message 2: " + str(message)) print(">>>>> Thread: " + threading.currentThread().getName()) print(">>>>> " + msg) print(msg) raise Exception(msg) self.doubled = message.doubled def setUp(self): self.envelopeRepository = PersistedEnvelopeRepository() self.groupRepository = RecipientGroupRepository() self.broker = RabbitMqMessageBroker("amqp://localhost:5672", "guest", "guest", TestExecutor.__EXCHANGE_NAME, self.envelopeRepository, self.groupRepository) self.publisher = Publisher(self.broker, TestExecutor.__QUEUE_NAME) self.subscriber = Subscriber(self.broker, TestExecutor.__QUEUE_NAME, "test") self.broker.drainLoggingQueue() self.broker.drainQueue(self.subscriber.getQueue()) def tearDown(self): self.broker.deleteQueue(self.subscriber.getQueue()) def doublerRunnable(self, doublerExecutor): try: doublerExecutor.run() except TimeLimitExceededException as e: print(">>> Timed out: " + str(e)) except Exception as e: raise Exception(e) def testDoubler(self): doubler = TestExecutor.Doubler() doublerExecutor = Executor(self.subscriber, doubler, 5000) doublerThread = Thread(target=self.doublerRunnable, args=(doublerExecutor, )) doublerThread.start() #Define the client for that Executor consumer = TestExecutor.TestMessageConsumer() client = ExecutorClient(self.publisher, consumer) #Client sends a request to double 1 request = DoubleRequest() consumer.doubled = None request.number = 1 client.call(request) #MessageBroker.sleep(1500); self.assertIsNotNone(consumer.doubled) print(">>> Received reply with number: " + str(consumer.doubled)) self.assertEqual(2, consumer.doubled) #Client sends a request to double 17 consumer.doubled = None request.number = 17 client.call(request) self.assertIsNotNone(consumer.doubled) print(">>> Received reply with number: " + str(consumer.doubled)) self.assertEqual(34, consumer.doubled) doublerThread.join()
class TestTokenSecurityRabbitMQ(unittest.TestCase): __QUEUE_NAME = "rock.stars" __SERVICE_NAME = "local" __EXCHANGE_NAME = "unit-test-exchange" def setUp(self): self.envelopeRepository = PersistedEnvelopeRepository() self.groupRepository = RecipientGroupRepository() self.broker = RabbitMqMessageBroker( exchangeName=TestTokenSecurityRabbitMQ.__EXCHANGE_NAME, envelopeRepository=self.envelopeRepository, groupRepository=self.groupRepository) #self.queue = self.broker.messageQueue(TestTokenSecurityRabbitMQ.__QUEUE_NAME) self.publisher = Publisher(self.broker, TestTokenSecurityRabbitMQ.__QUEUE_NAME) self.subscriber = Subscriber(self.broker, TestTokenSecurityRabbitMQ.__QUEUE_NAME, TestTokenSecurityRabbitMQ.__SERVICE_NAME) #Drain any existing messages in the logging queue self.broker.drainLoggingQueue() self.envelopeRepository.deleteAll() self.groupRepository.deleteAll() self.brian = TestMessage("Brian May", 71, True) #self.tokenFactory = MockedTokenFactory.getFactory(); self.tokenFactory = JWTFactory() self.broker.setTokenFactory(self.tokenFactory) def tearDown(self): self.subscriber.getQueue().delete() def testSendSecureReceive(self): self.publisher.publish(self.brian) out = self.subscriber.receive() self.assertIsNotNone(out) self.assertEqual(State.Received, out.getState()) self.assertIsNotNone(out.getReceivedTimestamp()) self.assertEqual(self.brian, out.getMessage()) self.assertEqual(out, out.getMessage().getEnvelope()) def testSendSecureReject(self): #inProps = {} #inProps["valid"] = "false" #token = self.tokenFactory.create(inProps) #self.broker.setSendToken(token) #self.publisher.publish(self.brian); token = self.tokenFactory.create() self.broker.setSendToken(token[0:-2]) e = self.publisher.publish(self.brian) messageLogListener = self.broker.getMessageArchiver() messageLogThread = messageLogListener messageLogThread.start() try: # time out right away because we # should see that the message was # rejected out = self.subscriber.receive(1000) except TimeLimitExceededException as e: pass #no-op, expected messageLogThread.join() #Give some time to the background thread to catch up MessageBroker.sleep(1000) _all = self.envelopeRepository.findAll() if _all is not None: for p in _all: state = p.asSimpleEnvelope().getState() print(">>> TestPersistence.envelope(): p: " + p + ", state: " + state) self.assertEqual(State.Rejected, state)
class PLDriver(): def __init__(self, progID, ousUID, recipe): self._progID = progID self._ousUID = ousUID self._recipe = recipe #Initialization code for pipeline execution # Make sure we know where we are self.location = os.environ.get( 'DRAWS_LOCATION' ) if self.location == None: raise RuntimeError( "DRAWS_LOCATION env variable is not defined" ) # Make sure we know where the local replicated cache directory is self.replicatedCache = os.environ.get( 'DRAWS_LOCAL_CACHE' ) if self.replicatedCache == None: raise RuntimeError( "DRAWS_LOCAL_CACHE env variable is not defined" ) self.pipelineScript = "pipeline.py" self.thisDirectory = os.getcwd() # Assume the pipeline script is in this same directory self.pipelineExecutable = self.thisDirectory + "/" + self.pipelineScript self.pipelineRunDirectory = tempfile.mkdtemp( prefix="drw-" ) self.workingDirectory = tempfile.mkdtemp( prefix="drw-" ) print( ">>> PipelineDriver: pipelineRunDirectory:", self.pipelineRunDirectory ) print( ">>> PipelineDriver: workingDirectory:", self.workingDirectory ) self._broker = RabbitMqMessageBroker() params = collections.OrderedDict() params['progID'] = self._progID params['ousUID'] = self._ousUID params['recipe'] = self._recipe params['location'] = self.location params['pipelineRunDir'] = self.pipelineRunDirectory params['replicatedCache'] = self.replicatedCache self._temp = Template(params) def __del__(self): self._broker.closeConnection() #In this method all the tasks are executed and are accounted for. def run(self): res = True cont = True stats = collections.OrderedDict() try: rep, res = self._temp.check() res = res or cont tstats, res = self._temp.preTasks(res, cont) res = res or cont stats.update(tstats) ts = TaskStats('pipeline', 'pip') stats['pipeline'] = ts if res or cont: res = self._pipeline(ts) res = res or cont (tstats, res) = self._temp.postTasks(res, cont) res = res or cont stats.update(tstats) except Exception as e: res = False print("Exception handled") raise e for ts in stats: stats[ts].report() #Specific method to isolate the pipeline execution and error handling. def _pipeline(self, ts): res = True try: ts.start(['comp','exec']) completed = subprocess.run( [self.pipelineExecutable, self._progID, self._ousUID, self._recipe], cwd=self.pipelineRunDirectory) ret = completed.returncode if ret != 0: ts.fail(['comp','exec']) res = False print( ">>> PipelineDriver: Pipeline returned:", ret ) if ret != 2: raise RuntimeError( ">>> PipelineDriver: Pipeline returned: %d" % ret ) # Pipeline returned 2 -- processing failed # Push the OUS to ProcessingProblem, and we're done setState = XTSSSetState(self._ousUID, "ProcessingProblem") xtsspub = Publisher(self._broker, "xtss.transition") envelope = xtsspub.publish(setState) except Exception as e: res = False ts.fail(['comp','exec']) ex = TaskException.handle(e) ts.setEx(ex) ts.stop(['comp','exec']) return res
def __init__(self): self.__broker = RabbitMqMessageBroker("amqp://localhost:5672", "guest", "guest")
def __init__(self): self._baseUrl = "http://localhost:5984" # CouchDB self._dbcon = DbConnection(self._baseUrl) self._dbName = "status-entities" self._broker = RabbitMqMessageBroker() self._subscriber = Subscriber(self._broker, 'xtss.transitions', 'xtss')