def __init__(self, progID, ousUID, recipe):
self._progID = progID
self._ousUID = ousUID
self._recipe = recipe
#Initialization code for pipeline execution
# Make sure we know where we are
self.location = os.environ.get( 'DRAWS_LOCATION' )
if self.location == None:
raise RuntimeError( "DRAWS_LOCATION env variable is not defined" )
# Make sure we know where the local replicated cache directory is
self.replicatedCache = os.environ.get( 'DRAWS_LOCAL_CACHE' )
if self.replicatedCache == None:
raise RuntimeError( "DRAWS_LOCAL_CACHE env variable is not defined" )
self.pipelineScript = "pipeline.py"
self.thisDirectory = os.getcwd() # Assume the pipeline script is in this same directory
self.pipelineExecutable = self.thisDirectory + "/" + self.pipelineScript
self.pipelineRunDirectory = tempfile.mkdtemp( prefix="drw-" )
self.workingDirectory = tempfile.mkdtemp( prefix="drw-" )
print( ">>> PipelineDriver: pipelineRunDirectory:", self.pipelineRunDirectory )
print( ">>> PipelineDriver: workingDirectory:", self.workingDirectory )
self._broker = RabbitMqMessageBroker()
params = collections.OrderedDict()
params['progID'] = self._progID
params['ousUID'] = self._ousUID
params['recipe'] = self._recipe
params['location'] = self.location
params['pipelineRunDir'] = self.pipelineRunDirectory
params['replicatedCache'] = self.replicatedCache
self._temp = Template(params)
class BasicSender(object):
#private Logger logger = LoggerFactory.getLogger( BasicReceiver.class );
def __init__(self):
self.__broker = RabbitMqMessageBroker("amqp://localhost:5672", "guest", "guest")
def run(self, args):
publisher = Publisher(self.__broker, args.qname)
freddie = Person("Freddie Mercury", 45, False)
print("Sending to " + args.qname)
envelope = publisher.publish(freddie)
print(">>> Sent to " + args.qname + ": " + str(envelope.getMessage()))
self.__broker.closeConnection()
def setUp(self):
self.envelopeRepository = PersistedEnvelopeRepository()
self.groupRepository = RecipientGroupRepository()
self.broker = RabbitMqMessageBroker("amqp://localhost:5672", "guest",
"guest",
TestExecutor.__EXCHANGE_NAME,
self.envelopeRepository,
self.groupRepository)
self.publisher = Publisher(self.broker, TestExecutor.__QUEUE_NAME)
self.subscriber = Subscriber(self.broker, TestExecutor.__QUEUE_NAME,
"test")
self.broker.drainLoggingQueue()
self.broker.drainQueue(self.subscriber.getQueue())
class BasicReceiver(object):
def __init__(self):
self.__broker = RabbitMqMessageBroker("amqp://localhost:5672", "guest",
"guest")
def run(self, args):
subscriber = Subscriber(self.__broker, args.qname, args.sname)
print("Waiting for message")
received = subscriber.receive(120 * 1000)
msg = args.sname + "(" + str(os.getpid()) + ") received: " + str(
received.getMessage())
self.__broker.closeConnection()
print(msg)
def __init__(self, location):
self._baseUrl = "http://localhost:5984" # CouchDB
self._dbconn = DbConnection(baseUrl)
self._xtss = ExecutorClient('localhost', 'msgq', 'xtss')
self._mq = MqConnection('localhost', 'msgq')
self._broker = RabbitMqMessageBroker()
self.location = location
def setUp(self):
self.envelopeRepository = PersistedEnvelopeRepository()
self.groupRepository = RecipientGroupRepository()
self.broker = RabbitMqMessageBroker(
exchangeName=TestTokenSecurityRabbitMQ.__EXCHANGE_NAME,
envelopeRepository=self.envelopeRepository,
groupRepository=self.groupRepository)
#self.queue = self.broker.messageQueue(TestTokenSecurityRabbitMQ.__QUEUE_NAME)
self.publisher = Publisher(self.broker,
TestTokenSecurityRabbitMQ.__QUEUE_NAME)
self.subscriber = Subscriber(self.broker,
TestTokenSecurityRabbitMQ.__QUEUE_NAME,
TestTokenSecurityRabbitMQ.__SERVICE_NAME)
#Drain any existing messages in the logging queue
self.broker.drainLoggingQueue()
self.envelopeRepository.deleteAll()
self.groupRepository.deleteAll()
self.brian = TestMessage("Brian May", 71, True)
#self.tokenFactory = MockedTokenFactory.getFactory();
self.tokenFactory = JWTFactory()
self.broker.setTokenFactory(self.tokenFactory)
class TestExecutor(unittest.TestCase):
__QUEUE_NAME = "test.executor.queue"
__EXCHANGE_NAME = "unit-test-exchange"
#Doubles its input
class Doubler(RequestProcessor):
def process(self, message):
request = message
print(">>> Received request with number: " + str(request.number))
response = DoubleResponse()
response.doubled = request.number + request.number
return response
class TestMessageConsumer(MessageConsumer):
def __init__(self):
self.doubled = None
def consume(self, message):
if not isinstance(message, DoubleResponse):
msg = "Not a " + DoubleResponse.__name__ + ": " + str(message)
print(">>>>> message 2: " + str(message))
print(">>>>> Thread: " + threading.currentThread().getName())
print(">>>>> " + msg)
print(msg)
raise Exception(msg)
self.doubled = message.doubled
def setUp(self):
self.envelopeRepository = PersistedEnvelopeRepository()
self.groupRepository = RecipientGroupRepository()
self.broker = RabbitMqMessageBroker("amqp://localhost:5672", "guest",
"guest",
TestExecutor.__EXCHANGE_NAME,
self.envelopeRepository,
self.groupRepository)
self.publisher = Publisher(self.broker, TestExecutor.__QUEUE_NAME)
self.subscriber = Subscriber(self.broker, TestExecutor.__QUEUE_NAME,
"test")
self.broker.drainLoggingQueue()
self.broker.drainQueue(self.subscriber.getQueue())
def tearDown(self):
self.broker.deleteQueue(self.subscriber.getQueue())
def doublerRunnable(self, doublerExecutor):
try:
doublerExecutor.run()
except TimeLimitExceededException as e:
print(">>> Timed out: " + str(e))
except Exception as e:
raise Exception(e)
def testDoubler(self):
doubler = TestExecutor.Doubler()
doublerExecutor = Executor(self.subscriber, doubler, 5000)
doublerThread = Thread(target=self.doublerRunnable,
args=(doublerExecutor, ))
doublerThread.start()
#Define the client for that Executor
consumer = TestExecutor.TestMessageConsumer()
client = ExecutorClient(self.publisher, consumer)
#Client sends a request to double 1
request = DoubleRequest()
consumer.doubled = None
request.number = 1
client.call(request)
#MessageBroker.sleep(1500);
self.assertIsNotNone(consumer.doubled)
print(">>> Received reply with number: " + str(consumer.doubled))
self.assertEqual(2, consumer.doubled)
#Client sends a request to double 17
consumer.doubled = None
request.number = 17
client.call(request)
self.assertIsNotNone(consumer.doubled)
print(">>> Received reply with number: " + str(consumer.doubled))
self.assertEqual(34, consumer.doubled)
doublerThread.join()
class TestTokenSecurityRabbitMQ(unittest.TestCase):
__QUEUE_NAME = "rock.stars"
__SERVICE_NAME = "local"
__EXCHANGE_NAME = "unit-test-exchange"
def setUp(self):
self.envelopeRepository = PersistedEnvelopeRepository()
self.groupRepository = RecipientGroupRepository()
self.broker = RabbitMqMessageBroker(
exchangeName=TestTokenSecurityRabbitMQ.__EXCHANGE_NAME,
envelopeRepository=self.envelopeRepository,
groupRepository=self.groupRepository)
#self.queue = self.broker.messageQueue(TestTokenSecurityRabbitMQ.__QUEUE_NAME)
self.publisher = Publisher(self.broker,
TestTokenSecurityRabbitMQ.__QUEUE_NAME)
self.subscriber = Subscriber(self.broker,
TestTokenSecurityRabbitMQ.__QUEUE_NAME,
TestTokenSecurityRabbitMQ.__SERVICE_NAME)
#Drain any existing messages in the logging queue
self.broker.drainLoggingQueue()
self.envelopeRepository.deleteAll()
self.groupRepository.deleteAll()
self.brian = TestMessage("Brian May", 71, True)
#self.tokenFactory = MockedTokenFactory.getFactory();
self.tokenFactory = JWTFactory()
self.broker.setTokenFactory(self.tokenFactory)
def tearDown(self):
self.subscriber.getQueue().delete()
def testSendSecureReceive(self):
self.publisher.publish(self.brian)
out = self.subscriber.receive()
self.assertIsNotNone(out)
self.assertEqual(State.Received, out.getState())
self.assertIsNotNone(out.getReceivedTimestamp())
self.assertEqual(self.brian, out.getMessage())
self.assertEqual(out, out.getMessage().getEnvelope())
def testSendSecureReject(self):
#inProps = {}
#inProps["valid"] = "false"
#token = self.tokenFactory.create(inProps)
#self.broker.setSendToken(token)
#self.publisher.publish(self.brian);
token = self.tokenFactory.create()
self.broker.setSendToken(token[0:-2])
e = self.publisher.publish(self.brian)
messageLogListener = self.broker.getMessageArchiver()
messageLogThread = messageLogListener
messageLogThread.start()
try:
# time out right away because we
# should see that the message was
# rejected
out = self.subscriber.receive(1000)
except TimeLimitExceededException as e:
pass #no-op, expected
messageLogThread.join()
#Give some time to the background thread to catch up
MessageBroker.sleep(1000)
_all = self.envelopeRepository.findAll()
if _all is not None:
for p in _all:
state = p.asSimpleEnvelope().getState()
print(">>> TestPersistence.envelope(): p: " + p + ", state: " +
state)
self.assertEqual(State.Rejected, state)
class PLDriver():
def __init__(self, progID, ousUID, recipe):
self._progID = progID
self._ousUID = ousUID
self._recipe = recipe
#Initialization code for pipeline execution
# Make sure we know where we are
self.location = os.environ.get( 'DRAWS_LOCATION' )
if self.location == None:
raise RuntimeError( "DRAWS_LOCATION env variable is not defined" )
# Make sure we know where the local replicated cache directory is
self.replicatedCache = os.environ.get( 'DRAWS_LOCAL_CACHE' )
if self.replicatedCache == None:
raise RuntimeError( "DRAWS_LOCAL_CACHE env variable is not defined" )
self.pipelineScript = "pipeline.py"
self.thisDirectory = os.getcwd() # Assume the pipeline script is in this same directory
self.pipelineExecutable = self.thisDirectory + "/" + self.pipelineScript
self.pipelineRunDirectory = tempfile.mkdtemp( prefix="drw-" )
self.workingDirectory = tempfile.mkdtemp( prefix="drw-" )
print( ">>> PipelineDriver: pipelineRunDirectory:", self.pipelineRunDirectory )
print( ">>> PipelineDriver: workingDirectory:", self.workingDirectory )
self._broker = RabbitMqMessageBroker()
params = collections.OrderedDict()
params['progID'] = self._progID
params['ousUID'] = self._ousUID
params['recipe'] = self._recipe
params['location'] = self.location
params['pipelineRunDir'] = self.pipelineRunDirectory
params['replicatedCache'] = self.replicatedCache
self._temp = Template(params)
def __del__(self):
self._broker.closeConnection()
#In this method all the tasks are executed and are accounted for.
def run(self):
res = True
cont = True
stats = collections.OrderedDict()
try:
rep, res = self._temp.check()
res = res or cont
tstats, res = self._temp.preTasks(res, cont)
res = res or cont
stats.update(tstats)
ts = TaskStats('pipeline', 'pip')
stats['pipeline'] = ts
if res or cont:
res = self._pipeline(ts)
res = res or cont
(tstats, res) = self._temp.postTasks(res, cont)
res = res or cont
stats.update(tstats)
except Exception as e:
res = False
print("Exception handled")
raise e
for ts in stats:
stats[ts].report()
#Specific method to isolate the pipeline execution and error handling.
def _pipeline(self, ts):
res = True
try:
ts.start(['comp','exec'])
completed = subprocess.run( [self.pipelineExecutable, self._progID, self._ousUID, self._recipe], cwd=self.pipelineRunDirectory)
ret = completed.returncode
if ret != 0:
ts.fail(['comp','exec'])
res = False
print( ">>> PipelineDriver: Pipeline returned:", ret )
if ret != 2:
raise RuntimeError( ">>> PipelineDriver: Pipeline returned: %d" % ret )
# Pipeline returned 2 -- processing failed
# Push the OUS to ProcessingProblem, and we're done
setState = XTSSSetState(self._ousUID, "ProcessingProblem")
xtsspub = Publisher(self._broker, "xtss.transition")
envelope = xtsspub.publish(setState)
except Exception as e:
res = False
ts.fail(['comp','exec'])
ex = TaskException.handle(e)
ts.setEx(ex)
ts.stop(['comp','exec'])
return res
def __init__(self):
self.__broker = RabbitMqMessageBroker("amqp://localhost:5672", "guest", "guest")
def __init__(self):
self._baseUrl = "http://localhost:5984" # CouchDB
self._dbcon = DbConnection(self._baseUrl)
self._dbName = "status-entities"
self._broker = RabbitMqMessageBroker()
self._subscriber = Subscriber(self._broker, 'xtss.transitions', 'xtss')