def testPurge(self): # Create a message queue, add some messages to it, then purge the data and # verify that it's empty mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Now add some messages bus.publish(mqName, "abc", persistent=True) bus.publish(mqName, "def", persistent=True) # Verify that the messages were added self.assertEqual(_getQueueMessageCount(mqName), 2) self.assertFalse(bus.isEmpty(mqName)) # Purge the queue bus.purge(mqName=mqName) # Verify that the message queue is now empty self.assertEqual(_getQueueMessageCount(mqName), 0) self.assertTrue(bus.isEmpty(mqName))
def testPublishManyMessages(self): numMessagesToPublish = 50 mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Now add a bunch of messages expectedContent = [str(i) for i in xrange(numMessagesToPublish)] _LOGGER.info("testPublishManyMessages: publishing %s tiny messages", numMessagesToPublish) for body in expectedContent: bus.publish(mqName, body, persistent=True) _LOGGER.info("testPublishManyMessages: done publishing %s tiny " "messages", numMessagesToPublish) # Verify that the messages were added self.assertEqual(_getQueueMessageCount(mqName), numMessagesToPublish) connParams = amqp.connection.getRabbitmqConnectionParameters() with amqp.synchronous_amqp_client.SynchronousAmqpClient(connParams) as ( amqpClient): actualContent = [] for i in xrange(numMessagesToPublish): msg = amqpClient.getOneMessage(mqName, noAck=False) actualContent.append(msg.body) msg.ack() self.assertSequenceEqual(actualContent, expectedContent)
def testPollOneMessage(self): # Verify that it can retrieve a message by polling mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName, durable=True) with bus.consume(mqName) as consumer: # Now add some messages msgBody1 = "a" * 100 msgBody2 = "b" * 100000 bus.publish(mqName, msgBody1, persistent=True) bus.publish(mqName, msgBody2, persistent=True) msg = consumer.pollOneMessage() msg.ack() self.assertEqual(msg.body, msgBody1) msg = consumer.pollOneMessage() msg.ack() self.assertEqual(msg.body, msgBody2) msg = consumer.pollOneMessage() self.assertIsNone(msg) # Verify that consumer's context manager cleaned up self.assertIsNone(consumer._channelMgr) # Verify that the message queue is empty now self.assertEqual(_getQueueMessageCount(mqName), 0)
def testConsumerIterable(self): # Create a message queue, publish some messages to it, and then use # the message consumer iterable to retrieve those messages numMessagesToPublish = 10 mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Now add a bunch of messages expectedContent = [] for i in xrange(numMessagesToPublish): expectedContent.append(str(i)) bus.publish(mqName, expectedContent[-1], persistent=True) # Verify that correct number of messages were published self.assertEqual(_getQueueMessageCount(mqName), numMessagesToPublish) # Now, create a consumer iterable and consume the messages # NOTE: we use a thread to avoid deadlocking the test runner in case # something is wrong with the iterable def runConsumerThread(mqName, numMessages, resultQ): try: with MessageBusConnector() as bus: with bus.consume(mqName=mqName) as consumer: it = iter(consumer) for _i in xrange(numMessages): msg = next(it) resultQ.put(msg.body) msg.ack() except: resultQ.put(dict(exception=sys.exc_info()[1])) raise resultQ = Queue.Queue() consumerThread = threading.Thread( target=runConsumerThread, args=(mqName, numMessagesToPublish, resultQ)) consumerThread.setDaemon(True) consumerThread.start() consumerThread.join(timeout=30) self.assertFalse(consumerThread.isAlive()) # Verify content actualContent = [] while True: try: actualContent.append(resultQ.get_nowait()) except Queue.Empty: break self.assertEqual(actualContent, expectedContent) # Verify that the message queue is now empty self.assertEqual(_getQueueMessageCount(mqName), 0)
def testStartMultipleModelRunnersAndStopThem(self): # Starts several ModelRunners and stops them gracefully # to confirm that they can all stop without conflicting with each other: # if ModelRunnerProxy doesn't configure subprocess.Popen with # `close_fds=True`, then graceful shutdown will fail because the stdin # of some child processes will be cloned into those that are started # after them and closing stding of an earlier ModelRunner child process # won't have the desired effect of terminating that process (since other # clones of that file descriptor will prevent it from fully closing) # # TODO send commands to models and verify output runners = [] modelIDs = tuple("abcdef" + str(i) for i in xrange(5)) with ModelSwapperInterface() as swapper: modelInputMQs = tuple( swapper._getModelInputQName(modelID=modelID) for modelID in modelIDs) with amqp_test_utils.managedQueueDeleter(modelInputMQs): with MessageBusConnector() as bus: for mq in modelInputMQs: bus.createMessageQueue(mq, durable=True) for modelID in modelIDs: runners.append( slot_agent.ModelRunnerProxy(modelID=modelID, onTermination=lambda: None, logger=_LOGGER)) returnCodes = [runner.stopGracefully() for runner in runners] self.assertEqual(returnCodes, [0] * len(runners))
def testPublish(self): # Publish messages and verify that they were published mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Now add some messages - a small and a large one msg1 = "a" * 100 msg2 = "b" * 100000 bus.publish(mqName, msg1, persistent=True) bus.publish(mqName, msg2, persistent=True) # Verify that the messages were added self.assertEqual(_getQueueMessageCount(mqName), 2) connParams = amqp.connection.getRabbitmqConnectionParameters() with amqp.synchronous_amqp_client.SynchronousAmqpClient(connParams) as ( amqpClient): msg = amqpClient.getOneMessage(mqName, noAck=False) self.assertEqual(msg.body, msg1) msg.ack() msg = amqpClient.getOneMessage(mqName, noAck=False) self.assertEqual(msg.body, msg2) msg.ack() self.assertEqual(_getQueueMessageCount(mqName), 0)
def testStartMultipleModelRunnersAndStopThem(self): # Starts several ModelRunners and stops them gracefully # to confirm that they can all stop without conflicting with each other: # if ModelRunnerProxy doesn't configure subprocess.Popen with # `close_fds=True`, then graceful shutdown will fail because the stdin # of some child processes will be cloned into those that are started # after them and closing stding of an earlier ModelRunner child process # won't have the desired effect of terminating that process (since other # clones of that file descriptor will prevent it from fully closing) # # TODO send commands to models and verify output runners = [] modelIDs = tuple("abcdef" + str(i) for i in xrange(5)) with ModelSwapperInterface() as swapper: modelInputMQs = tuple(swapper._getModelInputQName(modelID=modelID) for modelID in modelIDs) with amqp_test_utils.managedQueueDeleter(modelInputMQs): with MessageBusConnector() as bus: for mq in modelInputMQs: bus.createMessageQueue(mq, durable=True) for modelID in modelIDs: runners.append( slot_agent.ModelRunnerProxy( modelID=modelID, onTermination=lambda: None, logger=_LOGGER)) returnCodes = [runner.stopGracefully() for runner in runners] self.assertEqual(returnCodes, [0] * len(runners))
def testIsEmptyWithEmptyQueue(self): mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) self.assertTrue(bus.isEmpty(mqName))
def testPurgeWithEmptyQueue(self): # Verify that puring an empty queue doesn't raise an exception mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Purge the empty queue bus.purge(mqName=mqName) # Verify that the message queue exists and indeed has no messages self.assertEqual(_getQueueMessageCount(mqName), 0)
def testGetAllMessageQueues(self): durableMQ = self._getUniqueMessageQueueName() nonDurableMQ = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter((durableMQ, nonDurableMQ)): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=durableMQ, durable=True) bus.createMessageQueue(mqName=nonDurableMQ, durable=False) allQueues = bus.getAllMessageQueues() self.assertIn(durableMQ, allQueues) self.assertIn(nonDurableMQ, allQueues)
def testCreateDurableMessageQueue(self): # Create a durable message queue and verify that it exists # TODO Test that it's a Durable queue and auto-delete=false mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: bus.createMessageQueue(mqName=mqName, durable=True) # Check that MessageBusConnector's context manager cleaned up self.assertIsNone(bus._channelMgr) self.assertEqual(_getQueueMessageCount(mqName), 0)
def testPollOneMessageWithUnackedMessagesReturnedToQueue(self): # Verify that unacked messages retrieved by polling are returned to the # queue after closing the MessageBusConnector instance mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Publish messages to the queue expectedContent = [str(i) for i in xrange(10)] for body in expectedContent: bus.publish(mqName, body, persistent=True) # Retrive the published messages without acking them actualContent = [] with MessageBusConnector() as bus: with bus.consume(mqName) as consumer: for i in xrange(len(expectedContent)): msg = consumer.pollOneMessage() actualContent.append(msg.body) msg = consumer.pollOneMessage() self.assertIsNone(msg) self.assertEqual(actualContent, expectedContent) del actualContent # Now read them again, they should have been returned to the message queue # in the original order. # NOTE: RabbitMQ broker restores them back in original order, but this is # not mandated by AMQP 0.9.1 actualContent = [] with MessageBusConnector() as bus: with bus.consume(mqName) as consumer: for i in xrange(len(expectedContent)): msg = consumer.pollOneMessage() actualContent.append(msg.body) msg.ack() msg = consumer.pollOneMessage() self.assertIsNone(msg) self.assertEqual(actualContent, expectedContent) # Verify that the message queue is empty now self.assertEqual(_getQueueMessageCount(mqName), 0)
def testCreateNonDurableMessageQueueSecondTime(self): # Create a non-durable message queue and verify that repeating the create # call succeeds mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: bus.createMessageQueue(mqName=mqName, durable=False) self.assertEqual(_getQueueMessageCount(mqName), 0) # And one more time... bus.createMessageQueue(mqName=mqName, durable=False) self.assertEqual(_getQueueMessageCount(mqName), 0)
def testIsEmptyWithNonEmptyQueue(self): mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Now add some messages bus.publish(mqName, "abc", persistent=True) bus.publish(mqName, "def", persistent=True) # Verify that the messages were added self.assertEqual(_getQueueMessageCount(mqName), 2) self.assertFalse(bus.isEmpty(mqName))
def testStartModelRunnerAndStopIt(self): # Simple test that starts a ModelRunner and stops it gracefully # TODO send command to model and verify output modelID = "abcdef" with ModelSwapperInterface() as swapper: modelInputMQ = swapper._getModelInputQName(modelID=modelID) with amqp_test_utils.managedQueueDeleter(modelInputMQ): with MessageBusConnector() as bus: bus.createMessageQueue(modelInputMQ, durable=True) runner = slot_agent.ModelRunnerProxy(modelID=modelID, onTermination=lambda: None, logger=_LOGGER) returnCode = runner.stopGracefully() self.assertEqual(returnCode, 0)
def testDeleteMessageQueue(self): mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: bus.createMessageQueue(mqName=mqName, durable=True) self.assertEqual(_getQueueMessageCount(mqName), 0) bus.deleteMessageQueue(mqName=mqName) connParams = amqp.connection.getRabbitmqConnectionParameters() with amqp.synchronous_amqp_client.SynchronousAmqpClient(connParams) as ( amqpClient): with self.assertRaises(amqp.exceptions.AmqpChannelError) as excContext: r = amqpClient.declareQueue(mqName, passive=True) self.assertEqual(excContext.exception.code, amqp.constants.AMQPErrorCodes.NOT_FOUND)
def testMessageBusConnectorCleanupOfConsumerGenerators(self): # Verify that MessageBusConnector closes unclosed consumers numMessagesToPublish = 1 mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Now add messages expectedContent = [] for i in xrange(numMessagesToPublish): expectedContent.append(str(i)) bus.publish(mqName, expectedContent[-1], persistent=True) # Verify that correct number of messages were published self.assertEqual(_getQueueMessageCount(mqName), numMessagesToPublish) bus = MessageBusConnector() # Now, create a consumer iterable and start it by consuming the expected # messages # NOTE: we use a thread to avoid deadlocking the test runner in case # something is wrong with the iterable def runConsumerThread(bus, mqName, numMessages, resultQ): try: # NOTE: in this test, we intentionally don't close the consumer consumer = bus.consume(mqName=mqName) resultQ.put(consumer) it = iter(consumer) for _i in xrange(numMessages): msg = next(it) msg.ack() resultQ.put(msg.body) except: resultQ.put(dict(exception=sys.exc_info()[1])) raise resultQ = Queue.Queue() consumerThread = threading.Thread( target=runConsumerThread, args=(bus, mqName, numMessagesToPublish, resultQ)) consumerThread.setDaemon(True) consumerThread.start() # Wait for thread to stop consumerThread.join(timeout=30) self.assertFalse(consumerThread.isAlive()) # Reap the consumer consumer1 = resultQ.get_nowait() self.assertIsInstance(consumer1, message_bus_connector._QueueConsumer) # Verify content actualContent = [] while True: try: actualContent.append(resultQ.get_nowait()) except Queue.Empty: break self.assertEqual(actualContent, expectedContent) # Verify that this consumer isn't closed self.assertEqual(len(bus._consumers), 1) self.assertIn(consumer1, bus._consumers) self.assertIsNotNone(consumer1._channelMgr) # Create another consumer, but don't start it (by telling it to consume 0 # messages; we want to test cleanup of both started and unstarted # consumers resultQ = Queue.Queue() consumerThread = threading.Thread( target=runConsumerThread, args=(bus, mqName, 0, resultQ)) consumerThread.setDaemon(True) consumerThread.start() # Wait for thread to stop consumerThread.join(timeout=30) self.assertFalse(consumerThread.isAlive()) # Reap the consumer consumer2 = resultQ.get_nowait() self.assertIsInstance(consumer2, message_bus_connector._QueueConsumer) try: item = resultQ.get_nowait() except Queue.Empty: pass else: self.fail("Unexpected item in resultQ: %r" % (item,)) # Verify that this consumer isn't closed self.assertEqual(len(bus._consumers), 2) self.assertIn(consumer2, bus._consumers) self.assertIsNotNone(consumer2._channelMgr) # Verify that this consumer hasn't started self.assertIsNone(consumer2._channelMgr._client) # Verify that the first consumer is still there, too self.assertIn(consumer1, bus._consumers) # Finanly, close the MessageBusConnector and verify that both consumers # got closed, too bus.close() self.assertFalse(bus._consumers) self.assertIsNone(consumer1._channelMgr) self.assertIsNone(consumer1._bus) self.assertIsNone(consumer1._channelMgr) self.assertIsNone(consumer2._channelMgr) self.assertIsNone(consumer2._bus) self.assertIsNone(consumer2._channelMgr)
def testConsumerIterableWithUnackedMessagesReturnedToQueue(self): # Verify that it can retrieve a message by polling mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Add the number of messages equal to MessageBusConnector's prefetch # limit expectedContent = [str(i) for i in xrange(MessageBusConnector._PREFETCH_MAX)] for body in expectedContent: bus.publish(mqName, body, persistent=True) # Retrive the published messages without acking them def runConsumerThread(mqName, numMessages, resultQ): try: with MessageBusConnector() as bus: with bus.consume(mqName=mqName) as consumer: it = iter(consumer) for _i in xrange(numMessages): # Read and don't ack msg = next(it) resultQ.put(msg.body) except: resultQ.put(dict(exception=sys.exc_info()[1])) raise resultQ = Queue.Queue() consumerThread = threading.Thread( target=runConsumerThread, args=(mqName, MessageBusConnector._PREFETCH_MAX, resultQ)) consumerThread.setDaemon(True) consumerThread.start() consumerThread.join(timeout=30) self.assertFalse(consumerThread.isAlive()) # Verify the retrieved unacked content actualContent = [] while True: try: actualContent.append(resultQ.get_nowait()) except Queue.Empty: break self.assertEqual(actualContent, expectedContent) del actualContent # Now read them again, they should have been returned to the message queue # NOTE: RabbitMQ broker restores them back in original order actualContent = [] with MessageBusConnector() as bus: with bus.consume(mqName=mqName) as consumer: for i in xrange(MessageBusConnector._PREFETCH_MAX): msg = consumer.pollOneMessage() actualContent.append(msg.body) msg.ack() msg = consumer.pollOneMessage() self.assertIsNone(msg) self.assertEqual(actualContent, expectedContent) # Verify that the message queue is empty now self.assertEqual(_getQueueMessageCount(mqName), 0)