def __init__(self): """ Initialize the ModelSwapperInterface. This uses a lazy loading of the input and output queues with no pre-meditation. """ self._logger = _getLogger() config = ModelSwapperConfig() self._resultsQueueName = config.get(self._CONFIG_SECTION, self._RESULTS_Q_OPTION_NAME) # The name of a model's input message queue is the concatenation of this # prefix and the modelID self._modelInputQueueNamePrefix = config.get( self._CONFIG_SECTION, self._MODEL_INPUT_Q_PREFIX_OPTION_NAME) self._schedulerNotificationQueueName = config.get( self._CONFIG_SECTION, self._SCHEDULER_NOTIFICATION_Q_OPTION_NAME) # Message bus connector self._bus = MessageBusConnector() # Outstanding request and/or response consumer instances self._consumers = []
def testPollOneMessageWithUnackedMessagesReturnedToQueue(self): # Verify that unacked messages retrieved by polling are returned to the # queue after closing the MessageBusConnector instance mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Publish messages to the queue expectedContent = [str(i) for i in xrange(10)] for body in expectedContent: bus.publish(mqName, body, persistent=True) # Retrive the published messages without acking them actualContent = [] with MessageBusConnector() as bus: with bus.consume(mqName) as consumer: for i in xrange(len(expectedContent)): msg = consumer.pollOneMessage() actualContent.append(msg.body) msg = consumer.pollOneMessage() self.assertIsNone(msg) self.assertEqual(actualContent, expectedContent) del actualContent # Now read them again, they should have been returned to the message queue # in the original order. # NOTE: RabbitMQ broker restores them back in original order, but this is # not mandated by AMQP 0.9.1 actualContent = [] with MessageBusConnector() as bus: with bus.consume(mqName) as consumer: for i in xrange(len(expectedContent)): msg = consumer.pollOneMessage() actualContent.append(msg.body) msg.ack() msg = consumer.pollOneMessage() self.assertIsNone(msg) self.assertEqual(actualContent, expectedContent) # Verify that the message queue is empty now self.assertEqual(_getQueueMessageCount(mqName), 0)
def testStartMultipleModelRunnersAndStopThem(self): # Starts several ModelRunners and stops them gracefully # to confirm that they can all stop without conflicting with each other: # if ModelRunnerProxy doesn't configure subprocess.Popen with # `close_fds=True`, then graceful shutdown will fail because the stdin # of some child processes will be cloned into those that are started # after them and closing stding of an earlier ModelRunner child process # won't have the desired effect of terminating that process (since other # clones of that file descriptor will prevent it from fully closing) # # TODO send commands to models and verify output runners = [] modelIDs = tuple("abcdef" + str(i) for i in xrange(5)) with ModelSwapperInterface() as swapper: modelInputMQs = tuple( swapper._getModelInputQName(modelID=modelID) for modelID in modelIDs) with amqp_test_utils.managedQueueDeleter(modelInputMQs): with MessageBusConnector() as bus: for mq in modelInputMQs: bus.createMessageQueue(mq, durable=True) for modelID in modelIDs: runners.append( slot_agent.ModelRunnerProxy(modelID=modelID, onTermination=lambda: None, logger=_LOGGER)) returnCodes = [runner.stopGracefully() for runner in runners] self.assertEqual(returnCodes, [0] * len(runners))
def testPublishManyMessages(self): numMessagesToPublish = 50 mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Now add a bunch of messages expectedContent = [str(i) for i in xrange(numMessagesToPublish)] _LOGGER.info("testPublishManyMessages: publishing %s tiny messages", numMessagesToPublish) for body in expectedContent: bus.publish(mqName, body, persistent=True) _LOGGER.info("testPublishManyMessages: done publishing %s tiny " "messages", numMessagesToPublish) # Verify that the messages were added self.assertEqual(_getQueueMessageCount(mqName), numMessagesToPublish) connParams = amqp.connection.getRabbitmqConnectionParameters() with amqp.synchronous_amqp_client.SynchronousAmqpClient(connParams) as ( amqpClient): actualContent = [] for i in xrange(numMessagesToPublish): msg = amqpClient.getOneMessage(mqName, noAck=False) actualContent.append(msg.body) msg.ack() self.assertSequenceEqual(actualContent, expectedContent)
def testPollOneMessage(self): # Verify that it can retrieve a message by polling mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName, durable=True) with bus.consume(mqName) as consumer: # Now add some messages msgBody1 = "a" * 100 msgBody2 = "b" * 100000 bus.publish(mqName, msgBody1, persistent=True) bus.publish(mqName, msgBody2, persistent=True) msg = consumer.pollOneMessage() msg.ack() self.assertEqual(msg.body, msgBody1) msg = consumer.pollOneMessage() msg.ack() self.assertEqual(msg.body, msgBody2) msg = consumer.pollOneMessage() self.assertIsNone(msg) # Verify that consumer's context manager cleaned up self.assertIsNone(consumer._channelMgr) # Verify that the message queue is empty now self.assertEqual(_getQueueMessageCount(mqName), 0)
def testConsumerIterable(self): # Create a message queue, publish some messages to it, and then use # the message consumer iterable to retrieve those messages numMessagesToPublish = 10 mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Now add a bunch of messages expectedContent = [] for i in xrange(numMessagesToPublish): expectedContent.append(str(i)) bus.publish(mqName, expectedContent[-1], persistent=True) # Verify that correct number of messages were published self.assertEqual(_getQueueMessageCount(mqName), numMessagesToPublish) # Now, create a consumer iterable and consume the messages # NOTE: we use a thread to avoid deadlocking the test runner in case # something is wrong with the iterable def runConsumerThread(mqName, numMessages, resultQ): try: with MessageBusConnector() as bus: with bus.consume(mqName=mqName) as consumer: it = iter(consumer) for _i in xrange(numMessages): msg = next(it) resultQ.put(msg.body) msg.ack() except: resultQ.put(dict(exception=sys.exc_info()[1])) raise resultQ = Queue.Queue() consumerThread = threading.Thread( target=runConsumerThread, args=(mqName, numMessagesToPublish, resultQ)) consumerThread.setDaemon(True) consumerThread.start() consumerThread.join(timeout=30) self.assertFalse(consumerThread.isAlive()) # Verify content actualContent = [] while True: try: actualContent.append(resultQ.get_nowait()) except Queue.Empty: break self.assertEqual(actualContent, expectedContent) # Verify that the message queue is now empty self.assertEqual(_getQueueMessageCount(mqName), 0)
def testPublish(self): # Publish messages and verify that they were published mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Now add some messages - a small and a large one msg1 = "a" * 100 msg2 = "b" * 100000 bus.publish(mqName, msg1, persistent=True) bus.publish(mqName, msg2, persistent=True) # Verify that the messages were added self.assertEqual(_getQueueMessageCount(mqName), 2) connParams = amqp.connection.getRabbitmqConnectionParameters() with amqp.synchronous_amqp_client.SynchronousAmqpClient(connParams) as ( amqpClient): msg = amqpClient.getOneMessage(mqName, noAck=False) self.assertEqual(msg.body, msg1) msg.ack() msg = amqpClient.getOneMessage(mqName, noAck=False) self.assertEqual(msg.body, msg2) msg.ack() self.assertEqual(_getQueueMessageCount(mqName), 0)
def testPurge(self): # Create a message queue, add some messages to it, then purge the data and # verify that it's empty mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Now add some messages bus.publish(mqName, "abc", persistent=True) bus.publish(mqName, "def", persistent=True) # Verify that the messages were added self.assertEqual(_getQueueMessageCount(mqName), 2) self.assertFalse(bus.isEmpty(mqName)) # Purge the queue bus.purge(mqName=mqName) # Verify that the message queue is now empty self.assertEqual(_getQueueMessageCount(mqName), 0) self.assertTrue(bus.isEmpty(mqName))
def __init__(self): """ Initialize the ModelSwapperInterface. This uses a lazy loading of the input and output queues with no pre-meditation. """ self._logger = _getLogger() config = ModelSwapperConfig() self._resultsQueueName = config.get( self._CONFIG_SECTION, self._RESULTS_Q_OPTION_NAME) # The name of a model's input message queue is the concatenation of this # prefix and the modelID self._modelInputQueueNamePrefix = config.get( self._CONFIG_SECTION, self._MODEL_INPUT_Q_PREFIX_OPTION_NAME) self._schedulerNotificationQueueName = config.get( self._CONFIG_SECTION, self._SCHEDULER_NOTIFICATION_Q_OPTION_NAME) # Message bus connector self._bus = MessageBusConnector() # Outstanding request and/or response consumer instances self._consumers = []
def testPublishExgNotPublished(self): """ Test MessageBusConnector.publishExg returns false when failed to publish immediately """ exgName = "testPublishExgNotPublished" routingKey = "testPublishExgNotPublished-routing-key" # Create an exchange, but don't bind a queue to it connParams = amqp.connection.getRabbitmqConnectionParameters() with amqp.synchronous_amqp_client.SynchronousAmqpClient(connParams) as ( amqpClient): amqpClient.declareExchange(exgName, exchangeType="direct") # Now publish to that exchange via MessageBusConnector with MessageBusConnector() as bus: # Now attempt to publish a single message with mandatory=True published = bus.publishExg( exchange=exgName, routingKey=routingKey, body="testPublishExgNotPublished-body", properties=None, mandatory=True) # Verify that the message failed to publish self.assertFalse(published)
def testPublishWithQueueNotFound(self): # Verify that isEmpty on a non-existent message queue raises the expected # exception mqName = self._getUniqueMessageQueueName() with self.assertRaises(MessageQueueNotFound): with MessageBusConnector() as bus: bus.publish(mqName, "abc", persistent=True)
def testIsEmptyWithQueueNotFound(self): # Verify that isEmpty on a non-existent message queue raises the expected # exception mqName = self._getUniqueMessageQueueName() with MessageBusConnector() as bus: with self.assertRaises(MessageQueueNotFound): bus.isEmpty(mqName=mqName)
def testCreateDurableMessageQueueSecondTime(self): # Create a durable message queue and verify that repeating the create call # succeeds mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: bus.createMessageQueue(mqName=mqName, durable=True) self.assertEqual(_getQueueMessageCount(mqName), 0) # And one more time... with MessageBusConnector() as bus: bus.createMessageQueue(mqName=mqName, durable=True) self.assertEqual(_getQueueMessageCount(mqName), 0)
def testPollOneMessageWithQueueNotFound(self): # Verify that calling pollOneMessage on a non-existent queue raises the # expected exception mqName = self._getUniqueMessageQueueName() with MessageBusConnector() as bus: with self.assertRaises(MessageQueueNotFound): with bus.consume(mqName) as consumer: consumer.pollOneMessage()
def testIsEmptyWithEmptyQueue(self): mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) self.assertTrue(bus.isEmpty(mqName))
def runConsumerThread(mqName, resultQ): try: with MessageBusConnector() as bus: with bus.consume(mqName=mqName, blocking=False) as consumer: for msg in consumer: resultQ.put(msg.body) msg.ack() except: resultQ.put(dict(exception=sys.exc_info()[1])) raise
def runConsumerThread(mqName, resultQ): try: with MessageBusConnector() as bus: with bus.consume(mqName=mqName) as consumer: # NOTE: we actually don't expect any messages in this test for msg in consumer: msg.ack() except: # pylint: disable=W0702 # NOTE: this is what we expect in this test since the mq wasn't created resultQ.put(dict(exception=sys.exc_info()[1]))
def runConsumerThread(mqName, numMessages, resultQ): try: with MessageBusConnector() as bus: with bus.consume(mqName=mqName) as consumer: it = iter(consumer) for _i in xrange(numMessages): # Read and don't ack msg = next(it) resultQ.put(msg.body) except: resultQ.put(dict(exception=sys.exc_info()[1])) raise
def onTimeout(resultsQueueName): _LOGGER.error( "Timed out waiting to get results from models; numResults=%d; " "expected=%d", len(seenMetricIDs), len(allMetricIDs)) # HACK delete model swapper results queue to abort the consumer try: with MessageBusConnector() as bus: bus.deleteMessageQueue(resultsQueueName) except Exception: _LOGGER.exception("Failed to delete results mq=%s", resultsQueueName) raise
def testCreateDurableMessageQueue(self): # Create a durable message queue and verify that it exists # TODO Test that it's a Durable queue and auto-delete=false mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: bus.createMessageQueue(mqName=mqName, durable=True) # Check that MessageBusConnector's context manager cleaned up self.assertIsNone(bus._channelMgr) self.assertEqual(_getQueueMessageCount(mqName), 0)
def testDeleteMessageQueueThatDoesNotExist(self): # Verify that deleting a non-existent message queue doesn't raise an # exception mqName = self._getUniqueMessageQueueName() # NOTE: deleting an entity that doesn't exist used to result in # NOT_FOUND=404 channel error from RabbitMQ. However, more recent versions # of RabbitMQ changed that behavior such that it now completes with success. # Per https://www.rabbitmq.com/specification.html: "We have made # queue.delete into an idempotent assertion that the queue must not exist, # in the same way that queue.declare asserts that it must." with MessageBusConnector() as bus: bus.deleteMessageQueue(mqName=mqName)
def testPurgeWithEmptyQueue(self): # Verify that puring an empty queue doesn't raise an exception mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Purge the empty queue bus.purge(mqName=mqName) # Verify that the message queue exists and indeed has no messages self.assertEqual(_getQueueMessageCount(mqName), 0)
def testGetAllMessageQueues(self): durableMQ = self._getUniqueMessageQueueName() nonDurableMQ = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter((durableMQ, nonDurableMQ)): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=durableMQ, durable=True) bus.createMessageQueue(mqName=nonDurableMQ, durable=False) allQueues = bus.getAllMessageQueues() self.assertIn(durableMQ, allQueues) self.assertIn(nonDurableMQ, allQueues)
def testIsEmptyWithNonEmptyQueue(self): mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Now add some messages bus.publish(mqName, "abc", persistent=True) bus.publish(mqName, "def", persistent=True) # Verify that the messages were added self.assertEqual(_getQueueMessageCount(mqName), 2) self.assertFalse(bus.isEmpty(mqName))
def testDeleteMessageQueue(self): mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: bus.createMessageQueue(mqName=mqName, durable=True) self.assertEqual(_getQueueMessageCount(mqName), 0) bus.deleteMessageQueue(mqName=mqName) connParams = amqp.connection.getRabbitmqConnectionParameters() with amqp.synchronous_amqp_client.SynchronousAmqpClient(connParams) as ( amqpClient): with self.assertRaises(amqp.exceptions.AmqpChannelError) as excContext: r = amqpClient.declareQueue(mqName, passive=True) self.assertEqual(excContext.exception.code, amqp.constants.AMQPErrorCodes.NOT_FOUND)
def testStartModelRunnerAndStopIt(self): # Simple test that starts a ModelRunner and stops it gracefully # TODO send command to model and verify output modelID = "abcdef" with ModelSwapperInterface() as swapper: modelInputMQ = swapper._getModelInputQName(modelID=modelID) with amqp_test_utils.managedQueueDeleter(modelInputMQ): with MessageBusConnector() as bus: bus.createMessageQueue(modelInputMQ, durable=True) runner = slot_agent.ModelRunnerProxy(modelID=modelID, onTermination=lambda: None, logger=_LOGGER) returnCode = runner.stopGracefully() self.assertEqual(returnCode, 0)
def _cleanRabbitmq(): """Delete Taurus Engine-related message queues and exchanges""" g_log.info("Deleting Taurus Engine-related message queues and exchanges") appConfig = taurus.engine.config modelSwapperConfig = model_swapper.ModelSwapperConfig() # Delete queues belonging to Taurus taurusQueues = [ modelSwapperConfig.get("interface_bus", "results_queue"), modelSwapperConfig.get("interface_bus", "scheduler_notification_queue"), appConfig.get("metric_listener", "queue_name"), DynamoDBService._INPUT_QUEUE_NAME # pylint: disable=W0212 ] modelInputPrefix = modelSwapperConfig.get("interface_bus", "model_input_queue_prefix") with MessageBusConnector() as messageBus: for queue in messageBus.getAllMessageQueues(): if queue.startswith(modelInputPrefix) or queue in taurusQueues: messageBus.deleteMessageQueue(queue) # Delete exchanges belonging to Taurus taurusExchanges = [ appConfig.get("metric_streamer", "results_exchange_name"), appConfig.get("non_metric_data", "exchange_name") ] amqpClient = amqp.synchronous_amqp_client.SynchronousAmqpClient( connectionParams=amqp.connection.getRabbitmqConnectionParameters()) with amqpClient: for exg in taurusExchanges: g_log.info("Deleting Taurus exchange=%s", exg) amqpClient.deleteExchange(exchange=exg)
def testMessageBusIsAccessible(self): # pylint: disable=R0201 with MessageBusConnector() as bus: bus.isMessageQeueuePresent("")
def testMessageBusConnectorCleanupOfConsumerGenerators(self): # Verify that MessageBusConnector closes unclosed consumers numMessagesToPublish = 1 mqName = self._getUniqueMessageQueueName() with amqp_test_utils.managedQueueDeleter(mqName): with MessageBusConnector() as bus: # Create the queue bus.createMessageQueue(mqName=mqName, durable=True) # Now add messages expectedContent = [] for i in xrange(numMessagesToPublish): expectedContent.append(str(i)) bus.publish(mqName, expectedContent[-1], persistent=True) # Verify that correct number of messages were published self.assertEqual(_getQueueMessageCount(mqName), numMessagesToPublish) bus = MessageBusConnector() # Now, create a consumer iterable and start it by consuming the expected # messages # NOTE: we use a thread to avoid deadlocking the test runner in case # something is wrong with the iterable def runConsumerThread(bus, mqName, numMessages, resultQ): try: # NOTE: in this test, we intentionally don't close the consumer consumer = bus.consume(mqName=mqName) resultQ.put(consumer) it = iter(consumer) for _i in xrange(numMessages): msg = next(it) msg.ack() resultQ.put(msg.body) except: resultQ.put(dict(exception=sys.exc_info()[1])) raise resultQ = Queue.Queue() consumerThread = threading.Thread( target=runConsumerThread, args=(bus, mqName, numMessagesToPublish, resultQ)) consumerThread.setDaemon(True) consumerThread.start() # Wait for thread to stop consumerThread.join(timeout=30) self.assertFalse(consumerThread.isAlive()) # Reap the consumer consumer1 = resultQ.get_nowait() self.assertIsInstance(consumer1, message_bus_connector._QueueConsumer) # Verify content actualContent = [] while True: try: actualContent.append(resultQ.get_nowait()) except Queue.Empty: break self.assertEqual(actualContent, expectedContent) # Verify that this consumer isn't closed self.assertEqual(len(bus._consumers), 1) self.assertIn(consumer1, bus._consumers) self.assertIsNotNone(consumer1._channelMgr) # Create another consumer, but don't start it (by telling it to consume 0 # messages; we want to test cleanup of both started and unstarted # consumers resultQ = Queue.Queue() consumerThread = threading.Thread( target=runConsumerThread, args=(bus, mqName, 0, resultQ)) consumerThread.setDaemon(True) consumerThread.start() # Wait for thread to stop consumerThread.join(timeout=30) self.assertFalse(consumerThread.isAlive()) # Reap the consumer consumer2 = resultQ.get_nowait() self.assertIsInstance(consumer2, message_bus_connector._QueueConsumer) try: item = resultQ.get_nowait() except Queue.Empty: pass else: self.fail("Unexpected item in resultQ: %r" % (item,)) # Verify that this consumer isn't closed self.assertEqual(len(bus._consumers), 2) self.assertIn(consumer2, bus._consumers) self.assertIsNotNone(consumer2._channelMgr) # Verify that this consumer hasn't started self.assertIsNone(consumer2._channelMgr._client) # Verify that the first consumer is still there, too self.assertIn(consumer1, bus._consumers) # Finanly, close the MessageBusConnector and verify that both consumers # got closed, too bus.close() self.assertFalse(bus._consumers) self.assertIsNone(consumer1._channelMgr) self.assertIsNone(consumer1._bus) self.assertIsNone(consumer1._channelMgr) self.assertIsNone(consumer2._channelMgr) self.assertIsNone(consumer2._bus) self.assertIsNone(consumer2._channelMgr)
class ModelSwapperInterface(object): """ This is the interface class to connect the application layer to the Model Swapper. """ #_INPUT_Q_OPTION_NAME = "input_queue" #_INPUT_Q_ENV_VAR = ModelSwapperConfig.getEnvVarOverrideName( # configName=ModelSwapperConfig.CONFIG_NAME, # section=_CONFIG_SECTION, # option=_INPUT_Q_OPTION_NAME) #""" For testing: environment variable for overriding input queue name """ _CONFIG_SECTION = "interface_bus" _RESULTS_Q_OPTION_NAME = "results_queue" # For testing: environment variable for overriding output queue name _RESULTS_Q_ENV_VAR = ModelSwapperConfig()._getEnvVarOverrideName( configName=ModelSwapperConfig.CONFIG_NAME, section=_CONFIG_SECTION, option=_RESULTS_Q_OPTION_NAME) _SCHEDULER_NOTIFICATION_Q_OPTION_NAME = "scheduler_notification_queue" _MODEL_INPUT_Q_PREFIX_OPTION_NAME = "model_input_queue_prefix" def __init__(self): """ Initialize the ModelSwapperInterface. This uses a lazy loading of the input and output queues with no pre-meditation. """ self._logger = _getLogger() config = ModelSwapperConfig() self._resultsQueueName = config.get( self._CONFIG_SECTION, self._RESULTS_Q_OPTION_NAME) # The name of a model's input message queue is the concatenation of this # prefix and the modelID self._modelInputQueueNamePrefix = config.get( self._CONFIG_SECTION, self._MODEL_INPUT_Q_PREFIX_OPTION_NAME) self._schedulerNotificationQueueName = config.get( self._CONFIG_SECTION, self._SCHEDULER_NOTIFICATION_Q_OPTION_NAME) # Message bus connector self._bus = MessageBusConnector() # Outstanding request and/or response consumer instances self._consumers = [] def __enter__(self): """ Context Manager protocol method. Allows a ModelSwapperInterface instance to be used in a "with" statement for automatic clean-up Parameters: ------------------------------------------------------------------------ retval: self. """ return self def __exit__(self, _excType, _excVal, _excTb): """ Context Manager protocol method. Allows a ModelSwapperInterface instance to be used in a "with" statement for automatic cleanup :returns: False so as not to suppress the exception, if any """ self.close() return False def close(self): """ Gracefully close ModelSwapperInterface instance (e.g., tear down connections). If this is not called, the underlying connections will eventually timeout, but it is good practice to close explicitly. """ if self._consumers: self._logger.error( "While closing %s, discovered %s unclosed consumers; will " "attempt to close them now", self.__class__.__name__, len(self._consumers)) for consumer in tuple(self._consumers): consumer.close() assert not self._consumers try: self._bus.close() finally: self._bus = None def _onConsumerClosed(self, consumer): """ Called by consumer instance's close() method to remove the consumer from our outstanding consumers list """ self._consumers.remove(consumer) def _getModelInputQName(self, modelID): return self._modelInputQueueNamePrefix + modelID def _getModelIDFromInputQName(self, mqName): assert mqName.startswith(self._modelInputQueueNamePrefix), ( "mq=%s doesn't start with %s") % (mqName, self._modelInputQueueNamePrefix) return mqName[len(self._modelInputQueueNamePrefix):] def defineModel(self, modelID, args, commandID): """ Initialize model's input message queue and send the "defineModel" command. The ModelCommandResult will be delivered asynchronously, along with the corresponding commandID and no args, to the process that is consuming ModelSwapper results. :param modelID: a hex string that uniquely identifies the target model. :param args: dict with the following properties: "modelConfig": model config dict suitable for passing to OPF ModelFactory.create() "inferenceArgs": Model inference arguments suitable for passing to model.enableInference() "inputRecordSchema": a sequence of nupic.data.fieldmeta.FieldMetaInfo instances with field names/types/special as expected by the model and in the same order as they will appear in input records. This is needed in order to avoid the overhead of passing fields names with each and every input record, while permitting the necessary dictionaries to be constructed by ModelRunner for input to the OPF Model. :param commandID: a numeric or string id to associate with the command and result. """ # TODO: validate input args dict against schema mqName = self._getModelInputQName(modelID) self._bus.createMessageQueue(mqName, durable=True) self.submitRequests(modelID, (ModelCommand(commandID, "defineModel", args),)) def cloneModel(self, modelID, newModelID, commandID): """ Initiate cloning of an existing model. Initialize the new model's input message queue and send the "cloneModel" command to the source model. The ModelCommandResult will be delivered asynchronously, along with the corresponding commandID and no args, to the process that is consuming ModelSwapper results. :param modelID: a hex string that uniquely identifies the existing model. :param newModelID: a hex string that uniquely identifies the new model. :param commandID: a numeric or string id to associate with the command and result. :raises: ModelNotFound if the source model's input endpoint doesn't exist """ # Create the model input message queue for the new model self._bus.createMessageQueue(self._getModelInputQName(newModelID), durable=True) self.submitRequests( modelID, (ModelCommand(commandID, "cloneModel", args={"modelID": newModelID}),)) def deleteModel(self, modelID, commandID): """ Submit a request to delete a model. The ModelCommandResult will be delivered asynchronously, along with the corresponding commandID and no args, to the process that is consuming ModelSwapper results. This method is idempotent. :param modelID: a hex string that uniquely identifies the target model. :param commandID: a numeric or string id to associate with the command and result. """ # First, purge unread input messages for this model, if any, to avoid # unnecessary processing before the model is deleted mq = self._getModelInputQName(modelID) self._logger.info("deleteModel: purging mq=%s before submitting " "deleteModel command for model=%s", mq, modelID) try: self._bus.purge(mq) except message_bus_connector.MessageQueueNotFound: # deleteModel is an idempotent operation: assume this exception is # due to repeated attempt pass else: try: self.submitRequests(modelID, (ModelCommand(commandID, "deleteModel"),)) except ModelNotFound: # deleteModel is an idempotent operation: assume this exception is # due to repeated attempt pass def cleanUpAfterModelDeletion(self, modelID): """ For use by Engine's ModelRunner after it deletes a model: clean up resources that ModelSwapperInterface created to support the model, such as deleting the model's input message queue """ self._bus.deleteMessageQueue(self._getModelInputQName(modelID)) def modelInputPending(self, modelID): """ Check if input requests are pending for a model :param modelID: a string that uniquely identifies the target model. :returns: True if the model's input queue exists and is non-empty; False if the model's input queue is non-empty or doesn't exist """ try: return not self._bus.isEmpty(self._getModelInputQName(modelID)) except message_bus_connector.MessageQueueNotFound: return False def getModelsWithInputPending(self): """ Get model IDs of all models with pending input (non-empty input queues) :returns: (possibly empty) sequence of model IDs whose input streams are non-empty """ # NOTE: queues may be deleted as we're running through the list, so we need # to play it safe def safeIsInputPending(mq): try: return not self._bus.isEmpty(mq) except message_bus_connector.MessageQueueNotFound: return False prefix = self._modelInputQueueNamePrefix return tuple( self._getModelIDFromInputQName(mq) for mq in self._bus.getAllMessageQueues() if mq.startswith(prefix) and safeIsInputPending(mq)) def submitRequests(self, modelID, requests): """ Submit a batch of requests for processing by a model with the given modelID. NOTE: it's an error to submit requests for a model after calling deleteModel() Keyword arguments: :param modelID: a string that uniquely identifies the target model. :param requests: a sequence of ModelCommand and/or ModelInputRow instances. NOTE: To create or delete a model, call the createModel or deleteModel method instead of submitting the "defineModel" or "deleteModel" commands. Together, the sequence of requests constitutes a request "batch". :returns: UUID of the submitted batch (intended for test code only) :raises: ModelNotFound if model's input endpoint doesn't exist Requests for a specific model will be processed in the submitted order. The results will be delivered asynchronously, along with the corresponding requestIDs, to the process that is consuming ModelSwapper results. NOTE: This assumes retry logic will be handled by the underlying MQ implementation. """ batchID = uuid.uuid1().hex msg = RequestMessagePackager.marshal( batchID=batchID, batchState=BatchPackager.marshal(batch=requests)) mqName = self._getModelInputQName(modelID) try: self._bus.publish(mqName, msg, persistent=True) except message_bus_connector.MessageQueueNotFound as e: self._logger.warn( "App layer attempted to submit numRequests=%s to model=%s, but its " "input queue doesn't exist. Likely a race condition with model " "deletion path.", len(requests), modelID) raise ModelNotFound(repr(e)) except: self._logger.exception( "Failed to publish request batch=%s for model=%s via mq=%s; " "msgLen=%s; msgPrefix=%r. NOTE: it's an error to submit requests to a " "model after deleting it.", batchID, modelID, mqName, len(msg), msg[:32]) raise # Send a notification to Model Scheduler so it will schedule the model # for processing input try: self._bus.publish(self._schedulerNotificationQueueName, json.dumps(modelID), persistent=False) except message_bus_connector.MessageQueueNotFound: # If it's not fully up yet, its notification queue might not have been # created, which is ok self._logger.warn( "Couldn't send model data notification to Model Scheduler: mq=%s not " "found. Model Scheduler service not started or initialized the mq yet?", self._schedulerNotificationQueueName) return batchID def consumeRequests(self, modelID, blocking=True): """ Create an instance of the _MessageConsumer iterable for reading model requests, a batch at a time. The iterable yields _ConsumedRequestBatch instances. NOTE: This API is intended for Engine Model Runners. :param modelID: a string that uniquely identifies the target model. :param blocking: if True, the iterable will block until another batch becomes available; if False, the iterable will terminate iteration when no more batches are available in the queue. [defaults to True] :returns: an instance of model_swapper_interface._MessageConsumer iterable; IMPORTANT: the caller is responsible for closing it before closing this ModelSwapperInterface instance (hint: use the returned _MessageConsumer instance as Context Manager) :raises: ModelNotFound if model's input endpoint doesn't exist TODO: need tests for consumeRequests with ModelNotFound Example: with ModelSwapperInterface() as swapper: with swapper.consumeRequests(modelID) as consumer: for batch in consumer: processRequests(batchID=batch.batchID, requests=batch.objects) batch.ack() """ mq = self._getModelInputQName(modelID) def onQueueNotFound(): msg = ("Attempt to consume requests from model=%s is impossible because " "its input queue doesn't exist. Likely a race condition with " "model deletion path.") % (modelID,) self._logger.warn(msg) raise ModelNotFound(msg) consumer = _MessageConsumer(mqName=mq, blocking=blocking, decode=_ConsumedRequestBatch.decodeMessage, swapper=self, bus=self._bus, onQueueNotFound=onQueueNotFound) self._consumers.append(consumer) return consumer def _initResultsMessageQueue(self): self._bus.createMessageQueue(self._resultsQueueName, durable=True) def submitResults(self, modelID, results): """ Submit a batch of results (used by ModelSwapper layer) Keyword arguments: :param modelID: a string that uniquely identifies the target model. :param results: a sequence of ModelCommandResult and/or ModelInferenceResult instances NOTE: This assumes retry logic will be handled by the underlying MQ implementation. """ msg = ResultMessagePackager.marshal( modelID=modelID, batchState=BatchPackager.marshal(batch=results)) try: try: self._bus.publish(self._resultsQueueName, msg, persistent=True) except message_bus_connector.MessageQueueNotFound: self._logger.info("submitResults: results mq=%s didn't exist; " "declaring now and re-publishing message", self._resultsQueueName) self._initResultsMessageQueue() self._bus.publish(self._resultsQueueName, msg, persistent=True) except: self._logger.exception( "submitResults: Failed to publish results from model=%s via mq=%s; " "msgLen=%s; msgPrefix=%r", modelID, self._resultsQueueName, len(msg), msg[:32]) raise def consumeResults(self): """ Create an instance of the _MessageConsumer iterable for reading model results, a batch at a time. The iterable yields _ConsumedResultBatch instances. :returns: an instance of model_swapper_interface._MessageConsumer iterable; IMPORTANT: the caller is responsible for closing it before closing this ModelSwapperInterface instance (hint: use the returned _MessageConsumer instance as Context Manager) Example: with ModelSwapperInterface() as swapper: with swapper.consumeResults() as consumer: for batch in consumer: processResults(modelID=batch.modelID, results=batch.objects) batch.ack() """ consumer = _MessageConsumer(mqName=self._resultsQueueName, blocking=True, decode=_ConsumedResultBatch.decodeMessage, swapper=self, bus=self._bus, onQueueNotFound=self._initResultsMessageQueue) self._consumers.append(consumer) return consumer def initSchedulerNotification(self): """ Initialize Model Scheduler's notification message queue; for use by Model Scheduler. """ self._bus.createMessageQueue(self._schedulerNotificationQueueName, durable=False) def consumeModelSchedulerNotifications(self): """ Create an instance of the _MessageConsumer iterable for reading model scheduler notifications. The iterable yields _ConsumedNotification instances. :returns: an instance of model_swapper_interface._MessageConsumer iterable; IMPORTANT: the caller is responsible for closing it before closing this ModelSwapperInterface instance (hint: use the returned _MessageConsumer instance as Context Manager) Example: with ModelSwapperInterface() as swapper: with swapper.consumeModelSchedulerNotifications() as consumer: for notification in consumer: processNotification(notification.value) notification.ack() """ consumer = _MessageConsumer(mqName=self._schedulerNotificationQueueName, blocking=True, decode=_ConsumedNotification.decodeMessage, swapper=self, bus=self._bus) self._consumers.append(consumer) return consumer
def testModelSwapper(self): """Simple end-to-end test of the model swapper system.""" modelSchedulerSubprocess = self._startModelSchedulerSubprocess() self.addCleanup(lambda: modelSchedulerSubprocess.kill() if modelSchedulerSubprocess.returncode is None else None) modelID = "foobar" resultBatches = [] with ModelSwapperInterface() as swapperAPI: possibleModels = getScalarMetricWithTimeOfDayParams(metricData=[0], minVal=0, maxVal=1000) # Submit requests including a model creation command and two data rows. args = possibleModels[0] args["inputRecordSchema"] = ( FieldMetaInfo("c0", FieldMetaType.datetime, FieldMetaSpecial.timestamp), FieldMetaInfo("c1", FieldMetaType.float, FieldMetaSpecial.none), ) # Define the model _LOGGER.info("Defining the model") swapperAPI.defineModel(modelID=modelID, args=args, commandID="defineModelCmd1") # Attempt to define the same model again _LOGGER.info("Defining the model again") swapperAPI.defineModel(modelID=modelID, args=args, commandID="defineModelCmd2") # Send input rows to the model inputRows = [ ModelInputRow( rowID="rowfoo", data=[datetime.datetime(2013, 5, 23, 8, 13, 00), 5.3]), ModelInputRow( rowID="rowbar", data=[datetime.datetime(2013, 5, 23, 8, 13, 15), 2.4]), ] _LOGGER.info("Submitting batch of %d input rows...", len(inputRows)) swapperAPI.submitRequests(modelID=modelID, requests=inputRows) _LOGGER.info("These models have pending input: %s", swapperAPI.getModelsWithInputPending()) # Retrieve all results. # NOTE: We collect results via background thread to avoid # deadlocking the test runner in the event consuming blocks unexpectedly _LOGGER.info("Reading all batches of results...") numBatchesExpected = 3 resultBatches.extend( self._consumeResults(numBatchesExpected, timeout=20)) self.assertEqual(len(resultBatches), numBatchesExpected) with MessageBusConnector() as bus: # The results message queue should be empty now self.assertTrue(bus.isEmpty(swapperAPI._resultsQueueName)) # Delete the model _LOGGER.info("Deleting the model") swapperAPI.deleteModel(modelID=modelID, commandID="deleteModelCmd1") _LOGGER.info("Waiting for model deletion result") resultBatches.extend(self._consumeResults(1, timeout=20)) self.assertEqual(len(resultBatches), 4) with MessageBusConnector() as bus: # The results message queue should be empty now self.assertTrue(bus.isEmpty(swapperAPI._resultsQueueName)) # The model input queue should be deleted now self.assertFalse( bus.isMessageQeueuePresent( swapperAPI._getModelInputQName(modelID=modelID))) # Try deleting the model again, to make sure there are no exceptions _LOGGER.info("Attempting to delete the model again") swapperAPI.deleteModel(modelID=modelID, commandID="deleteModelCmd1") # Verify results # First result batch should be the first defineModel result batch = resultBatches[0] self.assertEqual(batch.modelID, modelID) self.assertEqual(len(batch.objects), 1) result = batch.objects[0] self.assertIsInstance(result, ModelCommandResult) self.assertEqual(result.method, "defineModel") self.assertEqual(result.status, htmengineerrno.SUCCESS) self.assertEqual(result.commandID, "defineModelCmd1") # The second result batch should for the second defineModel result for the # same model batch = resultBatches[1] self.assertEqual(batch.modelID, modelID) self.assertEqual(len(batch.objects), 1) result = batch.objects[0] self.assertIsInstance(result, ModelCommandResult) self.assertEqual(result.method, "defineModel") self.assertEqual(result.status, htmengineerrno.SUCCESS) self.assertEqual(result.commandID, "defineModelCmd2") # The third batch should be for the two input rows batch = resultBatches[2] self.assertEqual(batch.modelID, modelID) self.assertEqual(len(batch.objects), len(inputRows)) for inputRow, result in zip(inputRows, batch.objects): self.assertIsInstance(result, ModelInferenceResult) self.assertEqual(result.status, htmengineerrno.SUCCESS) self.assertEqual(result.rowID, inputRow.rowID) self.assertIsInstance(result.anomalyScore, float) # The fourth batch should be for the "deleteModel" batch = resultBatches[3] self.assertEqual(batch.modelID, modelID) self.assertEqual(len(batch.objects), 1) result = batch.objects[0] self.assertIsInstance(result, ModelCommandResult) self.assertEqual(result.method, "deleteModel") self.assertEqual(result.status, htmengineerrno.SUCCESS) self.assertEqual(result.commandID, "deleteModelCmd1") # Signal Model Scheduler Service subprocess to shut down and wait for it waitResult = dict() def runWaiterThread(): try: waitResult["returnCode"] = modelSchedulerSubprocess.wait() except: _LOGGER.exception( "Waiting for modelSchedulerSubprocess failed") waitResult["exceptionInfo"] = traceback.format_exc() raise return modelSchedulerSubprocess.terminate() waiterThread = threading.Thread(target=runWaiterThread) waiterThread.setDaemon(True) waiterThread.start() waiterThread.join(timeout=30) self.assertFalse(waiterThread.isAlive()) self.assertEqual(waitResult["returnCode"], 0, msg=repr(waitResult))
def run(self): """ Consumes pending results. Once result batch arrives, it will be dispatched to the correct model command result handler. :see: `_processModelCommandResult` and `_processModelInferenceResults` """ # Properties for publishing model command results on RabbitMQ exchange modelCommandResultProperties = MessageProperties( deliveryMode=amqp.constants.AMQPDeliveryModes.PERSISTENT_MESSAGE, headers=dict(dataType="model-cmd-result")) # Properties for publishing model inference results on RabbitMQ exchange modelInferenceResultProperties = MessageProperties( deliveryMode=amqp.constants.AMQPDeliveryModes.PERSISTENT_MESSAGE) # Declare an exchange for forwarding our results with amqp.synchronous_amqp_client.SynchronousAmqpClient( amqp.connection.getRabbitmqConnectionParameters( )) as amqpClient: amqpClient.declareExchange(self._modelResultsExchange, exchangeType="fanout", durable=True) with ModelSwapperInterface() as modelSwapper, MessageBusConnector( ) as bus: with modelSwapper.consumeResults() as consumer: for batch in consumer: if self._profiling: batchStartTime = time.time() inferenceResults = [] for result in batch.objects: try: if isinstance(result, ModelCommandResult): self._processModelCommandResult( batch.modelID, result) # Construct model command result message for consumption by # downstream processes try: cmdResultMessage = self._composeModelCommandResultMessage( modelID=batch.modelID, cmdResult=result) except (ObjectNotFoundError, MetricNotMonitoredError): pass else: bus.publishExg( exchange=self._modelResultsExchange, routingKey="", body=self._serializeModelResult( cmdResultMessage), properties=modelCommandResultProperties ) elif isinstance(result, ModelInferenceResult): inferenceResults.append(result) else: self._log.error("Unsupported ModelResult=%r", result) except ObjectNotFoundError: self._log.exception( "Error processing result=%r " "from model=%s", result, batch.modelID) if inferenceResults: result = self._processModelInferenceResults( inferenceResults, metricID=batch.modelID) if result is not None: # Construct model results payload for consumption by # downstream processes metricRow, dataRows = result resultsMessage = self._composeModelInferenceResultsMessage( metricRow, dataRows) payload = self._serializeModelResult( resultsMessage) bus.publishExg( exchange=self._modelResultsExchange, routingKey="", body=payload, properties=modelInferenceResultProperties) batch.ack() if self._profiling: if inferenceResults: if result is not None: # pylint: disable=W0633 metricRow, rows = result rowIdRange = ("%s..%s" % (rows[0].rowid, rows[-1].rowid) if len(rows) > 1 else str( rows[0].rowid)) self._log.info( "{TAG:ANOM.BATCH.INF.DONE} model=%s; " "numItems=%d; rows=[%s]; tailRowTS=%s; duration=%.4fs; " "ds=%s; name=%s", batch.modelID, len(batch.objects), rowIdRange, rows[-1].timestamp.isoformat() + "Z", time.time() - batchStartTime, metricRow.datasource, metricRow.name) else: self._log.info( "{TAG:ANOM.BATCH.CMD.DONE} model=%s; " "numItems=%d; duration=%.4fs", batch.modelID, len(batch.objects), time.time() - batchStartTime) self._log.info("Stopped processing model results")
class ModelSwapperInterface(object): """ This is the interface class to connect the application layer to the Model Swapper. """ #_INPUT_Q_OPTION_NAME = "input_queue" #_INPUT_Q_ENV_VAR = ModelSwapperConfig.getEnvVarOverrideName( # configName=ModelSwapperConfig.CONFIG_NAME, # section=_CONFIG_SECTION, # option=_INPUT_Q_OPTION_NAME) #""" For testing: environment variable for overriding input queue name """ _CONFIG_SECTION = "interface_bus" _RESULTS_Q_OPTION_NAME = "results_queue" # For testing: environment variable for overriding output queue name _RESULTS_Q_ENV_VAR = ModelSwapperConfig()._getEnvVarOverrideName( configName=ModelSwapperConfig.CONFIG_NAME, section=_CONFIG_SECTION, option=_RESULTS_Q_OPTION_NAME) _SCHEDULER_NOTIFICATION_Q_OPTION_NAME = "scheduler_notification_queue" _MODEL_INPUT_Q_PREFIX_OPTION_NAME = "model_input_queue_prefix" def __init__(self): """ Initialize the ModelSwapperInterface. This uses a lazy loading of the input and output queues with no pre-meditation. """ self._logger = _getLogger() config = ModelSwapperConfig() self._resultsQueueName = config.get(self._CONFIG_SECTION, self._RESULTS_Q_OPTION_NAME) # The name of a model's input message queue is the concatenation of this # prefix and the modelID self._modelInputQueueNamePrefix = config.get( self._CONFIG_SECTION, self._MODEL_INPUT_Q_PREFIX_OPTION_NAME) self._schedulerNotificationQueueName = config.get( self._CONFIG_SECTION, self._SCHEDULER_NOTIFICATION_Q_OPTION_NAME) # Message bus connector self._bus = MessageBusConnector() # Outstanding request and/or response consumer instances self._consumers = [] def __enter__(self): """ Context Manager protocol method. Allows a ModelSwapperInterface instance to be used in a "with" statement for automatic clean-up Parameters: ------------------------------------------------------------------------ retval: self. """ return self def __exit__(self, _excType, _excVal, _excTb): """ Context Manager protocol method. Allows a ModelSwapperInterface instance to be used in a "with" statement for automatic cleanup :returns: False so as not to suppress the exception, if any """ self.close() return False def close(self): """ Gracefully close ModelSwapperInterface instance (e.g., tear down connections). If this is not called, the underlying connections will eventually timeout, but it is good practice to close explicitly. """ if self._consumers: self._logger.error( "While closing %s, discovered %s unclosed consumers; will " "attempt to close them now", self.__class__.__name__, len(self._consumers)) for consumer in tuple(self._consumers): consumer.close() assert not self._consumers try: self._bus.close() finally: self._bus = None def _onConsumerClosed(self, consumer): """ Called by consumer instance's close() method to remove the consumer from our outstanding consumers list """ self._consumers.remove(consumer) def _getModelInputQName(self, modelID): return self._modelInputQueueNamePrefix + modelID def _getModelIDFromInputQName(self, mqName): assert mqName.startswith(self._modelInputQueueNamePrefix), ( "mq=%s doesn't start with %s") % (mqName, self._modelInputQueueNamePrefix) return mqName[len(self._modelInputQueueNamePrefix):] def defineModel(self, modelID, args, commandID): """ Initialize model's input message queue and send the "defineModel" command. The ModelCommandResult will be delivered asynchronously, along with the corresponding commandID and no args, to the process that is consuming ModelSwapper results. :param modelID: a hex string that uniquely identifies the target model. :param args: dict with the following properties: "modelConfig": model config dict suitable for passing to OPF ModelFactory.create() "inferenceArgs": Model inference arguments suitable for passing to model.enableInference() "inputRecordSchema": a sequence of nupic.data.fieldmeta.FieldMetaInfo instances with field names/types/special as expected by the model and in the same order as they will appear in input records. This is needed in order to avoid the overhead of passing fields names with each and every input record, while permitting the necessary dictionaries to be constructed by ModelRunner for input to the OPF Model. :param commandID: a numeric or string id to associate with the command and result. """ # TODO: validate input args dict against schema mqName = self._getModelInputQName(modelID) self._bus.createMessageQueue(mqName, durable=True) self.submitRequests(modelID, (ModelCommand(commandID, "defineModel", args), )) def cloneModel(self, modelID, newModelID, commandID): """ Initiate cloning of an existing model. Initialize the new model's input message queue and send the "cloneModel" command to the source model. The ModelCommandResult will be delivered asynchronously, along with the corresponding commandID and no args, to the process that is consuming ModelSwapper results. :param modelID: a hex string that uniquely identifies the existing model. :param newModelID: a hex string that uniquely identifies the new model. :param commandID: a numeric or string id to associate with the command and result. :raises: ModelNotFound if the source model's input endpoint doesn't exist """ # Create the model input message queue for the new model self._bus.createMessageQueue(self._getModelInputQName(newModelID), durable=True) self.submitRequests(modelID, (ModelCommand( commandID, "cloneModel", args={"modelID": newModelID}), )) def deleteModel(self, modelID, commandID): """ Submit a request to delete a model. The ModelCommandResult will be delivered asynchronously, along with the corresponding commandID and no args, to the process that is consuming ModelSwapper results. This method is idempotent. :param modelID: a hex string that uniquely identifies the target model. :param commandID: a numeric or string id to associate with the command and result. """ # First, purge unread input messages for this model, if any, to avoid # unnecessary processing before the model is deleted mq = self._getModelInputQName(modelID) self._logger.info( "deleteModel: purging mq=%s before submitting " "deleteModel command for model=%s", mq, modelID) try: self._bus.purge(mq) except message_bus_connector.MessageQueueNotFound: # deleteModel is an idempotent operation: assume this exception is # due to repeated attempt pass else: try: self.submitRequests(modelID, (ModelCommand(commandID, "deleteModel"), )) except ModelNotFound: # deleteModel is an idempotent operation: assume this exception is # due to repeated attempt pass def cleanUpAfterModelDeletion(self, modelID): """ For use by Engine's ModelRunner after it deletes a model: clean up resources that ModelSwapperInterface created to support the model, such as deleting the model's input message queue """ self._bus.deleteMessageQueue(self._getModelInputQName(modelID)) def modelInputPending(self, modelID): """ Check if input requests are pending for a model :param modelID: a string that uniquely identifies the target model. :returns: True if the model's input queue exists and is non-empty; False if the model's input queue is non-empty or doesn't exist """ try: return not self._bus.isEmpty(self._getModelInputQName(modelID)) except message_bus_connector.MessageQueueNotFound: return False def getModelsWithInputPending(self): """ Get model IDs of all models with pending input (non-empty input queues) :returns: (possibly empty) sequence of model IDs whose input streams are non-empty """ # NOTE: queues may be deleted as we're running through the list, so we need # to play it safe def safeIsInputPending(mq): try: return not self._bus.isEmpty(mq) except message_bus_connector.MessageQueueNotFound: return False prefix = self._modelInputQueueNamePrefix return tuple( self._getModelIDFromInputQName(mq) for mq in self._bus.getAllMessageQueues() if mq.startswith(prefix) and safeIsInputPending(mq)) def submitRequests(self, modelID, requests): """ Submit a batch of requests for processing by a model with the given modelID. NOTE: it's an error to submit requests for a model after calling deleteModel() Keyword arguments: :param modelID: a string that uniquely identifies the target model. :param requests: a sequence of ModelCommand and/or ModelInputRow instances. NOTE: To create or delete a model, call the createModel or deleteModel method instead of submitting the "defineModel" or "deleteModel" commands. Together, the sequence of requests constitutes a request "batch". :returns: UUID of the submitted batch (intended for test code only) :raises: ModelNotFound if model's input endpoint doesn't exist Requests for a specific model will be processed in the submitted order. The results will be delivered asynchronously, along with the corresponding requestIDs, to the process that is consuming ModelSwapper results. NOTE: This assumes retry logic will be handled by the underlying MQ implementation. """ batchID = uuid.uuid1().hex msg = RequestMessagePackager.marshal( batchID=batchID, batchState=BatchPackager.marshal(batch=requests)) mqName = self._getModelInputQName(modelID) try: self._bus.publish(mqName, msg, persistent=True) except message_bus_connector.MessageQueueNotFound as e: self._logger.warn( "App layer attempted to submit numRequests=%s to model=%s, but its " "input queue doesn't exist. Likely a race condition with model " "deletion path.", len(requests), modelID) raise ModelNotFound(repr(e)) except: self._logger.exception( "Failed to publish request batch=%s for model=%s via mq=%s; " "msgLen=%s; msgPrefix=%r. NOTE: it's an error to submit requests to a " "model after deleting it.", batchID, modelID, mqName, len(msg), msg[:32]) raise # Send a notification to Model Scheduler so it will schedule the model # for processing input try: self._bus.publish(self._schedulerNotificationQueueName, json.dumps(modelID), persistent=False) except message_bus_connector.MessageQueueNotFound: # If it's not fully up yet, its notification queue might not have been # created, which is ok self._logger.warn( "Couldn't send model data notification to Model Scheduler: mq=%s not " "found. Model Scheduler service not started or initialized the mq yet?", self._schedulerNotificationQueueName) return batchID def consumeRequests(self, modelID, blocking=True): """ Create an instance of the _MessageConsumer iterable for reading model requests, a batch at a time. The iterable yields _ConsumedRequestBatch instances. NOTE: This API is intended for Engine Model Runners. :param modelID: a string that uniquely identifies the target model. :param blocking: if True, the iterable will block until another batch becomes available; if False, the iterable will terminate iteration when no more batches are available in the queue. [defaults to True] :returns: an instance of model_swapper_interface._MessageConsumer iterable; IMPORTANT: the caller is responsible for closing it before closing this ModelSwapperInterface instance (hint: use the returned _MessageConsumer instance as Context Manager) :raises: ModelNotFound if model's input endpoint doesn't exist TODO: need tests for consumeRequests with ModelNotFound Example: with ModelSwapperInterface() as swapper: with swapper.consumeRequests(modelID) as consumer: for batch in consumer: processRequests(batchID=batch.batchID, requests=batch.objects) batch.ack() """ mq = self._getModelInputQName(modelID) def onQueueNotFound(): msg = ( "Attempt to consume requests from model=%s is impossible because " "its input queue doesn't exist. Likely a race condition with " "model deletion path.") % (modelID, ) self._logger.warn(msg) raise ModelNotFound(msg) consumer = _MessageConsumer(mqName=mq, blocking=blocking, decode=_ConsumedRequestBatch.decodeMessage, swapper=self, bus=self._bus, onQueueNotFound=onQueueNotFound) self._consumers.append(consumer) return consumer def _initResultsMessageQueue(self): self._bus.createMessageQueue(self._resultsQueueName, durable=True) def submitResults(self, modelID, results): """ Submit a batch of results (used by ModelSwapper layer) Keyword arguments: :param modelID: a string that uniquely identifies the target model. :param results: a sequence of ModelCommandResult and/or ModelInferenceResult instances NOTE: This assumes retry logic will be handled by the underlying MQ implementation. """ msg = ResultMessagePackager.marshal( modelID=modelID, batchState=BatchPackager.marshal(batch=results)) try: try: self._bus.publish(self._resultsQueueName, msg, persistent=True) except message_bus_connector.MessageQueueNotFound: self._logger.info( "submitResults: results mq=%s didn't exist; " "declaring now and re-publishing message", self._resultsQueueName) self._initResultsMessageQueue() self._bus.publish(self._resultsQueueName, msg, persistent=True) except: self._logger.exception( "submitResults: Failed to publish results from model=%s via mq=%s; " "msgLen=%s; msgPrefix=%r", modelID, self._resultsQueueName, len(msg), msg[:32]) raise def consumeResults(self): """ Create an instance of the _MessageConsumer iterable for reading model results, a batch at a time. The iterable yields _ConsumedResultBatch instances. :returns: an instance of model_swapper_interface._MessageConsumer iterable; IMPORTANT: the caller is responsible for closing it before closing this ModelSwapperInterface instance (hint: use the returned _MessageConsumer instance as Context Manager) Example: with ModelSwapperInterface() as swapper: with swapper.consumeResults() as consumer: for batch in consumer: processResults(modelID=batch.modelID, results=batch.objects) batch.ack() """ consumer = _MessageConsumer( mqName=self._resultsQueueName, blocking=True, decode=_ConsumedResultBatch.decodeMessage, swapper=self, bus=self._bus, onQueueNotFound=self._initResultsMessageQueue) self._consumers.append(consumer) return consumer def initSchedulerNotification(self): """ Initialize Model Scheduler's notification message queue; for use by Model Scheduler. """ self._bus.createMessageQueue(self._schedulerNotificationQueueName, durable=False) def consumeModelSchedulerNotifications(self): """ Create an instance of the _MessageConsumer iterable for reading model scheduler notifications. The iterable yields _ConsumedNotification instances. :returns: an instance of model_swapper_interface._MessageConsumer iterable; IMPORTANT: the caller is responsible for closing it before closing this ModelSwapperInterface instance (hint: use the returned _MessageConsumer instance as Context Manager) Example: with ModelSwapperInterface() as swapper: with swapper.consumeModelSchedulerNotifications() as consumer: for notification in consumer: processNotification(notification.value) notification.ack() """ consumer = _MessageConsumer( mqName=self._schedulerNotificationQueueName, blocking=True, decode=_ConsumedNotification.decodeMessage, swapper=self, bus=self._bus) self._consumers.append(consumer) return consumer