def testMarshalUnmarshal(self): inputBatch = [ ModelCommandResult(commandID="commandID", method="testMethod", status=1, errorMessage="errorMessage"), ModelInputRow(rowID="foo", data=[1, 2, "Sep 21 02:24:21 UTC 2013"]), ModelInputRow(rowID="bar", data=[9, 54, "Sep 21 02:24:38 UTC 2013"]), ] batchState = BatchPackager.marshal(batch=inputBatch) requestBatch = BatchPackager.unmarshal(batchState=batchState) self.assertEqual(requestBatch[0].commandID, inputBatch[0].commandID) self.assertEqual(requestBatch[1].rowID, inputBatch[1].rowID) self.assertEqual(requestBatch[2].rowID, inputBatch[2].rowID)
def testSubmitRequestsWithContextManager(self, messageBusConnectorClassMock): requests = [ ModelCommand(commandID="abc", method="defineModel", args={'key1': 4098, 'key2': 4139}), ModelInputRow(rowID="foo", data=[1, 2, "Sep 21 02:24:21 UTC 2013"]), ModelInputRow(rowID="bar", data=[9, 54, "Sep 21 02:24:38 UTC 2013"]) ] modelID = "foofar" messageBusConnectorMock = messageBusConnectorClassMock.return_value with ModelSwapperInterface() as interface: batchID = interface.submitRequests(modelID=modelID, requests=requests) modelMQName = interface._modelInputQueueNamePrefix + modelID notificationMQName = interface._schedulerNotificationQueueName # Verify self.assertIsInstance(batchID, str) msg = RequestMessagePackager.marshal( batchID=batchID, batchState=BatchPackager.marshal(batch=requests)) self.assertEqual(messageBusConnectorMock.publish.call_count, 2) messageBusConnectorMock.publish.assert_any_call( modelMQName, msg, persistent=True) messageBusConnectorMock.publish.assert_any_call( notificationMQName, json.dumps(modelID), persistent=False)
def testSubmitResultsFailure(self, messageBusConnectorClassMock): results = [ ModelCommandResult(commandID="abc", method="testMethod", status=0, args={'key1': 4098, 'key2': 4139}), ModelInferenceResult(rowID="foo", status=0, anomalyScore=1), ModelInferenceResult(rowID="bar", status=0, anomalyScore=2) ] messageBusConnectorMock = messageBusConnectorClassMock.return_value messageBusConnectorMock.publish.side_effect = ( message_bus_connector.MessageQueueNotFound("from mock test"), Exception("re-publish failed in mock test")) modelID = "foofar" with self.assertRaises(Exception) as raisesCM: with ModelSwapperInterface() as interface: mqName = interface._resultsQueueName interface.submitResults(modelID=modelID, results=results) # Verify self.assertEqual(raisesCM.exception.args[0], "re-publish failed in mock test") messageBusConnectorMock.createMessageQueue.assert_called_once_with( mqName, durable=True) self.assertEqual(messageBusConnectorMock.publish.call_count, 2) msg = ResultMessagePackager.marshal( modelID=modelID, batchState=BatchPackager.marshal(batch=results)) messageBusConnectorMock.publish.assert_called_with(mqName, msg, persistent=True)
def testSubmitResultsRecoveryFromMessageQueueNotFound( self, messageBusConnectorClassMock): # submitResults recovers from the initial MessageQueueNotFound by creating # the results message queue and re-publishing the message results = [ ModelCommandResult(commandID="abc", method="testMethod", status=0, args={'key1': 4098, 'key2': 4139}), ModelInferenceResult(rowID="foo", status=0, anomalyScore=1), ModelInferenceResult(rowID="bar", status=0, anomalyScore=2) ] messageBusConnectorMock = messageBusConnectorClassMock.return_value messageBusConnectorMock.publish.side_effect = ( message_bus_connector.MessageQueueNotFound("from mock test"), Mock()) modelID = "foofar" with ModelSwapperInterface() as interface: interface.submitResults(modelID=modelID, results=results) mqName = interface._resultsQueueName # Verify messageBusConnectorMock.createMessageQueue.assert_called_once_with( mqName, durable=True) self.assertEqual(messageBusConnectorMock.publish.call_count, 2) msg = ResultMessagePackager.marshal( modelID=modelID, batchState=BatchPackager.marshal(batch=results)) messageBusConnectorMock.publish.assert_called_with(mqName, msg, persistent=True)
def testConsumeRequestsNonBlocking(self, messageBusConnectorClassMock): expectedRequests = ( ModelCommand(commandID="abc", method="defineModel", args={'key1': 4098, 'key2': 4139}), ModelInputRow(rowID="foo", data=[1, 2, "Sep 21 02:24:21 UTC 2013"]), ModelInputRow(rowID="bar", data=[9, 54, "Sep 21 02:24:38 UTC 2013"]), ) modelID = "foobar" batchID = uuid.uuid1().hex msg = RequestMessagePackager.marshal( batchID=batchID, batchState=BatchPackager.marshal(batch=expectedRequests)) messageBusConnectorMock = messageBusConnectorClassMock.return_value ackMock = Mock(return_value=None) messageBusConnectorMock.consume.return_value = Mock( spec_set=message_bus_connector._QueueConsumer, __iter__=lambda *args, **kwargs: iter( [message_bus_connector._ConsumedMessage(body=msg, ack=ackMock)])) with ModelSwapperInterface() as interface: self.assertEqual(len(interface._consumers), 0) with interface.consumeRequests(modelID, blocking=False) as consumer: self.assertIsNotNone(consumer._mqConsumer) self.assertEqual(len(interface._consumers), 1) batch = next(iter(consumer)) self.assertEqual(batch.batchID, batchID) self.assertEqual(batch.objects, expectedRequests) self.assertTrue(callable(batch.ack)) self.assertIs(batch.ack, ackMock) # Make sure we didn't skip any fields self.assertEqual(set(batch._fields), set(["batchID", "objects", "ack"])) self.assertIsNone(consumer._mqConsumer) self.assertEqual(len(interface._consumers), 0) self.assertEqual(len(interface._consumers), 0) modelMQName = interface._modelInputQueueNamePrefix + modelID messageBusConnectorMock.consume.assert_called_once_with( modelMQName, blocking=False)
def testMarshalAndUnmarshal(self): resultBatch = [ ModelCommandResult(commandID="abc", method="testMethod", status=0, args={'key1': 4098, 'key2': 4139}), ModelInferenceResult(rowID="foo", status=0, anomalyScore=1), ModelInferenceResult(rowID="bar", status=0, anomalyScore=2) ] batchState = BatchPackager.marshal(batch=resultBatch) msg = ResultMessagePackager.marshal(modelID="foobar", batchState=batchState) r = ResultMessagePackager.unmarshal(msg) self.assertEqual(r.modelID, "foobar") self.assertEqual(r.batchState, batchState) # Make sure we aren't forgetting to test any returned fields self.assertEqual(set(["modelID", "batchState"]), set(r._fields))
def testMarshalAndUnmarshal(self): requestBatch = [ ModelCommand(commandID="abc", method="defineModel", args={'key1': 4098, 'key2': 4139}), ModelInputRow(rowID="foo", data=[1, 2, "Sep 21 02:24:21 UTC 2013"]), ModelInputRow(rowID="bar", data=[9, 54, "Sep 21 02:24:38 UTC 2013"]), ] batchState = BatchPackager.marshal(batch=requestBatch) msg = RequestMessagePackager.marshal(batchID="foobar", batchState=batchState) r = RequestMessagePackager.unmarshal(msg) self.assertEqual(r.batchID, "foobar") self.assertEqual(r.batchState, batchState) # Make sure we aren't forgetting to test any returned fields self.assertEqual(set(["batchID", "batchState"]), set(r._fields))
def testContextManagerAndConsumeResults(self, messageBusConnectorClassMock): expectedResults = ( ModelCommandResult(commandID="abc", method="testMethod", status=0, args={'key1': 4098, 'key2': 4139}), ModelInferenceResult(rowID="foo", status=0, anomalyScore=1.3), ModelInferenceResult(rowID="bar", status=0, anomalyScore=2.9) ) modelID = "foobar" msg = ResultMessagePackager.marshal( modelID=modelID, batchState=BatchPackager.marshal(batch=expectedResults)) messageBusConnectorMock = messageBusConnectorClassMock.return_value ackMock = Mock(return_value=None) messageBusConnectorMock.consume.return_value = Mock( spec_set=message_bus_connector._QueueConsumer, __iter__=lambda *args, **kwargs: iter( [message_bus_connector._ConsumedMessage( body=msg, ack=ackMock)])) with ModelSwapperInterface() as interface: self.assertEqual(len(interface._consumers), 0) consumer = interface.consumeResults() self.assertIsNotNone(consumer._mqConsumer) self.assertEqual(len(interface._consumers), 1) batch = next(iter(consumer)) self.assertEqual(batch.modelID, modelID) self.assertEqual(batch.objects, expectedResults) self.assertTrue(callable(batch.ack)) self.assertIs(batch.ack, ackMock) self.assertEqual(set(batch._fields), set(["modelID", "objects", "ack"])) consumer.close() self.assertIsNone(consumer._mqConsumer) self.assertEqual(len(interface._consumers), 0) self.assertEqual(len(interface._consumers), 0)
def testSubmitResults(self, messageBusConnectorClassMock): results = [ ModelCommandResult(commandID="abc", method="testMethod", status=0, args={'key1': 4098, 'key2': 4139}), ModelInferenceResult(rowID="foo", status=0, anomalyScore=1), ModelInferenceResult(rowID="bar", status=0, anomalyScore=2) ] messageBusConnectorMock = messageBusConnectorClassMock.return_value interface = ModelSwapperInterface() modelID = "foofar" msg = ResultMessagePackager.marshal( modelID=modelID, batchState=BatchPackager.marshal(batch=results)) interface.submitResults(modelID=modelID, results=results) mqName = interface._resultsQueueName messageBusConnectorMock.publish.assert_called_once_with(mqName, msg, persistent=True)