def testMarshalUnmarshal(self):
inputBatch = [
ModelCommandResult(commandID="commandID", method="testMethod", status=1,
errorMessage="errorMessage"),
ModelInputRow(rowID="foo", data=[1, 2, "Sep 21 02:24:21 UTC 2013"]),
ModelInputRow(rowID="bar", data=[9, 54, "Sep 21 02:24:38 UTC 2013"]),
]
batchState = BatchPackager.marshal(batch=inputBatch)
requestBatch = BatchPackager.unmarshal(batchState=batchState)
self.assertEqual(requestBatch[0].commandID, inputBatch[0].commandID)
self.assertEqual(requestBatch[1].rowID, inputBatch[1].rowID)
self.assertEqual(requestBatch[2].rowID, inputBatch[2].rowID)
def testSubmitRequestsWithContextManager(self, messageBusConnectorClassMock):
requests = [
ModelCommand(commandID="abc", method="defineModel",
args={'key1': 4098, 'key2': 4139}),
ModelInputRow(rowID="foo", data=[1, 2, "Sep 21 02:24:21 UTC 2013"]),
ModelInputRow(rowID="bar", data=[9, 54, "Sep 21 02:24:38 UTC 2013"])
]
modelID = "foofar"
messageBusConnectorMock = messageBusConnectorClassMock.return_value
with ModelSwapperInterface() as interface:
batchID = interface.submitRequests(modelID=modelID, requests=requests)
modelMQName = interface._modelInputQueueNamePrefix + modelID
notificationMQName = interface._schedulerNotificationQueueName
# Verify
self.assertIsInstance(batchID, str)
msg = RequestMessagePackager.marshal(
batchID=batchID,
batchState=BatchPackager.marshal(batch=requests))
self.assertEqual(messageBusConnectorMock.publish.call_count, 2)
messageBusConnectorMock.publish.assert_any_call(
modelMQName, msg, persistent=True)
messageBusConnectorMock.publish.assert_any_call(
notificationMQName, json.dumps(modelID), persistent=False)
def testSubmitResultsFailure(self, messageBusConnectorClassMock):
results = [
ModelCommandResult(commandID="abc", method="testMethod", status=0,
args={'key1': 4098, 'key2': 4139}),
ModelInferenceResult(rowID="foo", status=0, anomalyScore=1),
ModelInferenceResult(rowID="bar", status=0, anomalyScore=2)
]
messageBusConnectorMock = messageBusConnectorClassMock.return_value
messageBusConnectorMock.publish.side_effect = (
message_bus_connector.MessageQueueNotFound("from mock test"),
Exception("re-publish failed in mock test"))
modelID = "foofar"
with self.assertRaises(Exception) as raisesCM:
with ModelSwapperInterface() as interface:
mqName = interface._resultsQueueName
interface.submitResults(modelID=modelID, results=results)
# Verify
self.assertEqual(raisesCM.exception.args[0],
"re-publish failed in mock test")
messageBusConnectorMock.createMessageQueue.assert_called_once_with(
mqName, durable=True)
self.assertEqual(messageBusConnectorMock.publish.call_count, 2)
msg = ResultMessagePackager.marshal(
modelID=modelID,
batchState=BatchPackager.marshal(batch=results))
messageBusConnectorMock.publish.assert_called_with(mqName, msg,
persistent=True)
def testSubmitResultsRecoveryFromMessageQueueNotFound(
self, messageBusConnectorClassMock):
# submitResults recovers from the initial MessageQueueNotFound by creating
# the results message queue and re-publishing the message
results = [
ModelCommandResult(commandID="abc", method="testMethod", status=0,
args={'key1': 4098, 'key2': 4139}),
ModelInferenceResult(rowID="foo", status=0, anomalyScore=1),
ModelInferenceResult(rowID="bar", status=0, anomalyScore=2)
]
messageBusConnectorMock = messageBusConnectorClassMock.return_value
messageBusConnectorMock.publish.side_effect = (
message_bus_connector.MessageQueueNotFound("from mock test"),
Mock())
modelID = "foofar"
with ModelSwapperInterface() as interface:
interface.submitResults(modelID=modelID, results=results)
mqName = interface._resultsQueueName
# Verify
messageBusConnectorMock.createMessageQueue.assert_called_once_with(
mqName, durable=True)
self.assertEqual(messageBusConnectorMock.publish.call_count, 2)
msg = ResultMessagePackager.marshal(
modelID=modelID,
batchState=BatchPackager.marshal(batch=results))
messageBusConnectorMock.publish.assert_called_with(mqName, msg,
persistent=True)
def testConsumeRequestsNonBlocking(self, messageBusConnectorClassMock):
expectedRequests = (
ModelCommand(commandID="abc", method="defineModel",
args={'key1': 4098, 'key2': 4139}),
ModelInputRow(rowID="foo", data=[1, 2, "Sep 21 02:24:21 UTC 2013"]),
ModelInputRow(rowID="bar", data=[9, 54, "Sep 21 02:24:38 UTC 2013"]),
)
modelID = "foobar"
batchID = uuid.uuid1().hex
msg = RequestMessagePackager.marshal(
batchID=batchID,
batchState=BatchPackager.marshal(batch=expectedRequests))
messageBusConnectorMock = messageBusConnectorClassMock.return_value
ackMock = Mock(return_value=None)
messageBusConnectorMock.consume.return_value = Mock(
spec_set=message_bus_connector._QueueConsumer,
__iter__=lambda *args, **kwargs: iter(
[message_bus_connector._ConsumedMessage(body=msg, ack=ackMock)]))
with ModelSwapperInterface() as interface:
self.assertEqual(len(interface._consumers), 0)
with interface.consumeRequests(modelID, blocking=False) as consumer:
self.assertIsNotNone(consumer._mqConsumer)
self.assertEqual(len(interface._consumers), 1)
batch = next(iter(consumer))
self.assertEqual(batch.batchID, batchID)
self.assertEqual(batch.objects, expectedRequests)
self.assertTrue(callable(batch.ack))
self.assertIs(batch.ack, ackMock)
# Make sure we didn't skip any fields
self.assertEqual(set(batch._fields), set(["batchID", "objects", "ack"]))
self.assertIsNone(consumer._mqConsumer)
self.assertEqual(len(interface._consumers), 0)
self.assertEqual(len(interface._consumers), 0)
modelMQName = interface._modelInputQueueNamePrefix + modelID
messageBusConnectorMock.consume.assert_called_once_with(
modelMQName, blocking=False)
def testMarshalAndUnmarshal(self):
resultBatch = [
ModelCommandResult(commandID="abc", method="testMethod", status=0,
args={'key1': 4098, 'key2': 4139}),
ModelInferenceResult(rowID="foo", status=0, anomalyScore=1),
ModelInferenceResult(rowID="bar", status=0, anomalyScore=2)
]
batchState = BatchPackager.marshal(batch=resultBatch)
msg = ResultMessagePackager.marshal(modelID="foobar", batchState=batchState)
r = ResultMessagePackager.unmarshal(msg)
self.assertEqual(r.modelID, "foobar")
self.assertEqual(r.batchState, batchState)
# Make sure we aren't forgetting to test any returned fields
self.assertEqual(set(["modelID", "batchState"]), set(r._fields))
def testMarshalAndUnmarshal(self):
requestBatch = [
ModelCommand(commandID="abc", method="defineModel",
args={'key1': 4098, 'key2': 4139}),
ModelInputRow(rowID="foo", data=[1, 2, "Sep 21 02:24:21 UTC 2013"]),
ModelInputRow(rowID="bar", data=[9, 54, "Sep 21 02:24:38 UTC 2013"]),
]
batchState = BatchPackager.marshal(batch=requestBatch)
msg = RequestMessagePackager.marshal(batchID="foobar",
batchState=batchState)
r = RequestMessagePackager.unmarshal(msg)
self.assertEqual(r.batchID, "foobar")
self.assertEqual(r.batchState, batchState)
# Make sure we aren't forgetting to test any returned fields
self.assertEqual(set(["batchID", "batchState"]), set(r._fields))
def testContextManagerAndConsumeResults(self, messageBusConnectorClassMock):
expectedResults = (
ModelCommandResult(commandID="abc", method="testMethod", status=0,
args={'key1': 4098, 'key2': 4139}),
ModelInferenceResult(rowID="foo", status=0, anomalyScore=1.3),
ModelInferenceResult(rowID="bar", status=0, anomalyScore=2.9)
)
modelID = "foobar"
msg = ResultMessagePackager.marshal(
modelID=modelID,
batchState=BatchPackager.marshal(batch=expectedResults))
messageBusConnectorMock = messageBusConnectorClassMock.return_value
ackMock = Mock(return_value=None)
messageBusConnectorMock.consume.return_value = Mock(
spec_set=message_bus_connector._QueueConsumer,
__iter__=lambda *args, **kwargs: iter(
[message_bus_connector._ConsumedMessage(
body=msg, ack=ackMock)]))
with ModelSwapperInterface() as interface:
self.assertEqual(len(interface._consumers), 0)
consumer = interface.consumeResults()
self.assertIsNotNone(consumer._mqConsumer)
self.assertEqual(len(interface._consumers), 1)
batch = next(iter(consumer))
self.assertEqual(batch.modelID, modelID)
self.assertEqual(batch.objects, expectedResults)
self.assertTrue(callable(batch.ack))
self.assertIs(batch.ack, ackMock)
self.assertEqual(set(batch._fields), set(["modelID", "objects", "ack"]))
consumer.close()
self.assertIsNone(consumer._mqConsumer)
self.assertEqual(len(interface._consumers), 0)
self.assertEqual(len(interface._consumers), 0)
def testSubmitResults(self, messageBusConnectorClassMock):
results = [
ModelCommandResult(commandID="abc", method="testMethod", status=0,
args={'key1': 4098, 'key2': 4139}),
ModelInferenceResult(rowID="foo", status=0, anomalyScore=1),
ModelInferenceResult(rowID="bar", status=0, anomalyScore=2)
]
messageBusConnectorMock = messageBusConnectorClassMock.return_value
interface = ModelSwapperInterface()
modelID = "foofar"
msg = ResultMessagePackager.marshal(
modelID=modelID,
batchState=BatchPackager.marshal(batch=results))
interface.submitResults(modelID=modelID, results=results)
mqName = interface._resultsQueueName
messageBusConnectorMock.publish.assert_called_once_with(mqName, msg,
persistent=True)
