def testModelResultHandlerSkipsStaleBatch(
self, _amqpUtilsMock, deserializeModelResult, connectDynamoDB, _gracefulCreateTable
):
""" Given a stale batch of model inference results, verify that it isn't
saved to DynamoDB
"""
# We're going to mostly mock out all of the arguments to
# DynamoDBService.messageHandler() since it is normally called by amqp lib.
# Then simulate the process of handling an inbound batch of model inference
# results and assert that the appropriate put_item() calls are made at the
# other end.
message = amqp.messages.ConsumerMessage(
body=Mock(),
properties=Mock(headers=dict()),
methodInfo=amqp.messages.MessageDeliveryInfo(
consumerTag=Mock(), deliveryTag=Mock(), redelivered=False, exchange=Mock(), routingKey=""
),
ackImpl=Mock(),
nackImpl=Mock(),
)
# We will have to bypass the normal serialize/deserialize phases to avoid
# dependency on sqlalchemy rowproxy. Instead, we'll just mock out the
# AnomalyService.deserializeModelResult() call, returning an object that
# approximates a batch of model inference results as much as possible
ts = epochFromNaiveUTCDatetime(
datetime.utcnow().replace(microsecond=0) - timedelta(days=DynamoDBService._FRESH_DATA_THRESHOLD_DAYS + 1)
)
resultRow = dict(rowid=4790, ts=ts, value=9305.0, rawAnomaly=0.775, anomaly=0.999840891)
metricId = "3b035a5916994f2bb950f5717138f94b"
deserializeModelResult.return_value = dict(
metric=dict(
uid=metricId,
name="XIGNITE.AGN.VOLUME",
description="XIGNITE.AGN.VOLUME",
resource="Resource-of-XIGNITE.AGN.VOLUME",
location="",
datasource="custom",
spec=dict(userInfo=dict(symbol="AGN", metricType="StockVolume", metricTypeName="Stock Volume")),
),
results=[resultRow],
)
service = DynamoDBService()
publishMetricDataPatch = patch.object(service, "_publishMetricData", spec_set=service._publishMetricData)
publishInstancePatch = patch.object(
service, "_publishInstanceDataHourly", spec_set=service._publishInstanceDataHourly
)
with publishMetricDataPatch as publishMetricDataMock, publishInstancePatch as publishInstanceMock:
service.messageHandler(message)
deserializeModelResult.assert_called_once_with(message.body)
self.assertEqual(publishMetricDataMock.call_count, 0)
self.assertEqual(publishInstanceMock.call_count, 0)
def testMessageHandlerRoutesTweetDataToDynamoDB(self, _amqpUtilsMock, connectDynamoDB, _gracefulCreateTable):
""" Simple test for twitter interface
"""
## channel = Mock()
## method = Mock(routing_key="taurus.data.non-metric.twitter")
## properties = Mock()
tweetData = [
{
"metric_name": "Metric Name",
"tweet_uid": "3b035a5916994f2bb950f5717138f94b",
"created_at": "2015-02-19T19:43:24.870109",
"agg_ts": "2015-02-19T19:43:24.870118",
"text": "Tweet text",
"userid": "10",
"username": "******",
"retweet_count": "0",
}
]
message = amqp.messages.ConsumerMessage(
body=json.dumps(tweetData),
properties=Mock(),
methodInfo=amqp.messages.MessageDeliveryInfo(
consumerTag=Mock(),
deliveryTag=Mock(),
redelivered=False,
exchange=Mock(),
routingKey="taurus.data.non-metric.twitter",
),
ackImpl=Mock(),
nackImpl=Mock(),
)
service = DynamoDBService()
service.messageHandler(message)
(
service._metric_tweets.batch_write.return_value.__enter__.return_value.put_item.assert_called_once_with(
data=OrderedDict(
[
("metric_name_tweet_uid", "Metric Name-3b035a5916994f2bb950f5717138f94b"),
("metric_name", "Metric Name"),
("tweet_uid", "3b035a5916994f2bb950f5717138f94b"),
("created_at", "2015-02-19T19:43:24.870109"),
("agg_ts", "2015-02-19T19:43:24.870118"),
("text", "Tweet text"),
("userid", "10"),
("username", "Tweet username"),
("retweet_count", "0"),
]
),
overwrite=True,
)
)
def testMessageHandlerRoutesTweetDataToDynamoDB(self, _amqpUtilsMock,
connectDynamoDB,
_gracefulCreateTable):
""" Simple test for twitter interface
"""
## channel = Mock()
## method = Mock(routing_key="taurus.data.non-metric.twitter")
## properties = Mock()
tweetData = [{
"metric_name": "Metric Name",
"tweet_uid": "3b035a5916994f2bb950f5717138f94b",
"created_at": "2015-02-19T19:43:24.870109",
"agg_ts": "2015-02-19T19:43:24.870118",
"text": "Tweet text",
"userid": "10",
"username": "******",
"retweet_count": "0"
}]
message = amqp.messages.ConsumerMessage(
body=json.dumps(tweetData),
properties=Mock(),
methodInfo=amqp.messages.MessageDeliveryInfo(
consumerTag=Mock(),
deliveryTag=Mock(),
redelivered=False,
exchange=Mock(),
routingKey="taurus.data.non-metric.twitter"),
ackImpl=Mock(),
nackImpl=Mock())
service = DynamoDBService()
service.messageHandler(message)
(service._metric_tweets.batch_write.return_value.__enter__.
return_value.put_item.assert_called_once_with(data=OrderedDict([
("metric_name_tweet_uid",
"Metric Name-3b035a5916994f2bb950f5717138f94b"),
("metric_name", "Metric Name"),
("tweet_uid", "3b035a5916994f2bb950f5717138f94b"),
("created_at", "2015-02-19T19:43:24.870109"),
("agg_ts", "2015-02-19T19:43:24.870118"), ("text", "Tweet text"),
("userid", "10"), ("username", "Tweet username"),
("retweet_count", "0")
]),
overwrite=True))
def testMessageHandlerRoutesMetricDataToDynamoDB(
self, _amqpUtilsMock,
deserializeModelResult, connectDynamoDB, _gracefulCreateTable):
""" Given a batch of model inference results, send the appropriate data to
DynamoDB tables according to design in an environment where both rabbitmq
and dynamodb are mocked out
"""
# We're going to mostly mock out all of the arguments to
# DynamoDBService.messageHandler() since it is normally called by amqp lib.
# Then simulate the process of handling an inbound batch of model inference
# results and assert that the appropriate put_item() calls are made at the
# other end.
message = amqp.messages.ConsumerMessage(
body=Mock(),
properties=Mock(headers=dict()),
methodInfo=amqp.messages.MessageDeliveryInfo(consumerTag=Mock(),
deliveryTag=Mock(),
redelivered=False,
exchange=Mock(),
routingKey=""),
ackImpl=Mock(),
nackImpl=Mock())
# We will have to bypass the normal serialize/deserialize phases to avoid
# dependency on sqlalchemy rowproxy. Instead, we'll just mock out the
# AnomalyService.deserializeModelResult() call, returning an object that
# approximates a batch of model inference results as much as possible
now = int(time.time())
resultRow = dict(
rowid=4790,
ts=now,
value=9305.0,
rawAnomaly=0.775,
anomaly=0.999840891
)
metricId = "3b035a5916994f2bb950f5717138f94b"
deserializeModelResult.return_value = dict(
metric=dict(
uid=metricId,
name="XIGNITE.AGN.VOLUME",
description="XIGNITE.AGN.VOLUME",
resource="Resource-of-XIGNITE.AGN.VOLUME",
location = "",
datasource = "custom",
spec=dict(
userInfo=dict(
symbol="AGN",
metricType="StockVolume",
metricTypeName="Stock Volume"
)
)
),
results=[resultRow]
)
service = DynamoDBService()
service.messageHandler(message)
deserializeModelResult.assert_called_once_with(message.body)
mockMetricDataPutItem = (
service._metric_data.batch_write.return_value.__enter__
.return_value.put_item)
data = dynamodb_service.convertInferenceResultRowToMetricDataItem(
metricId, resultRow)
mockMetricDataPutItem.assert_called_once_with(data=data._asdict(),
overwrite=True)
self.assertFalse(service._metric_tweets.batch_write.called)
# Make sure that a model command result doesn't get mistaken for an
# inference result batch
deserializeModelResult.return_value = Mock()
message.properties = Mock(headers=dict(dataType="model-cmd-result"))
message.body = Mock()
service = DynamoDBService()
with patch.object(service, "_handleModelCommandResult",
spec_set=service._handleModelCommandResult):
service.messageHandler(message)
service._handleModelCommandResult.assert_called_once_with(message.body)
def testModelResultHandlerSkipsStaleBatch(self, _amqpUtilsMock,
deserializeModelResult,
connectDynamoDB,
_gracefulCreateTable):
""" Given a stale batch of model inference results, verify that it isn't
saved to DynamoDB
"""
# We're going to mostly mock out all of the arguments to
# DynamoDBService.messageHandler() since it is normally called by amqp lib.
# Then simulate the process of handling an inbound batch of model inference
# results and assert that the appropriate put_item() calls are made at the
# other end.
message = amqp.messages.ConsumerMessage(
body=Mock(),
properties=Mock(headers=dict()),
methodInfo=amqp.messages.MessageDeliveryInfo(consumerTag=Mock(),
deliveryTag=Mock(),
redelivered=False,
exchange=Mock(),
routingKey=""),
ackImpl=Mock(),
nackImpl=Mock())
# We will have to bypass the normal serialize/deserialize phases to avoid
# dependency on sqlalchemy rowproxy. Instead, we'll just mock out the
# AnomalyService.deserializeModelResult() call, returning an object that
# approximates a batch of model inference results as much as possible
ts = epochFromNaiveUTCDatetime(
datetime.utcnow().replace(microsecond=0) -
timedelta(days=DynamoDBService._FRESH_DATA_THRESHOLD_DAYS + 1))
resultRow = dict(rowid=4790,
ts=ts,
value=9305.0,
rawAnomaly=0.775,
anomaly=0.999840891)
metricId = "3b035a5916994f2bb950f5717138f94b"
deserializeModelResult.return_value = dict(metric=dict(
uid=metricId,
name="XIGNITE.AGN.VOLUME",
description="XIGNITE.AGN.VOLUME",
resource="Resource-of-XIGNITE.AGN.VOLUME",
location="",
datasource="custom",
spec=dict(userInfo=dict(symbol="AGN",
metricType="StockVolume",
metricTypeName="Stock Volume"))),
results=[resultRow])
service = DynamoDBService()
publishMetricDataPatch = patch.object(
service, "_publishMetricData", spec_set=service._publishMetricData)
publishInstancePatch = patch.object(
service,
"_publishInstanceDataHourly",
spec_set=service._publishInstanceDataHourly)
with publishMetricDataPatch as publishMetricDataMock, \
publishInstancePatch as publishInstanceMock:
service.messageHandler(message)
deserializeModelResult.assert_called_once_with(message.body)
self.assertEqual(publishMetricDataMock.call_count, 0)
self.assertEqual(publishInstanceMock.call_count, 0)
def testMessageHandlerRoutesMetricDataToDynamoDB(self, _amqpUtilsMock,
deserializeModelResult,
connectDynamoDB,
_gracefulCreateTable):
""" Given a batch of model inference results, send the appropriate data to
DynamoDB tables according to design in an environment where both rabbitmq
and dynamodb are mocked out
"""
# We're going to mostly mock out all of the arguments to
# DynamoDBService.messageHandler() since it is normally called by amqp lib.
# Then simulate the process of handling an inbound batch of model inference
# results and assert that the appropriate put_item() calls are made at the
# other end.
message = amqp.messages.ConsumerMessage(
body=Mock(),
properties=Mock(headers=dict()),
methodInfo=amqp.messages.MessageDeliveryInfo(consumerTag=Mock(),
deliveryTag=Mock(),
redelivered=False,
exchange=Mock(),
routingKey=""),
ackImpl=Mock(),
nackImpl=Mock())
# We will have to bypass the normal serialize/deserialize phases to avoid
# dependency on sqlalchemy rowproxy. Instead, we'll just mock out the
# AnomalyService.deserializeModelResult() call, returning an object that
# approximates a batch of model inference results as much as possible
now = int(time.time())
resultRow = dict(rowid=4790,
ts=now,
value=9305.0,
rawAnomaly=0.775,
anomaly=0.999840891)
metricId = "3b035a5916994f2bb950f5717138f94b"
deserializeModelResult.return_value = dict(metric=dict(
uid=metricId,
name="XIGNITE.AGN.VOLUME",
description="XIGNITE.AGN.VOLUME",
resource="Resource-of-XIGNITE.AGN.VOLUME",
location="",
datasource="custom",
spec=dict(userInfo=dict(symbol="AGN",
metricType="StockVolume",
metricTypeName="Stock Volume"))),
results=[resultRow])
service = DynamoDBService()
service.messageHandler(message)
deserializeModelResult.assert_called_once_with(message.body)
mockMetricDataPutItem = (service._metric_data.batch_write.return_value.
__enter__.return_value.put_item)
data = dynamodb_service.convertInferenceResultRowToMetricDataItem(
metricId, resultRow)
mockMetricDataPutItem.assert_called_once_with(data=data._asdict(),
overwrite=True)
self.assertFalse(service._metric_tweets.batch_write.called)
# Make sure that a model command result doesn't get mistaken for an
# inference result batch
deserializeModelResult.return_value = Mock()
message.properties = Mock(headers=dict(dataType="model-cmd-result"))
message.body = Mock()
service = DynamoDBService()
with patch.object(service,
"_handleModelCommandResult",
spec_set=service._handleModelCommandResult):
service.messageHandler(message)
service._handleModelCommandResult.assert_called_once_with(
message.body)
