def testModelResultHandlerSkipsStaleBatch(
self, _amqpUtilsMock,
deserializeModelResult, connectDynamoDB, _gracefulCreateTable):
""" Given a stale batch of model inference results, verify that it isn't
saved to DynamoDB
"""
# We're going to mostly mock out all of the arguments to
# DynamoDBService.messageHandler() since it is normally called by amqp lib.
# Then simulate the process of handling an inbound batch of model inference
# results and assert that the appropriate put_item() calls are made at the
# other end.
message = amqp.messages.ConsumerMessage(
body=Mock(),
properties=Mock(headers=dict()),
methodInfo=amqp.messages.MessageDeliveryInfo(consumerTag=Mock(),
deliveryTag=Mock(),
redelivered=False,
exchange=Mock(),
routingKey=""),
ackImpl=Mock(),
nackImpl=Mock())
# We will have to bypass the normal serialize/deserialize phases to avoid
# dependency on sqlalchemy rowproxy. Instead, we'll just mock out the
# AnomalyService.deserializeModelResult() call, returning an object that
# approximates a batch of model inference results as much as possible
ts = epochFromNaiveUTCDatetime(
datetime.utcnow().replace(microsecond=0) -
timedelta(days=DynamoDBService._FRESH_DATA_THRESHOLD_DAYS + 1))
resultRow = dict(
rowid=4790,
ts=ts,
value=9305.0,
rawAnomaly=0.775,
anomaly=0.999840891
)
metricId = "3b035a5916994f2bb950f5717138f94b"
deserializeModelResult.return_value = dict(
metric=dict(
uid=metricId,
name="XIGNITE.AGN.VOLUME",
description="XIGNITE.AGN.VOLUME",
resource="Resource-of-XIGNITE.AGN.VOLUME",
location = "",
datasource = "custom",
spec=dict(
userInfo=dict(
symbol="AGN",
metricType="StockVolume",
metricTypeName="Stock Volume"
)
)
),
results=[resultRow]
)
service = DynamoDBService()
publishMetricDataPatch = patch.object(
service, "_publishMetricData",
spec_set=service._publishMetricData)
publishInstancePatch = patch.object(
service, "_publishInstanceDataHourly",
spec_set=service._publishInstanceDataHourly)
with publishMetricDataPatch as publishMetricDataMock, \
publishInstancePatch as publishInstanceMock:
service.messageHandler(message)
deserializeModelResult.assert_called_once_with(message.body)
self.assertEqual(publishMetricDataMock.call_count, 0)
self.assertEqual(publishInstanceMock.call_count, 0)
def testMessageHandlerRoutesTweetDataToDynamoDB(
self, _amqpUtilsMock,
connectDynamoDB, _gracefulCreateTable):
""" Simple test for twitter interface
"""
## channel = Mock()
## method = Mock(routing_key="taurus.data.non-metric.twitter")
## properties = Mock()
tweetData = [
{
"metric_name": "Metric Name",
"tweet_uid": "3b035a5916994f2bb950f5717138f94b",
"created_at": "2015-02-19T19:43:24.870109",
"agg_ts": "2015-02-19T19:43:24.870118",
"text": "Tweet text",
"userid": "10",
"username": "******",
"retweet_count": "0"
}
]
message = amqp.messages.ConsumerMessage(
body=json.dumps(tweetData),
properties=Mock(),
methodInfo=amqp.messages.MessageDeliveryInfo(
consumerTag=Mock(),
deliveryTag=Mock(),
redelivered=False,
exchange=Mock(),
routingKey="taurus.data.non-metric.twitter"),
ackImpl=Mock(),
nackImpl=Mock())
service = DynamoDBService()
service.messageHandler(message)
(service
._metric_tweets
.batch_write
.return_value
.__enter__
.return_value
.put_item
.assert_called_once_with(
data=OrderedDict(
[
("metric_name_tweet_uid",
"Metric Name-3b035a5916994f2bb950f5717138f94b"),
("metric_name", "Metric Name"),
("tweet_uid", "3b035a5916994f2bb950f5717138f94b"),
("created_at", "2015-02-19T19:43:24.870109"),
("agg_ts", "2015-02-19T19:43:24.870118"),
("text", "Tweet text"),
("userid", "10"),
("username", "Tweet username"),
("retweet_count", "0")
]
),
overwrite=True))
def testMessageHandlerRoutesMetricDataToDynamoDB(
self, _amqpUtilsMock,
deserializeModelResult, connectDynamoDB, _gracefulCreateTable):
""" Given a batch of model inference results, send the appropriate data to
DynamoDB tables according to design in an environment where both rabbitmq
and dynamodb are mocked out
"""
# We're going to mostly mock out all of the arguments to
# DynamoDBService.messageHandler() since it is normally called by amqp lib.
# Then simulate the process of handling an inbound batch of model inference
# results and assert that the appropriate put_item() calls are made at the
# other end.
message = amqp.messages.ConsumerMessage(
body=Mock(),
properties=Mock(headers=dict()),
methodInfo=amqp.messages.MessageDeliveryInfo(consumerTag=Mock(),
deliveryTag=Mock(),
redelivered=False,
exchange=Mock(),
routingKey=""),
ackImpl=Mock(),
nackImpl=Mock())
# We will have to bypass the normal serialize/deserialize phases to avoid
# dependency on sqlalchemy rowproxy. Instead, we'll just mock out the
# AnomalyService.deserializeModelResult() call, returning an object that
# approximates a batch of model inference results as much as possible
now = int(time.time())
resultRow = dict(
rowid=4790,
ts=now,
value=9305.0,
rawAnomaly=0.775,
anomaly=0.999840891
)
metricId = "3b035a5916994f2bb950f5717138f94b"
deserializeModelResult.return_value = dict(
metric=dict(
uid=metricId,
name="XIGNITE.AGN.VOLUME",
description="XIGNITE.AGN.VOLUME",
resource="Resource-of-XIGNITE.AGN.VOLUME",
location = "",
datasource = "custom",
spec=dict(
userInfo=dict(
symbol="AGN",
metricType="StockVolume",
metricTypeName="Stock Volume"
)
)
),
results=[resultRow]
)
service = DynamoDBService()
service.messageHandler(message)
deserializeModelResult.assert_called_once_with(message.body)
mockMetricDataPutItem = (
service._metric_data.batch_write.return_value.__enter__
.return_value.put_item)
data = dynamodb_service.convertInferenceResultRowToMetricDataItem(
metricId, resultRow)
mockMetricDataPutItem.assert_called_once_with(data=data._asdict(),
overwrite=True)
self.assertFalse(service._metric_tweets.batch_write.called)
# Make sure that a model command result doesn't get mistaken for an
# inference result batch
deserializeModelResult.return_value = Mock()
message.properties = Mock(headers=dict(dataType="model-cmd-result"))
message.body = Mock()
service = DynamoDBService()
with patch.object(service, "_handleModelCommandResult",
spec_set=service._handleModelCommandResult):
service.messageHandler(message)
service._handleModelCommandResult.assert_called_once_with(message.body)