def testScrubDataSamplesOutOfOrder(self, _repositoryMock):
streamer = metric_streamer_util.MetricStreamer()
now = datetime.utcnow()
getTailMetricRowTimestampPatch = patch.object(
streamer,
"_getTailMetricRowTimestamp",
autospec=True,
return_value=now)
oneInterval = timedelta(seconds=300)
taintedSamples = [
(now - oneInterval, -1.0),
(now + oneInterval * 1, 2.0),
(now, -3.0),
(now, -3.0),
(now + oneInterval * 2, 4.0),
(now + oneInterval * 2, -4.0),
(now + oneInterval * 3, 5.0),
]
expectedPassingSamples = [(now + oneInterval * 1, 2.0),
(now + oneInterval * 2, 4.0),
(now + oneInterval * 3, 5.0)]
with getTailMetricRowTimestampPatch:
passingData = streamer._scrubDataSamples(
data=taintedSamples,
metricID=Mock(name="MetricID"),
conn=Mock(name="SqlalchemyConnection"),
lastDataRowID=Mock(name="lastDataRowID"))
self.assertSequenceEqual(passingData, expectedPassingSamples)
def testSendInputRowsToModelSubmitRequestsOtherError(self):
""" Test MetricStreamer._sendInputRowsToModel with error other than
ModelNotFound from ModelSwapperInterface.submitRequests
"""
metricDataOutputChunkSize = metric_streamer_util.config.getint(
"metric_streamer", "chunk_size")
now = datetime.utcnow()
inputRows = [
model_swapper_interface.ModelInputRow(
rowID=1 + i, data=(
now + timedelta(seconds=60 * i),
i,
)) for i in xrange(metricDataOutputChunkSize * 3)
]
class OtherError(Exception):
pass
modelSwapper = Mock(
spec_set=model_swapper_interface.ModelSwapperInterface)
modelSwapper.submitRequests.side_effect = OtherError
metricID = "abcdef"
streamer = metric_streamer_util.MetricStreamer()
with self.assertRaises(OtherError):
streamer._sendInputRowsToModel(inputRows=inputRows,
metricID=metricID,
modelSwapper=modelSwapper)
self.assertEqual(modelSwapper.submitRequests.call_count, 1)
self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[0][0][1],
inputRows[:metricDataOutputChunkSize])
# And one more with just one row
modelSwapper = Mock(
spec_set=model_swapper_interface.ModelSwapperInterface)
modelSwapper.submitRequests.side_effect = OtherError
with self.assertRaises(OtherError):
streamer._sendInputRowsToModel(inputRows=inputRows[:1],
metricID=metricID,
modelSwapper=modelSwapper)
self.assertEqual(modelSwapper.submitRequests.call_count, 1)
self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[0][0][1], inputRows[:1])
def testSendInputRowsToModelModelNotFoundError(self):
""" Test MetricStreamer._sendInputRowsToModel with ModelNotFound error
from ModelSwapperInterface.submitRequests
"""
metricDataOutputChunkSize = metric_streamer_util.config.getint(
"metric_streamer", "chunk_size")
now = datetime.utcnow()
inputRows = [
model_swapper_interface.ModelInputRow(
rowID=1 + i, data=(
now + timedelta(seconds=60 * i),
i,
)) for i in xrange(metricDataOutputChunkSize * 3)
]
modelSwapper = Mock(
spec_set=model_swapper_interface.ModelSwapperInterface)
modelSwapper.submitRequests.side_effect = (
model_swapper_interface.ModelNotFound)
metricID = "abcdef"
streamer = metric_streamer_util.MetricStreamer()
streamer._sendInputRowsToModel(inputRows=inputRows,
metricID=metricID,
modelSwapper=modelSwapper)
self.assertEqual(modelSwapper.submitRequests.call_count, 1)
self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[0][0][1],
inputRows[:metricDataOutputChunkSize])
def testSendInputRowsToModel(self):
""" Test MetricStreamer._sendInputRowsToModel """
metricDataOutputChunkSize = metric_streamer_util.config.getint(
"metric_streamer", "chunk_size")
now = datetime.utcnow()
expectedBatch1 = [
model_swapper_interface.ModelInputRow(
rowID=1 + i, data=(
now + timedelta(seconds=60 * i),
i,
)) for i in xrange(metricDataOutputChunkSize)
]
expectedBatch2 = [
model_swapper_interface.ModelInputRow(
rowID=1 + i, data=(
now + timedelta(seconds=60 * i),
i,
)) for i in xrange(i + 1, metricDataOutputChunkSize + i + 1)
]
expectedBatch3 = [
model_swapper_interface.ModelInputRow(
rowID=1 + i, data=(
now + timedelta(seconds=60 * i),
i,
))
for i in xrange(i + 1, metricDataOutputChunkSize // 2 + i + 1)
]
inputRows = expectedBatch1 + expectedBatch2 + expectedBatch3
modelSwapper = Mock(
spec_set=model_swapper_interface.ModelSwapperInterface)
metricID = "abcdef"
streamer = metric_streamer_util.MetricStreamer()
streamer._sendInputRowsToModel(inputRows=inputRows,
metricID=metricID,
modelSwapper=modelSwapper)
self.assertEqual(modelSwapper.submitRequests.call_count, 3)
self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[0][0][1],
expectedBatch1)
self.assertEqual(modelSwapper.submitRequests.call_args_list[1][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[1][0][1],
expectedBatch2)
self.assertEqual(modelSwapper.submitRequests.call_args_list[2][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[2][0][1],
expectedBatch3)
# And one more time with just one input row
modelSwapper = Mock(
spec_set=model_swapper_interface.ModelSwapperInterface)
streamer._sendInputRowsToModel(inputRows=inputRows[:1],
metricID=metricID,
modelSwapper=modelSwapper)
self.assertEqual(modelSwapper.submitRequests.call_count, 1)
self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[0][0][1], inputRows[:1])