def sendBacklogDataToModel(conn, metricId, logger):
""" Send backlog data to OPF/CLA model. Do not call this before starting the
model.
:param conn: SQLAlchemy Connection object for executing SQL
:type conn: sqlalchemy.engine.Connection
:param metricId: unique identifier of the metric row
:param logger: logger object
"""
backlogData = tuple(
model_swapper_interface.ModelInputRow(
rowID=md.rowid, data=(md.timestamp, md.metric_value,))
for md in repository.getMetricData(
conn,
metricId,
fields=[schema.metric_data.c.rowid,
schema.metric_data.c.timestamp,
schema.metric_data.c.metric_value]))
if backlogData:
with model_swapper_interface.ModelSwapperInterface() as modelSwapper:
model_data_feeder.sendInputRowsToModel(
modelId=metricId,
inputRows=backlogData,
batchSize=config.getint("metric_streamer", "chunk_size"),
modelSwapper=modelSwapper,
logger=logger,
profiling=(config.getboolean("debugging", "profiling") or
logger.isEnabledFor(logging.DEBUG)))
logger.info("sendBacklogDataToModel: sent %d backlog data rows to model=%s",
len(backlogData), metricId)
def testSendInputRowsToModelSubmitRequestsOtherError(self):
""" Test MetricStreamer._sendInputRowsToModel with error other than
ModelNotFound from ModelSwapperInterface.submitRequests
"""
metricDataOutputChunkSize = metric_streamer_util.config.getint(
"metric_streamer", "chunk_size")
now = datetime.utcnow()
inputRows = [
model_swapper_interface.ModelInputRow(
rowID=1 + i, data=(
now + timedelta(seconds=60 * i),
i,
)) for i in xrange(metricDataOutputChunkSize * 3)
]
class OtherError(Exception):
pass
modelSwapper = Mock(
spec_set=model_swapper_interface.ModelSwapperInterface)
modelSwapper.submitRequests.side_effect = OtherError
metricID = "abcdef"
streamer = metric_streamer_util.MetricStreamer()
with self.assertRaises(OtherError):
streamer._sendInputRowsToModel(inputRows=inputRows,
metricID=metricID,
modelSwapper=modelSwapper)
self.assertEqual(modelSwapper.submitRequests.call_count, 1)
self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[0][0][1],
inputRows[:metricDataOutputChunkSize])
# And one more with just one row
modelSwapper = Mock(
spec_set=model_swapper_interface.ModelSwapperInterface)
modelSwapper.submitRequests.side_effect = OtherError
with self.assertRaises(OtherError):
streamer._sendInputRowsToModel(inputRows=inputRows[:1],
metricID=metricID,
modelSwapper=modelSwapper)
self.assertEqual(modelSwapper.submitRequests.call_count, 1)
self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[0][0][1], inputRows[:1])
def testSendInputRowsToModelModelNotFoundError(self):
""" Test MetricStreamer._sendInputRowsToModel with ModelNotFound error
from ModelSwapperInterface.submitRequests
"""
metricDataOutputChunkSize = metric_streamer_util.config.getint(
"metric_streamer", "chunk_size")
now = datetime.utcnow()
inputRows = [
model_swapper_interface.ModelInputRow(
rowID=1 + i, data=(
now + timedelta(seconds=60 * i),
i,
)) for i in xrange(metricDataOutputChunkSize * 3)
]
modelSwapper = Mock(
spec_set=model_swapper_interface.ModelSwapperInterface)
modelSwapper.submitRequests.side_effect = (
model_swapper_interface.ModelNotFound)
metricID = "abcdef"
streamer = metric_streamer_util.MetricStreamer()
streamer._sendInputRowsToModel(inputRows=inputRows,
metricID=metricID,
modelSwapper=modelSwapper)
self.assertEqual(modelSwapper.submitRequests.call_count, 1)
self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[0][0][1],
inputRows[:metricDataOutputChunkSize])
def testSendInputRowsToModel(self):
""" Test MetricStreamer._sendInputRowsToModel """
metricDataOutputChunkSize = metric_streamer_util.config.getint(
"metric_streamer", "chunk_size")
now = datetime.utcnow()
expectedBatch1 = [
model_swapper_interface.ModelInputRow(
rowID=1 + i, data=(
now + timedelta(seconds=60 * i),
i,
)) for i in xrange(metricDataOutputChunkSize)
]
expectedBatch2 = [
model_swapper_interface.ModelInputRow(
rowID=1 + i, data=(
now + timedelta(seconds=60 * i),
i,
)) for i in xrange(i + 1, metricDataOutputChunkSize + i + 1)
]
expectedBatch3 = [
model_swapper_interface.ModelInputRow(
rowID=1 + i, data=(
now + timedelta(seconds=60 * i),
i,
))
for i in xrange(i + 1, metricDataOutputChunkSize // 2 + i + 1)
]
inputRows = expectedBatch1 + expectedBatch2 + expectedBatch3
modelSwapper = Mock(
spec_set=model_swapper_interface.ModelSwapperInterface)
metricID = "abcdef"
streamer = metric_streamer_util.MetricStreamer()
streamer._sendInputRowsToModel(inputRows=inputRows,
metricID=metricID,
modelSwapper=modelSwapper)
self.assertEqual(modelSwapper.submitRequests.call_count, 3)
self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[0][0][1],
expectedBatch1)
self.assertEqual(modelSwapper.submitRequests.call_args_list[1][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[1][0][1],
expectedBatch2)
self.assertEqual(modelSwapper.submitRequests.call_args_list[2][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[2][0][1],
expectedBatch3)
# And one more time with just one input row
modelSwapper = Mock(
spec_set=model_swapper_interface.ModelSwapperInterface)
streamer._sendInputRowsToModel(inputRows=inputRows[:1],
metricID=metricID,
modelSwapper=modelSwapper)
self.assertEqual(modelSwapper.submitRequests.call_count, 1)
self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0],
metricID)
self.assertSequenceEqual(
modelSwapper.submitRequests.call_args_list[0][0][1], inputRows[:1])