def sendBacklogDataToModel(conn, metricId, logger): """ Send backlog data to OPF/CLA model. Do not call this before starting the model. :param conn: SQLAlchemy Connection object for executing SQL :type conn: sqlalchemy.engine.Connection :param metricId: unique identifier of the metric row :param logger: logger object """ backlogData = tuple( model_swapper_interface.ModelInputRow( rowID=md.rowid, data=(md.timestamp, md.metric_value,)) for md in repository.getMetricData( conn, metricId, fields=[schema.metric_data.c.rowid, schema.metric_data.c.timestamp, schema.metric_data.c.metric_value])) if backlogData: with model_swapper_interface.ModelSwapperInterface() as modelSwapper: model_data_feeder.sendInputRowsToModel( modelId=metricId, inputRows=backlogData, batchSize=config.getint("metric_streamer", "chunk_size"), modelSwapper=modelSwapper, logger=logger, profiling=(config.getboolean("debugging", "profiling") or logger.isEnabledFor(logging.DEBUG))) logger.info("sendBacklogDataToModel: sent %d backlog data rows to model=%s", len(backlogData), metricId)
def testSendInputRowsToModelSubmitRequestsOtherError(self): """ Test MetricStreamer._sendInputRowsToModel with error other than ModelNotFound from ModelSwapperInterface.submitRequests """ metricDataOutputChunkSize = metric_streamer_util.config.getint( "metric_streamer", "chunk_size") now = datetime.utcnow() inputRows = [ model_swapper_interface.ModelInputRow( rowID=1 + i, data=( now + timedelta(seconds=60 * i), i, )) for i in xrange(metricDataOutputChunkSize * 3) ] class OtherError(Exception): pass modelSwapper = Mock( spec_set=model_swapper_interface.ModelSwapperInterface) modelSwapper.submitRequests.side_effect = OtherError metricID = "abcdef" streamer = metric_streamer_util.MetricStreamer() with self.assertRaises(OtherError): streamer._sendInputRowsToModel(inputRows=inputRows, metricID=metricID, modelSwapper=modelSwapper) self.assertEqual(modelSwapper.submitRequests.call_count, 1) self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0], metricID) self.assertSequenceEqual( modelSwapper.submitRequests.call_args_list[0][0][1], inputRows[:metricDataOutputChunkSize]) # And one more with just one row modelSwapper = Mock( spec_set=model_swapper_interface.ModelSwapperInterface) modelSwapper.submitRequests.side_effect = OtherError with self.assertRaises(OtherError): streamer._sendInputRowsToModel(inputRows=inputRows[:1], metricID=metricID, modelSwapper=modelSwapper) self.assertEqual(modelSwapper.submitRequests.call_count, 1) self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0], metricID) self.assertSequenceEqual( modelSwapper.submitRequests.call_args_list[0][0][1], inputRows[:1])
def testSendInputRowsToModelModelNotFoundError(self): """ Test MetricStreamer._sendInputRowsToModel with ModelNotFound error from ModelSwapperInterface.submitRequests """ metricDataOutputChunkSize = metric_streamer_util.config.getint( "metric_streamer", "chunk_size") now = datetime.utcnow() inputRows = [ model_swapper_interface.ModelInputRow( rowID=1 + i, data=( now + timedelta(seconds=60 * i), i, )) for i in xrange(metricDataOutputChunkSize * 3) ] modelSwapper = Mock( spec_set=model_swapper_interface.ModelSwapperInterface) modelSwapper.submitRequests.side_effect = ( model_swapper_interface.ModelNotFound) metricID = "abcdef" streamer = metric_streamer_util.MetricStreamer() streamer._sendInputRowsToModel(inputRows=inputRows, metricID=metricID, modelSwapper=modelSwapper) self.assertEqual(modelSwapper.submitRequests.call_count, 1) self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0], metricID) self.assertSequenceEqual( modelSwapper.submitRequests.call_args_list[0][0][1], inputRows[:metricDataOutputChunkSize])
def testSendInputRowsToModel(self): """ Test MetricStreamer._sendInputRowsToModel """ metricDataOutputChunkSize = metric_streamer_util.config.getint( "metric_streamer", "chunk_size") now = datetime.utcnow() expectedBatch1 = [ model_swapper_interface.ModelInputRow( rowID=1 + i, data=( now + timedelta(seconds=60 * i), i, )) for i in xrange(metricDataOutputChunkSize) ] expectedBatch2 = [ model_swapper_interface.ModelInputRow( rowID=1 + i, data=( now + timedelta(seconds=60 * i), i, )) for i in xrange(i + 1, metricDataOutputChunkSize + i + 1) ] expectedBatch3 = [ model_swapper_interface.ModelInputRow( rowID=1 + i, data=( now + timedelta(seconds=60 * i), i, )) for i in xrange(i + 1, metricDataOutputChunkSize // 2 + i + 1) ] inputRows = expectedBatch1 + expectedBatch2 + expectedBatch3 modelSwapper = Mock( spec_set=model_swapper_interface.ModelSwapperInterface) metricID = "abcdef" streamer = metric_streamer_util.MetricStreamer() streamer._sendInputRowsToModel(inputRows=inputRows, metricID=metricID, modelSwapper=modelSwapper) self.assertEqual(modelSwapper.submitRequests.call_count, 3) self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0], metricID) self.assertSequenceEqual( modelSwapper.submitRequests.call_args_list[0][0][1], expectedBatch1) self.assertEqual(modelSwapper.submitRequests.call_args_list[1][0][0], metricID) self.assertSequenceEqual( modelSwapper.submitRequests.call_args_list[1][0][1], expectedBatch2) self.assertEqual(modelSwapper.submitRequests.call_args_list[2][0][0], metricID) self.assertSequenceEqual( modelSwapper.submitRequests.call_args_list[2][0][1], expectedBatch3) # And one more time with just one input row modelSwapper = Mock( spec_set=model_swapper_interface.ModelSwapperInterface) streamer._sendInputRowsToModel(inputRows=inputRows[:1], metricID=metricID, modelSwapper=modelSwapper) self.assertEqual(modelSwapper.submitRequests.call_count, 1) self.assertEqual(modelSwapper.submitRequests.call_args_list[0][0][0], metricID) self.assertSequenceEqual( modelSwapper.submitRequests.call_args_list[0][0][1], inputRows[:1])