def testModelInputRowConstructorFromSerializableState(self):
rowID = 1
data = [1, 2, datetime.datetime.utcnow()]
inputRow = ModelInputRow(rowID=rowID, data=copy.copy(data))
self.assertEqual(inputRow.rowID, rowID)
self.assertEqual(inputRow.data, data)
self.assertIn("ModelInputRow<", str(inputRow))
self.assertIn("ModelInputRow<", repr(inputRow))
inputRow2 = _ModelRequestResultBase.__createFromState__(
inputRow.__getstate__())
self.assertEqual(inputRow2.rowID, rowID)
self.assertEqual(inputRow2.data, data)
self.assertIn("ModelInputRow<", str(inputRow2))
self.assertIn("ModelInputRow<", repr(inputRow2))
def _storeDataSamples(self, data, metricID, conn):
""" Store the given metric data samples in metric_data table
:param data: A sequence of data samples; each data sample is a pair:
(datetime.datetime, float)
:param metricID: unique metric id
:param sqlalchemy.engine.Connection conn: A sqlalchemy connection object
:returns: a (possibly empty) tuple of ModelInputRow objects corresponding
to the samples that were stored; ordered by rowid.
:rtype: tuple of model_swapper_interface.ModelInputRow objects
"""
if data:
# repository.addMetricData expects samples as pairs of (value, timestamp)
data = tuple((value, ts) for (ts, value) in data)
# Save new metric data in metric table
rows = repository.addMetricData(conn, metricID, data)
# Update tail metric data timestamp cache for metrics stored by us
self._tailInputMetricDataTimestamps[metricID] = rows[-1]["timestamp"]
# Add newly-stored records to batch for sending to CLA model
modelInputRows = tuple(
ModelInputRow(rowID=row["rowid"], data=(timestamp, metricValue,))
for (metricValue, timestamp), row
in itertools.izip_longest(data, rows))
else:
modelInputRows = tuple()
self._log.warning("_storeDataSamples called with empty data")
return modelInputRows
def testModelSwapper(self):
"""Simple end-to-end test of the model swapper system."""
modelSchedulerSubprocess = self._startModelSchedulerSubprocess()
self.addCleanup(lambda: modelSchedulerSubprocess.kill() if
modelSchedulerSubprocess.returncode is None else None)
modelID = "foobar"
resultBatches = []
with ModelSwapperInterface() as swapperAPI:
possibleModels = getScalarMetricWithTimeOfDayParams(metricData=[0],
minVal=0,
maxVal=1000)
# Submit requests including a model creation command and two data rows.
args = possibleModels[0]
args["inputRecordSchema"] = (
FieldMetaInfo("c0", FieldMetaType.datetime,
FieldMetaSpecial.timestamp),
FieldMetaInfo("c1", FieldMetaType.float,
FieldMetaSpecial.none),
)
# Define the model
_LOGGER.info("Defining the model")
swapperAPI.defineModel(modelID=modelID,
args=args,
commandID="defineModelCmd1")
# Attempt to define the same model again
_LOGGER.info("Defining the model again")
swapperAPI.defineModel(modelID=modelID,
args=args,
commandID="defineModelCmd2")
# Send input rows to the model
inputRows = [
ModelInputRow(
rowID="rowfoo",
data=[datetime.datetime(2013, 5, 23, 8, 13, 00), 5.3]),
ModelInputRow(
rowID="rowbar",
data=[datetime.datetime(2013, 5, 23, 8, 13, 15), 2.4]),
]
_LOGGER.info("Submitting batch of %d input rows...",
len(inputRows))
swapperAPI.submitRequests(modelID=modelID, requests=inputRows)
_LOGGER.info("These models have pending input: %s",
swapperAPI.getModelsWithInputPending())
# Retrieve all results.
# NOTE: We collect results via background thread to avoid
# deadlocking the test runner in the event consuming blocks unexpectedly
_LOGGER.info("Reading all batches of results...")
numBatchesExpected = 3
resultBatches.extend(
self._consumeResults(numBatchesExpected, timeout=20))
self.assertEqual(len(resultBatches), numBatchesExpected)
with MessageBusConnector() as bus:
# The results message queue should be empty now
self.assertTrue(bus.isEmpty(swapperAPI._resultsQueueName))
# Delete the model
_LOGGER.info("Deleting the model")
swapperAPI.deleteModel(modelID=modelID,
commandID="deleteModelCmd1")
_LOGGER.info("Waiting for model deletion result")
resultBatches.extend(self._consumeResults(1, timeout=20))
self.assertEqual(len(resultBatches), 4)
with MessageBusConnector() as bus:
# The results message queue should be empty now
self.assertTrue(bus.isEmpty(swapperAPI._resultsQueueName))
# The model input queue should be deleted now
self.assertFalse(
bus.isMessageQeueuePresent(
swapperAPI._getModelInputQName(modelID=modelID)))
# Try deleting the model again, to make sure there are no exceptions
_LOGGER.info("Attempting to delete the model again")
swapperAPI.deleteModel(modelID=modelID,
commandID="deleteModelCmd1")
# Verify results
# First result batch should be the first defineModel result
batch = resultBatches[0]
self.assertEqual(batch.modelID, modelID)
self.assertEqual(len(batch.objects), 1)
result = batch.objects[0]
self.assertIsInstance(result, ModelCommandResult)
self.assertEqual(result.method, "defineModel")
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.commandID, "defineModelCmd1")
# The second result batch should for the second defineModel result for the
# same model
batch = resultBatches[1]
self.assertEqual(batch.modelID, modelID)
self.assertEqual(len(batch.objects), 1)
result = batch.objects[0]
self.assertIsInstance(result, ModelCommandResult)
self.assertEqual(result.method, "defineModel")
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.commandID, "defineModelCmd2")
# The third batch should be for the two input rows
batch = resultBatches[2]
self.assertEqual(batch.modelID, modelID)
self.assertEqual(len(batch.objects), len(inputRows))
for inputRow, result in zip(inputRows, batch.objects):
self.assertIsInstance(result, ModelInferenceResult)
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.rowID, inputRow.rowID)
self.assertIsInstance(result.anomalyScore, float)
# The fourth batch should be for the "deleteModel"
batch = resultBatches[3]
self.assertEqual(batch.modelID, modelID)
self.assertEqual(len(batch.objects), 1)
result = batch.objects[0]
self.assertIsInstance(result, ModelCommandResult)
self.assertEqual(result.method, "deleteModel")
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.commandID, "deleteModelCmd1")
# Signal Model Scheduler Service subprocess to shut down and wait for it
waitResult = dict()
def runWaiterThread():
try:
waitResult["returnCode"] = modelSchedulerSubprocess.wait()
except:
_LOGGER.exception(
"Waiting for modelSchedulerSubprocess failed")
waitResult["exceptionInfo"] = traceback.format_exc()
raise
return
modelSchedulerSubprocess.terminate()
waiterThread = threading.Thread(target=runWaiterThread)
waiterThread.setDaemon(True)
waiterThread.start()
waiterThread.join(timeout=30)
self.assertFalse(waiterThread.isAlive())
self.assertEqual(waitResult["returnCode"], 0, msg=repr(waitResult))
def _auxTestRunModelWithFullThenIncrementalCheckpoints(
self, classifierEnabled):
modelID = "foobar"
checkpointMgr = model_checkpoint_mgr.ModelCheckpointMgr()
args = getScalarMetricWithTimeOfDayAnomalyParams(metricData=[0],
minVal=0,
maxVal=1000)
args["modelConfig"]["modelParams"]["clEnable"] = classifierEnabled
# Submit requests including a model creation command and two data rows.
args["inputRecordSchema"] = (
FieldMetaInfo("c0", FieldMetaType.datetime,
FieldMetaSpecial.timestamp),
FieldMetaInfo("c1", FieldMetaType.float, FieldMetaSpecial.none),
)
with ModelSwapperInterface() as swapperAPI:
# Define the model
_LOGGER.info("Defining the model")
swapperAPI.defineModel(modelID=modelID,
args=args,
commandID="defineModelCmd1")
# Send input rows to the model
inputRows = [
ModelInputRow(
rowID="rowfoo",
data=[datetime.datetime(2014, 5, 23, 8, 13, 00), 5.3]),
ModelInputRow(
rowID="rowbar",
data=[datetime.datetime(2014, 5, 23, 8, 13, 15), 2.4]),
]
_LOGGER.info(
"Submitting batch of %d input rows with ids=[%s..%s]...",
len(inputRows), inputRows[0].rowID, inputRows[-1].rowID)
swapperAPI.submitRequests(modelID=modelID, requests=inputRows)
# Run model_runner and collect results
with self._startModelRunnerSubprocess(
modelID) as modelRunnerProcess:
resultBatches = self._consumeResults(numExpectedBatches=2,
timeout=15)
self._waitForProcessToStopAndCheck(modelRunnerProcess)
with MessageBusConnector() as bus:
# The results message queue should be empty now
self.assertTrue(bus.isEmpty(swapperAPI._resultsQueueName))
self.assertEqual(len(resultBatches), 2, repr(resultBatches))
# First result batch should be the first defineModel result
batch = resultBatches[0]
self.assertEqual(batch.modelID, modelID)
self.assertEqual(len(batch.objects), 1)
result = batch.objects[0]
self.assertIsInstance(result, ModelCommandResult)
self.assertEqual(result.method, "defineModel")
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.commandID, "defineModelCmd1")
# The second result batch should be for the two input rows
batch = resultBatches[1]
self.assertEqual(batch.modelID, modelID)
self.assertEqual(len(batch.objects), len(inputRows))
for inputRow, result in zip(inputRows, batch.objects):
self.assertIsInstance(result, ModelInferenceResult)
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.rowID, inputRow.rowID)
self.assertIsInstance(result.anomalyScore, float)
if classifierEnabled:
self.assertIsInstance(result.multiStepBestPredictions,
dict)
else:
self.assertIsNone(result.multiStepBestPredictions)
# Verify model checkpoint
model = checkpointMgr.load(modelID)
del model
attrs = checkpointMgr.loadCheckpointAttributes(modelID)
self.assertIn(
model_runner._ModelArchiver._BATCH_IDS_CHECKPOINT_ATTR_NAME,
attrs,
msg=repr(attrs))
self.assertEqual(len(attrs[
model_runner._ModelArchiver._BATCH_IDS_CHECKPOINT_ATTR_NAME]),
2,
msg=repr(attrs))
self.assertNotIn(model_runner._ModelArchiver.
_INPUT_SAMPLES_SINCE_CHECKPOINT_ATTR_NAME,
attrs,
msg=repr(attrs))
# Now, check incremental checkpointing
inputRows2 = [
ModelInputRow(
rowID=2,
data=[datetime.datetime(2014, 5, 23, 8, 13, 20), 2.7]),
ModelInputRow(
rowID=3,
data=[datetime.datetime(2014, 5, 23, 8, 13, 25), 3.9]),
]
_LOGGER.info(
"Submitting batch of %d input rows with ids=[%s..%s]...",
len(inputRows2), inputRows2[0].rowID, inputRows2[-1].rowID)
inputBatchID = swapperAPI.submitRequests(modelID=modelID,
requests=inputRows2)
with self._startModelRunnerSubprocess(
modelID) as modelRunnerProcess:
resultBatches = self._consumeResults(numExpectedBatches=1,
timeout=15)
self._waitForProcessToStopAndCheck(modelRunnerProcess)
with MessageBusConnector() as bus:
self.assertTrue(bus.isEmpty(swapperAPI._resultsQueueName))
batch = resultBatches[0]
self.assertEqual(batch.modelID, modelID)
self.assertEqual(len(batch.objects), len(inputRows2))
for inputRow, result in zip(inputRows2, batch.objects):
self.assertIsInstance(result, ModelInferenceResult)
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.rowID, inputRow.rowID)
self.assertIsInstance(result.anomalyScore, float)
if classifierEnabled:
self.assertIsInstance(result.multiStepBestPredictions,
dict)
else:
self.assertIsNone(result.multiStepBestPredictions)
model = checkpointMgr.load(modelID)
del model
attrs = checkpointMgr.loadCheckpointAttributes(modelID)
self.assertIn(
model_runner._ModelArchiver._BATCH_IDS_CHECKPOINT_ATTR_NAME,
attrs,
msg=repr(attrs))
self.assertSequenceEqual(attrs[
model_runner._ModelArchiver._BATCH_IDS_CHECKPOINT_ATTR_NAME],
[inputBatchID],
msg=repr(attrs))
self.assertIn(model_runner._ModelArchiver.
_INPUT_SAMPLES_SINCE_CHECKPOINT_ATTR_NAME,
attrs,
msg=repr(attrs))
self.assertSequenceEqual(
model_runner._ModelArchiver._decodeDataSamples(
attrs[model_runner._ModelArchiver.
_INPUT_SAMPLES_SINCE_CHECKPOINT_ATTR_NAME]),
[row.data for row in inputRows2],
msg=repr(attrs))
# Final run with incremental checkpointing
inputRows3 = [
ModelInputRow(
rowID=4,
data=[datetime.datetime(2014, 5, 23, 8, 13, 30), 4.7]),
ModelInputRow(
rowID=5,
data=[datetime.datetime(2014, 5, 23, 8, 13, 35), 5.9]),
]
_LOGGER.info(
"Submitting batch of %d input rows with ids=[%s..%s]...",
len(inputRows3), inputRows3[0].rowID, inputRows3[-1].rowID)
inputBatchID = swapperAPI.submitRequests(modelID=modelID,
requests=inputRows3)
with self._startModelRunnerSubprocess(
modelID) as modelRunnerProcess:
resultBatches = self._consumeResults(numExpectedBatches=1,
timeout=15)
self._waitForProcessToStopAndCheck(modelRunnerProcess)
with MessageBusConnector() as bus:
self.assertTrue(bus.isEmpty(swapperAPI._resultsQueueName))
batch = resultBatches[0]
self.assertEqual(batch.modelID, modelID)
self.assertEqual(len(batch.objects), len(inputRows3))
for inputRow, result in zip(inputRows3, batch.objects):
self.assertIsInstance(result, ModelInferenceResult)
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.rowID, inputRow.rowID)
self.assertIsInstance(result.anomalyScore, float)
if classifierEnabled:
self.assertIsInstance(result.multiStepBestPredictions,
dict)
else:
self.assertIsNone(result.multiStepBestPredictions)
model = checkpointMgr.load(modelID)
del model
attrs = checkpointMgr.loadCheckpointAttributes(modelID)
self.assertIn(
model_runner._ModelArchiver._BATCH_IDS_CHECKPOINT_ATTR_NAME,
attrs,
msg=repr(attrs))
self.assertSequenceEqual(attrs[
model_runner._ModelArchiver._BATCH_IDS_CHECKPOINT_ATTR_NAME],
[inputBatchID],
msg=repr(attrs))
self.assertIn(model_runner._ModelArchiver.
_INPUT_SAMPLES_SINCE_CHECKPOINT_ATTR_NAME,
attrs,
msg=repr(attrs))
self.assertSequenceEqual(
model_runner._ModelArchiver._decodeDataSamples(
attrs[model_runner._ModelArchiver.
_INPUT_SAMPLES_SINCE_CHECKPOINT_ATTR_NAME]),
[row.data for row in itertools.chain(inputRows2, inputRows3)],
msg=repr(attrs))
# Delete the model
_LOGGER.info("Deleting the model=%s", modelID)
swapperAPI.deleteModel(modelID=modelID,
commandID="deleteModelCmd1")
with self._startModelRunnerSubprocess(
modelID) as modelRunnerProcess:
resultBatches = self._consumeResults(numExpectedBatches=1,
timeout=15)
self._waitForProcessToStopAndCheck(modelRunnerProcess)
self.assertEqual(len(resultBatches), 1, repr(resultBatches))
# First result batch should be the first defineModel result
batch = resultBatches[0]
self.assertEqual(batch.modelID, modelID)
self.assertEqual(len(batch.objects), 1)
result = batch.objects[0]
self.assertIsInstance(result, ModelCommandResult)
self.assertEqual(result.method, "deleteModel")
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.commandID, "deleteModelCmd1")
with MessageBusConnector() as bus:
self.assertTrue(bus.isEmpty(swapperAPI._resultsQueueName))
# The model input queue should be deleted now
self.assertFalse(
bus.isMessageQeueuePresent(
swapperAPI._getModelInputQName(modelID=modelID)))
# The model checkpoint should be gone too
with self.assertRaises(model_checkpoint_mgr.ModelNotFound):
checkpointMgr.load(modelID)
with self.assertRaises(model_checkpoint_mgr.ModelNotFound):
checkpointMgr.loadModelDefinition(modelID)
with self.assertRaises(model_checkpoint_mgr.ModelNotFound):
checkpointMgr.loadCheckpointAttributes(modelID)
with self.assertRaises(model_checkpoint_mgr.ModelNotFound):
checkpointMgr.remove(modelID)
def testCloneModel(self):
modelSchedulerSubprocess = self._startModelSchedulerSubprocess()
self.addCleanup(lambda: modelSchedulerSubprocess.kill() if
modelSchedulerSubprocess.returncode is None else None)
modelID = "abc"
destModelID = "def"
resultBatches = []
with ModelSwapperInterface() as swapperAPI:
args = getScalarMetricWithTimeOfDayAnomalyParams(metricData=[0],
minVal=0,
maxVal=1000)
# Submit requests including a model creation command and two data rows.
args["inputRecordSchema"] = (
FieldMetaInfo("c0", FieldMetaType.datetime,
FieldMetaSpecial.timestamp),
FieldMetaInfo("c1", FieldMetaType.float,
FieldMetaSpecial.none),
)
# Define the model
_LOGGER.info("Defining the model")
swapperAPI.defineModel(modelID=modelID,
args=args,
commandID="defineModelCmd1")
resultBatches.extend(self._consumeResults(1, timeout=20))
self.assertEqual(len(resultBatches), 1)
# Clone the just-defined model
_LOGGER.info("Cloning model")
swapperAPI.cloneModel(modelID,
destModelID,
commandID="cloneModelCmd1")
resultBatches.extend(self._consumeResults(1, timeout=20))
self.assertEqual(len(resultBatches), 2)
# Send input rows to the clone
inputRows = [
ModelInputRow(
rowID="rowfoo",
data=[datetime.datetime(2013, 5, 23, 8, 13, 00), 5.3]),
ModelInputRow(
rowID="rowbar",
data=[datetime.datetime(2013, 5, 23, 8, 13, 15), 2.4]),
]
_LOGGER.info("Submitting batch of %d input rows...",
len(inputRows))
swapperAPI.submitRequests(modelID=destModelID, requests=inputRows)
_LOGGER.info("These models have pending input: %s",
swapperAPI.getModelsWithInputPending())
resultBatches.extend(self._consumeResults(1, timeout=20))
self.assertEqual(len(resultBatches), 3)
with MessageBusConnector() as bus:
# The results message queue should be empty now
self.assertTrue(bus.isEmpty(swapperAPI._resultsQueueName))
# Delete the model
_LOGGER.info("Deleting the model")
swapperAPI.deleteModel(modelID=destModelID,
commandID="deleteModelCmd1")
_LOGGER.info("Waiting for model deletion result")
resultBatches.extend(self._consumeResults(1, timeout=20))
self.assertEqual(len(resultBatches), 4)
with MessageBusConnector() as bus:
# The results message queue should be empty now
self.assertTrue(bus.isEmpty(swapperAPI._resultsQueueName))
# The model input queue should be deleted now
self.assertFalse(
bus.isMessageQeueuePresent(
swapperAPI._getModelInputQName(modelID=destModelID)))
# Verify results
# First result batch should be the defineModel result
batch = resultBatches[0]
self.assertEqual(batch.modelID, modelID)
self.assertEqual(len(batch.objects), 1)
result = batch.objects[0]
self.assertIsInstance(result, ModelCommandResult)
self.assertEqual(result.method, "defineModel")
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.commandID, "defineModelCmd1")
# The second result batch should for the cloneModel result
batch = resultBatches[1]
self.assertEqual(batch.modelID, modelID)
self.assertEqual(len(batch.objects), 1)
result = batch.objects[0]
self.assertIsInstance(result, ModelCommandResult)
self.assertEqual(result.method, "cloneModel")
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.commandID, "cloneModelCmd1")
# The third batch should be for the two input rows
batch = resultBatches[2]
self.assertEqual(batch.modelID, destModelID)
self.assertEqual(len(batch.objects), len(inputRows))
for inputRow, result in zip(inputRows, batch.objects):
self.assertIsInstance(result, ModelInferenceResult)
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.rowID, inputRow.rowID)
self.assertIsInstance(result.anomalyScore, float)
self.assertIsInstance(result.multiStepBestPredictions, dict)
# The fourth batch should be for the "deleteModel"
batch = resultBatches[3]
self.assertEqual(batch.modelID, destModelID)
self.assertEqual(len(batch.objects), 1)
result = batch.objects[0]
self.assertIsInstance(result, ModelCommandResult)
self.assertEqual(result.method, "deleteModel")
self.assertEqual(result.status, htmengineerrno.SUCCESS)
self.assertEqual(result.commandID, "deleteModelCmd1")
# Signal Model Scheduler Service subprocess to shut down and wait for it
waitResult = dict()
def runWaiterThread():
try:
waitResult["returnCode"] = modelSchedulerSubprocess.wait()
except:
_LOGGER.exception(
"Waiting for modelSchedulerSubprocess failed")
waitResult["exceptionInfo"] = traceback.format_exc()
raise
return
modelSchedulerSubprocess.terminate()
waiterThread = threading.Thread(target=runWaiterThread)
waiterThread.setDaemon(True)
waiterThread.start()
waiterThread.join(timeout=30)
self.assertFalse(waiterThread.isAlive())
self.assertEqual(waitResult["returnCode"], 0, msg=repr(waitResult))