def testExportImport(self):
metricName = "test-" + uuid.uuid1().hex
adapter = datasource_adapter_factory.createCustomDatasourceAdapter()
g_log.info("Creating htmengine custom metric; name=%s", metricName)
metricId = adapter.createMetric(metricName)
self.addCleanup(adapter.deleteMetricByName, metricName)
# Add some data
# NOTE: we discard the fractional part because it gets eliminated
# in the database, and we will want to compare against retrieved
# items later.
now = datetime.datetime.utcnow().replace(microsecond=0)
data = [(0, now - datetime.timedelta(minutes=5)), (100, now)]
with self.engine.connect() as conn:
repository.addMetricData(conn, metricId, data)
# Turn on monitoring
modelSpec = {
"datasource": "custom",
"metricSpec": {
"metric": metricName
},
}
adapter.monitorMetric(modelSpec)
def checkExportSpec(exportSpec):
self.assertEqual(exportSpec["datasource"], modelSpec["datasource"])
self.assertEqual(exportSpec["metricSpec"], modelSpec["metricSpec"])
self.assertSequenceEqual(exportSpec["data"], data)
# Export
exportSpec = adapter.exportModel(metricId)
checkExportSpec(exportSpec)
# Delete metric
adapter.deleteMetricByName(metricName)
self.checkModelDeleted(metricId)
# Import
metricId = adapter.importModel(
htmengine.utils.jsonDecode(htmengine.utils.jsonEncode(exportSpec)))
with self.engine.connect() as conn:
metricObj = repository.getMetric(
conn,
metricId,
fields=[schema.metric.c.parameters, schema.metric.c.status])
self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA)
self.assertEqual(json.loads(metricObj.parameters), modelSpec)
self._validateModelSpec(json.loads(metricObj.parameters))
# Export again
exportSpec = adapter.exportModel(metricId)
checkExportSpec(exportSpec)
def testActivateModelClassifierEnabled(self):
""" Test activateModel with classifier enabled in model spec. """
metricName = "test-" + uuid.uuid1().hex
adapter = datasource_adapter_factory.createCustomDatasourceAdapter()
g_log.info("Creating htmengine custom metric; name=%s", metricName)
metricId = adapter.createMetric(metricName)
self.addCleanup(adapter.deleteMetricByName, metricName)
# Turn on monitoring
modelSpec = {
"datasource": "custom",
"metricSpec": {
"metric": metricName
},
"modelParams": {
"enableClassifier": True
}
}
adapter.monitorMetric(modelSpec)
with self.engine.connect() as conn:
metricObj = repository.getMetric(conn,
metricId,
fields=[schema.metric.c.status])
self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA)
# Add some data
data = [
(0, datetime.datetime.utcnow() - datetime.timedelta(minutes=5)),
(100, datetime.datetime.utcnow())
]
with self.engine.connect() as conn:
repository.addMetricData(conn, metricId, data)
# Activate model
adapter.activateModel(metricId)
with self.engine.connect() as conn:
metricObj = repository.getMetric(conn,
metricId,
fields=[schema.metric.c.status,
schema.metric.c.model_params])
self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING,
MetricStatus.ACTIVE))
self._assertClassifierStatusInModelParams(metricObj.model_params,
classifierEnabled=True)
g_log.info("Waiting for model to become active")
self.checkModelIsActive(metricId)
g_log.info("Waiting at least one model result")
self.checkModelResultsSize(metricId, 1, atLeast=True)
def testActivateModelClassifierEnabled(self):
""" Test activateModel with classifier enabled in model spec. """
metricName = "test-" + uuid.uuid1().hex
adapter = datasource_adapter_factory.createCustomDatasourceAdapter()
g_log.info("Creating htmengine custom metric; name=%s", metricName)
metricId = adapter.createMetric(metricName)
self.addCleanup(adapter.deleteMetricByName, metricName)
# Turn on monitoring
modelSpec = {
"datasource": "custom",
"metricSpec": {
"metric": metricName
},
"modelParams": {
"enableClassifier": True
}
}
adapter.monitorMetric(modelSpec)
with self.engine.connect() as conn:
metricObj = repository.getMetric(conn,
metricId,
fields=[schema.metric.c.status])
self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA)
# Add some data
data = [(0,
datetime.datetime.utcnow() - datetime.timedelta(minutes=5)),
(100, datetime.datetime.utcnow())]
with self.engine.connect() as conn:
repository.addMetricData(conn, metricId, data)
# Activate model
adapter.activateModel(metricId)
with self.engine.connect() as conn:
metricObj = repository.getMetric(
conn,
metricId,
fields=[schema.metric.c.status, schema.metric.c.model_params])
self.assertIn(metricObj.status,
(MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE))
self._assertClassifierStatusInModelParams(metricObj.model_params,
classifierEnabled=True)
g_log.info("Waiting for model to become active")
self.checkModelIsActive(metricId)
g_log.info("Waiting at least one model result")
self.checkModelResultsSize(metricId, 1, atLeast=True)
def testMonitorMetricWithEnoughDataForStats(self):
""" monitorMetric should create a model when there is enough data rows """
metricName = "test-" + uuid.uuid1().hex
adapter = datasource_adapter_factory.createCustomDatasourceAdapter()
g_log.info("Creating htmengine custom metric; name=%s", metricName)
metricId = adapter.createMetric(metricName)
self.addCleanup(adapter.deleteMetricByName, metricName)
# Add enough data to force activation of model
data = [
(offset, datetime.datetime.utcnow() + datetime.timedelta(minutes=offset))
for offset in xrange(
0,
scalar_metric_utils.MODEL_CREATION_RECORD_THRESHOLD * 5,
5)
]
self.assertEqual(len(data),
scalar_metric_utils.MODEL_CREATION_RECORD_THRESHOLD)
with self.engine.connect() as conn:
repository.addMetricData(conn, metricId, data)
# Turn on monitoring
modelSpec = {
"datasource": "custom",
"metricSpec": {
"metric": metricName
},
}
adapter.monitorMetric(modelSpec)
with self.engine.connect() as conn:
metricObj = repository.getMetric(conn,
metricId,
fields=[schema.metric.c.status])
self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING,
MetricStatus.ACTIVE))
g_log.info("Waiting for model to become active")
self.checkModelIsActive(metricId)
g_log.info("Waiting at least one model result")
self.checkModelResultsSize(metricId, 1, atLeast=True)
def _storeDataSamples(self, data, metricID, conn):
""" Store the given metric data samples in metric_data table
:param data: A sequence of data samples; each data sample is a pair:
(datetime.datetime, float)
:param metricID: unique metric id
:param sqlalchemy.engine.Connection conn: A sqlalchemy connection object
:returns: a (possibly empty) tuple of ModelInputRow objects corresponding
to the samples that were stored; ordered by rowid.
:rtype: tuple of model_swapper_interface.ModelInputRow objects
"""
if data:
# repository.addMetricData expects samples as pairs of (value, timestamp)
data = tuple((value, ts) for (ts, value) in data)
# Save new metric data in metric table
rows = repository.addMetricData(conn, metricID, data)
# Update tail metric data timestamp cache for metrics stored by us
self._tailInputMetricDataTimestamps[metricID] = rows[-1]["timestamp"]
# Add newly-stored records to batch for sending to CLA model
modelInputRows = tuple(
ModelInputRow(rowID=row["rowid"], data=(timestamp, metricValue,))
for (metricValue, timestamp), row
in itertools.izip_longest(data, rows))
else:
modelInputRows = tuple()
self._log.warning("_storeDataSamples called with empty data")
return modelInputRows
def _storeDataSamples(self, data, metricID, conn):
""" Store the given metric data samples in metric_data table
:param data: A sequence of data samples; each data sample is a pair:
(datetime.datetime, float)
:param metricID: unique metric id
:param sqlalchemy.engine.Connection conn: A sqlalchemy connection object
:returns: a (possibly empty) tuple of ModelInputRow objects corresponding
to the samples that were stored; ordered by rowid.
:rtype: tuple of model_swapper_interface.ModelInputRow objects
"""
if data:
# repository.addMetricData expects samples as pairs of (value, timestamp)
data = tuple((value, ts) for (ts, value) in data)
# Save new metric data in metric table
rows = repository.addMetricData(conn, metricID, data)
# Update tail metric data timestamp cache for metrics stored by us
self._tailInputMetricDataTimestamps[metricID] = rows[-1]["timestamp"]
# Add newly-stored records to batch for sending to CLA model
modelInputRows = tuple(
ModelInputRow(rowID=row["rowid"], data=(timestamp, metricValue,))
for (metricValue, timestamp), row
in itertools.izip_longest(data, rows))
else:
modelInputRows = tuple()
self._log.warning("_storeDataSamples called with empty data")
return modelInputRows
def testMonitorMetricWithEnoughDataForStats(self):
""" monitorMetric should create a model when there is enough data rows """
metricName = "test-" + uuid.uuid1().hex
adapter = datasource_adapter_factory.createCustomDatasourceAdapter()
g_log.info("Creating htmengine custom metric; name=%s", metricName)
metricId = adapter.createMetric(metricName)
self.addCleanup(adapter.deleteMetricByName, metricName)
# Add enough data to force activation of model
data = [
(offset,
datetime.datetime.utcnow() + datetime.timedelta(minutes=offset))
for offset in xrange(
0, scalar_metric_utils.MODEL_CREATION_RECORD_THRESHOLD * 5, 5)
]
self.assertEqual(len(data),
scalar_metric_utils.MODEL_CREATION_RECORD_THRESHOLD)
with self.engine.connect() as conn:
repository.addMetricData(conn, metricId, data)
# Turn on monitoring
modelSpec = {
"datasource": "custom",
"metricSpec": {
"metric": metricName
},
}
adapter.monitorMetric(modelSpec)
with self.engine.connect() as conn:
metricObj = repository.getMetric(conn,
metricId,
fields=[schema.metric.c.status])
self.assertIn(metricObj.status,
(MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE))
g_log.info("Waiting for model to become active")
self.checkModelIsActive(metricId)
g_log.info("Waiting at least one model result")
self.checkModelResultsSize(metricId, 1, atLeast=True)
def testExportImportCompleteModelParams(self):
metricName = "test-" + uuid.uuid1().hex
adapter = datasource_adapter_factory.createCustomDatasourceAdapter()
g_log.info("Creating htmengine custom metric; name=%s", metricName)
metricId = adapter.createMetric(metricName)
self.addCleanup(adapter.deleteMetricByName, metricName)
# Add some data
# NOTE: we discard the fractional part because it gets eliminated
# in the database, and we will want to compare against retrieved
# items later.
now = datetime.datetime.utcnow().replace(microsecond=0)
data = [
(0, now - datetime.timedelta(minutes=5)),
(100, now)
]
with self.engine.connect() as conn:
repository.addMetricData(conn, metricId, data)
fileName = "custom_datasource_adapter_test_model_config.json"
with self._openTestDataFile(fileName) as modelConfigFile:
modelConfig = json.load(modelConfigFile)
# Turn on monitoring
modelSpec = {
"datasource": "custom",
"metricSpec": {
"metric": metricName
},
"completeModelParams": {
"modelConfig": modelConfig,
"inferenceArgs": {"predictionSteps": [1], "predictedField": "bar",
"inputPredictedField": "auto"},
"timestampFieldName": "foo",
"valueFieldName": "bar"
}
}
adapter.monitorMetric(modelSpec)
def checkExportSpec(exportSpec):
self.assertEqual(exportSpec["datasource"], modelSpec["datasource"])
self.assertEqual(exportSpec["metricSpec"], modelSpec["metricSpec"])
self.assertSequenceEqual(exportSpec["data"], data)
# Export
exportSpec = adapter.exportModel(metricId)
checkExportSpec(exportSpec)
# Delete metric
adapter.deleteMetricByName(metricName)
self.checkModelDeleted(metricId)
# Import
metricId = adapter.importModel(
htmengine.utils.jsonDecode(htmengine.utils.jsonEncode(exportSpec)))
with self.engine.connect() as conn:
metricObj = repository.getMetric(conn,
metricId,
fields=[schema.metric.c.parameters,
schema.metric.c.status])
self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING,
MetricStatus.ACTIVE))
self.assertEqual(json.loads(metricObj.parameters), modelSpec)
self._validateModelSpec(json.loads(metricObj.parameters))
# Export again
exportSpec = adapter.exportModel(metricId)
checkExportSpec(exportSpec)
def testExportImport(self):
metricName = "test-" + uuid.uuid1().hex
adapter = datasource_adapter_factory.createCustomDatasourceAdapter()
g_log.info("Creating htmengine custom metric; name=%s", metricName)
metricId = adapter.createMetric(metricName)
self.addCleanup(adapter.deleteMetricByName, metricName)
# Add some data
# NOTE: we discard the fractional part because it gets eliminated
# in the database, and we will want to compare against retrieved
# items later.
now = datetime.datetime.utcnow().replace(microsecond=0)
data = [
(0, now - datetime.timedelta(minutes=5)),
(100, now)
]
with self.engine.connect() as conn:
repository.addMetricData(conn, metricId, data)
# Turn on monitoring
modelSpec = {
"datasource": "custom",
"metricSpec": {
"metric": metricName
},
}
adapter.monitorMetric(modelSpec)
def checkExportSpec(exportSpec):
self.assertEqual(exportSpec["datasource"], modelSpec["datasource"])
self.assertEqual(exportSpec["metricSpec"], modelSpec["metricSpec"])
self.assertSequenceEqual(exportSpec["data"], data)
# Export
exportSpec = adapter.exportModel(metricId)
checkExportSpec(exportSpec)
# Delete metric
adapter.deleteMetricByName(metricName)
self.checkModelDeleted(metricId)
# Import
metricId = adapter.importModel(
htmengine.utils.jsonDecode(htmengine.utils.jsonEncode(exportSpec)))
with self.engine.connect() as conn:
metricObj = repository.getMetric(conn,
metricId,
fields=[schema.metric.c.parameters,
schema.metric.c.status])
self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA)
self.assertEqual(json.loads(metricObj.parameters), modelSpec)
self._validateModelSpec(json.loads(metricObj.parameters))
# Export again
exportSpec = adapter.exportModel(metricId)
checkExportSpec(exportSpec)
def testExportImportCompleteModelParams(self):
metricName = "test-" + uuid.uuid1().hex
adapter = datasource_adapter_factory.createCustomDatasourceAdapter()
g_log.info("Creating htmengine custom metric; name=%s", metricName)
metricId = adapter.createMetric(metricName)
self.addCleanup(adapter.deleteMetricByName, metricName)
# Add some data
# NOTE: we discard the fractional part because it gets eliminated
# in the database, and we will want to compare against retrieved
# items later.
now = datetime.datetime.utcnow().replace(microsecond=0)
data = [(0, now - datetime.timedelta(minutes=5)), (100, now)]
with self.engine.connect() as conn:
repository.addMetricData(conn, metricId, data)
fileName = "custom_datasource_adapter_test_model_config.json"
with self._openTestDataFile(fileName) as modelConfigFile:
modelConfig = json.load(modelConfigFile)
# Turn on monitoring
modelSpec = {
"datasource": "custom",
"metricSpec": {
"metric": metricName
},
"completeModelParams": {
"modelConfig": modelConfig,
"inferenceArgs": {
"predictionSteps": [1],
"predictedField": "bar",
"inputPredictedField": "auto"
},
"timestampFieldName": "foo",
"valueFieldName": "bar"
}
}
adapter.monitorMetric(modelSpec)
def checkExportSpec(exportSpec):
self.assertEqual(exportSpec["datasource"], modelSpec["datasource"])
self.assertEqual(exportSpec["metricSpec"], modelSpec["metricSpec"])
self.assertSequenceEqual(exportSpec["data"], data)
# Export
exportSpec = adapter.exportModel(metricId)
checkExportSpec(exportSpec)
# Delete metric
adapter.deleteMetricByName(metricName)
self.checkModelDeleted(metricId)
# Import
metricId = adapter.importModel(
htmengine.utils.jsonDecode(htmengine.utils.jsonEncode(exportSpec)))
with self.engine.connect() as conn:
metricObj = repository.getMetric(
conn,
metricId,
fields=[schema.metric.c.parameters, schema.metric.c.status])
self.assertIn(metricObj.status,
(MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE))
self.assertEqual(json.loads(metricObj.parameters), modelSpec)
self._validateModelSpec(json.loads(metricObj.parameters))
# Export again
exportSpec = adapter.exportModel(metricId)
checkExportSpec(exportSpec)