def testMonitorMetricCompleteModelParamsNoInferenceArgs(self): """ Test monitorMetric() raises ValueError with completeModelParams but not inferenceArgs. """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() fileName = "custom_datasource_adapter_test_model_config.json" with self._openTestDataFile(fileName) as modelConfigFile: modelConfig = json.load(modelConfigFile) modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "completeModelParams": { "modelConfig": modelConfig, "timestampFieldName": "jon", "valueFieldName": "snow" } } with self.assertRaises(ValueError) as excCtx: adapter.monitorMetric(modelSpec) excArgZero = excCtx.exception.args[0] initialMsg = excArgZero[0:len(scalar_metric_utils. _NO_INFERENCE_ARGS_MSG)] self.assertEqual(initialMsg, scalar_metric_utils._NO_INFERENCE_ARGS_MSG)
def testUnmonitorMetricPendingData(self): """ Test unmonitorMetric on metric in PENDING_DATA state """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName } } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric( conn, metricId, fields=[schema.metric.c.parameters, schema.metric.c.status]) self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA) self.assertEqual(json.loads(metricObj.parameters), modelSpec) self._validateModelSpec(json.loads(metricObj.parameters)) # Turn off monitoring adapter.unmonitorMetric(metricId) self.checkMetricUnmonitoredById(metricId)
def testUnmonitorMetricWithModel(self): """ Test unmonitorMetric on metric with active model """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric: name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "modelParams": { "min": 0, # optional "max": 100 # optional } } adapter.monitorMetric(modelSpec) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId) # Turn off monitoring g_log.info( "Unmonitoring htmengine custom metric with active model: " "name=%s", metricName) adapter.unmonitorMetric(metricId) self.checkMetricUnmonitoredById(metricId)
def testMonitorMetricCompleteModelParamsNoValueFieldName(self): """ Test monitorMetric() raises ValueError with completeModelParams but not valueFieldName. """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() fileName = "custom_datasource_adapter_test_model_config.json" with self._openTestDataFile(fileName) as modelConfigFile: modelConfig = json.load(modelConfigFile) modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "completeModelParams": { "modelConfig": modelConfig, "inferenceArgs": { "predictionSteps": [1], "predictedField": "bachman", "inputPredictedField": "auto" }, "timestampFieldName": "erlich" } } with self.assertRaises(ValueError) as excCtx: adapter.monitorMetric(modelSpec) excArgZero = excCtx.exception.args[0] initialMsg = excArgZero[0:len(scalar_metric_utils. _NO_VALUE_FIELD_NAME_MSG)] self.assertEqual(initialMsg, scalar_metric_utils._NO_VALUE_FIELD_NAME_MSG)
def testMonitorMetricWithResource(self): """Test monitorMetric that includes an explicit resource string.""" metricName = "test-" + uuid.uuid1().hex resource = "Test Resource" adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName, "resource": resource, } } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.parameters, schema.metric.c.status, schema.metric.c.server]) self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA) self.assertEqual(json.loads(metricObj.parameters), modelSpec) self.assertEqual(metricObj.server, resource) self._validateModelSpec(json.loads(metricObj.parameters))
def _addMetric(engine, metricName): """Add the new metric to the database.""" if metricName in gCustomMetrics: try: # Attempt to reload the metric metricId = gCustomMetrics[metricName][0].uid with engine.connect() as conn: gCustomMetrics[metricName][0] = repository.getMetric( conn, metricId) return except htmengine.exceptions.ObjectNotFoundError: # Do nothing, we will create new metric and update cache below pass # Use the adapter to create the metric try: metricId = createCustomDatasourceAdapter().createMetric(metricName) except htmengine.exceptions.MetricAlreadyExists as e: metricId = e.uid with engine.connect() as conn: metric = repository.getMetric(conn, metricId) # Add it to our cache gCustomMetrics[metricName] = [metric, datetime.datetime.utcnow()] _trimMetricCache()
def testMonitorMetricNameMismatch(self): """ Test monitorMetric() raises ValueError when inferenceArgs-predictedField doesn't match valueFieldName. """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() fileName = "custom_datasource_adapter_test_model_config.json" with self._openTestDataFile(fileName) as modelConfigFile: modelConfig = json.load(modelConfigFile) modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "completeModelParams": { "modelConfig": modelConfig, "inferenceArgs": {"predictionSteps": [1], "predictedField": "baz", "inputPredictedField": "auto"}, "timestampFieldName": "snorf", "valueFieldName": "bar" } } with self.assertRaises(ValueError) as excCtx: adapter.monitorMetric(modelSpec) excArgZero = excCtx.exception.args[0] initialMsg = excArgZero[0: len( scalar_metric_utils._INCONSISTENT_PREDICTED_FIELD_NAME_MSG)] self.assertEqual(initialMsg, scalar_metric_utils._INCONSISTENT_PREDICTED_FIELD_NAME_MSG)
def testMonitorMetricCompleteModelParamsNoInferenceArgs(self): """ Test monitorMetric() raises ValueError with completeModelParams but not inferenceArgs. """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() fileName = "custom_datasource_adapter_test_model_config.json" with self._openTestDataFile(fileName) as modelConfigFile: modelConfig = json.load(modelConfigFile) modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "completeModelParams": { "modelConfig": modelConfig, "timestampFieldName": "jon", "valueFieldName": "snow" } } with self.assertRaises(ValueError) as excCtx: adapter.monitorMetric(modelSpec) excArgZero = excCtx.exception.args[0] initialMsg = excArgZero[0: len( scalar_metric_utils._NO_INFERENCE_ARGS_MSG)] self.assertEqual(initialMsg, scalar_metric_utils._NO_INFERENCE_ARGS_MSG)
def testUnmonitorMetricPendingData(self): """ Test unmonitorMetric on metric in PENDING_DATA state """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName } } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.parameters, schema.metric.c.status]) self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA) self.assertEqual(json.loads(metricObj.parameters), modelSpec) self._validateModelSpec(json.loads(metricObj.parameters)) # Turn off monitoring adapter.unmonitorMetric(metricId) self.checkMetricUnmonitoredById(metricId)
def testMonitorMetricThatIsAlreadyMonitored(self): """ monitorMetric should raise if already monitored """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName } } modelId = adapter.monitorMetric(modelSpec) with self.assertRaises(app_exceptions.MetricAlreadyMonitored) as cm: adapter.monitorMetric(modelSpec) self.assertEqual(cm.exception.uid, modelId)
def testUnmonitorMetricWithModel(self): """ Test unmonitorMetric on metric with active model """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric: name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "modelParams": { "min": 0, # optional "max": 100 # optional } } adapter.monitorMetric(modelSpec) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId) # Turn off monitoring g_log.info("Unmonitoring htmengine custom metric with active model: " "name=%s", metricName) adapter.unmonitorMetric(metricId) self.checkMetricUnmonitoredById(metricId)
def _addMetric(engine, metricName): """Add the new metric to the database.""" if metricName in gCustomMetrics: try: # Attempt to reload the metric metricId = gCustomMetrics[metricName][0].uid with engine.connect() as conn: gCustomMetrics[metricName][0] = repository.getMetric(conn, metricId) return except htmengine.exceptions.ObjectNotFoundError: # Do nothing, we will create new metric and update cache below pass # Use the adapter to create the metric try: metricId = createCustomDatasourceAdapter().createMetric(metricName) except htmengine.exceptions.MetricAlreadyExists as e: metricId = e.uid with engine.connect() as conn: metric = repository.getMetric(conn, metricId) # Add it to our cache gCustomMetrics[metricName] = [metric, datetime.datetime.utcnow()] _trimMetricCache()
def testExportImport(self): metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Add some data # NOTE: we discard the fractional part because it gets eliminated # in the database, and we will want to compare against retrieved # items later. now = datetime.datetime.utcnow().replace(microsecond=0) data = [(0, now - datetime.timedelta(minutes=5)), (100, now)] with self.engine.connect() as conn: repository.addMetricData(conn, metricId, data) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, } adapter.monitorMetric(modelSpec) def checkExportSpec(exportSpec): self.assertEqual(exportSpec["datasource"], modelSpec["datasource"]) self.assertEqual(exportSpec["metricSpec"], modelSpec["metricSpec"]) self.assertSequenceEqual(exportSpec["data"], data) # Export exportSpec = adapter.exportModel(metricId) checkExportSpec(exportSpec) # Delete metric adapter.deleteMetricByName(metricName) self.checkModelDeleted(metricId) # Import metricId = adapter.importModel( htmengine.utils.jsonDecode(htmengine.utils.jsonEncode(exportSpec))) with self.engine.connect() as conn: metricObj = repository.getMetric( conn, metricId, fields=[schema.metric.c.parameters, schema.metric.c.status]) self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA) self.assertEqual(json.loads(metricObj.parameters), modelSpec) self._validateModelSpec(json.loads(metricObj.parameters)) # Export again exportSpec = adapter.exportModel(metricId) checkExportSpec(exportSpec)
def testActivateModelClassifierEnabled(self): """ Test activateModel with classifier enabled in model spec. """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "modelParams": { "enableClassifier": True } } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.status]) self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA) # Add some data data = [ (0, datetime.datetime.utcnow() - datetime.timedelta(minutes=5)), (100, datetime.datetime.utcnow()) ] with self.engine.connect() as conn: repository.addMetricData(conn, metricId, data) # Activate model adapter.activateModel(metricId) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.status, schema.metric.c.model_params]) self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE)) self._assertClassifierStatusInModelParams(metricObj.model_params, classifierEnabled=True) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId) g_log.info("Waiting at least one model result") self.checkModelResultsSize(metricId, 1, atLeast=True)
def testActivateModelClassifierEnabled(self): """ Test activateModel with classifier enabled in model spec. """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "modelParams": { "enableClassifier": True } } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.status]) self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA) # Add some data data = [(0, datetime.datetime.utcnow() - datetime.timedelta(minutes=5)), (100, datetime.datetime.utcnow())] with self.engine.connect() as conn: repository.addMetricData(conn, metricId, data) # Activate model adapter.activateModel(metricId) with self.engine.connect() as conn: metricObj = repository.getMetric( conn, metricId, fields=[schema.metric.c.status, schema.metric.c.model_params]) self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE)) self._assertClassifierStatusInModelParams(metricObj.model_params, classifierEnabled=True) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId) g_log.info("Waiting at least one model result") self.checkModelResultsSize(metricId, 1, atLeast=True)
def testDeleteMetricByNameUnmonitored(self): """ Test deletion of unmonitored metric """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating grok custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) g_log.info("Deleteing unmonitored grok custom metric; name=%s", metricName) adapter.deleteMetricByName(metricName) g_log.info("Waiting for model to complete deletion") self.checkModelDeleted(metricId)
def testDeleteMetricByNameUnmonitored(self): """ Test deletion of unmonitored metric """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) g_log.info("Deleteing unmonitored htmengine custom metric; name=%s", metricName) adapter.deleteMetricByName(metricName) g_log.info("Waiting for model to complete deletion") self.checkModelDeleted(metricId)
def testCreateMetricThatAlreadyExists(self): """ Creating a custom metric with name that already exists should raise """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) with self.assertRaises(app_exceptions.MetricAlreadyExists) as cm: adapter.createMetric(metricName) self.assertEqual(cm.exception.uid, metricId)
def testMonitorMetricWithCompleteModelParams(self): """ Test monitorMetric with complete set of user-provided model parameters that activates a model """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) fileName = "custom_datasource_adapter_test_model_config.json" with self._openTestDataFile(fileName) as modelConfigFile: modelConfig = json.load(modelConfigFile) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "completeModelParams": { "modelConfig": modelConfig, "inferenceArgs": { "predictionSteps": [1], "predictedField": "bar", "inputPredictedField": "auto" }, "timestampFieldName": "foo", "valueFieldName": "bar" } } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric( conn, metricId, fields=[schema.metric.c.status, schema.metric.c.parameters]) self._validateModelSpec(json.loads(metricObj.parameters)) self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE)) self.assertEqual(json.loads(metricObj.parameters), modelSpec) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId)
def testMonitorMetricWithEnoughDataForStats(self): """ monitorMetric should create a model when there is enough data rows """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Add enough data to force activation of model data = [ (offset, datetime.datetime.utcnow() + datetime.timedelta(minutes=offset)) for offset in xrange( 0, scalar_metric_utils.MODEL_CREATION_RECORD_THRESHOLD * 5, 5) ] self.assertEqual(len(data), scalar_metric_utils.MODEL_CREATION_RECORD_THRESHOLD) with self.engine.connect() as conn: repository.addMetricData(conn, metricId, data) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.status]) self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE)) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId) g_log.info("Waiting at least one model result") self.checkModelResultsSize(metricId, 1, atLeast=True)
def testMonitorMetricWithMinResolution(self): """ Test monitorMetric with user-provided min/max and minResolution that activates a model. Make sure resolution doesn't drop below minResolution. """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "modelParams": { "min": 0, # optional "max": 1, # optional "minResolution": 0.5 # optional } } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.status, schema.metric.c.parameters]) self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE)) self.assertEqual(json.loads(metricObj.parameters), modelSpec) #print metricObj.parameters self._validateModelSpec(json.loads(metricObj.parameters)) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId) self.checkEncoderResolution(metricId, 0, 1, minResolution=0.5)
def testMonitorMetricWithCompleteModelParams(self): """ Test monitorMetric with complete set of user-provided model parameters that activates a model """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) fileName = "custom_datasource_adapter_test_model_config.json" with self._openTestDataFile(fileName) as modelConfigFile: modelConfig = json.load(modelConfigFile) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "completeModelParams": { "modelConfig": modelConfig, "inferenceArgs": {"predictionSteps": [1], "predictedField": "bar", "inputPredictedField": "auto"}, "timestampFieldName": "foo", "valueFieldName": "bar" } } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.status, schema.metric.c.parameters]) self._validateModelSpec(json.loads(metricObj.parameters)) self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE)) self.assertEqual(json.loads(metricObj.parameters), modelSpec) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId)
def testMonitorMetricClassifierEnabled(self): """ Test monitorMetric with request for enabled classifier in model params """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "modelParams": { "min": 0, # optional "max": 100, # optional "enableClassifier": True } } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[ schema.metric.c.status, schema.metric.c.parameters, schema.metric.c.model_params ]) self.assertEqual(metricObj.status, MetricStatus.CREATE_PENDING) self.assertEqual(json.loads(metricObj.parameters), modelSpec) self._assertClassifierStatusInModelParams(metricObj.model_params, classifierEnabled=True) self._validateModelSpec(json.loads(metricObj.parameters)) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId) self.checkEncoderResolution(metricId, 0, 100)
def testMonitorMetricClassifierEnabled(self): """ Test monitorMetric with request for enabled classifier in model params """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "modelParams": { "min": 0, # optional "max": 100, # optional "enableClassifier": True } } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.status, schema.metric.c.parameters, schema.metric.c.model_params]) self.assertEqual(metricObj.status, MetricStatus.CREATE_PENDING) self.assertEqual(json.loads(metricObj.parameters), modelSpec) self._assertClassifierStatusInModelParams(metricObj.model_params, classifierEnabled=True) self._validateModelSpec(json.loads(metricObj.parameters)) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId) self.checkEncoderResolution(metricId, 0, 100)
def testMonitorMetricWithMinResolution(self): """ Test monitorMetric with user-provided min/max and minResolution that activates a model. Make sure resolution doesn't drop below minResolution. """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "modelParams": { "min": 0, # optional "max": 1, # optional "minResolution": 0.5 # optional } } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric( conn, metricId, fields=[schema.metric.c.status, schema.metric.c.parameters]) self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE)) self.assertEqual(json.loads(metricObj.parameters), modelSpec) #print metricObj.parameters self._validateModelSpec(json.loads(metricObj.parameters)) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId) self.checkEncoderResolution(metricId, 0, 1, minResolution=0.5)
def testCreateMetric(self): """ Test creation of custom metric """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.name, schema.metric.c.datasource, schema.metric.c.status]) self.assertEqual(metricObj.name, metricName) self.assertEqual(metricObj.datasource, "custom") self.assertEqual(metricObj.status, MetricStatus.UNMONITORED)
def testCreateMetric(self): """ Test creation of custom metric """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[ schema.metric.c.name, schema.metric.c.datasource, schema.metric.c.status ]) self.assertEqual(metricObj.name, metricName) self.assertEqual(metricObj.datasource, "custom") self.assertEqual(metricObj.status, MetricStatus.UNMONITORED)
def testMonitorMetricModelParamsAndCompleteModelParams(self): """ Test monitorMetric() raises ValueError for mutually exclusive model params input options. """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() fileName = "custom_datasource_adapter_test_model_config.json" with self._openTestDataFile(fileName) as modelConfigFile: modelConfig = json.load(modelConfigFile) modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "completeModelParams": { "modelConfig": modelConfig, "inferenceArgs": { "predictionSteps": [1], "predictedField": "bar", "inputPredictedField": "auto" }, "timestampFieldName": "foo", "valueFieldName": "bar" }, "modelParams": { "min": 0, "max": 100 } } with self.assertRaises(ValueError) as excCtx: adapter.monitorMetric(modelSpec) excArgZero = excCtx.exception.args[0] initialMsg = excArgZero[0:len(scalar_metric_utils._MUTEX_MODEL_SPEC_MSG )] self.assertEqual(initialMsg, scalar_metric_utils._MUTEX_MODEL_SPEC_MSG)
def testDeleteMetricWithModel(self): """ Test monitorMetric with user-provided min/max that activates a model """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric: name=%s", metricName) metricId = adapter.createMetric(metricName) try: # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "modelParams": { "min": 0, # optional "max": 100 # optional } } adapter.monitorMetric(modelSpec) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId) g_log.info("Deleteing htmengine custom metric with active model: " "name=%s", metricName) adapter.deleteMetricByName(metricName) g_log.info("Waiting for model to complete deletion") self.checkModelDeleted(metricId) except: # pylint: disable=W0702 g_log.exception("Something went wrong") adapter.deleteMetricByName(metricName)
def testMonitorMetricModelParamsAndCompleteModelParams(self): """ Test monitorMetric() raises ValueError for mutually exclusive model params input options. """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() fileName = "custom_datasource_adapter_test_model_config.json" with self._openTestDataFile(fileName) as modelConfigFile: modelConfig = json.load(modelConfigFile) modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "completeModelParams": { "modelConfig": modelConfig, "inferenceArgs": {"predictionSteps": [1], "predictedField": "bar", "inputPredictedField": "auto"}, "timestampFieldName": "foo", "valueFieldName": "bar" }, "modelParams": { "min": 0, "max": 100 } } with self.assertRaises(ValueError) as excCtx: adapter.monitorMetric(modelSpec) excArgZero = excCtx.exception.args[0] initialMsg = excArgZero[0: len( scalar_metric_utils._MUTEX_MODEL_SPEC_MSG)] self.assertEqual(initialMsg, scalar_metric_utils._MUTEX_MODEL_SPEC_MSG)
def testMonitorMetricWithUserInfo(self): """Test monitorMetric that includes an explicit userInfo property in metricSpec. """ metricName = "test-" + uuid.uuid1().hex userInfo = {"symbol": "test-user-info"} adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName, "userInfo": userInfo } } adapter.monitorMetric(modelSpec) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[ schema.metric.c.parameters, schema.metric.c.status, schema.metric.c.server ]) self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA) self.assertEqual(json.loads(metricObj.parameters), modelSpec) self._validateModelSpec(json.loads(metricObj.parameters))
def testDeleteMetricWithModel(self): """ Test monitorMetric with user-provided min/max that activates a model """ metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric: name=%s", metricName) metricId = adapter.createMetric(metricName) try: # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "modelParams": { "min": 0, # optional "max": 100 # optional } } adapter.monitorMetric(modelSpec) g_log.info("Waiting for model to become active") self.checkModelIsActive(metricId) g_log.info( "Deleting htmengine custom metric with active model: " "name=%s", metricName) adapter.deleteMetricByName(metricName) g_log.info("Waiting for model to complete deletion") self.checkModelDeleted(metricId) except: # pylint: disable=W0702 g_log.exception("Something went wrong") adapter.deleteMetricByName(metricName)
def testExportImport(self): metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Add some data # NOTE: we discard the fractional part because it gets eliminated # in the database, and we will want to compare against retrieved # items later. now = datetime.datetime.utcnow().replace(microsecond=0) data = [ (0, now - datetime.timedelta(minutes=5)), (100, now) ] with self.engine.connect() as conn: repository.addMetricData(conn, metricId, data) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, } adapter.monitorMetric(modelSpec) def checkExportSpec(exportSpec): self.assertEqual(exportSpec["datasource"], modelSpec["datasource"]) self.assertEqual(exportSpec["metricSpec"], modelSpec["metricSpec"]) self.assertSequenceEqual(exportSpec["data"], data) # Export exportSpec = adapter.exportModel(metricId) checkExportSpec(exportSpec) # Delete metric adapter.deleteMetricByName(metricName) self.checkModelDeleted(metricId) # Import metricId = adapter.importModel( htmengine.utils.jsonDecode(htmengine.utils.jsonEncode(exportSpec))) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.parameters, schema.metric.c.status]) self.assertEqual(metricObj.status, MetricStatus.PENDING_DATA) self.assertEqual(json.loads(metricObj.parameters), modelSpec) self._validateModelSpec(json.loads(metricObj.parameters)) # Export again exportSpec = adapter.exportModel(metricId) checkExportSpec(exportSpec)
def testExportImportCompleteModelParams(self): metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Add some data # NOTE: we discard the fractional part because it gets eliminated # in the database, and we will want to compare against retrieved # items later. now = datetime.datetime.utcnow().replace(microsecond=0) data = [ (0, now - datetime.timedelta(minutes=5)), (100, now) ] with self.engine.connect() as conn: repository.addMetricData(conn, metricId, data) fileName = "custom_datasource_adapter_test_model_config.json" with self._openTestDataFile(fileName) as modelConfigFile: modelConfig = json.load(modelConfigFile) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "completeModelParams": { "modelConfig": modelConfig, "inferenceArgs": {"predictionSteps": [1], "predictedField": "bar", "inputPredictedField": "auto"}, "timestampFieldName": "foo", "valueFieldName": "bar" } } adapter.monitorMetric(modelSpec) def checkExportSpec(exportSpec): self.assertEqual(exportSpec["datasource"], modelSpec["datasource"]) self.assertEqual(exportSpec["metricSpec"], modelSpec["metricSpec"]) self.assertSequenceEqual(exportSpec["data"], data) # Export exportSpec = adapter.exportModel(metricId) checkExportSpec(exportSpec) # Delete metric adapter.deleteMetricByName(metricName) self.checkModelDeleted(metricId) # Import metricId = adapter.importModel( htmengine.utils.jsonDecode(htmengine.utils.jsonEncode(exportSpec))) with self.engine.connect() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.parameters, schema.metric.c.status]) self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE)) self.assertEqual(json.loads(metricObj.parameters), modelSpec) self._validateModelSpec(json.loads(metricObj.parameters)) # Export again exportSpec = adapter.exportModel(metricId) checkExportSpec(exportSpec)
def testExportImportCompleteModelParams(self): metricName = "test-" + uuid.uuid1().hex adapter = datasource_adapter_factory.createCustomDatasourceAdapter() g_log.info("Creating htmengine custom metric; name=%s", metricName) metricId = adapter.createMetric(metricName) self.addCleanup(adapter.deleteMetricByName, metricName) # Add some data # NOTE: we discard the fractional part because it gets eliminated # in the database, and we will want to compare against retrieved # items later. now = datetime.datetime.utcnow().replace(microsecond=0) data = [(0, now - datetime.timedelta(minutes=5)), (100, now)] with self.engine.connect() as conn: repository.addMetricData(conn, metricId, data) fileName = "custom_datasource_adapter_test_model_config.json" with self._openTestDataFile(fileName) as modelConfigFile: modelConfig = json.load(modelConfigFile) # Turn on monitoring modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "completeModelParams": { "modelConfig": modelConfig, "inferenceArgs": { "predictionSteps": [1], "predictedField": "bar", "inputPredictedField": "auto" }, "timestampFieldName": "foo", "valueFieldName": "bar" } } adapter.monitorMetric(modelSpec) def checkExportSpec(exportSpec): self.assertEqual(exportSpec["datasource"], modelSpec["datasource"]) self.assertEqual(exportSpec["metricSpec"], modelSpec["metricSpec"]) self.assertSequenceEqual(exportSpec["data"], data) # Export exportSpec = adapter.exportModel(metricId) checkExportSpec(exportSpec) # Delete metric adapter.deleteMetricByName(metricName) self.checkModelDeleted(metricId) # Import metricId = adapter.importModel( htmengine.utils.jsonDecode(htmengine.utils.jsonEncode(exportSpec))) with self.engine.connect() as conn: metricObj = repository.getMetric( conn, metricId, fields=[schema.metric.c.parameters, schema.metric.c.status]) self.assertIn(metricObj.status, (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE)) self.assertEqual(json.loads(metricObj.parameters), modelSpec) self._validateModelSpec(json.loads(metricObj.parameters)) # Export again exportSpec = adapter.exportModel(metricId) checkExportSpec(exportSpec)