def activateModel(self, metricId): """ Start a model that is PENDING_DATA, creating the OPF/CLA model NOTE: used by MetricStreamer when model is in PENDING_DATA state and sufficient data samples are available to get statistics and complete model creation. :param metricId: unique identifier of the metric row :raises grok.app.exceptions.ObjectNotFoundError: if metric with the referenced metric uid doesn't exist :raises grok.app.exceptions.MetricStatisticsNotReadyError: """ with self.connectionFactory() as conn: # TODO: This function is identical to custom metric activateModel() metricObj = repository.getMetric( conn, metricId, fields=[schema.metric.c.datasource, schema.metric.c.parameters] ) if metricObj.datasource != self._DATASOURCE: raise TypeError("activateModel: not a cloudwatch metric=%r" % (metricObj,)) if metricObj.parameters: parameters = htmengine.utils.jsonDecode(metricObj.parameters) else: parameters = {} stats = self._getMetricStatistics(parameters["metricSpec"]) self._log.info("activateModel: metric=%s, stats=%r", metricId, stats) swarmParams = scalar_metric_utils.generateSwarmParams(stats) scalar_metric_utils.startModel(metricId, swarmParams=swarmParams, logger=self._log)
def activateModel(self, metricId): """ Start a model that is PENDING_DATA, creating the OPF/CLA model NOTE: used by MetricStreamer when model is in PENDING_DATA state and sufficient data samples are available to get statistics and complete model creation. :param metricId: unique identifier of the metric row :raises htmengine.exceptions.ObjectNotFoundError: if metric with the referenced metric uid doesn't exist :raises htmengine.exceptions.MetricStatisticsNotReadyError: """ # Load the existing metric with self.connectionFactory() as conn: metricObj = repository.getMetric( conn, metricId, fields=[schema.metric.c.datasource]) if metricObj.datasource != self._DATASOURCE: raise TypeError( "activateModel: not an HTM metric=%s; datasource=%s" % (metricId, metricObj.datasource)) stats = self._getMetricStatistics(metricId) swarmParams = scalar_metric_utils.generateSwarmParams(stats) scalar_metric_utils.startModel(metricId, swarmParams=swarmParams, logger=self._log)
def activateModel(self, metricId): """ Start a model that is PENDING_DATA, creating the OPF/CLA model NOTE: used by MetricStreamer when model is in PENDING_DATA state and sufficient data samples are available to get statistics and complete model creation. :param metricId: unique identifier of the metric row :raises htmengine.exceptions.ObjectNotFoundError: if metric with the referenced metric uid doesn't exist :raises htmengine.exceptions.MetricStatisticsNotReadyError: """ # Load the existing metric with self.connectionFactory() as conn: metricObj = repository.getMetric(conn, metricId, fields=[schema.metric.c.datasource]) if metricObj.datasource != self._DATASOURCE: raise TypeError( "activateModel: not an HTM metric=%s; datasource=%s" % (metricId, metricObj.datasource)) stats = self._getMetricStatistics(metricId) swarmParams = scalar_metric_utils.generateSwarmParams(stats) scalar_metric_utils.startModel(metricId, swarmParams=swarmParams, logger=self._log)
def activateModel(self, metricId): """ Start a model that is PENDING_DATA, creating the OPF/CLA model NOTE: used by MetricStreamer when model is in PENDING_DATA state and sufficient data samples are available to get statistics and complete model creation. :param metricId: unique identifier of the metric row :raises htm.it.app.exceptions.ObjectNotFoundError: if metric with the referenced metric uid doesn't exist :raises htm.it.app.exceptions.MetricStatisticsNotReadyError: """ with self.connectionFactory() as conn: # TODO: This function is identical to custom metric activateModel() metricObj = repository.getMetric(conn, metricId, fields=[ schema.metric.c.datasource, schema.metric.c.parameters ]) if metricObj.datasource != self._DATASOURCE: raise TypeError("activateModel: not a cloudwatch metric=%r" % (metricObj, )) if metricObj.parameters: parameters = htmengine.utils.jsonDecode(metricObj.parameters) else: parameters = {} stats = self._getMetricStatistics(parameters["metricSpec"]) self._log.info("activateModel: metric=%s, stats=%r", metricId, stats) swarmParams = scalar_metric_utils.generateSwarmParams(stats) scalar_metric_utils.startModel(metricId, swarmParams=swarmParams, logger=self._log)
def monitorMetric(self, modelSpec): """ Start monitoring a metric; create a "cloudwatch model" DAO object for the given model specification. :param modelSpec: model specification for Cloudwatch-based model :type modelSpec: dict :: { "datasource": "cloudwatch", "metricSpec": { "region": "us-west-2", "namespace": "AWS/EC2", "metric": "CPUUtilization", "dimensions": { "InstanceId": "i-12d67826" } }, # optional "modelParams": { "min": 0, # optional "max": 100 # optional } } :returns: datasource-specific unique model identifier :raises grok.app.exceptions.ObjectNotFoundError: if referenced metric doesn't exist :raises grok.app.exceptions.MetricNotSupportedError: if requested metric isn't supported :raises grok.app.exceptions.MetricAlreadyMonitored: if the metric is already being monitored """ metricSpec = modelSpec["metricSpec"] metricAdapter = AWSResourceAdapterBase.createMetricAdapter(metricSpec) # NOTE: getResourceName may be slow (AWS query) # TODO MER-3430: would be handy to use caching to speed things up a lot resourceName = metricAdapter.getResourceName() canonicalResourceName = self.getInstanceNameForModelSpec(modelSpec) resourceLocation = metricAdapter.getResourceLocation() metricName = metricAdapter.getMetricName() metricPeriod = metricAdapter.getMetricPeriod() metricDescription = metricAdapter.getMetricSummary() nameColumnValue = self._composeMetricNameColumnValue( metricName=metricName, metricNamespace=metricSpec["namespace"] ) # Determine if the model should be started. This will happen if the # nativeMetric input includes both "min" and "max" or we have default values # for both "min" and "max" defaultMin = metricAdapter.getMetricDefaultMin() defaultMax = metricAdapter.getMetricDefaultMax() if defaultMin is None or defaultMax is None: defaultMin = defaultMax = None # Get user-provided min/max, if any modelParams = modelSpec.get("modelParams", dict()) explicitMin = modelParams.get("min") explicitMax = modelParams.get("max") if (explicitMin is None) != (explicitMax is None): raise ValueError("min and max params must both be None or non-None; modelSpec=%r" % (modelSpec,)) minVal = explicitMin if explicitMin is not None else defaultMin maxVal = explicitMax if explicitMax is not None else defaultMax stats = {"min": minVal, "max": maxVal} swarmParams = scalar_metric_utils.generateSwarmParams(stats) # Perform the start-monitoring operation atomically/reliably @repository.retryOnTransientErrors def startMonitoringWithRetries(): """ :returns: metricId """ with self.connectionFactory() as conn: with conn.begin(): repository.lockOperationExclusive(conn, repository.OperationLock.METRICS) # Check if the metric is already monitored matchingMetrics = repository.getCloudwatchMetricsForNameAndServer( conn, nameColumnValue, canonicalResourceName, fields=[schema.metric.c.uid, schema.metric.c.parameters], ) for m in matchingMetrics: parameters = htmengine.utils.jsonDecode(m.parameters) if parameters["metricSpec"]["dimensions"] == metricSpec["dimensions"]: msg = ( "monitorMetric: Cloudwatch modelId=%s is already " "monitoring metric=%s on resource=%s; model=%r" % (m.uid, nameColumnValue, canonicalResourceName, m) ) self._log.warning(msg) raise grok.app.exceptions.MetricAlreadyMonitored(msg, uid=m.uid) # Add a metric row for the requested metric metricDict = repository.addMetric( conn, name=nameColumnValue, description=metricDescription, server=canonicalResourceName, location=resourceLocation, poll_interval=metricPeriod, status=MetricStatus.UNMONITORED, datasource=self._DATASOURCE, parameters=htmengine.utils.jsonEncode(modelSpec), tag_name=resourceName, ) metricId = metricDict["uid"] self._log.info("monitorMetric: metric=%s, stats=%r", metricId, stats) # Start monitoring scalar_metric_utils.startMonitoring( conn=conn, metricId=metricId, swarmParams=swarmParams, logger=self._log ) return metricId return startMonitoringWithRetries()
def monitorMetric(self, modelSpec): """ Start monitoring a metric; create a model linked to an existing Autostack :param modelSpec: model specification for an Autostack-based model :type modelSpec: dict :: { "datasource": "autostack", "metricSpec": { # TODO [MER-3533]: This should be autostack name instead "autostackId": "a858c6990a444cd8a07466ec7f3cae16", "slaveDatasource": "cloudwatch", "slaveMetric": { # specific to slaveDatasource "namespace": "AWS/EC2", "metric": "CPUUtilization" }, "period": 300 # aggregation period; seconds }, "modelParams": { # optional; specific to slave metric "min": 0, # optional "max": 100 # optional } } :returns: datasource-specific unique model identifier :raises YOMP.app.exceptions.ObjectNotFoundError: if referenced autostack doesn't exist :raises YOMP.app.exceptions.MetricNotSupportedError: if requested metric isn't supported :raises YOMP.app.exceptions.MetricAlreadyMonitored: if the metric is already being monitored """ metricSpec = modelSpec["metricSpec"] autostackId = metricSpec["autostackId"] with self.connectionFactory() as conn: autostack = repository.getAutostack(conn, autostackId) slaveDatasource = metricSpec["slaveDatasource"] slaveMetric = metricSpec["slaveMetric"] canonicalResourceName = self.getInstanceNameForModelSpec(modelSpec) metricAdapter = AutostackMetricAdapterBase.getMetricAdapter( slaveDatasource) nameColumnValue = metricAdapter.getMetricName(slaveMetric) metricDescription = metricAdapter.getMetricDescription( slaveMetric, autostack) queryParams = metricAdapter.getQueryParams(nameColumnValue) defaultMin = queryParams["min"] defaultMax = queryParams["max"] defaultPeriod = queryParams["period"] modelParams = modelSpec.get("modelParams", dict()) explicitMin = modelParams.get("min") explicitMax = modelParams.get("max") explicitPeriod = metricSpec.get("period") if (explicitMin is None) != (explicitMax is None): raise ValueError( "min and max params must both be None or non-None; modelSpec=%r" % (modelSpec, )) minVal = explicitMin if explicitMin is not None else defaultMin maxVal = explicitMax if explicitMax is not None else defaultMax period = explicitPeriod if explicitPeriod is not None else defaultPeriod stats = {"min": minVal, "max": maxVal} if minVal is None or maxVal is None: raise ValueError("Expected min and max to be set") swarmParams = scalar_metric_utils.generateSwarmParams(stats) @repository.retryOnTransientErrors def startMonitoringWithRetries(): """ :returns: metricId """ with self.connectionFactory() as conn: with conn.begin(): repository.lockOperationExclusive( conn, repository.OperationLock.METRICS) # Check if the metric is already monitored matchingMetrics = repository.getAutostackMetricsWithMetricName( conn, autostackId, nameColumnValue, fields=[schema.metric.c.uid]) matchingMetric = next(iter(matchingMetrics), None) if matchingMetric: msg = ( "monitorMetric: Autostack modelId=%s is already " "monitoring metric=%s on resource=%s; model=%r" % (matchingMetric.uid, nameColumnValue, canonicalResourceName, matchingMetric)) self._log.warning(msg) raise YOMP.app.exceptions.MetricAlreadyMonitored( msg, uid=matchingMetric.uid) # Add a metric row for the requested metric metricDict = repository.addMetric( conn, datasource=self._DATASOURCE, name=nameColumnValue, description=metricDescription, server=canonicalResourceName, location=autostack.region, tag_name=autostack.name, parameters=htmengine.utils.jsonEncode(modelSpec), poll_interval=period, status=MetricStatus.UNMONITORED) metricId = metricDict["uid"] repository.addMetricToAutostack(conn, autostackId, metricId) # Start monitoring scalar_metric_utils.startMonitoring( conn=conn, metricId=metricId, swarmParams=swarmParams, logger=self._log) self._log.info("monitorMetric: monitoring metric=%s, stats=%r", metricId, stats) return metricId return startMonitoringWithRetries()
def monitorMetric(self, modelSpec): """ Start monitoring a metric; create a "cloudwatch model" DAO object for the given model specification. :param modelSpec: model specification for Cloudwatch-based model :type modelSpec: dict :: { "datasource": "cloudwatch", "metricSpec": { "region": "us-west-2", "namespace": "AWS/EC2", "metric": "CPUUtilization", "dimensions": { "InstanceId": "i-12d67826" } }, # optional "modelParams": { "min": 0, # optional "max": 100 # optional } } :returns: datasource-specific unique model identifier :raises grok.app.exceptions.ObjectNotFoundError: if referenced metric doesn't exist :raises grok.app.exceptions.MetricNotSupportedError: if requested metric isn't supported :raises grok.app.exceptions.MetricAlreadyMonitored: if the metric is already being monitored """ metricSpec = modelSpec["metricSpec"] metricAdapter = AWSResourceAdapterBase.createMetricAdapter(metricSpec) # NOTE: getResourceName may be slow (AWS query) # TODO MER-3430: would be handy to use caching to speed things up a lot resourceName = metricAdapter.getResourceName() canonicalResourceName = self.getInstanceNameForModelSpec(modelSpec) resourceLocation = metricAdapter.getResourceLocation() metricName = metricAdapter.getMetricName() metricPeriod = metricAdapter.getMetricPeriod() metricDescription = metricAdapter.getMetricSummary() nameColumnValue = self._composeMetricNameColumnValue( metricName=metricName, metricNamespace=metricSpec["namespace"]) # Determine if the model should be started. This will happen if the # nativeMetric input includes both "min" and "max" or we have default values # for both "min" and "max" defaultMin = metricAdapter.getMetricDefaultMin() defaultMax = metricAdapter.getMetricDefaultMax() if defaultMin is None or defaultMax is None: defaultMin = defaultMax = None # Get user-provided min/max, if any modelParams = modelSpec.get("modelParams", dict()) explicitMin = modelParams.get("min") explicitMax = modelParams.get("max") if (explicitMin is None) != (explicitMax is None): raise ValueError( "min and max params must both be None or non-None; modelSpec=%r" % (modelSpec,)) minVal = explicitMin if explicitMin is not None else defaultMin maxVal = explicitMax if explicitMax is not None else defaultMax stats = {"min": minVal, "max": maxVal} swarmParams = scalar_metric_utils.generateSwarmParams(stats) # Perform the start-monitoring operation atomically/reliably @repository.retryOnTransientErrors def startMonitoringWithRetries(): """ :returns: metricId """ with self.connectionFactory() as conn: with conn.begin(): repository.lockOperationExclusive(conn, repository.OperationLock.METRICS) # Check if the metric is already monitored matchingMetrics = repository.getCloudwatchMetricsForNameAndServer( conn, nameColumnValue, canonicalResourceName, fields=[schema.metric.c.uid, schema.metric.c.parameters]) for m in matchingMetrics: parameters = htmengine.utils.jsonDecode(m.parameters) if (parameters["metricSpec"]["dimensions"] == metricSpec["dimensions"]): msg = ("monitorMetric: Cloudwatch modelId=%s is already " "monitoring metric=%s on resource=%s; model=%r" % (m.uid, nameColumnValue, canonicalResourceName, m)) self._log.warning(msg) raise grok.app.exceptions.MetricAlreadyMonitored(msg, uid=m.uid) # Add a metric row for the requested metric metricDict = repository.addMetric( conn, name=nameColumnValue, description=metricDescription, server=canonicalResourceName, location=resourceLocation, poll_interval=metricPeriod, status=MetricStatus.UNMONITORED, datasource=self._DATASOURCE, parameters=htmengine.utils.jsonEncode(modelSpec), tag_name=resourceName) metricId = metricDict["uid"] self._log.info("monitorMetric: metric=%s, stats=%r", metricId, stats) # Start monitoring scalar_metric_utils.startMonitoring( conn=conn, metricId=metricId, swarmParams=swarmParams, logger=self._log) return metricId return startMonitoringWithRetries()
def monitorMetric(self, modelSpec): """ Start monitoring a metric; create a model linked to an existing Autostack :param modelSpec: model specification for an Autostack-based model :type modelSpec: dict :: { "datasource": "autostack", "metricSpec": { # TODO [MER-3533]: This should be autostack name instead "autostackId": "a858c6990a444cd8a07466ec7f3cae16", "slaveDatasource": "cloudwatch", "slaveMetric": { # specific to slaveDatasource "namespace": "AWS/EC2", "metric": "CPUUtilization" }, "period": 300 # aggregation period; seconds }, "modelParams": { # optional; specific to slave metric "min": 0, # optional "max": 100 # optional } } :returns: datasource-specific unique model identifier :raises YOMP.app.exceptions.ObjectNotFoundError: if referenced autostack doesn't exist :raises YOMP.app.exceptions.MetricNotSupportedError: if requested metric isn't supported :raises YOMP.app.exceptions.MetricAlreadyMonitored: if the metric is already being monitored """ metricSpec = modelSpec["metricSpec"] autostackId = metricSpec["autostackId"] with self.connectionFactory() as conn: autostack = repository.getAutostack(conn, autostackId) slaveDatasource = metricSpec["slaveDatasource"] slaveMetric = metricSpec["slaveMetric"] canonicalResourceName = self.getInstanceNameForModelSpec(modelSpec) metricAdapter = AutostackMetricAdapterBase.getMetricAdapter(slaveDatasource) nameColumnValue = metricAdapter.getMetricName(slaveMetric) metricDescription = metricAdapter.getMetricDescription(slaveMetric, autostack) queryParams = metricAdapter.getQueryParams(nameColumnValue) defaultMin = queryParams["min"] defaultMax = queryParams["max"] defaultPeriod = queryParams["period"] modelParams = modelSpec.get("modelParams", dict()) explicitMin = modelParams.get("min") explicitMax = modelParams.get("max") explicitPeriod = metricSpec.get("period") if (explicitMin is None) != (explicitMax is None): raise ValueError( "min and max params must both be None or non-None; modelSpec=%r" % (modelSpec,)) minVal = explicitMin if explicitMin is not None else defaultMin maxVal = explicitMax if explicitMax is not None else defaultMax period = explicitPeriod if explicitPeriod is not None else defaultPeriod stats = {"min": minVal, "max": maxVal} if minVal is None or maxVal is None: raise ValueError("Expected min and max to be set") swarmParams = scalar_metric_utils.generateSwarmParams(stats) @repository.retryOnTransientErrors def startMonitoringWithRetries(): """ :returns: metricId """ with self.connectionFactory() as conn: with conn.begin(): repository.lockOperationExclusive(conn, repository.OperationLock.METRICS) # Check if the metric is already monitored matchingMetrics = repository.getAutostackMetricsWithMetricName( conn, autostackId, nameColumnValue, fields=[schema.metric.c.uid]) matchingMetric = next(iter(matchingMetrics), None) if matchingMetric: msg = ("monitorMetric: Autostack modelId=%s is already " "monitoring metric=%s on resource=%s; model=%r" % (matchingMetric.uid, nameColumnValue, canonicalResourceName, matchingMetric)) self._log.warning(msg) raise YOMP.app.exceptions.MetricAlreadyMonitored( msg, uid=matchingMetric.uid) # Add a metric row for the requested metric metricDict = repository.addMetric( conn, datasource=self._DATASOURCE, name=nameColumnValue, description=metricDescription, server=canonicalResourceName, location=autostack.region, tag_name=autostack.name, parameters=htmengine.utils.jsonEncode(modelSpec), poll_interval=period, status=MetricStatus.UNMONITORED) metricId = metricDict["uid"] repository.addMetricToAutostack(conn, autostackId, metricId) # Start monitoring scalar_metric_utils.startMonitoring( conn=conn, metricId=metricId, swarmParams=swarmParams, logger=self._log) self._log.info("monitorMetric: monitoring metric=%s, stats=%r", metricId, stats) return metricId return startMonitoringWithRetries()
def monitorMetric(self, modelSpec): """ Start monitoring a metric; perform model creation logic specific to custom metrics. Start the model if possible: this will happen if modelParams includes both "min" and "max" or there is enough data to estimate them. Alternatively, users may pass in a full set of model parameters. :param modelSpec: model specification for HTM model; per ``model_spec_schema.json`` with the ``metricSpec`` property per ``custom_metric_spec_schema.json`` :type modelSpec: dict :: 1st variant: `uid` is the unique id of an existing metric; # TODO: it would be preferable to punt on this variant, and refer # to custom metric by name in modelSpec for consistency with # import/export. Web GUI uses metric name; some tests use this variant, # though. { "datasource": "custom", "metricSpec": { "uid": "4a833e2294494b4fbc5004e03bad45b6", "unit": "Count", # optional "resource": "prod.web1", # optional "userInfo": {"symbol": "<TICKER_SYMBOL>"} # optional }, # Optional model params "modelParams": { "min": min-value, # optional "max": max-value # optional } } :: 2nd variant: `metric` is the unique name of the metric; a new custom metric row will be created with this name, if it doesn't exit { "datasource": "custom", "metricSpec": { "metric": "prod.web.14.memory", "unit": "Count", # optional "resource": "prod.web1", # optional "userInfo": {"symbol": "<TICKER_SYMBOL>"} # optional }, # Optional model params "modelParams": { "min": min-value, # optional "max": max-value # optional } } :returns: datasource-specific unique model identifier :raises ValueError: if finds something invalid in arg :raises TypeError: if metric with the referenced uid is not a custom metric :raises htmengine.exceptions.ObjectNotFoundError: if referenced metric doesn't exist :raises htmengine.exceptions.MetricNotSupportedError: if requested metric isn't supported :raises htmengine.exceptions.MetricAlreadyMonitored: if the metric is already being monitored """ metricSpec = modelSpec["metricSpec"] with self.connectionFactory() as conn: if "uid" in metricSpec: # Via metric ID metricId = metricSpec["uid"] # Convert modelSpec to canonical form modelSpec = copy.deepcopy(modelSpec) modelSpec["metricSpec"].pop("uid") modelSpec["metricSpec"]["metric"] = ( repository.retryOnTransientErrors(repository.getMetric)( conn, metricId).name) elif "metric" in metricSpec: # Via metric name try: # Create the metric, if needed metricId = repository.retryOnTransientErrors(self._createMetric)( conn, metricSpec["metric"]) except app_exceptions.MetricAlreadyExists as e: # It already existed metricId = e.uid else: raise ValueError( "Neither uid nor metric name present in metricSpec; modelSpec=%r" % (modelSpec,)) if "completeModelParams" in modelSpec: # Attempt to build swarm params from complete model params if present swarmParams = ( scalar_metric_utils.generateSwarmParamsFromCompleteModelParams( modelSpec)) else: # Generate swarm params from specified metric min and max or estimate them modelParams = modelSpec.get("modelParams", dict()) minVal = modelParams.get("min") maxVal = modelParams.get("max") minResolution = modelParams.get("minResolution") enableClassifier = modelParams.get("enableClassifier", False) if (minVal is None) != (maxVal is None): raise ValueError( "min and max params must both be None or non-None; metric=%s; " "modelSpec=%r" % (metricId, modelSpec,)) # Start monitoring if minVal is None or maxVal is None: minVal = maxVal = None with self.connectionFactory() as conn: numDataRows = repository.retryOnTransientErrors( repository.getMetricDataCount)(conn, metricId) if numDataRows >= scalar_metric_utils.MODEL_CREATION_RECORD_THRESHOLD: try: stats = self._getMetricStatistics(metricId) self._log.info("monitorMetric: trigger numDataRows=%d, stats=%s", numDataRows, stats) minVal = stats["min"] maxVal = stats["max"] except app_exceptions.MetricStatisticsNotReadyError: pass stats = {"min": minVal, "max": maxVal, "minResolution": minResolution} self._log.debug("monitorMetric: metric=%s, stats=%r", metricId, stats) swarmParams = scalar_metric_utils.generateSwarmParams(stats, enableClassifier) self._startMonitoringWithRetries(metricId, modelSpec, swarmParams) return metricId
def monitorMetric(self, modelSpec): """ Start monitoring a metric; perform model creation logic specific to custom metrics. Start the model if possible: this will happen if modelParams includes both "min" and "max" or there is enough data to estimate them. :param modelSpec: model specification for HTM model; per ``model_spec_schema.json`` with the ``metricSpec`` property per ``custom_metric_spec_schema.json`` :type modelSpec: dict :: 1st variant: `uid` is the unique id of an existing metric; # TODO: it would be preferable to punt on this variant, and refer # to custom metric by name in modelSpec for consistency with # import/export. Web GUI uses metric name; some tests use this variant, # though. { "datasource": "custom", "metricSpec": { "uid": "4a833e2294494b4fbc5004e03bad45b6", "unit": "Count", # optional "resource": "prod.web1", # optional "userInfo": {"symbol": "<TICKER_SYMBOL>"} # optional }, # Optional model params "modelParams": { "min": min-value, # optional "max": max-value # optional } } :: 2nd variant: `metric` is the unique name of the metric; a new custom metric row will be created with this name, if it doesn't exit { "datasource": "custom", "metricSpec": { "metric": "prod.web.14.memory", "unit": "Count", # optional "resource": "prod.web1", # optional "userInfo": {"symbol": "<TICKER_SYMBOL>"} # optional }, # Optional model params "modelParams": { "min": min-value, # optional "max": max-value # optional } } :returns: datasource-specific unique model identifier :raises ValueError: if finds something invalid in arg :raises TypeError: if metric with the referenced uid is not a custom metric :raises htmengine.exceptions.ObjectNotFoundError: if referenced metric doesn't exist :raises htmengine.exceptions.MetricNotSupportedError: if requested metric isn't supported :raises htmengine.exceptions.MetricAlreadyMonitored: if the metric is already being monitored """ metricSpec = modelSpec["metricSpec"] with self.connectionFactory() as conn: if "uid" in metricSpec: # Via metric ID metricId = metricSpec["uid"] # Convert modelSpec to canonical form modelSpec = copy.deepcopy(modelSpec) modelSpec["metricSpec"].pop("uid") modelSpec["metricSpec"]["metric"] = ( repository.retryOnTransientErrors(repository.getMetric)( conn, metricId).name) elif "metric" in metricSpec: # Via metric name try: # Crete the metric, if needed metricId = repository.retryOnTransientErrors( self._createMetric)(conn, metricSpec["metric"]) except app_exceptions.MetricAlreadyExists as e: # It already existed metricId = e.uid else: raise ValueError( "Neither uid nor metric name present in metricSpec; modelSpec=%r" % (modelSpec, )) modelParams = modelSpec.get("modelParams", dict()) minVal = modelParams.get("min") maxVal = modelParams.get("max") minResolution = modelParams.get("minResolution") if (minVal is None) != (maxVal is None): raise ValueError( "min and max params must both be None or non-None; metric=%s; " "modelSpec=%r" % ( metricId, modelSpec, )) # Start monitoring if minVal is None or maxVal is None: minVal = maxVal = None with self.connectionFactory() as conn: numDataRows = repository.retryOnTransientErrors( repository.getMetricDataCount)(conn, metricId) if numDataRows >= scalar_metric_utils.MODEL_CREATION_RECORD_THRESHOLD: try: stats = self._getMetricStatistics(metricId) self._log.info( "monitorMetric: trigger numDataRows=%d, stats=%s", numDataRows, stats) minVal = stats["min"] maxVal = stats["max"] except app_exceptions.MetricStatisticsNotReadyError: pass stats = {"min": minVal, "max": maxVal, "minResolution": minResolution} self._log.debug("monitorMetric: metric=%s, stats=%r", metricId, stats) swarmParams = scalar_metric_utils.generateSwarmParams(stats) self._startMonitoringWithRetries(metricId, modelSpec, swarmParams) return metricId