def createAutostackDatasourceAdapter(): """ Convenience function for creating a datasource adapter for Grok Autostack Metrics :returns: datasource adapter for Grok Autostack Metrics :rtype: grok.app.adapters.datasource.cloudwatch._AutostackDatasourceAdapter """ return createDatasourceAdapter("autostack")
def createCloudwatchDatasourceAdapter(): """ Convenience function for creating a datasource adapter for Cloudwatch Metrics :returns: datasource adapter for Cloudwatch Metrics :rtype: grok.app.adapters.datasource.cloudwatch._CloudwatchDatasourceAdapter """ return createDatasourceAdapter("cloudwatch")
def createAutostackDatasourceAdapter(): """ Convenience function for creating a datasource adapter for HTM-IT Autostack Metrics :returns: datasource adapter for HTM-IT Autostack Metrics :rtype: htm.it.app.adapters.datasource.cloudwatch._AutostackDatasourceAdapter """ return createDatasourceAdapter("autostack")
def createCloudwatchDatasourceAdapter(): """ Convenience function for creating a datasource adapter for Cloudwatch Metrics :returns: datasource adapter for Cloudwatch Metrics :rtype: htm.it.app.adapters.datasource.cloudwatch._CloudwatchDatasourceAdapter """ return createDatasourceAdapter("cloudwatch")
def deleteModel(metricId): try: with web.ctx.connFactory() as conn: metricRow = repository.getMetric(conn, metricId) except app_exceptions.ObjectNotFoundError: raise web.notfound("ObjectNotFoundError Metric not found: Metric ID: %s" % metricId) log.debug("Deleting model for %s metric=%s", metricRow.datasource, metricId) # NOTE: this is the new way using datasource adapters try: createDatasourceAdapter(metricRow.datasource).unmonitorMetric(metricId) except app_exceptions.ObjectNotFoundError: raise web.notfound( "ObjectNotFoundError Metric not found: Metric ID: %s" % (metricId,)) return utils.jsonEncode({'result': 'success'})
def DELETE(self, metricName): adapter = createDatasourceAdapter("custom") try: adapter.deleteMetricByName(metricName) except app_exceptions.ObjectNotFoundError: raise web.notfound("Metric not found. Metric name=%s" % (metricName,)) self.addStandardHeaders() return json.dumps({"result": "success"})
def DELETE(self, metricName): adapter = createDatasourceAdapter("custom") try: adapter.deleteMetricByName(metricName) except app_exceptions.ObjectNotFoundError: raise web.notfound("Metric not found. Metric name=%s" % (metricName,)) self.addStandardHeaders() return json.dumps({'result': 'success'})
def _createModel(self, nativeMetric): adapter = createDatasourceAdapter("custom") try: metricId = adapter.monitorMetric(nativeMetric) except MetricAlreadyMonitored as e: metricId = e.uid engine = repository.engineFactory(config=self.config) with engine.begin() as conn: return repository.getMetric(conn, metricId)
def testExport(self): """Tests exporting custom metrics.""" metricName = "testBatchSend.%i" % int(time.time()) LOGGER.info("Running test with metric name: %s", metricName) self.addCleanup(self._deleteMetric, metricName) # Get the timestamp for 14 days ago (the limit for exporting data) baseTimestamp = int(time.time()) - (60 * 60 * 24 * 14) ts1 = baseTimestamp - (60 * 5) dt1 = datetime.datetime.utcfromtimestamp(ts1).strftime("%Y-%m-%d %H:%M:%S") ts2 = baseTimestamp + (60 * 5) dt2 = datetime.datetime.utcfromtimestamp(ts2).strftime("%Y-%m-%d %H:%M:%S") ts3 = baseTimestamp + (60 * 10) dt3 = datetime.datetime.utcfromtimestamp(ts3).strftime("%Y-%m-%d %H:%M:%S") # Add custom metric data sock = socket.socket() sock.connect(("localhost", self.plaintextPort)) sock.sendall("%s 5.0 %i\n" % (metricName, ts1)) sock.sendall("%s 6.0 %i\n" % (metricName, ts2)) sock.sendall("%s 7.0 %i\n" % (metricName, ts3)) self.gracefullyCloseSocket(sock) uid = self.checkMetricCreated(metricName) # Save the uid for later LOGGER.info("Metric %s has uid: %s", metricName, uid) # Send model creation request nativeMetric = {"datasource": "custom", "metricSpec": {"uid": uid}, "modelParams": {"min": 0.0, "max": 100.0}} model = self._createModel(nativeMetric) self.assertEqual(model.uid, uid) self.assertEqual(model.name, metricName) self.assertEqual(model.server, metricName) self.checkModelResults(uid, [[dt3, 7.0, 0.0, 3], [dt2, 6.0, 0.0, 2], [dt1, 5.0, 0.0, 1]]) exported = createDatasourceAdapter("custom").exportModel(uid) self.assertEqual(exported["metricSpec"]["metric"], metricName) self.assertEqual(exported["datasource"], "custom") self.assertEqual(len(exported["data"]), 2) self.assertSequenceEqual(exported["data"][0], [6.0, datetime.datetime.utcfromtimestamp(ts2)]) self.assertSequenceEqual(exported["data"][1], [7.0, datetime.datetime.utcfromtimestamp(ts3)])
def createModel(cls, modelSpec=None): """ NOTE MER-3479: this code path is presently incorrectly used for two purposes: * Importing of all types of metrics (not desirable; there should be a separate endpoint or an import-specific flag in this endpoint for importing that facilitates slightly different behavior, such as suppressing certain errors to allow for re-import in case of tranisent error part way through the prior import) """ if not modelSpec: # Metric data is missing log.error( "Data is missing in request, raising BadRequest exception") raise InvalidRequestResponse({"result": "Metric data is missing"}) # TODO MER-3479: import using import-specific endpoint # NOTE: pending MER-3479, this is presently a hack for exercising # the adapter import API importing = False if modelSpec.get("datasource") == "custom": if "data" in modelSpec: importing = True try: adapter = createDatasourceAdapter(modelSpec["datasource"]) try: # TODO: Maybe initialize transaction and commit here if importing: # TODO MER-3479: import using import-specific endpoint # NOTE: pending MER-3479, this is presently a hack for exercising # the adapter import API metricId = adapter.importModel(modelSpec) else: metricId = adapter.monitorMetric(modelSpec) except app_exceptions.MetricAlreadyMonitored as e: metricId = e.uid with web.ctx.connFactory() as conn: return repository.getMetric(conn, metricId) except (ValueError, app_exceptions.MetricNotSupportedError) as e: raise InvalidRequestResponse({"result": repr(e)})
def createModel(cls, modelSpec=None): """ NOTE MER-3479: this code path is presently incorrectly used for two purposes: * Importing of all types of metrics (not desirable; there should be a separate endpoint or an import-specific flag in this endpoint for importing that facilitates slightly different behavior, such as suppressing certain errors to allow for re-import in case of tranisent error part way through the prior import) """ if not modelSpec: # Metric data is missing log.error("Data is missing in request, raising BadRequest exception") raise InvalidRequestResponse({"result": "Metric data is missing"}) # TODO MER-3479: import using import-specific endpoint # NOTE: pending MER-3479, this is presently a hack for exercising # the adapter import API importing = False if modelSpec.get("datasource") == "custom": if "data" in modelSpec: importing = True try: adapter = createDatasourceAdapter(modelSpec["datasource"]) try: # TODO: Maybe initialize transaction and commit here if importing: # TODO MER-3479: import using import-specific endpoint # NOTE: pending MER-3479, this is presently a hack for exercising # the adapter import API metricId = adapter.importModel(modelSpec) else: metricId = adapter.monitorMetric(modelSpec) except app_exceptions.MetricAlreadyMonitored as e: metricId = e.uid with web.ctx.connFactory() as conn: return repository.getMetric(conn, metricId) except (ValueError, app_exceptions.MetricNotSupportedError) as e: raise InvalidRequestResponse({"result": repr(e)})
def _deleteMetric(self, metricName): adapter = createDatasourceAdapter("custom") adapter.deleteMetricByName(metricName)
def handler(environ, start_response): metricName = environ["PATH_INFO"] if environ["REQUEST_METHOD"] == "PUT": # Trigger model creation... modelSpec = { "datasource": "custom", "metricSpec": { "metric": metricName }, "modelParams": {} } try: modelSpec["modelParams"].update(json.load(environ["wsgi.input"])) except Exception as e: start_response("400 Bad Request", [("Content-Type", "text/html")]) yield "Unable to parse request" adapter = createDatasourceAdapter(modelSpec["datasource"]) try: modelId = adapter.monitorMetric(modelSpec) start_response("201 Created", [("Content-Type", "text/html")]) yield "Created %s\n" % modelId except MetricAlreadyMonitored: start_response("400 Bad Request", [("Content-Type", "text/html")]) yield "Model already exists for %s" % metricName elif environ["REQUEST_METHOD"] == "POST": # Send data... start_response("200 OK", [("Content-Type", "text/html")]) for sample in environ["wsgi.input"]: value, ts = sample.split(" ") sendSample(bus, metricName=metricName, value=float(value), epochTimestamp=int(ts)) yield "Saved %s %f @ %d\n" % (metricName, float(value), int(ts)) elif environ["REQUEST_METHOD"] == "GET": with repository.engineFactory(appConfig).connect() as conn: fields = (schema.metric_data.c.metric_value, schema.metric_data.c.timestamp, schema.metric_data.c.rowid, schema.metric_data.c.anomaly_score) sort = schema.metric_data.c.timestamp.asc() metricObj = repository.getCustomMetricByName( conn, metricName, fields=[schema.metric.c.uid]) result = repository.getMetricData(conn, metricId=metricObj.uid, fields=fields, sort=sort) start_response("200 OK", [("Content-Type", "text/html")]) for row in result: yield " ".join( (metricName, str(row.metric_value), str(calendar.timegm(row.timestamp.timetuple())), str(row.anomaly_score))) + "\n"
def _deleteModel(self, metricId): adapter = createDatasourceAdapter("custom") adapter.unmonitorMetric(metricId)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see http://www.gnu.org/licenses. # # http://numenta.org/licenses/ # ---------------------------------------------------------------------- """Create the cpu percent model. See also send_cpu.py and README.md.""" from htmengine.adapters.datasource import createDatasourceAdapter modelSpec = { "datasource": "custom", "metricSpec": { "metric": "cpu_percent" }, "modelParams": { "min": 0, # optional "max": 100 # optional } } adapter = createDatasourceAdapter(modelSpec["datasource"]) modelId = adapter.monitorMetric(modelSpec) print "Model", modelId, "created..."
#!/usr/bin/env python# ----------------------------------------------------------------------# Numenta Platform for Intelligent Computing (NuPIC)# Copyright (C) 2015, Numenta, Inc. Unless you have purchased from# Numenta, Inc. a separate commercial license for this software code, the# following terms and conditions apply:## This program is free software: you can redistribute it and/or modify# it under the terms of the GNU General Public License version 3 as# published by the Free Software Foundation.## This program is distributed in the hope that it will be useful,# but WITHOUT ANY WARRANTY; without even the implied warranty of# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.# See the GNU General Public License for more details.## You should have received a copy of the GNU General Public License# along with this program. If not, see http://www.gnu.org/licenses.## http://numenta.org/licenses/# ---------------------------------------------------------------------- """Create the cpu percent model. See also send_cpu.py and README.md.""" from htmengine.adapters.datasource import createDatasourceAdapter modelSpec = { "datasource": "custom", "metricSpec": { "metric": "cpu_percent" }, "modelParams": { "min": 0, # optional "max": 100 # optional }} adapter = createDatasourceAdapter(modelSpec["datasource"])modelId = adapter.monitorMetric(modelSpec) print "Model", modelId, "created..."
def _deleteModel(metricId): adapter = createDatasourceAdapter("custom") adapter.unmonitorMetric(metricId)
def _deleteMetric(metricName): adapter = createDatasourceAdapter("custom") adapter.deleteMetricByName(metricName)
def handler(environ, start_response): metricName = environ["PATH_INFO"] if environ["REQUEST_METHOD"] == "PUT": # Trigger model creation... modelSpec = {"datasource": "custom", "metricSpec": {"metric": metricName}, "modelParams": {}} try: modelSpec["modelParams"].update(json.load(environ["wsgi.input"])) except Exception as e: print e start_response("400 Bad Request", [("Content-Type", "text/html")]) yield "Unable to parse request" adapter = createDatasourceAdapter(modelSpec["datasource"]) try: modelId = adapter.monitorMetric(modelSpec) start_response("201 Created", [("Content-Type", "text/html")]) yield "Created %s\n" % modelId except MetricAlreadyMonitored: start_response("400 Bad Request", [("Content-Type", "text/html")]) yield "Model already exists for %s" % metricName elif environ["REQUEST_METHOD"] == "POST": # Send data... start_response("200 OK", [("Content-Type", "text/html")]) for sample in environ["wsgi.input"]: value, ts = sample.split(" ") sendSample(bus, metricName=metricName, value=float(value), epochTimestamp=int(ts)) yield "Saved %s %f @ %d\n" % (metricName, float(value), int(ts)) elif environ["REQUEST_METHOD"] == "GET": # parameters = parse_qs(environ.get('QUERY_STRING', '')) # print parameters # if 'since' in parameters: # since = parameters['since'][0] with repository.engineFactory(appConfig).connect() as conn: fields = ( schema.metric_data.c.metric_value, schema.metric_data.c.timestamp, schema.metric_data.c.rowid, schema.metric_data.c.anomaly_score, ) sort = schema.metric_data.c.timestamp.asc() metricObj = repository.getCustomMetricByName(conn, metricName, fields=[schema.metric.c.uid]) result = repository.getMetricData(conn, metricId=metricObj.uid, fields=fields, sort=sort) start_response("200 OK", [("Content-Type", "text/html")]) for row in result: yield " ".join( ( metricName, str(row.metric_value), str(calendar.timegm(row.timestamp.timetuple())), str(row.anomaly_score), ) ) + "\n"
def _exportNativeMetric(metric): return createDatasourceAdapter(metric.datasource).exportModel(metric.uid)
def streamMetricData(self, data, metricID, modelSwapper): """ Store the data samples in metric_data table if needed, and stream the data samples to the model associated with the metric if the metric is monitored. If the metric is in PENDING_DATA state: if there are now enough data samples to start a model, start it and stream it the entire backlog of data samples; if there are still not enough data samples, suppress streaming of the data samples. :param data: A sequence of data samples; each data sample is a three-tuple: (datetime.datetime, float, None) OR (datetime.datetime, float, rowid) The third item in each three-tuple is either None in all elements of the sequence or is a valid rowid in all elements of the sequence. If it's None, as is the case with metrics collected from AWS CloudWatch, the row is added to the metric_data table before sending it to the model. If rowid is not None, as is presently the case with HTM metrics, the data samples are assumed to be stored already. :param metricID: unique id of the HTM metric :param modelSwapper: ModelSwapper object for sending data to models :type modelSwapper: an instance of ModelSwapperInterface :raises: ObjectNotFoundError when the metric for the data doesn't exist """ if not data: self._log.warn("Empty input metric data batch for metric=%s", metricID) return @repository.retryOnTransientErrors def storeDataWithRetries(): """ :returns: a three-tuple <modelInputRows, datasource, metricStatus>; modelInputRows: None if model was in state not suitable for streaming; otherwise a (possibly empty) tuple of ModelInputRow objects corresponding to the samples that were stored; ordered by rowid """ with repository.engineFactory(config).connect() as conn: with conn.begin(): # Syncrhonize with adapter's monitorMetric metricObj = repository.getMetricWithUpdateLock( conn, metricID, fields=[ schema.metric.c.status, schema.metric.c.last_rowid, schema.metric.c.datasource ]) if (metricObj.status != MetricStatus.UNMONITORED and metricObj.status != MetricStatus.ACTIVE and metricObj.status != MetricStatus.PENDING_DATA and metricObj.status != MetricStatus.CREATE_PENDING): self._log.error( "Can't stream: metric=%s has unexpected status=%s", metricID, metricObj.status) modelInputRows = None else: # TODO: unit-test passingSamples = self._scrubDataSamples( data, metricID, conn, metricObj.last_rowid) if passingSamples: modelInputRows = self._storeDataSamples( passingSamples, metricID, conn) else: modelInputRows = tuple() return (modelInputRows, metricObj.datasource, metricObj.status) (modelInputRows, datasource, metricStatus) = storeDataWithRetries() if modelInputRows is None: # Metric was in state not suitable for streaming return if not modelInputRows: # TODO: unit-test # Nothing was added, so nothing further to do self._log.error("No records to stream to model=%s", metricID) return if metricStatus == MetricStatus.UNMONITORED: # Metric was not monitored during storage, so we're done #self._log.info("Status of metric=%s is UNMONITORED; not forwarding " # "%d rows: rowids[%s..%s]; data=[%s..%s]", # metricID, len(modelInputRows), # modelInputRows[0].rowID, modelInputRows[-1].rowID, # modelInputRows[0].data, modelInputRows[-1].data) return lastDataRowID = modelInputRows[-1].rowID # Check models that are waiting for activation upon sufficient data if metricStatus == MetricStatus.PENDING_DATA: if lastDataRowID >= MODEL_CREATION_RECORD_THRESHOLD: try: # Activate metric that is supported by Datasource Adapter createDatasourceAdapter(datasource).activateModel(metricID) except (MetricStatisticsNotReadyError, MetricStatusChangedError) as ex: # Perhaps the metric status changed in the meantime. We can just # ignore this and it will sort itself out if additional records come # in (e.g., HTM Metric). self._log.error("Couldn't start model=%s: %r", metricID, ex) return # Stream data if model is activated # TODO: unit-test if metricStatus in (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE): self._sendInputRowsToModel(inputRows=modelInputRows, metricID=metricID, modelSwapper=modelSwapper) self._log.debug("Streamed numRecords=%d to model=%s", len(modelInputRows), metricID)
def streamMetricData(self, data, metricID, modelSwapper): """ Store the data samples in metric_data table if needed, and stream the data samples to the model associated with the metric if the metric is monitored. If the metric is in PENDING_DATA state: if there are now enough data samples to start a model, start it and stream it the entire backlog of data samples; if there are still not enough data samples, suppress streaming of the data samples. :param data: A sequence of data samples; each data sample is a three-tuple: (datetime.datetime, float, None) OR (datetime.datetime, float, rowid) The third item in each three-tuple is either None in all elements of the sequence or is a valid rowid in all elements of the sequence. If it's None, as is the case with metrics collected from AWS CloudWatch, the row is added to the metric_data table before sending it to the model. If rowid is not None, as is presently the case with HTM metrics, the data samples are assumed to be stored already. :param metricID: unique id of the HTM metric :param modelSwapper: ModelSwapper object for sending data to models :type modelSwapper: an instance of ModelSwapperInterface :raises: ObjectNotFoundError when the metric for the data doesn't exist """ if not data: self._log.warn("Empty input metric data batch for metric=%s", metricID) return @repository.retryOnTransientErrors def storeDataWithRetries(): """ :returns: a three-tuple <modelInputRows, datasource, metricStatus>; modelInputRows: None if model was in state not suitable for streaming; otherwise a (possibly empty) tuple of ModelInputRow objects corresponding to the samples that were stored; ordered by rowid """ with repository.engineFactory(config).connect() as conn: with conn.begin(): # Synchronize with adapter's monitorMetric metricObj = repository.getMetricWithUpdateLock( conn, metricID, fields=[schema.metric.c.status, schema.metric.c.last_rowid, schema.metric.c.datasource]) if (metricObj.status != MetricStatus.UNMONITORED and metricObj.status != MetricStatus.ACTIVE and metricObj.status != MetricStatus.PENDING_DATA and metricObj.status != MetricStatus.CREATE_PENDING): self._log.error("Can't stream: metric=%s has unexpected status=%s", metricID, metricObj.status) modelInputRows = None else: # TODO: unit-test passingSamples = self._scrubDataSamples(data, metricID, conn, metricObj.last_rowid) if passingSamples: modelInputRows = self._storeDataSamples(passingSamples, metricID, conn) else: modelInputRows = tuple() return (modelInputRows, metricObj.datasource, metricObj.status) (modelInputRows, datasource, metricStatus) = storeDataWithRetries() if modelInputRows is None: # Metric was in state not suitable for streaming return if not modelInputRows: # TODO: unit-test # Nothing was added, so nothing further to do self._log.error("No records to stream to model=%s", metricID) return if metricStatus == MetricStatus.UNMONITORED: # Metric was not monitored during storage, so we're done #self._log.info("Status of metric=%s is UNMONITORED; not forwarding " # "%d rows: rowids[%s..%s]; data=[%s..%s]", # metricID, len(modelInputRows), # modelInputRows[0].rowID, modelInputRows[-1].rowID, # modelInputRows[0].data, modelInputRows[-1].data) return lastDataRowID = modelInputRows[-1].rowID # Check models that are waiting for activation upon sufficient data if metricStatus == MetricStatus.PENDING_DATA: if lastDataRowID >= MODEL_CREATION_RECORD_THRESHOLD: try: # Activate metric that is supported by Datasource Adapter createDatasourceAdapter(datasource).activateModel(metricID) except (MetricStatisticsNotReadyError, MetricStatusChangedError) as ex: # Perhaps the metric status changed in the meantime. We can just # ignore this and it will sort itself out if additional records come # in (e.g., HTM Metric). self._log.error("Couldn't start model=%s: %r", metricID, ex) return # Stream data if model is activated # TODO: unit-test if metricStatus in (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE): self._sendInputRowsToModel( inputRows=modelInputRows, metricID=metricID, modelSwapper=modelSwapper) self._log.debug("Streamed numRecords=%d to model=%s", len(modelInputRows), metricID)
def _exportNativeMetric(metric): return createDatasourceAdapter(metric.datasource).exportModel( metric.uid)
# This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # See the GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see http://www.gnu.org/licenses. # # http://numenta.org/licenses/ # ---------------------------------------------------------------------- """Create the cpu percent model. See also send_cpu.py and README.md.""" from htmengine.adapters.datasource import createDatasourceAdapter modelSpec = { "datasource": "custom", "metricSpec": {"metric": "cpu_percent"}, "modelParams": {"min": 0, "max": 100}, # optional # optional } adapter = createDatasourceAdapter(modelSpec["datasource"]) modelId = adapter.monitorMetric(modelSpec) print "Model", modelId, "created..."