Beispiel #1
0
def createAutostackDatasourceAdapter():
    """ Convenience function for creating a datasource adapter for Grok Autostack
  Metrics

  :returns: datasource adapter for Grok Autostack Metrics
  :rtype: grok.app.adapters.datasource.cloudwatch._AutostackDatasourceAdapter
  """
    return createDatasourceAdapter("autostack")
Beispiel #2
0
def createCloudwatchDatasourceAdapter():
    """ Convenience function for creating a datasource adapter for Cloudwatch
  Metrics

  :returns: datasource adapter for Cloudwatch Metrics
  :rtype: grok.app.adapters.datasource.cloudwatch._CloudwatchDatasourceAdapter
  """
    return createDatasourceAdapter("cloudwatch")
Beispiel #3
0
def createAutostackDatasourceAdapter():
  """ Convenience function for creating a datasource adapter for HTM-IT Autostack
  Metrics

  :returns: datasource adapter for HTM-IT Autostack Metrics
  :rtype: htm.it.app.adapters.datasource.cloudwatch._AutostackDatasourceAdapter
  """
  return createDatasourceAdapter("autostack")
Beispiel #4
0
def createCloudwatchDatasourceAdapter():
  """ Convenience function for creating a datasource adapter for Cloudwatch
  Metrics

  :returns: datasource adapter for Cloudwatch Metrics
  :rtype: htm.it.app.adapters.datasource.cloudwatch._CloudwatchDatasourceAdapter
  """
  return createDatasourceAdapter("cloudwatch")
Beispiel #5
0
  def deleteModel(metricId):
    try:
      with web.ctx.connFactory() as conn:
        metricRow = repository.getMetric(conn, metricId)
    except app_exceptions.ObjectNotFoundError:
      raise web.notfound("ObjectNotFoundError Metric not found: Metric ID: %s"
                         % metricId)

    log.debug("Deleting model for %s metric=%s", metricRow.datasource,
              metricId)
    # NOTE: this is the new way using datasource adapters
    try:
      createDatasourceAdapter(metricRow.datasource).unmonitorMetric(metricId)
    except app_exceptions.ObjectNotFoundError:
      raise web.notfound(
        "ObjectNotFoundError Metric not found: Metric ID: %s" % (metricId,))

    return utils.jsonEncode({'result': 'success'})
Beispiel #6
0
    def DELETE(self, metricName):
        adapter = createDatasourceAdapter("custom")
        try:
            adapter.deleteMetricByName(metricName)
        except app_exceptions.ObjectNotFoundError:
            raise web.notfound("Metric not found. Metric name=%s" % (metricName,))

        self.addStandardHeaders()
        return json.dumps({"result": "success"})
Beispiel #7
0
  def DELETE(self, metricName):
    adapter = createDatasourceAdapter("custom")
    try:
      adapter.deleteMetricByName(metricName)
    except app_exceptions.ObjectNotFoundError:
      raise web.notfound("Metric not found. Metric name=%s" % (metricName,))

    self.addStandardHeaders()
    return json.dumps({'result': 'success'})
Beispiel #8
0
  def deleteModel(metricId):
    try:
      with web.ctx.connFactory() as conn:
        metricRow = repository.getMetric(conn, metricId)
    except app_exceptions.ObjectNotFoundError:
      raise web.notfound("ObjectNotFoundError Metric not found: Metric ID: %s"
                         % metricId)

    log.debug("Deleting model for %s metric=%s", metricRow.datasource,
              metricId)
    # NOTE: this is the new way using datasource adapters
    try:
      createDatasourceAdapter(metricRow.datasource).unmonitorMetric(metricId)
    except app_exceptions.ObjectNotFoundError:
      raise web.notfound(
        "ObjectNotFoundError Metric not found: Metric ID: %s" % (metricId,))

    return utils.jsonEncode({'result': 'success'})
  def _createModel(self, nativeMetric):
    adapter = createDatasourceAdapter("custom")
    try:
      metricId = adapter.monitorMetric(nativeMetric)
    except MetricAlreadyMonitored as e:
      metricId = e.uid

    engine = repository.engineFactory(config=self.config)

    with engine.begin() as conn:
      return repository.getMetric(conn, metricId)
Beispiel #10
0
    def _createModel(self, nativeMetric):
        adapter = createDatasourceAdapter("custom")
        try:
            metricId = adapter.monitorMetric(nativeMetric)
        except MetricAlreadyMonitored as e:
            metricId = e.uid

        engine = repository.engineFactory(config=self.config)

        with engine.begin() as conn:
            return repository.getMetric(conn, metricId)
  def testExport(self):
    """Tests exporting custom metrics."""
    metricName = "testBatchSend.%i" % int(time.time())
    LOGGER.info("Running test with metric name: %s", metricName)

    self.addCleanup(self._deleteMetric, metricName)

    # Get the timestamp for 14 days ago (the limit for exporting data)
    baseTimestamp = int(time.time()) - (60 * 60 * 24 * 14)
    ts1 = baseTimestamp - (60 * 5)
    dt1 = datetime.datetime.utcfromtimestamp(ts1).strftime("%Y-%m-%d %H:%M:%S")
    ts2 = baseTimestamp + (60 * 5)
    dt2 = datetime.datetime.utcfromtimestamp(ts2).strftime("%Y-%m-%d %H:%M:%S")
    ts3 = baseTimestamp + (60 * 10)
    dt3 = datetime.datetime.utcfromtimestamp(ts3).strftime("%Y-%m-%d %H:%M:%S")

    # Add custom metric data
    sock = socket.socket()
    sock.connect(("localhost", self.plaintextPort))
    sock.sendall("%s 5.0 %i\n" % (metricName, ts1))
    sock.sendall("%s 6.0 %i\n" % (metricName, ts2))
    sock.sendall("%s 7.0 %i\n" % (metricName, ts3))
    self.gracefullyCloseSocket(sock)

    uid = self.checkMetricCreated(metricName)

    # Save the uid for later
    LOGGER.info("Metric %s has uid: %s", metricName, uid)

    # Send model creation request
    nativeMetric = {"datasource": "custom",
                    "metricSpec": {"uid": uid},
                    "modelParams": {"min": 0.0,
                                    "max": 100.0}}

    model = self._createModel(nativeMetric)

    self.assertEqual(model.uid, uid)
    self.assertEqual(model.name, metricName)
    self.assertEqual(model.server, metricName)

    self.checkModelResults(uid, [[dt3, 7.0, 0.0, 3], [dt2, 6.0, 0.0, 2],
                                 [dt1, 5.0, 0.0, 1]])

    exported = createDatasourceAdapter("custom").exportModel(uid)

    self.assertEqual(exported["metricSpec"]["metric"], metricName)
    self.assertEqual(exported["datasource"], "custom")
    self.assertEqual(len(exported["data"]), 2)
    self.assertSequenceEqual(exported["data"][0],
                             [6.0, datetime.datetime.utcfromtimestamp(ts2)])
    self.assertSequenceEqual(exported["data"][1],
                             [7.0, datetime.datetime.utcfromtimestamp(ts3)])
Beispiel #12
0
  def testExport(self):
    """Tests exporting custom metrics."""
    metricName = "testBatchSend.%i" % int(time.time())
    LOGGER.info("Running test with metric name: %s", metricName)

    self.addCleanup(self._deleteMetric, metricName)

    # Get the timestamp for 14 days ago (the limit for exporting data)
    baseTimestamp = int(time.time()) - (60 * 60 * 24 * 14)
    ts1 = baseTimestamp - (60 * 5)
    dt1 = datetime.datetime.utcfromtimestamp(ts1).strftime("%Y-%m-%d %H:%M:%S")
    ts2 = baseTimestamp + (60 * 5)
    dt2 = datetime.datetime.utcfromtimestamp(ts2).strftime("%Y-%m-%d %H:%M:%S")
    ts3 = baseTimestamp + (60 * 10)
    dt3 = datetime.datetime.utcfromtimestamp(ts3).strftime("%Y-%m-%d %H:%M:%S")

    # Add custom metric data
    sock = socket.socket()
    sock.connect(("localhost", self.plaintextPort))
    sock.sendall("%s 5.0 %i\n" % (metricName, ts1))
    sock.sendall("%s 6.0 %i\n" % (metricName, ts2))
    sock.sendall("%s 7.0 %i\n" % (metricName, ts3))
    self.gracefullyCloseSocket(sock)

    uid = self.checkMetricCreated(metricName)

    # Save the uid for later
    LOGGER.info("Metric %s has uid: %s", metricName, uid)

    # Send model creation request
    nativeMetric = {"datasource": "custom",
                    "metricSpec": {"uid": uid},
                    "modelParams": {"min": 0.0,
                                    "max": 100.0}}

    model = self._createModel(nativeMetric)

    self.assertEqual(model.uid, uid)
    self.assertEqual(model.name, metricName)
    self.assertEqual(model.server, metricName)

    self.checkModelResults(uid, [[dt3, 7.0, 0.0, 3], [dt2, 6.0, 0.0, 2],
                                 [dt1, 5.0, 0.0, 1]])

    exported = createDatasourceAdapter("custom").exportModel(uid)

    self.assertEqual(exported["metricSpec"]["metric"], metricName)
    self.assertEqual(exported["datasource"], "custom")
    self.assertEqual(len(exported["data"]), 2)
    self.assertSequenceEqual(exported["data"][0],
                             [6.0, datetime.datetime.utcfromtimestamp(ts2)])
    self.assertSequenceEqual(exported["data"][1],
                             [7.0, datetime.datetime.utcfromtimestamp(ts3)])
Beispiel #13
0
    def createModel(cls, modelSpec=None):
        """
    NOTE MER-3479: this code path is presently incorrectly used for two
      purposes:
        * Importing of all types of metrics (not desirable; there should be a
          separate endpoint or an import-specific flag in this endpoint for
          importing that facilitates slightly different behavior, such as
          suppressing certain errors to allow for re-import in case of tranisent
          error part way through the prior import)
    """

        if not modelSpec:
            # Metric data is missing
            log.error(
                "Data is missing in request, raising BadRequest exception")
            raise InvalidRequestResponse({"result": "Metric data is missing"})

        # TODO MER-3479: import using import-specific endpoint
        # NOTE: pending MER-3479, this is presently a hack for exercising
        #   the adapter import API
        importing = False

        if modelSpec.get("datasource") == "custom":
            if "data" in modelSpec:
                importing = True

        try:
            adapter = createDatasourceAdapter(modelSpec["datasource"])
            try:
                # TODO: Maybe initialize transaction and commit here
                if importing:
                    # TODO MER-3479: import using import-specific endpoint
                    # NOTE: pending MER-3479, this is presently a hack for exercising
                    #   the adapter import API
                    metricId = adapter.importModel(modelSpec)
                else:
                    metricId = adapter.monitorMetric(modelSpec)
            except app_exceptions.MetricAlreadyMonitored as e:
                metricId = e.uid

            with web.ctx.connFactory() as conn:
                return repository.getMetric(conn, metricId)
        except (ValueError, app_exceptions.MetricNotSupportedError) as e:
            raise InvalidRequestResponse({"result": repr(e)})
Beispiel #14
0
  def createModel(cls, modelSpec=None):
    """
    NOTE MER-3479: this code path is presently incorrectly used for two
      purposes:
        * Importing of all types of metrics (not desirable; there should be a
          separate endpoint or an import-specific flag in this endpoint for
          importing that facilitates slightly different behavior, such as
          suppressing certain errors to allow for re-import in case of tranisent
          error part way through the prior import)
    """

    if not modelSpec:
      # Metric data is missing
      log.error("Data is missing in request, raising BadRequest exception")
      raise InvalidRequestResponse({"result": "Metric data is missing"})

    # TODO MER-3479: import using import-specific endpoint
    # NOTE: pending MER-3479, this is presently a hack for exercising
    #   the adapter import API
    importing = False

    if modelSpec.get("datasource") == "custom":
      if "data" in modelSpec:
        importing = True

    try:
      adapter = createDatasourceAdapter(modelSpec["datasource"])
      try:
        # TODO: Maybe initialize transaction and commit here
        if importing:
          # TODO MER-3479: import using import-specific endpoint
          # NOTE: pending MER-3479, this is presently a hack for exercising
          #   the adapter import API
          metricId = adapter.importModel(modelSpec)
        else:
          metricId = adapter.monitorMetric(modelSpec)
      except app_exceptions.MetricAlreadyMonitored as e:
        metricId = e.uid

      with web.ctx.connFactory() as conn:
        return repository.getMetric(conn, metricId)
    except (ValueError, app_exceptions.MetricNotSupportedError) as e:
      raise InvalidRequestResponse({"result": repr(e)})
Beispiel #15
0
 def _deleteMetric(self, metricName):
     adapter = createDatasourceAdapter("custom")
     adapter.deleteMetricByName(metricName)
Beispiel #16
0
def handler(environ, start_response):
    metricName = environ["PATH_INFO"]

    if environ["REQUEST_METHOD"] == "PUT":
        # Trigger model creation...

        modelSpec = {
            "datasource": "custom",
            "metricSpec": {
                "metric": metricName
            },
            "modelParams": {}
        }

        try:
            modelSpec["modelParams"].update(json.load(environ["wsgi.input"]))
        except Exception as e:
            start_response("400 Bad Request", [("Content-Type", "text/html")])
            yield "Unable to parse request"

        adapter = createDatasourceAdapter(modelSpec["datasource"])
        try:
            modelId = adapter.monitorMetric(modelSpec)
            start_response("201 Created", [("Content-Type", "text/html")])
            yield "Created %s\n" % modelId

        except MetricAlreadyMonitored:
            start_response("400 Bad Request", [("Content-Type", "text/html")])
            yield "Model already exists for %s" % metricName
    elif environ["REQUEST_METHOD"] == "POST":
        # Send data...

        start_response("200 OK", [("Content-Type", "text/html")])

        for sample in environ["wsgi.input"]:
            value, ts = sample.split(" ")
            sendSample(bus,
                       metricName=metricName,
                       value=float(value),
                       epochTimestamp=int(ts))

            yield "Saved %s %f @ %d\n" % (metricName, float(value), int(ts))
    elif environ["REQUEST_METHOD"] == "GET":
        with repository.engineFactory(appConfig).connect() as conn:
            fields = (schema.metric_data.c.metric_value,
                      schema.metric_data.c.timestamp,
                      schema.metric_data.c.rowid,
                      schema.metric_data.c.anomaly_score)
            sort = schema.metric_data.c.timestamp.asc()

            metricObj = repository.getCustomMetricByName(
                conn, metricName, fields=[schema.metric.c.uid])

            result = repository.getMetricData(conn,
                                              metricId=metricObj.uid,
                                              fields=fields,
                                              sort=sort)

            start_response("200 OK", [("Content-Type", "text/html")])

            for row in result:
                yield " ".join(
                    (metricName, str(row.metric_value),
                     str(calendar.timegm(row.timestamp.timetuple())),
                     str(row.anomaly_score))) + "\n"
Beispiel #17
0
 def _deleteModel(self, metricId):
     adapter = createDatasourceAdapter("custom")
     adapter.unmonitorMetric(metricId)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Create the cpu percent model.  See also send_cpu.py and README.md."""

from htmengine.adapters.datasource import createDatasourceAdapter



modelSpec = {
  "datasource": "custom",
  "metricSpec": {
    "metric": "cpu_percent"
  },
  "modelParams": {
    "min": 0,  # optional
    "max": 100  # optional
  }
}

adapter = createDatasourceAdapter(modelSpec["datasource"])
modelId = adapter.monitorMetric(modelSpec)

print "Model", modelId, "created..."
Beispiel #19
0
#!/usr/bin/env python# ----------------------------------------------------------------------# Numenta Platform for Intelligent Computing (NuPIC)# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from# Numenta, Inc. a separate commercial license for this software code, the# following terms and conditions apply:## This program is free software: you can redistribute it and/or modify# it under the terms of the GNU General Public License version 3 as# published by the Free Software Foundation.## This program is distributed in the hope that it will be useful,# but WITHOUT ANY WARRANTY; without even the implied warranty of# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.# See the GNU General Public License for more details.## You should have received a copy of the GNU General Public License# along with this program.  If not, see http://www.gnu.org/licenses.## http://numenta.org/licenses/# ----------------------------------------------------------------------
"""Create the cpu percent model.  See also send_cpu.py and README.md."""
from htmengine.adapters.datasource import createDatasourceAdapter


modelSpec = { "datasource": "custom", "metricSpec": { "metric": "cpu_percent"  }, "modelParams": { "min": 0,  # optional "max": 100 # optional  }}
adapter = createDatasourceAdapter(modelSpec["datasource"])modelId = adapter.monitorMetric(modelSpec)
print "Model", modelId, "created..."
 def _deleteModel(metricId):
   adapter = createDatasourceAdapter("custom")
   adapter.unmonitorMetric(metricId)
 def _deleteMetric(metricName):
   adapter = createDatasourceAdapter("custom")
   adapter.deleteMetricByName(metricName)
def handler(environ, start_response):
    metricName = environ["PATH_INFO"]

    if environ["REQUEST_METHOD"] == "PUT":
        # Trigger model creation...

        modelSpec = {"datasource": "custom", "metricSpec": {"metric": metricName}, "modelParams": {}}

        try:
            modelSpec["modelParams"].update(json.load(environ["wsgi.input"]))
        except Exception as e:
            print e
            start_response("400 Bad Request", [("Content-Type", "text/html")])
            yield "Unable to parse request"

        adapter = createDatasourceAdapter(modelSpec["datasource"])
        try:
            modelId = adapter.monitorMetric(modelSpec)
            start_response("201 Created", [("Content-Type", "text/html")])
            yield "Created %s\n" % modelId

        except MetricAlreadyMonitored:
            start_response("400 Bad Request", [("Content-Type", "text/html")])
            yield "Model already exists for %s" % metricName
    elif environ["REQUEST_METHOD"] == "POST":
        # Send data...

        start_response("200 OK", [("Content-Type", "text/html")])

        for sample in environ["wsgi.input"]:
            value, ts = sample.split(" ")
            sendSample(bus, metricName=metricName, value=float(value), epochTimestamp=int(ts))

            yield "Saved %s %f @ %d\n" % (metricName, float(value), int(ts))
    elif environ["REQUEST_METHOD"] == "GET":
        # parameters = parse_qs(environ.get('QUERY_STRING', ''))
        # print parameters
        # if 'since' in parameters:
        #   since = parameters['since'][0]
        with repository.engineFactory(appConfig).connect() as conn:
            fields = (
                schema.metric_data.c.metric_value,
                schema.metric_data.c.timestamp,
                schema.metric_data.c.rowid,
                schema.metric_data.c.anomaly_score,
            )
            sort = schema.metric_data.c.timestamp.asc()

            metricObj = repository.getCustomMetricByName(conn, metricName, fields=[schema.metric.c.uid])

            result = repository.getMetricData(conn, metricId=metricObj.uid, fields=fields, sort=sort)

            start_response("200 OK", [("Content-Type", "text/html")])

            for row in result:
                yield " ".join(
                    (
                        metricName,
                        str(row.metric_value),
                        str(calendar.timegm(row.timestamp.timetuple())),
                        str(row.anomaly_score),
                    )
                ) + "\n"
Beispiel #23
0
 def _exportNativeMetric(metric):
   return createDatasourceAdapter(metric.datasource).exportModel(metric.uid)
    def streamMetricData(self, data, metricID, modelSwapper):
        """ Store the data samples in metric_data table if needed, and stream the
    data samples to the model associated with the metric if the metric is
    monitored.

    If the metric is in PENDING_DATA state: if there are now enough data samples
    to start a model, start it and stream it the entire backlog of data samples;
    if there are still not enough data samples, suppress streaming of the data
    samples.

    :param data: A sequence of data samples; each data sample is a three-tuple:
                  (datetime.datetime, float, None)
                     OR
                  (datetime.datetime, float, rowid)
      The third item in each three-tuple is either None in all elements of the
      sequence or is a valid rowid in all elements of the sequence. If it's
      None, as is the case with metrics collected from AWS
      CloudWatch, the row is added to the metric_data table before sending it
      to the model. If rowid is not None, as is presently the case with HTM
      metrics, the data samples are assumed to be stored already.

    :param metricID: unique id of the HTM metric

    :param modelSwapper: ModelSwapper object for sending data to models
    :type modelSwapper: an instance of ModelSwapperInterface

    :raises: ObjectNotFoundError when the metric for the data doesn't exist
    """
        if not data:
            self._log.warn("Empty input metric data batch for metric=%s",
                           metricID)
            return

        @repository.retryOnTransientErrors
        def storeDataWithRetries():
            """
      :returns: a three-tuple <modelInputRows, datasource, metricStatus>;
        modelInputRows: None if model was in state not suitable for streaming;
          otherwise a (possibly empty) tuple of ModelInputRow objects
          corresponding to the samples that were stored; ordered by rowid
      """
            with repository.engineFactory(config).connect() as conn:
                with conn.begin():
                    # Syncrhonize with adapter's monitorMetric
                    metricObj = repository.getMetricWithUpdateLock(
                        conn,
                        metricID,
                        fields=[
                            schema.metric.c.status, schema.metric.c.last_rowid,
                            schema.metric.c.datasource
                        ])

                    if (metricObj.status != MetricStatus.UNMONITORED
                            and metricObj.status != MetricStatus.ACTIVE
                            and metricObj.status != MetricStatus.PENDING_DATA
                            and
                            metricObj.status != MetricStatus.CREATE_PENDING):
                        self._log.error(
                            "Can't stream: metric=%s has unexpected status=%s",
                            metricID, metricObj.status)
                        modelInputRows = None
                    else:
                        # TODO: unit-test
                        passingSamples = self._scrubDataSamples(
                            data, metricID, conn, metricObj.last_rowid)
                        if passingSamples:
                            modelInputRows = self._storeDataSamples(
                                passingSamples, metricID, conn)
                        else:
                            modelInputRows = tuple()

            return (modelInputRows, metricObj.datasource, metricObj.status)

        (modelInputRows, datasource, metricStatus) = storeDataWithRetries()

        if modelInputRows is None:
            # Metric was in state not suitable for streaming
            return

        if not modelInputRows:
            # TODO: unit-test
            # Nothing was added, so nothing further to do
            self._log.error("No records to stream to model=%s", metricID)
            return

        if metricStatus == MetricStatus.UNMONITORED:
            # Metric was not monitored during storage, so we're done
            #self._log.info("Status of metric=%s is UNMONITORED; not forwarding "
            #               "%d rows: rowids[%s..%s]; data=[%s..%s]",
            #               metricID, len(modelInputRows),
            #               modelInputRows[0].rowID, modelInputRows[-1].rowID,
            #               modelInputRows[0].data, modelInputRows[-1].data)
            return

        lastDataRowID = modelInputRows[-1].rowID

        # Check models that are waiting for activation upon sufficient data
        if metricStatus == MetricStatus.PENDING_DATA:
            if lastDataRowID >= MODEL_CREATION_RECORD_THRESHOLD:
                try:
                    # Activate metric that is supported by Datasource Adapter
                    createDatasourceAdapter(datasource).activateModel(metricID)
                except (MetricStatisticsNotReadyError,
                        MetricStatusChangedError) as ex:
                    # Perhaps the metric status changed in the meantime. We can just
                    # ignore this and it will sort itself out if additional records come
                    # in (e.g., HTM Metric).
                    self._log.error("Couldn't start model=%s: %r", metricID,
                                    ex)
            return

        # Stream data if model is activated
        # TODO: unit-test
        if metricStatus in (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE):
            self._sendInputRowsToModel(inputRows=modelInputRows,
                                       metricID=metricID,
                                       modelSwapper=modelSwapper)

            self._log.debug("Streamed numRecords=%d to model=%s",
                            len(modelInputRows), metricID)
  def streamMetricData(self, data, metricID, modelSwapper):
    """ Store the data samples in metric_data table if needed, and stream the
    data samples to the model associated with the metric if the metric is
    monitored.

    If the metric is in PENDING_DATA state: if there are now enough data samples
    to start a model, start it and stream it the entire backlog of data samples;
    if there are still not enough data samples, suppress streaming of the data
    samples.

    :param data: A sequence of data samples; each data sample is a three-tuple:
                  (datetime.datetime, float, None)
                     OR
                  (datetime.datetime, float, rowid)
      The third item in each three-tuple is either None in all elements of the
      sequence or is a valid rowid in all elements of the sequence. If it's
      None, as is the case with metrics collected from AWS
      CloudWatch, the row is added to the metric_data table before sending it
      to the model. If rowid is not None, as is presently the case with HTM
      metrics, the data samples are assumed to be stored already.

    :param metricID: unique id of the HTM metric

    :param modelSwapper: ModelSwapper object for sending data to models
    :type modelSwapper: an instance of ModelSwapperInterface

    :raises: ObjectNotFoundError when the metric for the data doesn't exist
    """
    if not data:
      self._log.warn("Empty input metric data batch for metric=%s", metricID)
      return

    @repository.retryOnTransientErrors
    def storeDataWithRetries():
      """
      :returns: a three-tuple <modelInputRows, datasource, metricStatus>;
        modelInputRows: None if model was in state not suitable for streaming;
          otherwise a (possibly empty) tuple of ModelInputRow objects
          corresponding to the samples that were stored; ordered by rowid
      """
      with repository.engineFactory(config).connect() as conn:
        with conn.begin():
          # Synchronize with adapter's monitorMetric
          metricObj = repository.getMetricWithUpdateLock(
            conn,
            metricID,
            fields=[schema.metric.c.status,
                    schema.metric.c.last_rowid,
                    schema.metric.c.datasource])

          if (metricObj.status != MetricStatus.UNMONITORED and
              metricObj.status != MetricStatus.ACTIVE and
              metricObj.status != MetricStatus.PENDING_DATA and
              metricObj.status != MetricStatus.CREATE_PENDING):
            self._log.error("Can't stream: metric=%s has unexpected status=%s",
                            metricID, metricObj.status)
            modelInputRows = None
          else:
            # TODO: unit-test
            passingSamples = self._scrubDataSamples(data,
                                                    metricID,
                                                    conn,
                                                    metricObj.last_rowid)
            if passingSamples:
              modelInputRows = self._storeDataSamples(passingSamples, metricID,
                                                      conn)
            else:
              modelInputRows = tuple()

      return (modelInputRows, metricObj.datasource, metricObj.status)


    (modelInputRows,
     datasource,
     metricStatus) = storeDataWithRetries()

    if modelInputRows is None:
      # Metric was in state not suitable for streaming
      return

    if not modelInputRows:
      # TODO: unit-test
      # Nothing was added, so nothing further to do
      self._log.error("No records to stream to model=%s", metricID)
      return

    if metricStatus == MetricStatus.UNMONITORED:
      # Metric was not monitored during storage, so we're done
      #self._log.info("Status of metric=%s is UNMONITORED; not forwarding "
      #               "%d rows: rowids[%s..%s]; data=[%s..%s]",
      #               metricID, len(modelInputRows),
      #               modelInputRows[0].rowID, modelInputRows[-1].rowID,
      #               modelInputRows[0].data, modelInputRows[-1].data)
      return

    lastDataRowID = modelInputRows[-1].rowID

    # Check models that are waiting for activation upon sufficient data
    if metricStatus == MetricStatus.PENDING_DATA:
      if lastDataRowID >= MODEL_CREATION_RECORD_THRESHOLD:
        try:
          # Activate metric that is supported by Datasource Adapter
          createDatasourceAdapter(datasource).activateModel(metricID)
        except (MetricStatisticsNotReadyError,
                MetricStatusChangedError) as ex:
          # Perhaps the metric status changed in the meantime. We can just
          # ignore this and it will sort itself out if additional records come
          # in (e.g., HTM Metric).
          self._log.error("Couldn't start model=%s: %r", metricID, ex)
      return

    # Stream data if model is activated
    # TODO: unit-test
    if metricStatus in (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE):
      self._sendInputRowsToModel(
        inputRows=modelInputRows,
        metricID=metricID,
        modelSwapper=modelSwapper)

      self._log.debug("Streamed numRecords=%d to model=%s",
                      len(modelInputRows), metricID)
Beispiel #26
0
 def _exportNativeMetric(metric):
     return createDatasourceAdapter(metric.datasource).exportModel(
         metric.uid)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""Create the cpu percent model.  See also send_cpu.py and README.md."""

from htmengine.adapters.datasource import createDatasourceAdapter


modelSpec = {
    "datasource": "custom",
    "metricSpec": {"metric": "cpu_percent"},
    "modelParams": {"min": 0, "max": 100},  # optional  # optional
}

adapter = createDatasourceAdapter(modelSpec["datasource"])
modelId = adapter.monitorMetric(modelSpec)

print "Model", modelId, "created..."