def createAutostackDatasourceAdapter():
""" Convenience function for creating a datasource adapter for Grok Autostack
Metrics
:returns: datasource adapter for Grok Autostack Metrics
:rtype: grok.app.adapters.datasource.cloudwatch._AutostackDatasourceAdapter
"""
return createDatasourceAdapter("autostack")
def createCloudwatchDatasourceAdapter():
""" Convenience function for creating a datasource adapter for Cloudwatch
Metrics
:returns: datasource adapter for Cloudwatch Metrics
:rtype: grok.app.adapters.datasource.cloudwatch._CloudwatchDatasourceAdapter
"""
return createDatasourceAdapter("cloudwatch")
def createAutostackDatasourceAdapter():
""" Convenience function for creating a datasource adapter for HTM-IT Autostack
Metrics
:returns: datasource adapter for HTM-IT Autostack Metrics
:rtype: htm.it.app.adapters.datasource.cloudwatch._AutostackDatasourceAdapter
"""
return createDatasourceAdapter("autostack")
def createCloudwatchDatasourceAdapter():
""" Convenience function for creating a datasource adapter for Cloudwatch
Metrics
:returns: datasource adapter for Cloudwatch Metrics
:rtype: htm.it.app.adapters.datasource.cloudwatch._CloudwatchDatasourceAdapter
"""
return createDatasourceAdapter("cloudwatch")
def deleteModel(metricId):
try:
with web.ctx.connFactory() as conn:
metricRow = repository.getMetric(conn, metricId)
except app_exceptions.ObjectNotFoundError:
raise web.notfound("ObjectNotFoundError Metric not found: Metric ID: %s"
% metricId)
log.debug("Deleting model for %s metric=%s", metricRow.datasource,
metricId)
# NOTE: this is the new way using datasource adapters
try:
createDatasourceAdapter(metricRow.datasource).unmonitorMetric(metricId)
except app_exceptions.ObjectNotFoundError:
raise web.notfound(
"ObjectNotFoundError Metric not found: Metric ID: %s" % (metricId,))
return utils.jsonEncode({'result': 'success'})
def DELETE(self, metricName):
adapter = createDatasourceAdapter("custom")
try:
adapter.deleteMetricByName(metricName)
except app_exceptions.ObjectNotFoundError:
raise web.notfound("Metric not found. Metric name=%s" % (metricName,))
self.addStandardHeaders()
return json.dumps({"result": "success"})
def DELETE(self, metricName):
adapter = createDatasourceAdapter("custom")
try:
adapter.deleteMetricByName(metricName)
except app_exceptions.ObjectNotFoundError:
raise web.notfound("Metric not found. Metric name=%s" % (metricName,))
self.addStandardHeaders()
return json.dumps({'result': 'success'})
def deleteModel(metricId):
try:
with web.ctx.connFactory() as conn:
metricRow = repository.getMetric(conn, metricId)
except app_exceptions.ObjectNotFoundError:
raise web.notfound("ObjectNotFoundError Metric not found: Metric ID: %s"
% metricId)
log.debug("Deleting model for %s metric=%s", metricRow.datasource,
metricId)
# NOTE: this is the new way using datasource adapters
try:
createDatasourceAdapter(metricRow.datasource).unmonitorMetric(metricId)
except app_exceptions.ObjectNotFoundError:
raise web.notfound(
"ObjectNotFoundError Metric not found: Metric ID: %s" % (metricId,))
return utils.jsonEncode({'result': 'success'})
def _createModel(self, nativeMetric):
adapter = createDatasourceAdapter("custom")
try:
metricId = adapter.monitorMetric(nativeMetric)
except MetricAlreadyMonitored as e:
metricId = e.uid
engine = repository.engineFactory(config=self.config)
with engine.begin() as conn:
return repository.getMetric(conn, metricId)
def _createModel(self, nativeMetric):
adapter = createDatasourceAdapter("custom")
try:
metricId = adapter.monitorMetric(nativeMetric)
except MetricAlreadyMonitored as e:
metricId = e.uid
engine = repository.engineFactory(config=self.config)
with engine.begin() as conn:
return repository.getMetric(conn, metricId)
def testExport(self):
"""Tests exporting custom metrics."""
metricName = "testBatchSend.%i" % int(time.time())
LOGGER.info("Running test with metric name: %s", metricName)
self.addCleanup(self._deleteMetric, metricName)
# Get the timestamp for 14 days ago (the limit for exporting data)
baseTimestamp = int(time.time()) - (60 * 60 * 24 * 14)
ts1 = baseTimestamp - (60 * 5)
dt1 = datetime.datetime.utcfromtimestamp(ts1).strftime("%Y-%m-%d %H:%M:%S")
ts2 = baseTimestamp + (60 * 5)
dt2 = datetime.datetime.utcfromtimestamp(ts2).strftime("%Y-%m-%d %H:%M:%S")
ts3 = baseTimestamp + (60 * 10)
dt3 = datetime.datetime.utcfromtimestamp(ts3).strftime("%Y-%m-%d %H:%M:%S")
# Add custom metric data
sock = socket.socket()
sock.connect(("localhost", self.plaintextPort))
sock.sendall("%s 5.0 %i\n" % (metricName, ts1))
sock.sendall("%s 6.0 %i\n" % (metricName, ts2))
sock.sendall("%s 7.0 %i\n" % (metricName, ts3))
self.gracefullyCloseSocket(sock)
uid = self.checkMetricCreated(metricName)
# Save the uid for later
LOGGER.info("Metric %s has uid: %s", metricName, uid)
# Send model creation request
nativeMetric = {"datasource": "custom",
"metricSpec": {"uid": uid},
"modelParams": {"min": 0.0,
"max": 100.0}}
model = self._createModel(nativeMetric)
self.assertEqual(model.uid, uid)
self.assertEqual(model.name, metricName)
self.assertEqual(model.server, metricName)
self.checkModelResults(uid, [[dt3, 7.0, 0.0, 3], [dt2, 6.0, 0.0, 2],
[dt1, 5.0, 0.0, 1]])
exported = createDatasourceAdapter("custom").exportModel(uid)
self.assertEqual(exported["metricSpec"]["metric"], metricName)
self.assertEqual(exported["datasource"], "custom")
self.assertEqual(len(exported["data"]), 2)
self.assertSequenceEqual(exported["data"][0],
[6.0, datetime.datetime.utcfromtimestamp(ts2)])
self.assertSequenceEqual(exported["data"][1],
[7.0, datetime.datetime.utcfromtimestamp(ts3)])
def testExport(self):
"""Tests exporting custom metrics."""
metricName = "testBatchSend.%i" % int(time.time())
LOGGER.info("Running test with metric name: %s", metricName)
self.addCleanup(self._deleteMetric, metricName)
# Get the timestamp for 14 days ago (the limit for exporting data)
baseTimestamp = int(time.time()) - (60 * 60 * 24 * 14)
ts1 = baseTimestamp - (60 * 5)
dt1 = datetime.datetime.utcfromtimestamp(ts1).strftime("%Y-%m-%d %H:%M:%S")
ts2 = baseTimestamp + (60 * 5)
dt2 = datetime.datetime.utcfromtimestamp(ts2).strftime("%Y-%m-%d %H:%M:%S")
ts3 = baseTimestamp + (60 * 10)
dt3 = datetime.datetime.utcfromtimestamp(ts3).strftime("%Y-%m-%d %H:%M:%S")
# Add custom metric data
sock = socket.socket()
sock.connect(("localhost", self.plaintextPort))
sock.sendall("%s 5.0 %i\n" % (metricName, ts1))
sock.sendall("%s 6.0 %i\n" % (metricName, ts2))
sock.sendall("%s 7.0 %i\n" % (metricName, ts3))
self.gracefullyCloseSocket(sock)
uid = self.checkMetricCreated(metricName)
# Save the uid for later
LOGGER.info("Metric %s has uid: %s", metricName, uid)
# Send model creation request
nativeMetric = {"datasource": "custom",
"metricSpec": {"uid": uid},
"modelParams": {"min": 0.0,
"max": 100.0}}
model = self._createModel(nativeMetric)
self.assertEqual(model.uid, uid)
self.assertEqual(model.name, metricName)
self.assertEqual(model.server, metricName)
self.checkModelResults(uid, [[dt3, 7.0, 0.0, 3], [dt2, 6.0, 0.0, 2],
[dt1, 5.0, 0.0, 1]])
exported = createDatasourceAdapter("custom").exportModel(uid)
self.assertEqual(exported["metricSpec"]["metric"], metricName)
self.assertEqual(exported["datasource"], "custom")
self.assertEqual(len(exported["data"]), 2)
self.assertSequenceEqual(exported["data"][0],
[6.0, datetime.datetime.utcfromtimestamp(ts2)])
self.assertSequenceEqual(exported["data"][1],
[7.0, datetime.datetime.utcfromtimestamp(ts3)])
def createModel(cls, modelSpec=None):
"""
NOTE MER-3479: this code path is presently incorrectly used for two
purposes:
* Importing of all types of metrics (not desirable; there should be a
separate endpoint or an import-specific flag in this endpoint for
importing that facilitates slightly different behavior, such as
suppressing certain errors to allow for re-import in case of tranisent
error part way through the prior import)
"""
if not modelSpec:
# Metric data is missing
log.error(
"Data is missing in request, raising BadRequest exception")
raise InvalidRequestResponse({"result": "Metric data is missing"})
# TODO MER-3479: import using import-specific endpoint
# NOTE: pending MER-3479, this is presently a hack for exercising
# the adapter import API
importing = False
if modelSpec.get("datasource") == "custom":
if "data" in modelSpec:
importing = True
try:
adapter = createDatasourceAdapter(modelSpec["datasource"])
try:
# TODO: Maybe initialize transaction and commit here
if importing:
# TODO MER-3479: import using import-specific endpoint
# NOTE: pending MER-3479, this is presently a hack for exercising
# the adapter import API
metricId = adapter.importModel(modelSpec)
else:
metricId = adapter.monitorMetric(modelSpec)
except app_exceptions.MetricAlreadyMonitored as e:
metricId = e.uid
with web.ctx.connFactory() as conn:
return repository.getMetric(conn, metricId)
except (ValueError, app_exceptions.MetricNotSupportedError) as e:
raise InvalidRequestResponse({"result": repr(e)})
def createModel(cls, modelSpec=None):
"""
NOTE MER-3479: this code path is presently incorrectly used for two
purposes:
* Importing of all types of metrics (not desirable; there should be a
separate endpoint or an import-specific flag in this endpoint for
importing that facilitates slightly different behavior, such as
suppressing certain errors to allow for re-import in case of tranisent
error part way through the prior import)
"""
if not modelSpec:
# Metric data is missing
log.error("Data is missing in request, raising BadRequest exception")
raise InvalidRequestResponse({"result": "Metric data is missing"})
# TODO MER-3479: import using import-specific endpoint
# NOTE: pending MER-3479, this is presently a hack for exercising
# the adapter import API
importing = False
if modelSpec.get("datasource") == "custom":
if "data" in modelSpec:
importing = True
try:
adapter = createDatasourceAdapter(modelSpec["datasource"])
try:
# TODO: Maybe initialize transaction and commit here
if importing:
# TODO MER-3479: import using import-specific endpoint
# NOTE: pending MER-3479, this is presently a hack for exercising
# the adapter import API
metricId = adapter.importModel(modelSpec)
else:
metricId = adapter.monitorMetric(modelSpec)
except app_exceptions.MetricAlreadyMonitored as e:
metricId = e.uid
with web.ctx.connFactory() as conn:
return repository.getMetric(conn, metricId)
except (ValueError, app_exceptions.MetricNotSupportedError) as e:
raise InvalidRequestResponse({"result": repr(e)})
def _deleteMetric(self, metricName):
adapter = createDatasourceAdapter("custom")
adapter.deleteMetricByName(metricName)
def handler(environ, start_response):
metricName = environ["PATH_INFO"]
if environ["REQUEST_METHOD"] == "PUT":
# Trigger model creation...
modelSpec = {
"datasource": "custom",
"metricSpec": {
"metric": metricName
},
"modelParams": {}
}
try:
modelSpec["modelParams"].update(json.load(environ["wsgi.input"]))
except Exception as e:
start_response("400 Bad Request", [("Content-Type", "text/html")])
yield "Unable to parse request"
adapter = createDatasourceAdapter(modelSpec["datasource"])
try:
modelId = adapter.monitorMetric(modelSpec)
start_response("201 Created", [("Content-Type", "text/html")])
yield "Created %s\n" % modelId
except MetricAlreadyMonitored:
start_response("400 Bad Request", [("Content-Type", "text/html")])
yield "Model already exists for %s" % metricName
elif environ["REQUEST_METHOD"] == "POST":
# Send data...
start_response("200 OK", [("Content-Type", "text/html")])
for sample in environ["wsgi.input"]:
value, ts = sample.split(" ")
sendSample(bus,
metricName=metricName,
value=float(value),
epochTimestamp=int(ts))
yield "Saved %s %f @ %d\n" % (metricName, float(value), int(ts))
elif environ["REQUEST_METHOD"] == "GET":
with repository.engineFactory(appConfig).connect() as conn:
fields = (schema.metric_data.c.metric_value,
schema.metric_data.c.timestamp,
schema.metric_data.c.rowid,
schema.metric_data.c.anomaly_score)
sort = schema.metric_data.c.timestamp.asc()
metricObj = repository.getCustomMetricByName(
conn, metricName, fields=[schema.metric.c.uid])
result = repository.getMetricData(conn,
metricId=metricObj.uid,
fields=fields,
sort=sort)
start_response("200 OK", [("Content-Type", "text/html")])
for row in result:
yield " ".join(
(metricName, str(row.metric_value),
str(calendar.timegm(row.timestamp.timetuple())),
str(row.anomaly_score))) + "\n"
def _deleteModel(self, metricId):
adapter = createDatasourceAdapter("custom")
adapter.unmonitorMetric(metricId)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
"""Create the cpu percent model. See also send_cpu.py and README.md."""
from htmengine.adapters.datasource import createDatasourceAdapter
modelSpec = {
"datasource": "custom",
"metricSpec": {
"metric": "cpu_percent"
},
"modelParams": {
"min": 0, # optional
"max": 100 # optional
}
}
adapter = createDatasourceAdapter(modelSpec["datasource"])
modelId = adapter.monitorMetric(modelSpec)
print "Model", modelId, "created..."
#!/usr/bin/env python# ----------------------------------------------------------------------# Numenta Platform for Intelligent Computing (NuPIC)# Copyright (C) 2015, Numenta, Inc. Unless you have purchased from# Numenta, Inc. a separate commercial license for this software code, the# following terms and conditions apply:## This program is free software: you can redistribute it and/or modify# it under the terms of the GNU General Public License version 3 as# published by the Free Software Foundation.## This program is distributed in the hope that it will be useful,# but WITHOUT ANY WARRANTY; without even the implied warranty of# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.# See the GNU General Public License for more details.## You should have received a copy of the GNU General Public License# along with this program. If not, see http://www.gnu.org/licenses.## http://numenta.org/licenses/# ----------------------------------------------------------------------
"""Create the cpu percent model. See also send_cpu.py and README.md."""
from htmengine.adapters.datasource import createDatasourceAdapter
modelSpec = { "datasource": "custom", "metricSpec": { "metric": "cpu_percent" }, "modelParams": { "min": 0, # optional "max": 100 # optional }}
adapter = createDatasourceAdapter(modelSpec["datasource"])modelId = adapter.monitorMetric(modelSpec)
print "Model", modelId, "created..."
def _deleteModel(metricId):
adapter = createDatasourceAdapter("custom")
adapter.unmonitorMetric(metricId)
def _deleteMetric(metricName):
adapter = createDatasourceAdapter("custom")
adapter.deleteMetricByName(metricName)
def handler(environ, start_response):
metricName = environ["PATH_INFO"]
if environ["REQUEST_METHOD"] == "PUT":
# Trigger model creation...
modelSpec = {"datasource": "custom", "metricSpec": {"metric": metricName}, "modelParams": {}}
try:
modelSpec["modelParams"].update(json.load(environ["wsgi.input"]))
except Exception as e:
print e
start_response("400 Bad Request", [("Content-Type", "text/html")])
yield "Unable to parse request"
adapter = createDatasourceAdapter(modelSpec["datasource"])
try:
modelId = adapter.monitorMetric(modelSpec)
start_response("201 Created", [("Content-Type", "text/html")])
yield "Created %s\n" % modelId
except MetricAlreadyMonitored:
start_response("400 Bad Request", [("Content-Type", "text/html")])
yield "Model already exists for %s" % metricName
elif environ["REQUEST_METHOD"] == "POST":
# Send data...
start_response("200 OK", [("Content-Type", "text/html")])
for sample in environ["wsgi.input"]:
value, ts = sample.split(" ")
sendSample(bus, metricName=metricName, value=float(value), epochTimestamp=int(ts))
yield "Saved %s %f @ %d\n" % (metricName, float(value), int(ts))
elif environ["REQUEST_METHOD"] == "GET":
# parameters = parse_qs(environ.get('QUERY_STRING', ''))
# print parameters
# if 'since' in parameters:
# since = parameters['since'][0]
with repository.engineFactory(appConfig).connect() as conn:
fields = (
schema.metric_data.c.metric_value,
schema.metric_data.c.timestamp,
schema.metric_data.c.rowid,
schema.metric_data.c.anomaly_score,
)
sort = schema.metric_data.c.timestamp.asc()
metricObj = repository.getCustomMetricByName(conn, metricName, fields=[schema.metric.c.uid])
result = repository.getMetricData(conn, metricId=metricObj.uid, fields=fields, sort=sort)
start_response("200 OK", [("Content-Type", "text/html")])
for row in result:
yield " ".join(
(
metricName,
str(row.metric_value),
str(calendar.timegm(row.timestamp.timetuple())),
str(row.anomaly_score),
)
) + "\n"
def _exportNativeMetric(metric): return createDatasourceAdapter(metric.datasource).exportModel(metric.uid)
def streamMetricData(self, data, metricID, modelSwapper):
""" Store the data samples in metric_data table if needed, and stream the
data samples to the model associated with the metric if the metric is
monitored.
If the metric is in PENDING_DATA state: if there are now enough data samples
to start a model, start it and stream it the entire backlog of data samples;
if there are still not enough data samples, suppress streaming of the data
samples.
:param data: A sequence of data samples; each data sample is a three-tuple:
(datetime.datetime, float, None)
OR
(datetime.datetime, float, rowid)
The third item in each three-tuple is either None in all elements of the
sequence or is a valid rowid in all elements of the sequence. If it's
None, as is the case with metrics collected from AWS
CloudWatch, the row is added to the metric_data table before sending it
to the model. If rowid is not None, as is presently the case with HTM
metrics, the data samples are assumed to be stored already.
:param metricID: unique id of the HTM metric
:param modelSwapper: ModelSwapper object for sending data to models
:type modelSwapper: an instance of ModelSwapperInterface
:raises: ObjectNotFoundError when the metric for the data doesn't exist
"""
if not data:
self._log.warn("Empty input metric data batch for metric=%s",
metricID)
return
@repository.retryOnTransientErrors
def storeDataWithRetries():
"""
:returns: a three-tuple <modelInputRows, datasource, metricStatus>;
modelInputRows: None if model was in state not suitable for streaming;
otherwise a (possibly empty) tuple of ModelInputRow objects
corresponding to the samples that were stored; ordered by rowid
"""
with repository.engineFactory(config).connect() as conn:
with conn.begin():
# Syncrhonize with adapter's monitorMetric
metricObj = repository.getMetricWithUpdateLock(
conn,
metricID,
fields=[
schema.metric.c.status, schema.metric.c.last_rowid,
schema.metric.c.datasource
])
if (metricObj.status != MetricStatus.UNMONITORED
and metricObj.status != MetricStatus.ACTIVE
and metricObj.status != MetricStatus.PENDING_DATA
and
metricObj.status != MetricStatus.CREATE_PENDING):
self._log.error(
"Can't stream: metric=%s has unexpected status=%s",
metricID, metricObj.status)
modelInputRows = None
else:
# TODO: unit-test
passingSamples = self._scrubDataSamples(
data, metricID, conn, metricObj.last_rowid)
if passingSamples:
modelInputRows = self._storeDataSamples(
passingSamples, metricID, conn)
else:
modelInputRows = tuple()
return (modelInputRows, metricObj.datasource, metricObj.status)
(modelInputRows, datasource, metricStatus) = storeDataWithRetries()
if modelInputRows is None:
# Metric was in state not suitable for streaming
return
if not modelInputRows:
# TODO: unit-test
# Nothing was added, so nothing further to do
self._log.error("No records to stream to model=%s", metricID)
return
if metricStatus == MetricStatus.UNMONITORED:
# Metric was not monitored during storage, so we're done
#self._log.info("Status of metric=%s is UNMONITORED; not forwarding "
# "%d rows: rowids[%s..%s]; data=[%s..%s]",
# metricID, len(modelInputRows),
# modelInputRows[0].rowID, modelInputRows[-1].rowID,
# modelInputRows[0].data, modelInputRows[-1].data)
return
lastDataRowID = modelInputRows[-1].rowID
# Check models that are waiting for activation upon sufficient data
if metricStatus == MetricStatus.PENDING_DATA:
if lastDataRowID >= MODEL_CREATION_RECORD_THRESHOLD:
try:
# Activate metric that is supported by Datasource Adapter
createDatasourceAdapter(datasource).activateModel(metricID)
except (MetricStatisticsNotReadyError,
MetricStatusChangedError) as ex:
# Perhaps the metric status changed in the meantime. We can just
# ignore this and it will sort itself out if additional records come
# in (e.g., HTM Metric).
self._log.error("Couldn't start model=%s: %r", metricID,
ex)
return
# Stream data if model is activated
# TODO: unit-test
if metricStatus in (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE):
self._sendInputRowsToModel(inputRows=modelInputRows,
metricID=metricID,
modelSwapper=modelSwapper)
self._log.debug("Streamed numRecords=%d to model=%s",
len(modelInputRows), metricID)
def streamMetricData(self, data, metricID, modelSwapper):
""" Store the data samples in metric_data table if needed, and stream the
data samples to the model associated with the metric if the metric is
monitored.
If the metric is in PENDING_DATA state: if there are now enough data samples
to start a model, start it and stream it the entire backlog of data samples;
if there are still not enough data samples, suppress streaming of the data
samples.
:param data: A sequence of data samples; each data sample is a three-tuple:
(datetime.datetime, float, None)
OR
(datetime.datetime, float, rowid)
The third item in each three-tuple is either None in all elements of the
sequence or is a valid rowid in all elements of the sequence. If it's
None, as is the case with metrics collected from AWS
CloudWatch, the row is added to the metric_data table before sending it
to the model. If rowid is not None, as is presently the case with HTM
metrics, the data samples are assumed to be stored already.
:param metricID: unique id of the HTM metric
:param modelSwapper: ModelSwapper object for sending data to models
:type modelSwapper: an instance of ModelSwapperInterface
:raises: ObjectNotFoundError when the metric for the data doesn't exist
"""
if not data:
self._log.warn("Empty input metric data batch for metric=%s", metricID)
return
@repository.retryOnTransientErrors
def storeDataWithRetries():
"""
:returns: a three-tuple <modelInputRows, datasource, metricStatus>;
modelInputRows: None if model was in state not suitable for streaming;
otherwise a (possibly empty) tuple of ModelInputRow objects
corresponding to the samples that were stored; ordered by rowid
"""
with repository.engineFactory(config).connect() as conn:
with conn.begin():
# Synchronize with adapter's monitorMetric
metricObj = repository.getMetricWithUpdateLock(
conn,
metricID,
fields=[schema.metric.c.status,
schema.metric.c.last_rowid,
schema.metric.c.datasource])
if (metricObj.status != MetricStatus.UNMONITORED and
metricObj.status != MetricStatus.ACTIVE and
metricObj.status != MetricStatus.PENDING_DATA and
metricObj.status != MetricStatus.CREATE_PENDING):
self._log.error("Can't stream: metric=%s has unexpected status=%s",
metricID, metricObj.status)
modelInputRows = None
else:
# TODO: unit-test
passingSamples = self._scrubDataSamples(data,
metricID,
conn,
metricObj.last_rowid)
if passingSamples:
modelInputRows = self._storeDataSamples(passingSamples, metricID,
conn)
else:
modelInputRows = tuple()
return (modelInputRows, metricObj.datasource, metricObj.status)
(modelInputRows,
datasource,
metricStatus) = storeDataWithRetries()
if modelInputRows is None:
# Metric was in state not suitable for streaming
return
if not modelInputRows:
# TODO: unit-test
# Nothing was added, so nothing further to do
self._log.error("No records to stream to model=%s", metricID)
return
if metricStatus == MetricStatus.UNMONITORED:
# Metric was not monitored during storage, so we're done
#self._log.info("Status of metric=%s is UNMONITORED; not forwarding "
# "%d rows: rowids[%s..%s]; data=[%s..%s]",
# metricID, len(modelInputRows),
# modelInputRows[0].rowID, modelInputRows[-1].rowID,
# modelInputRows[0].data, modelInputRows[-1].data)
return
lastDataRowID = modelInputRows[-1].rowID
# Check models that are waiting for activation upon sufficient data
if metricStatus == MetricStatus.PENDING_DATA:
if lastDataRowID >= MODEL_CREATION_RECORD_THRESHOLD:
try:
# Activate metric that is supported by Datasource Adapter
createDatasourceAdapter(datasource).activateModel(metricID)
except (MetricStatisticsNotReadyError,
MetricStatusChangedError) as ex:
# Perhaps the metric status changed in the meantime. We can just
# ignore this and it will sort itself out if additional records come
# in (e.g., HTM Metric).
self._log.error("Couldn't start model=%s: %r", metricID, ex)
return
# Stream data if model is activated
# TODO: unit-test
if metricStatus in (MetricStatus.CREATE_PENDING, MetricStatus.ACTIVE):
self._sendInputRowsToModel(
inputRows=modelInputRows,
metricID=metricID,
modelSwapper=modelSwapper)
self._log.debug("Streamed numRecords=%d to model=%s",
len(modelInputRows), metricID)
def _exportNativeMetric(metric):
return createDatasourceAdapter(metric.datasource).exportModel(
metric.uid)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
"""Create the cpu percent model. See also send_cpu.py and README.md."""
from htmengine.adapters.datasource import createDatasourceAdapter
modelSpec = {
"datasource": "custom",
"metricSpec": {"metric": "cpu_percent"},
"modelParams": {"min": 0, "max": 100}, # optional # optional
}
adapter = createDatasourceAdapter(modelSpec["datasource"])
modelId = adapter.monitorMetric(modelSpec)
print "Model", modelId, "created..."
