def testImportModel(self):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
autostack = adapter.createAutostack(self.stackSpec)
modelSpec = self.getModelSpec("cloudwatch", "CPUUtilization", autostack)
modelId = adapter.monitorMetric(modelSpec)
spec = adapter.exportModel(modelId)
adapter.unmonitorMetric(modelId)
modelId = adapter.importModel(spec)
self.validateModel(modelId, modelSpec, autostack)
with self.engine.connect() as conn:
metrics = repository.getAutostackMetrics(conn, autostack.uid)
self.assertEqual(len([metricObj for metricObj in metrics]), 1)
# Ensure that import can create an autostack if it doesn't exist
repository.deleteAutostack(conn, autostack.uid)
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
modelId = adapter.importModel(spec)
newModelSpec = dict(modelSpec)
with self.engine.connect() as conn:
repository.getMetric(conn, modelId)
autostack = repository.getAutostackFromMetric(conn, modelId)
self.addCleanup(self._deleteAutostack, autostack.uid)
newModelSpec["metricSpec"]["autostackId"] = autostack.uid
self.validateModel(modelId, modelSpec, autostack)
def testImportModel(self):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
autostack = adapter.createAutostack(self.stackSpec)
modelSpec = self.getModelSpec("cloudwatch", "CPUUtilization",
autostack)
modelId = adapter.monitorMetric(modelSpec)
spec = adapter.exportModel(modelId)
adapter.unmonitorMetric(modelId)
modelId = adapter.importModel(spec)
self.validateModel(modelId, modelSpec, autostack)
with self.engine.connect() as conn:
metrics = repository.getAutostackMetrics(conn, autostack.uid)
self.assertEqual(len([metricObj for metricObj in metrics]), 1)
# Ensure that import can create an autostack if it doesn't exist
repository.deleteAutostack(conn, autostack.uid)
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
modelId = adapter.importModel(spec)
newModelSpec = dict(modelSpec)
with self.engine.connect() as conn:
repository.getMetric(conn, modelId)
autostack = repository.getAutostackFromMetric(conn, modelId)
self.addCleanup(self._deleteAutostack, autostack.uid)
newModelSpec["metricSpec"]["autostackId"] = autostack.uid
self.validateModel(modelId, modelSpec, autostack)
def testMonitorMetric(self):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
autostack = adapter.createAutostack(self.stackSpec)
self.addCleanup(self._deleteAutostack, autostack.uid)
modelSpec = self.getModelSpec("cloudwatch", "CPUUtilization", autostack)
modelId = adapter.monitorMetric(modelSpec)
self.validateModel(modelId, modelSpec, autostack)
modelSpec = self.getModelSpec("cloudwatch", "DiskReadBytes", autostack)
modelId = adapter.monitorMetric(modelSpec)
self.validateModel(modelId, modelSpec, autostack)
modelSpec = self.getModelSpec("cloudwatch", "DiskWriteBytes", autostack)
modelId = adapter.monitorMetric(modelSpec)
self.validateModel(modelId, modelSpec, autostack)
modelSpec = self.getModelSpec("cloudwatch", "NetworkIn", autostack)
modelId = adapter.monitorMetric(modelSpec)
self.validateModel(modelId, modelSpec, autostack)
modelSpec = self.getModelSpec("cloudwatch", "NetworkOut", autostack)
modelId = adapter.monitorMetric(modelSpec)
self.validateModel(modelId, modelSpec, autostack)
modelSpec = self.getModelSpec("autostack", "InstanceCount", autostack)
modelId = adapter.monitorMetric(modelSpec)
self.validateModel(modelId, modelSpec, autostack)
with self.engine.connect() as conn:
metrics = repository.getAutostackMetrics(conn, autostack.uid)
self.assertEqual(len([metricObj for metricObj in metrics]), 6)
def testMonitorMetric(self):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
autostack = adapter.createAutostack(self.stackSpec)
self.addCleanup(self._deleteAutostack, autostack.uid)
modelSpec = self.getModelSpec("cloudwatch", "CPUUtilization",
autostack)
modelId = adapter.monitorMetric(modelSpec)
self.validateModel(modelId, modelSpec, autostack)
modelSpec = self.getModelSpec("cloudwatch", "DiskReadBytes", autostack)
modelId = adapter.monitorMetric(modelSpec)
self.validateModel(modelId, modelSpec, autostack)
modelSpec = self.getModelSpec("cloudwatch", "DiskWriteBytes",
autostack)
modelId = adapter.monitorMetric(modelSpec)
self.validateModel(modelId, modelSpec, autostack)
modelSpec = self.getModelSpec("cloudwatch", "NetworkIn", autostack)
modelId = adapter.monitorMetric(modelSpec)
self.validateModel(modelId, modelSpec, autostack)
modelSpec = self.getModelSpec("cloudwatch", "NetworkOut", autostack)
modelId = adapter.monitorMetric(modelSpec)
self.validateModel(modelId, modelSpec, autostack)
modelSpec = self.getModelSpec("autostack", "InstanceCount", autostack)
modelId = adapter.monitorMetric(modelSpec)
self.validateModel(modelId, modelSpec, autostack)
with self.engine.connect() as conn:
metrics = repository.getAutostackMetrics(conn, autostack.uid)
self.assertEqual(len([metricObj for metricObj in metrics]), 6)
def testCreateAutostackDatasourceAdapter(self):
""" Make sure createAutostackDatasourceAdapter returns the expected adapter
"""
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
self.assertEqual(adapter._DATASOURCE, "autostack")
self.assertEqual(adapter.__class__.__name__,
"_AutostackDatasourceAdapter")
def testCreateAutostack(self):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
autostack = adapter.createAutostack(self.stackSpec)
self.addCleanup(self._deleteAutostack, autostack.uid)
self.assertIsNotNone(autostack)
self.assertEqual(autostack.name, self.stackSpec["name"])
self.assertEqual(autostack.region, self.stackSpec["aggSpec"]["region"])
self.assertEqual(htmengine.utils.jsonDecode(autostack.filters),
self.stackSpec["aggSpec"]["filters"])
def testUnmonitorMetric(self):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
autostack = adapter.createAutostack(self.stackSpec)
self.addCleanup(self._deleteAutostack, autostack.uid)
modelSpec = self.getModelSpec("cloudwatch", "CPUUtilization", autostack)
modelId = adapter.monitorMetric(modelSpec)
adapter.unmonitorMetric(modelId)
self.checkModelDeleted(modelId)
def testCreateAutostack(self):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
autostack = adapter.createAutostack(self.stackSpec)
self.addCleanup(self._deleteAutostack, autostack.uid)
self.assertIsNotNone(autostack)
self.assertEqual(autostack.name, self.stackSpec["name"])
self.assertEqual(autostack.region, self.stackSpec["aggSpec"]["region"])
self.assertEqual(htmengine.utils.jsonDecode(autostack.filters),
self.stackSpec["aggSpec"]["filters"])
def testUnmonitorMetric(self):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
autostack = adapter.createAutostack(self.stackSpec)
self.addCleanup(self._deleteAutostack, autostack.uid)
modelSpec = self.getModelSpec("cloudwatch", "CPUUtilization",
autostack)
modelId = adapter.monitorMetric(modelSpec)
adapter.unmonitorMetric(modelId)
self.checkModelDeleted(modelId)
def testMonitorMetricThatIsAlreadyMonitored(self):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
autostack = adapter.createAutostack(self.stackSpec)
self.addCleanup(self._deleteAutostack, autostack.uid)
modelSpec = self.getModelSpec("cloudwatch", "CPUUtilization", autostack)
modelId = adapter.monitorMetric(modelSpec)
with self.assertRaises(htm.it.app.exceptions.MetricAlreadyMonitored) as cm:
adapter.monitorMetric(modelSpec)
self.assertEqual(cm.exception.uid, modelId)
def testCreateAutostackNoName(self, _mockEngineFactory):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
stackSpec = {
"aggSpec": {
"datasource": "cloudwatch",
"region": "us-west-2",
"resourceType": "AWS::EC2::Instance",
"filters": {
"tag:Name": ["*a*"]
}
}
}
self.assertRaises(ValueError, adapter.createAutostack, stackSpec)
def testCreateAutostackNoName(self, _mockEngineFactory):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
stackSpec = {
"aggSpec": {
"datasource": "cloudwatch",
"region": "us-west-2",
"resourceType": "AWS::EC2::Instance",
"filters": {
"tag:Name":["*a*"]
}
}
}
self.assertRaises(ValueError, adapter.createAutostack, stackSpec)
def testMonitorMetricThatIsAlreadyMonitored(self):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
autostack = adapter.createAutostack(self.stackSpec)
self.addCleanup(self._deleteAutostack, autostack.uid)
modelSpec = self.getModelSpec("cloudwatch", "CPUUtilization",
autostack)
modelId = adapter.monitorMetric(modelSpec)
with self.assertRaises(
htm.it.app.exceptions.MetricAlreadyMonitored) as cm:
adapter.monitorMetric(modelSpec)
self.assertEqual(cm.exception.uid, modelId)
def testExportModel(self):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
autostack = adapter.createAutostack(self.stackSpec)
self.addCleanup(self._deleteAutostack, autostack.uid)
modelSpec = self.getModelSpec("cloudwatch", "CPUUtilization", autostack)
modelId = adapter.monitorMetric(modelSpec)
expectedSpec = {
"datasource": "autostack",
"stackSpec": self.stackSpec,
"modelSpec": modelSpec
}
del expectedSpec["modelSpec"]["metricSpec"]["autostackId"]
spec = adapter.exportModel(modelId)
self.assertEqual(spec, expectedSpec)
def testExportModel(self):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
autostack = adapter.createAutostack(self.stackSpec)
self.addCleanup(self._deleteAutostack, autostack.uid)
modelSpec = self.getModelSpec("cloudwatch", "CPUUtilization",
autostack)
modelId = adapter.monitorMetric(modelSpec)
expectedSpec = {
"datasource": "autostack",
"stackSpec": self.stackSpec,
"modelSpec": modelSpec
}
del expectedSpec["modelSpec"]["metricSpec"]["autostackId"]
spec = adapter.exportModel(modelId)
self.assertEqual(spec, expectedSpec)
def testMonitorMetricNoExistingAutostack(self, getAutostackMock,
_mockEngineFactory):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
modelSpec = {
"datasource": "autostack",
"metricSpec": {
"autostackId": "9y2wn39y823nw9y8",
"slaveDatasource": "cloudwatch",
"slaveMetric": {
"namespace": "AWS/EC2",
"metric": "CPUUtilization",
"dimensions": {
"InstanceId": None
},
"period": 300
}
}
}
getAutostackMock.side_effect = ObjectNotFoundError()
self.assertRaises(ObjectNotFoundError, adapter.monitorMetric, modelSpec)
def testMonitorMetricNoExistingAutostack(self, getAutostackMock,
_mockEngineFactory):
adapter = datasource_adapter_factory.createAutostackDatasourceAdapter()
modelSpec = {
"datasource": "autostack",
"metricSpec": {
"autostackId": "9y2wn39y823nw9y8",
"slaveDatasource": "cloudwatch",
"slaveMetric": {
"namespace": "AWS/EC2",
"metric": "CPUUtilization",
"dimensions": {
"InstanceId": None
},
"period": 300
}
}
}
getAutostackMock.side_effect = ObjectNotFoundError()
self.assertRaises(ObjectNotFoundError, adapter.monitorMetric,
modelSpec)
def POST(self, autostackId, data=None): # pylint: disable=C0103,R0201
"""
Create one or more Autostack Metric(s)
::
POST /_autostacks/{autostackId}/metrics
[
{
"namespace": "AWS/EC2",
"metric": "CPUUtilization"
},
...
]
Request body is a list of items, each of which are a subset of the
standard cloudwatch native metric, specifying only:
:param namespace: AWS Namespace
:type namespace: str
:param metric: AWS Metric name
:type str:
`datasource`, `region`, and `dimensions` normally required when creating
models are not necessary.
"""
try:
self.addStandardHeaders()
with web.ctx.connFactory() as conn:
autostackRow = repository.getAutostack(conn,
autostackId)
data = data or utils.jsonDecode(web.data())
for nativeMetric in data:
try:
if nativeMetric["namespace"] == "Autostacks":
slaveDatasource = "autostack"
else:
slaveDatasource = "cloudwatch" # only support cloudwatch for now
modelParams = {}
if "min" and "max" in nativeMetric:
modelParams["min"] = nativeMetric["min"]
modelParams["max"] = nativeMetric["max"]
modelSpec = {
"datasource": "autostack",
"metricSpec": {
"autostackId": autostackRow.uid,
"slaveDatasource": slaveDatasource,
"slaveMetric": nativeMetric
},
"modelParams": modelParams
}
metricId = (createAutostackDatasourceAdapter()
.monitorMetric(modelSpec))
with web.ctx.connFactory() as conn:
metricRow = repository.getMetric(conn, metricId)
metricDict = convertMetricRowToMetricDict(metricRow)
except KeyError:
raise web.badrequest("Missing details in request")
except ValueError:
response = {"result": "failure"}
raise web.badrequest(utils.jsonEncode(response))
response = {"result": "success", "metric": metricDict}
raise web.created(utils.jsonEncode(response))
except ObjectNotFoundError:
raise web.notfound("Autostack not found: Autostack ID: %s" % autostackId)
except (web.HTTPError) as ex:
if bool(re.match(r"([45][0-9][0-9])\s?", web.ctx.status)):
# Log 400-599 status codes as errors, ignoring 200-399
log.error(str(ex) or repr(ex))
raise
except Exception as ex:
log.exception("POST Failed")
raise web.internalerror(str(ex) or repr(ex))
def testCreateAutostackDatasourceAdapter(self): """ Make sure createAutostackDatasourceAdapter returns the expected adapter """ adapter = datasource_adapter_factory.createAutostackDatasourceAdapter() self.assertEqual(adapter._DATASOURCE, "autostack") self.assertEqual(adapter.__class__.__name__, "_AutostackDatasourceAdapter")
def POST(self): # pylint: disable=C0103
r"""
Create an Autostack
::
POST /_autostacks
{
"name": {name},
"region": {region},
"filters": {
"tag:{Name}": ["{value}", "{value}", ...],
"tag:{Description}": ["{value}", "{value}", ...],
"tag:{etc}": ["{value}", "{value}", ...]
}
}
Request body must be a dictionary that includes:
:param name: Unique autostack name
:type name: str
:param region: AWS region
:type region: str
:param filters: AWS Tag value pattern
:type filters: dict
The creation request will be rejected if the filters match more than
MAX_INSTANCES_PER_AUTOSTACK.
From http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html:
::
You can also use wildcards with the filter values. An asterisk (*)
matches zero or more characters, and a question mark (?) matches
exactly one character. For example, you can use *database* as a filter
value to get all EBS snapshots that include database in the
description. If you were to specify database as the filter value, then
only snapshots whose description equals database would be returned.
Filter values are case sensitive. We support only exact string
matching, or substring matching (with wildcards).
Tip
Your search can include the literal values of the wildcard
characters; you just need to escape them with a backslash before the
character. For example, a value of \*numenta\?\\ searches for the
literal string *numenta?\.
"""
try:
self.addStandardHeaders()
data = web.data()
if not data:
raise web.badrequest("Metric data is missing")
nativeMetric = utils.jsonDecode(data)
try:
stackSpec = {
"name": nativeMetric["name"],
"aggSpec": {
"datasource":
"cloudwatch", # only support cloudwatch for now
"region": nativeMetric["region"],
"resourceType":
"AWS::EC2::Instance", # only support EC2 for now
"filters": nativeMetric["filters"]
}
}
adapter = createAutostackDatasourceAdapter()
with web.ctx.connFactory() as conn:
checkQuotaForInstanceAndRaise(conn, None)
autostack = adapter.createAutostack(stackSpec)
result = dict(autostack.items())
except DuplicateRecordError:
# TODO [MER-3543]: Make sure this actually gets hit
raise web.internalerror(
"The name you are trying to use, '%s', is already in use in AWS "
"region '%s'. Please enter a unique Autostack name." %
(nativeMetric.get(
"name", "None"), nativeMetric.get("region", "None")))
raise web.created(utils.jsonEncode(result))
except (web.HTTPError, QuotaError) as ex:
if bool(re.match(r"([45][0-9][0-9])\s?", web.ctx.status)):
# Log 400-599 status codes as errors, ignoring 200-399
log.error(str(ex) or repr(ex))
raise
except Exception as ex:
log.exception("POST Failed")
raise web.internalerror(str(ex) or repr(ex))
def POST(self, autostackId, data=None): # pylint: disable=C0103,R0201
"""
Create one or more Autostack Metric(s)
::
POST /_autostacks/{autostackId}/metrics
[
{
"namespace": "AWS/EC2",
"metric": "CPUUtilization"
},
...
]
Request body is a list of items, each of which are a subset of the
standard cloudwatch native metric, specifying only:
:param namespace: AWS Namespace
:type namespace: str
:param metric: AWS Metric name
:type str:
`datasource`, `region`, and `dimensions` normally required when creating
models are not necessary.
"""
try:
self.addStandardHeaders()
with web.ctx.connFactory() as conn:
autostackRow = repository.getAutostack(conn, autostackId)
data = data or utils.jsonDecode(web.data())
for nativeMetric in data:
try:
if nativeMetric["namespace"] == "Autostacks":
slaveDatasource = "autostack"
else:
slaveDatasource = "cloudwatch" # only support cloudwatch for now
modelParams = {}
if "min" and "max" in nativeMetric:
modelParams["min"] = nativeMetric["min"]
modelParams["max"] = nativeMetric["max"]
modelSpec = {
"datasource": "autostack",
"metricSpec": {
"autostackId": autostackRow.uid,
"slaveDatasource": slaveDatasource,
"slaveMetric": nativeMetric
},
"modelParams": modelParams
}
metricId = (createAutostackDatasourceAdapter().
monitorMetric(modelSpec))
with web.ctx.connFactory() as conn:
metricRow = repository.getMetric(conn, metricId)
metricDict = convertMetricRowToMetricDict(metricRow)
except KeyError:
raise web.badrequest("Missing details in request")
except ValueError:
response = {"result": "failure"}
raise web.badrequest(utils.jsonEncode(response))
response = {"result": "success", "metric": metricDict}
raise web.created(utils.jsonEncode(response))
except ObjectNotFoundError:
raise web.notfound("Autostack not found: Autostack ID: %s" %
autostackId)
except (web.HTTPError) as ex:
if bool(re.match(r"([45][0-9][0-9])\s?", web.ctx.status)):
# Log 400-599 status codes as errors, ignoring 200-399
log.error(str(ex) or repr(ex))
raise
except Exception as ex:
log.exception("POST Failed")
raise web.internalerror(str(ex) or repr(ex))
def POST(self): # pylint: disable=C0103
r"""
Create an Autostack
::
POST /_autostacks
{
"name": {name},
"region": {region},
"filters": {
"tag:{Name}": ["{value}", "{value}", ...],
"tag:{Description}": ["{value}", "{value}", ...],
"tag:{etc}": ["{value}", "{value}", ...]
}
}
Request body must be a dictionary that includes:
:param name: Unique autostack name
:type name: str
:param region: AWS region
:type region: str
:param filters: AWS Tag value pattern
:type filters: dict
The creation request will be rejected if the filters match more than
MAX_INSTANCES_PER_AUTOSTACK.
From http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Filtering.html:
::
You can also use wildcards with the filter values. An asterisk (*)
matches zero or more characters, and a question mark (?) matches
exactly one character. For example, you can use *database* as a filter
value to get all EBS snapshots that include database in the
description. If you were to specify database as the filter value, then
only snapshots whose description equals database would be returned.
Filter values are case sensitive. We support only exact string
matching, or substring matching (with wildcards).
Tip
Your search can include the literal values of the wildcard
characters; you just need to escape them with a backslash before the
character. For example, a value of \*numenta\?\\ searches for the
literal string *numenta?\.
"""
try:
self.addStandardHeaders()
data = web.data()
if not data:
raise web.badrequest("Metric data is missing")
nativeMetric = utils.jsonDecode(data)
try:
stackSpec = {
"name": nativeMetric["name"],
"aggSpec": {
"datasource": "cloudwatch", # only support cloudwatch for now
"region": nativeMetric["region"],
"resourceType": "AWS::EC2::Instance", # only support EC2 for now
"filters": nativeMetric["filters"]
}
}
adapter = createAutostackDatasourceAdapter()
with web.ctx.connFactory() as conn:
checkQuotaForInstanceAndRaise(conn, None)
autostack = adapter.createAutostack(stackSpec)
result = dict(autostack.items())
except DuplicateRecordError:
# TODO [MER-3543]: Make sure this actually gets hit
raise web.internalerror(
"The name you are trying to use, '%s', is already in use in AWS "
"region '%s'. Please enter a unique Autostack name." % (
nativeMetric.get("name", "None"),
nativeMetric.get("region", "None")))
raise web.created(utils.jsonEncode(result))
except (web.HTTPError, QuotaError) as ex:
if bool(re.match(r"([45][0-9][0-9])\s?", web.ctx.status)):
# Log 400-599 status codes as errors, ignoring 200-399
log.error(str(ex) or repr(ex))
raise
except Exception as ex:
log.exception("POST Failed")
raise web.internalerror(str(ex) or repr(ex))