def testMetricDataQueryParams(uid):
'''
This test makes MetricDataHandler GET calls with various params :
_models/<uid>/data?from=<>&to=<>&anomaly=<>
'''
with repository.engineFactory().connect() as conn:
firstMetricData = conn.execute(
"SELECT * FROM `metric_data` WHERE `uid`='%s' "
"and abs(`anomaly_score` - 0) > 1e-5 "
"ORDER BY `timestamp` ASC LIMIT 1" % uid).fetchall()
lastMetricData = conn.execute(
"SELECT * FROM `metric_data` WHERE `uid`='%s' "
"and abs(`anomaly_score` - 0) > 1e-5 "
"ORDER BY `timestamp` DESC LIMIT 1" % uid).fetchall()
firstTimeStamp = firstMetricData[0].timestamp
lastTimeStamp = lastMetricData[0].timestamp
anomalyScore = firstMetricData[0].anomaly_score
response = self.app.get("/%s/data?from=%s&to=%s&anomaly=%s"
% (uid, firstTimeStamp, lastTimeStamp, anomalyScore),
headers=self.headers)
assertions.assertSuccess(self, response)
getAllModelsResult = utils.jsonDecode(response.body)
for metricData in getAllModelsResult['data']:
self.assertGreaterEqual(metricData[2], anomalyScore)
self.assertGreaterEqual(datetime.strptime(metricData[0],
'%Y-%m-%d %H:%M:%S'), firstTimeStamp)
self.assertLessEqual(datetime.strptime(metricData[0],
'%Y-%m-%d %H:%M:%S'), lastTimeStamp)
def testGETSpecificInstanceFromRegion(self):
"""
Test for Get
'/_metrics/cloudwatch/<region-name>/AWS/<namespace>/instances/<InstancdId>'
response is validated for appropriate headers, body and status
Test is currently using ec2 box for jenkins-master, this test also
validates for retriving all supported metrics with dimensions
"""
supportedMetrics = (
createCloudwatchDatasourceAdapter().describeSupportedMetrics())
ec2Metrics = supportedMetrics[ResourceTypeNames.EC2_INSTANCE].keys()
# Instance used for following test is jenkins-master node
response = self.app.get("/us-west-2/AWS/EC2/instances/%s"
% VALID_EC2_INSTANCE["InstanceId"], headers=self.headers)
assertions.assertSuccess(self, response)
result = app_utils.jsonDecode(response.body)
self.assertIsInstance(result, list)
self.assertGreater(len(ec2Metrics), 0)
self.assertGreater(len(result), 0)
self.assertEqual(len(ec2Metrics), len(result))
for res in result:
self.assertEqual(res["region"], "us-west-2")
self.assertEqual(res["namespace"], "AWS/EC2")
self.assertEqual(res["datasource"], "cloudwatch")
self.assertIn(res["metric"], ec2Metrics)
self.assertIsInstance(res["dimensions"], dict)
self.assertEqual(res["dimensions"]["InstanceId"],
VALID_EC2_INSTANCE["InstanceId"])
ec2Metrics.remove(res["metric"])
self.assertEqual(ec2Metrics, [])
def testMetricDataTimeStampQueryParams(uid):
'''
This test makes MetricDataHandler GET calls with from and to params :
_models/<uid>/data?from=<>&to=<>
'''
with repository.engineFactory().connect() as conn:
firstMetricData = conn.execute(
sql.select([schema.metric_data])
.where(schema.metric_data.c.uid == uid)
.order_by(sql.expression.asc(schema.metric_data.c.timestamp))
.limit(1)).fetchall()
lastMetricData = conn.execute(
sql.select([schema.metric_data])
.where(schema.metric_data.c.uid == uid)
.order_by(sql.expression.desc(schema.metric_data.c.timestamp))
.limit(1)).fetchall()
firstTimeStamp = firstMetricData[0].timestamp
lastTimeStamp = lastMetricData[0].timestamp
response = self.app.get("/%s/data?from=%s&to=%s"
% (uid, firstTimeStamp, lastTimeStamp), headers=self.headers)
assertions.assertSuccess(self, response)
getAllModelsResult = utils.jsonDecode(response.body)
for metricData in getAllModelsResult['data']:
self.assertGreaterEqual(datetime.strptime(metricData[0],
'%Y-%m-%d %H:%M:%S'), firstTimeStamp)
self.assertLessEqual(datetime.strptime(metricData[0],
'%Y-%m-%d %H:%M:%S'), lastTimeStamp)
def testMetricDataForRandomRowID(uid):
'''
This tests if the metric data returned by the GET call :
_models/<uid>/data
has anomaly_score consistent with what is there in the actual
database by asserting it against a dao.MetricData.get() call
It repeats the process for 5 random sample rows for each uid
in the database.
Algorithm :
- Query the MetricDataHandler GET call for a certain uid
- Check if response is OK
- Find the last row id for the uid
- Select a random row between 1 and last row id
- Find the anomaly score for that row id
- Assert on the anomaly score
'''
response = self.app.get("/%s/data" %uid, headers=self.headers)
assertions.assertSuccess(self, response)
getAllModelsResult = utils.jsonDecode(response.body)
with repository.engineFactory().connect() as conn:
lastRowID = repository.getMetric(conn, uid).last_rowid
for _ in range(5):
randomRowID = randrange(1, lastRowID)
with repository.engineFactory().connect() as conn:
singleMetricData = repository.getMetricData(
conn,
uid,
rowid=randomRowID).first()
metricData = getMetricDataWithRowID(getAllModelsResult['data'],
randomRowID)
self.assertEqual(metricData[2], singleMetricData.anomaly_score)
self.assertEqual(datetime.strptime(metricData[0],
'%Y-%m-%d %H:%M:%S'), singleMetricData.timestamp)
def testGETListRegions(self):
"""
Test for Get '/_metrics/cloudwatch/regions'
response is validated for appropriate headers, body and status
response is validated against json for supported regions
"""
response = self.app.get("/regions", headers=self.headers)
assertions.assertSuccess(self, response)
result = app_utils.jsonDecode(response.body)
self.assertIsInstance(result, dict)
self.assertEqual(json.loads(response.body), self.regions)
def testGETListInstancesForRegion(self):
"""
Test for
Get '/_metrics/cloudwatch/<region-name>/AWS/<namespace>/instances'
response is validated for appropriate headers, body and status
response is validated against response
"""
response = self.app.get("/us-west-2/AWS/EC2",
headers=self.headers)
assertions.assertSuccess(self, response)
result = app_utils.jsonDecode(response.body)
self.assertIsInstance(result, list)
def testGETSpecificNamespace(self):
"""
Test for Get '/_metrics/cloudwatch/AWS/<namespace>'
response is validated for appropriate headers, body and status
response is validated against available metrics for each
namespaces supported
"""
for namespace in self._supportedAWSNamespaces():
response = self.app.get("/%s" % namespace, headers=self.headers)
assertions.assertSuccess(self, response)
result = app_utils.jsonDecode(response.body)
self.assertIsInstance(result, dict)
def testGETNamespaces(self):
"""
Test for Get '/_metrics/cloudwatch/namespaces'
response is validated for appropriate headers, body and status
response is validated against known namespaces
"""
response = self.app.get("/namespaces", headers=self.headers)
assertions.assertSuccess(self, response)
result = app_utils.jsonDecode(response.body)
self.assertIsInstance(result, dict)
supportedNamespaces = self._supportedAWSNamespaces() | set(["Autostacks"])
self.assertEqual(supportedNamespaces, set(result.keys()))
def testMetricDataAnomalyAsQueryParams(uid):
'''
This test makes MetricDataHandler GET calls with anomaly param :
_models/<uid>/data?anomaly=<>
'''
queryString = ("SELECT * FROM metric_data WHERE uid='%s' "
" and abs(anomaly_score - 0) > 1e-5 LIMIT 1") % uid
with repository.engineFactory().connect() as conn:
sampleMetricData = conn.execute(queryString).first()
anomalyScore = sampleMetricData.anomaly_score
response = self.app.get("/%s/data?anomaly=%s"
% (uid, anomalyScore), headers=self.headers)
assertions.assertSuccess(self, response)
getAllModelsResult = utils.jsonDecode(response.body)
for metricData in getAllModelsResult['data']:
self.assertGreaterEqual(metricData[2], anomalyScore)
def testGETInstanceForInvalidInstance(self):
"""
Test for Get
'/_metrics/cloudwatch/<region-name>/AWS/<namespace>/instances/<InstancdId>'
with invalid InstancceId
response is validated for appropriate headers, body and status
Expect a 200 OK even when attempting to GET from an invalid instance,
this saves the overhead of asking AWS if we're dealing with a valid
instance every GET.
We expect the CLI user to know what instance ID he/she is looking for.
"""
response = self.app.get("/us-west-2/AWS/EC2/instances/abcd1234",
status="*", headers=self.headers)
assertions.assertSuccess(self, response)
def testGETSpecificRegion(self):
"""
Test for Get '/_metrics/cloudwatch/regions/<region-name>'
response is validated for appropriate headers, body and status
response is validated against expected information in response
"""
response = self.app.get("/regions/us-west-2", headers=self.headers)
assertions.assertSuccess(self, response)
result = app_utils.jsonDecode(response.body)
self.assertIsInstance(result, list)
attributes = ["name", "region", "namespace", "datasource", "identifier", \
"metric", "dimensions"]
for res in result:
self.assertEqual(res["region"], "us-west-2")
self.assertEqual(res["datasource"], "cloudwatch")
self.assertItemsEqual(res.keys(), attributes)
def _testGETCloudWatchImpl(self, url):
response = self.app.get(url, headers=self.headers)
assertions.assertSuccess(self, response)
result = app_utils.jsonDecode(response.body)
self.assertIsInstance(result, dict)
supportedMetrics = createCloudwatchDatasourceAdapter().describeSupportedMetrics()
for metrics in supportedMetrics.values():
for metric, keys in metrics.items():
self.assertIn(keys["namespace"],
result["namespaces"],
"Expected namespace (%s) not found in response." % (
keys["namespace"]))
self.assertIn(metric,
result["namespaces"][keys["namespace"]]["metrics"],
"Expected metric (%s, %s) not found in response." % (
keys["namespace"], metric))
def testGETMetrics(self):
"""
Test for
Get '/_metrics/cloudwatch/<region-name>/AWS/<namespace>/metricName'
response is validated for appropriate headers, body and status
and response is validated with supported namespace and metric details
"""
response = self.app.get("/us-west-2/AWS/EC2/CPUUtilization",
headers=self.headers)
assertions.assertSuccess(self, response)
result = app_utils.jsonDecode(response.body)
self.assertIsInstance(result, list)
attributes = ["region", "namespace", "datasource", "metric", "dimensions"]
for res in result:
self.assertEqual(res["region"], "us-west-2")
self.assertEqual(res["datasource"], "cloudwatch")
self.assertEqual(res["metric"], "CPUUtilization")
self.assertItemsEqual(res.keys(), attributes)
self.assertIsInstance(res["dimensions"], dict)
def testModelInferencesLifeCycle(self):
startTime = time()
for model in sorted(self.data):
#create a model; post is forwarded to put
print "Creating metric for %s : " % model
response = self.app.put("/", json.dumps(model),
headers=self.headers)
assertions.assertSuccess(self, response, code=201)
response = self.app.get("/", headers=self.headers)
assertions.assertSuccess(self, response)
getAllModelsResult = utils.jsonDecode(response.body)
totalMetricCount = len(getAllModelsResult)
self.assertEqual(totalMetricCount, len(self.data))
#Get the uids of all the metrics created.
uids = [metric['uid'] for metric in getAllModelsResult]
while True:
with repository.engineFactory().connect() as conn:
initialModelCount = conn.execute(
sql.select([sql.func.count()], from_obj=schema.metric_data)
.where(schema.metric_data.c.rowid == 1)).scalar()
if initialModelCount == totalMetricCount:
print "Done creating all the initial models."
break
# Exit the test with some non-zero status if the test has run for more
# than 20 minutes to just create the initial models.
# Should not take more than that.
currentElapsedTime = (time() - startTime) / 60
print "Current elapsed time %s" % currentElapsedTime
if currentElapsedTime > 20:
print "More than 20 minutes has elapsed. Timing out."
sys.exit(42)
print "%s initial models created." % initialModelCount
print "Creating initial models for rest of the %s metrics" \
"..." % (totalMetricCount - initialModelCount)
sleep(60)
#Sleep for a long time.
minutes = 15
print "Sleeping for %s minutes to let things settled down." % minutes
while minutes > 0:
print "Resume in %s minutes." % minutes
minutes -= 1
sleep(60)
modelCreationDuration = (time() - startTime) / 60
with repository.engineFactory().connect() as conn:
lastRowIds = {uid: repository.getMetric(conn, uid).last_rowid
for uid in uids}
modelInferenceWithNonNullAnomalyScore = []
modelIds = lastRowIds.keys()
while True:
print set(modelInferenceWithNonNullAnomalyScore)
if len(modelIds) == len(set(modelInferenceWithNonNullAnomalyScore)):
print "Model inferences created for last_rowids for all the models."
break
for uid in modelIds:
with repository.engineFactory().connect() as conn:
anomalyNullCount = conn.execute(
sql.select([sql.func.count()], from_obj=schema.metric_data)
.where(schema.metric_data.c.rowid == lastRowIds[uid])
.where(schema.metric_data.c.uid == uid)
.where(schema.metric_data.c.anomaly_score == None)).scalar()
print "Model (%s) - Last Row ID (%s) : %s" \
% (uid, lastRowIds[uid], anomalyNullCount)
if anomalyNullCount == 0:
modelInferenceWithNonNullAnomalyScore.append(uid)
# Exit the test with some non-zero status if the test has run for more
# than 2 hours
currentElapsedTime = (time() - startTime) / 60
print "Current elapsed time %s" % currentElapsedTime
if currentElapsedTime > 120:
print "More than 2 hours has elapsed. Timing out."
sys.exit(42)
print "Going back to sleep for 60s..."
sleep(60)
self.assertEqual(anomalyNullCount, 0)
timeToCalculateAllInferences = time()
def getMetricDataWithRowID(metricDataList, rowid):
'''
Helper method to get the metric data of the nth row for a certain uid
'''
for metricData in metricDataList:
if metricData[3] == rowid:
return metricData
def testMetricDataForRandomRowID(uid):
'''
This tests if the metric data returned by the GET call :
_models/<uid>/data
has anomaly_score consistent with what is there in the actual
database by asserting it against a dao.MetricData.get() call
It repeats the process for 5 random sample rows for each uid
in the database.
Algorithm :
- Query the MetricDataHandler GET call for a certain uid
- Check if response is OK
- Find the last row id for the uid
- Select a random row between 1 and last row id
- Find the anomaly score for that row id
- Assert on the anomaly score
'''
response = self.app.get("/%s/data" %uid, headers=self.headers)
assertions.assertSuccess(self, response)
getAllModelsResult = utils.jsonDecode(response.body)
with repository.engineFactory().connect() as conn:
lastRowID = repository.getMetric(conn, uid).last_rowid
for _ in range(5):
randomRowID = randrange(1, lastRowID)
with repository.engineFactory().connect() as conn:
singleMetricData = repository.getMetricData(
conn,
uid,
rowid=randomRowID).first()
metricData = getMetricDataWithRowID(getAllModelsResult['data'],
randomRowID)
self.assertEqual(metricData[2], singleMetricData.anomaly_score)
self.assertEqual(datetime.strptime(metricData[0],
'%Y-%m-%d %H:%M:%S'), singleMetricData.timestamp)
map(testMetricDataForRandomRowID, uids)
def testMetricDataAnomalyAsQueryParams(uid):
'''
This test makes MetricDataHandler GET calls with anomaly param :
_models/<uid>/data?anomaly=<>
'''
queryString = ("SELECT * FROM metric_data WHERE uid='%s' "
" and abs(anomaly_score - 0) > 1e-5 LIMIT 1") % uid
with repository.engineFactory().connect() as conn:
sampleMetricData = conn.execute(queryString).first()
anomalyScore = sampleMetricData.anomaly_score
response = self.app.get("/%s/data?anomaly=%s"
% (uid, anomalyScore), headers=self.headers)
assertions.assertSuccess(self, response)
getAllModelsResult = utils.jsonDecode(response.body)
for metricData in getAllModelsResult['data']:
self.assertGreaterEqual(metricData[2], anomalyScore)
map(testMetricDataAnomalyAsQueryParams, uids)
def testMetricDataTimeStampQueryParams(uid):
'''
This test makes MetricDataHandler GET calls with from and to params :
_models/<uid>/data?from=<>&to=<>
'''
with repository.engineFactory().connect() as conn:
firstMetricData = conn.execute(
sql.select([schema.metric_data])
.where(schema.metric_data.c.uid == uid)
.order_by(sql.expression.asc(schema.metric_data.c.timestamp))
.limit(1)).fetchall()
lastMetricData = conn.execute(
sql.select([schema.metric_data])
.where(schema.metric_data.c.uid == uid)
.order_by(sql.expression.desc(schema.metric_data.c.timestamp))
.limit(1)).fetchall()
firstTimeStamp = firstMetricData[0].timestamp
lastTimeStamp = lastMetricData[0].timestamp
response = self.app.get("/%s/data?from=%s&to=%s"
% (uid, firstTimeStamp, lastTimeStamp), headers=self.headers)
assertions.assertSuccess(self, response)
getAllModelsResult = utils.jsonDecode(response.body)
for metricData in getAllModelsResult['data']:
self.assertGreaterEqual(datetime.strptime(metricData[0],
'%Y-%m-%d %H:%M:%S'), firstTimeStamp)
self.assertLessEqual(datetime.strptime(metricData[0],
'%Y-%m-%d %H:%M:%S'), lastTimeStamp)
map(testMetricDataTimeStampQueryParams, uids)
def testMetricDataQueryParams(uid):
'''
This test makes MetricDataHandler GET calls with various params :
_models/<uid>/data?from=<>&to=<>&anomaly=<>
'''
with repository.engineFactory().connect() as conn:
firstMetricData = conn.execute(
"SELECT * FROM `metric_data` WHERE `uid`='%s' "
"and abs(`anomaly_score` - 0) > 1e-5 "
"ORDER BY `timestamp` ASC LIMIT 1" % uid).fetchall()
lastMetricData = conn.execute(
"SELECT * FROM `metric_data` WHERE `uid`='%s' "
"and abs(`anomaly_score` - 0) > 1e-5 "
"ORDER BY `timestamp` DESC LIMIT 1" % uid).fetchall()
firstTimeStamp = firstMetricData[0].timestamp
lastTimeStamp = lastMetricData[0].timestamp
anomalyScore = firstMetricData[0].anomaly_score
response = self.app.get("/%s/data?from=%s&to=%s&anomaly=%s"
% (uid, firstTimeStamp, lastTimeStamp, anomalyScore),
headers=self.headers)
assertions.assertSuccess(self, response)
getAllModelsResult = utils.jsonDecode(response.body)
for metricData in getAllModelsResult['data']:
self.assertGreaterEqual(metricData[2], anomalyScore)
self.assertGreaterEqual(datetime.strptime(metricData[0],
'%Y-%m-%d %H:%M:%S'), firstTimeStamp)
self.assertLessEqual(datetime.strptime(metricData[0],
'%Y-%m-%d %H:%M:%S'), lastTimeStamp)
map(testMetricDataQueryParams, uids)
endTime = (time() - startTime) / 60
print "Test started at : %s" % \
strftime('%Y-%m-%d %H:%M:%S', localtime(startTime))
print "Test finished at : %s" % \
strftime('%Y-%m-%d %H:%M:%S', localtime(endTime))
print "Total metric count : %s" % totalMetricCount
print "Initial models created : %s" % initialModelCount
print "Approximate time taken to create inital models : %s minutes" \
% modelCreationDuration
print "Approximate time taken to calculate all inferences : %s minutes" \
% ((timeToCalculateAllInferences - startTime) / 60)
print "Approximate time taken for all the tests to finish : %s minutes" \
% ((time() - startTime) / 60)