def testManagerAndMeter(self):
manager = counters._CounterManager()
total = counters._TotalCounter('test.mytotal', 'Example total counter')
delta = counters._DeltaCounter('test.mydelta', 'Example delta counter')
manager.register(total)
manager.register(delta)
# Test that counters are accessible from manager object via dot notation.
self.assertIs(manager.test.mytotal, total)
self.assertIs(manager.test.mydelta, delta)
# Test for namespace errors when registering a counter with the manager.
badcounter = counters._RateCounter('test', 'Bad namespace')
with self.assertRaises(KeyError):
manager.register(badcounter)
badcounter = counters._RateCounter('test.mydelta.rate', 'Existing counter')
with self.assertRaises(KeyError):
manager.register(badcounter)
# Test basic functionality of meter class in conjunction with a meter.
meter = counters.Meter(manager.test)
total.increment()
total.increment(5)
delta.increment(6)
x = meter.sample()
self.assertEqual(6, x.test.mytotal)
self.assertEqual(6, x.test.mydelta)
total.increment(5)
delta.increment(6)
x = meter.sample()
self.assertEqual(11, x.test.mytotal)
self.assertEqual(6, x.test.mydelta)
# Test that namespace selection using a meter has the appropriate behavior.
d = meter.describe()
self.assertEqual(d.test.mytotal, 'Example total counter')
def testHealthReport(self):
"""Verify the health report generation and retrieval."""
# Test constants
cluster_name = 'test'
interval = MetricInterval('testint', 60)
group_key = Metric.EncodeGroupKey(cluster_name, interval)
num_machines = 5
num_samples = TREND_SAMPLE_SIZE + 1
# Outer variables
fake_time = 0
def fake_time_func():
return fake_time
managers = []
criteria_list = []
criteria_called = [0, 0]
# Test criteria #1
def _deltaCriteria(totals, deltas):
criteria_called[0] += 1
alerts = []
warnings = []
if len(totals['machine_data'].keys()) != len(deltas['machine_data'].keys()):
alerts.append('CLUSTER')
warnings = [m for m, v in deltas['machine_data'].iteritems() if v > 3]
return alerts, warnings
# Test criteria #2
def _rateCriteria(rates):
criteria_called[1] += 1
warnings = []
for m, v in rates['machine_data'].iteritems():
m_int = int(m[len('machine'):])
if v == m_int / interval.length:
warnings.append(m)
return [], warnings
def _OnGetReport(repeat, report):
# Verify that the criteria were actually executed.
for i in criteria_called:
self.assertEqual(num_samples, i)
# Verify that the criteria generated the expected warnings.
self.assertEqual(len(report.alerts), num_machines)
self.assertFalse('deltaCrit:CLUSTER' in report.alerts)
for i in range(1, num_machines + 1):
m_name = 'machine%d' % i
self.assertTrue(('rateCrit:' + m_name) in report.warnings)
self.assertTrue(('rateCrit:' + m_name) in report.alerts)
if i > 3:
self.assertTrue(('deltaCrit:' + m_name) in report.warnings)
else:
self.assertFalse(('deltaCrit:' + m_name) in report.warnings)
if repeat:
# Repeat the call to GetHealthReport to verify that criteria are only
# run when the report is generated.
HealthReport.GetHealthReport(self._client, cluster_name, interval, num_samples * interval.length,
partial(_OnGetReport, False), managers[0], criteria_list)
else:
self.stop()
def _OnMetricsUploaded():
# Create a criteria list and request a health report based on those criteria.
criteria_list.append(HealthCriteria('deltaCrit',
'Description',
_deltaCriteria,
[managers[0].aggtest.total, managers[0].aggtest.delta],
0))
criteria_list.append(HealthCriteria('rateCrit',
'Description',
_rateCriteria,
[managers[0].aggtest.rate],
5))
HealthReport.GetHealthReport(self._client, cluster_name, interval, num_samples * interval.length,
partial(_OnGetReport, True), managers[0], criteria_list)
with util.Barrier(_OnMetricsUploaded) as b:
# Generate metrics.
for m in range(1, num_machines + 1):
cm = counters._CounterManager()
cm.register(counters._TotalCounter('aggtest.total', 'Test Total'))
cm.register(counters._DeltaCounter('aggtest.delta', 'Test Delta'))
cm.register(counters._RateCounter('aggtest.rate', 'Test Rate', time_func=fake_time_func))
cm.register(counters._AverageCounter('aggtest.avg', 'Test Average'))
fake_time = 0
meter = counters.Meter(cm)
managers.append(cm)
for s in range(num_samples):
cm.aggtest.total.increment(m)
cm.aggtest.delta.increment(m)
cm.aggtest.rate.increment(m)
cm.aggtest.avg.add(m)
cm.aggtest.avg.add(m)
fake_time += interval.length
sample = json.dumps(meter.sample())
metric = Metric.Create(group_key, 'machine%d' % m, fake_time, sample)
metric.Update(self._client, b.Callback())
def testEmptyHealthReport(self):
"""Verify the health reports with no data are properly saved to the db.
"""
# Test constants
cluster_name = 'test'
interval = MetricInterval('testint', 60)
group_key = Metric.EncodeGroupKey(cluster_name, interval)
num_machines = 5
num_samples = TREND_SAMPLE_SIZE + 1
# Outer variables
managers = []
criteria_list = []
criteria_called = [0]
# Test criteria #1
def _blankCriteria():
criteria_called[0] += 1
return [], []
def _OnDirectQuery(reports):
self.assertEqual(len(reports), num_samples)
self.stop()
def _OnGetReport(report):
# Verify that the criteria were actually executed.
for i in criteria_called:
self.assertEqual(num_samples, i)
# Verify that the criteria generated the expected warnings.
self.assertEqual(len(report.alerts), 0)
self.assertEqual(len(report.warnings), 0)
HealthReport.QueryTimespan(self._client, group_key, interval.length, interval.length * num_samples,
_OnDirectQuery)
def _OnMetricsUploaded():
# Create a criteria list and request a health report based on those criteria.
criteria_list.append(HealthCriteria('blankCrit',
'Description',
_blankCriteria,
[],
0))
HealthReport.GetHealthReport(self._client, cluster_name, interval, num_samples * interval.length,
_OnGetReport, managers[0], criteria_list)
with util.Barrier(_OnMetricsUploaded) as b:
# Generate metrics.
for m in range(1, num_machines + 1):
cm = counters._CounterManager()
cm.register(counters._TotalCounter('aggtest.total', 'Test Total'))
fake_time = 0
meter = counters.Meter(cm)
managers.append(cm)
for s in range(num_samples):
fake_time += interval.length
sample = json.dumps(meter.sample())
metric = Metric.Create(group_key, 'machine%d' % m, fake_time, sample)
metric.Update(self._client, b.Callback())
def testHealthReport(self):
"""Verify the health report generation and retrieval."""
# Test constants
cluster_name = 'test'
interval = MetricInterval('testint', 60)
group_key = Metric.EncodeGroupKey(cluster_name, interval)
num_machines = 5
num_samples = TREND_SAMPLE_SIZE + 1
# Outer variables
fake_time = 0
def fake_time_func():
return fake_time
managers = []
criteria_list = []
criteria_called = [0, 0]
# Test criteria #1
def _deltaCriteria(totals, deltas):
criteria_called[0] += 1
alerts = []
warnings = []
if len(totals['machine_data'].keys()) != len(
deltas['machine_data'].keys()):
alerts.append('CLUSTER')
warnings = [
m for m, v in deltas['machine_data'].iteritems() if v > 3
]
return alerts, warnings
# Test criteria #2
def _rateCriteria(rates):
criteria_called[1] += 1
warnings = []
for m, v in rates['machine_data'].iteritems():
m_int = int(m[len('machine'):])
if v == m_int / interval.length:
warnings.append(m)
return [], warnings
def _OnGetReport(repeat, report):
# Verify that the criteria were actually executed.
for i in criteria_called:
self.assertEqual(num_samples, i)
# Verify that the criteria generated the expected warnings.
self.assertEqual(len(report.alerts), num_machines)
self.assertFalse('deltaCrit:CLUSTER' in report.alerts)
for i in range(1, num_machines + 1):
m_name = 'machine%d' % i
self.assertTrue(('rateCrit:' + m_name) in report.warnings)
self.assertTrue(('rateCrit:' + m_name) in report.alerts)
if i > 3:
self.assertTrue(('deltaCrit:' + m_name) in report.warnings)
else:
self.assertFalse(('deltaCrit:' +
m_name) in report.warnings)
if repeat:
# Repeat the call to GetHealthReport to verify that criteria are only
# run when the report is generated.
HealthReport.GetHealthReport(self._client, cluster_name,
interval,
num_samples * interval.length,
partial(_OnGetReport, False),
managers[0], criteria_list)
else:
self.stop()
def _OnMetricsUploaded():
# Create a criteria list and request a health report based on those criteria.
criteria_list.append(
HealthCriteria(
'deltaCrit', 'Description', _deltaCriteria,
[managers[0].aggtest.total, managers[0].aggtest.delta], 0))
criteria_list.append(
HealthCriteria('rateCrit', 'Description', _rateCriteria,
[managers[0].aggtest.rate], 5))
HealthReport.GetHealthReport(self._client, cluster_name, interval,
num_samples * interval.length,
partial(_OnGetReport, True),
managers[0], criteria_list)
with util.Barrier(_OnMetricsUploaded) as b:
# Generate metrics.
for m in range(1, num_machines + 1):
cm = counters._CounterManager()
cm.register(
counters._TotalCounter('aggtest.total', 'Test Total'))
cm.register(
counters._DeltaCounter('aggtest.delta', 'Test Delta'))
cm.register(
counters._RateCounter('aggtest.rate',
'Test Rate',
time_func=fake_time_func))
cm.register(
counters._AverageCounter('aggtest.avg', 'Test Average'))
fake_time = 0
meter = counters.Meter(cm)
managers.append(cm)
for s in range(num_samples):
cm.aggtest.total.increment(m)
cm.aggtest.delta.increment(m)
cm.aggtest.rate.increment(m)
cm.aggtest.avg.add(m)
cm.aggtest.avg.add(m)
fake_time += interval.length
sample = json.dumps(meter.sample())
metric = Metric.Create(group_key, 'machine%d' % m,
fake_time, sample)
metric.Update(self._client, b.Callback())
def testEmptyHealthReport(self):
"""Verify the health reports with no data are properly saved to the db.
"""
# Test constants
cluster_name = 'test'
interval = MetricInterval('testint', 60)
group_key = Metric.EncodeGroupKey(cluster_name, interval)
num_machines = 5
num_samples = TREND_SAMPLE_SIZE + 1
# Outer variables
managers = []
criteria_list = []
criteria_called = [0]
# Test criteria #1
def _blankCriteria():
criteria_called[0] += 1
return [], []
def _OnDirectQuery(reports):
self.assertEqual(len(reports), num_samples)
self.stop()
def _OnGetReport(report):
# Verify that the criteria were actually executed.
for i in criteria_called:
self.assertEqual(num_samples, i)
# Verify that the criteria generated the expected warnings.
self.assertEqual(len(report.alerts), 0)
self.assertEqual(len(report.warnings), 0)
HealthReport.QueryTimespan(self._client, group_key,
interval.length,
interval.length * num_samples,
_OnDirectQuery)
def _OnMetricsUploaded():
# Create a criteria list and request a health report based on those criteria.
criteria_list.append(
HealthCriteria('blankCrit', 'Description', _blankCriteria, [],
0))
HealthReport.GetHealthReport(self._client, cluster_name, interval,
num_samples * interval.length,
_OnGetReport, managers[0],
criteria_list)
with util.Barrier(_OnMetricsUploaded) as b:
# Generate metrics.
for m in range(1, num_machines + 1):
cm = counters._CounterManager()
cm.register(
counters._TotalCounter('aggtest.total', 'Test Total'))
fake_time = 0
meter = counters.Meter(cm)
managers.append(cm)
for s in range(num_samples):
fake_time += interval.length
sample = json.dumps(meter.sample())
metric = Metric.Create(group_key, 'machine%d' % m,
fake_time, sample)
metric.Update(self._client, b.Callback())
def testMetricsAggregator(self):
"""Test metrics aggregation with data from multiple machines"""
num_machines = 5
num_samples = 15
sample_duration = 60.0
group_key = 'agg_test_group_key'
fake_time = 0
managers = []
def fake_time_func():
return fake_time
def _OnAggregation(aggregator):
base_sum = sum(range(1, num_machines + 1))
base_avg = base_sum / num_machines
agg_total = aggregator.counter_data['aggtest.total']
agg_delta = aggregator.counter_data['aggtest.delta']
agg_rate = aggregator.counter_data['aggtest.rate']
agg_avg = aggregator.counter_data['aggtest.avg']
for s in range(num_samples):
# Check timestamp values
for cd in aggregator.counter_data.itervalues():
self.assertEqual(cd.cluster_total[s][0], sample_duration * (s + 1))
self.assertEqual(cd.cluster_avg[s][0], sample_duration * (s + 1))
# Check aggregate total
self.assertEqual(agg_total.cluster_total[s][1], base_sum * (s + 1))
self.assertEqual(agg_total.cluster_avg[s][1], base_avg * (s + 1))
# Check aggregate delta
self.assertEqual(agg_delta.cluster_total[s][1], base_sum)
self.assertEqual(agg_delta.cluster_avg[s][1], base_avg)
# Check aggregate rate
self.assertEqual(agg_rate.cluster_total[s][1], base_sum / sample_duration)
self.assertEqual(agg_rate.cluster_avg[s][1], base_avg / sample_duration)
# Check aggregate avg
self.assertEqual(agg_avg.cluster_total[s][1], base_sum)
self.assertEqual(agg_avg.cluster_avg[s][1], base_avg)
for m in range(1, num_machines + 1):
machine_name = 'machine%d' % m
# Check per-machine total
mtotal = agg_total.machine_data[machine_name]
self.assertEqual(mtotal[s][1], m * (s + 1))
# Check per-machine delta
mdelta = agg_delta.machine_data[machine_name]
self.assertEqual(mdelta[s][1], m)
# Check per-machine rate
mrate = agg_rate.machine_data[machine_name]
self.assertEqual(mrate[s][1], m / sample_duration)
# Check per-machine avg
mavg = agg_avg.machine_data[machine_name]
self.assertEqual(mavg[s][1], m)
self.stop()
def _OnMetricsUploaded():
AggregatedMetric.CreateAggregateForTimespan(self._client, group_key, 0, sample_duration * num_samples,
managers[0], _OnAggregation)
with util.Barrier(_OnMetricsUploaded) as b:
for m in range(1, num_machines + 1):
cm = counters._CounterManager()
cm.register(counters._TotalCounter('aggtest.total', 'Test Total'))
cm.register(counters._DeltaCounter('aggtest.delta', 'Test Delta'))
cm.register(counters._RateCounter('aggtest.rate', 'Test Rate', time_func=fake_time_func))
cm.register(counters._AverageCounter('aggtest.avg', 'Test Average'))
fake_time = 0
meter = counters.Meter(cm)
managers.append(cm)
for s in range(num_samples):
cm.aggtest.total.increment(m)
cm.aggtest.delta.increment(m)
cm.aggtest.rate.increment(m)
cm.aggtest.avg.add(m)
cm.aggtest.avg.add(m)
fake_time += sample_duration
sample = json.dumps(meter.sample())
metric = Metric.Create(group_key, 'machine%d' % m, fake_time, sample)
metric.Update(self._client, b.Callback())