def test_populate_buckets(self):
pb = metrics_pb2.MetricsData()
m = metrics.DistributionMetric('test')
d = distribution.Distribution(distribution.FixedWidthBucketer(10))
d.add(5)
d.add(15)
d.add(35)
d.add(65)
m._populate_value(pb, d, 1234)
self.assertEquals([1, 1, -1, 1, -2, 1], pb.distribution.bucket)
self.assertEquals(0, pb.distribution.underflow)
self.assertEquals(0, pb.distribution.overflow)
self.assertEquals(30, pb.distribution.mean)
pb = metrics_pb2.MetricsData()
d = distribution.Distribution(
distribution.FixedWidthBucketer(10, num_finite_buckets=1))
d.add(5)
d.add(15)
d.add(25)
m._populate_value(pb, d, 1234)
self.assertEquals([1], pb.distribution.bucket)
self.assertEquals(0, pb.distribution.underflow)
self.assertEquals(2, pb.distribution.overflow)
self.assertEquals(15, pb.distribution.mean)
def modify_fn(dist, delta):
# This is the same as the modify_fn in _DistributionMetricBase's add().
if dist == 0:
dist = distribution.Distribution(
distribution.GeometricBucketer())
dist.add(delta)
return dist
def test_dangerously_enable_cumulative_set(self):
cd = metrics.CumulativeDistributionMetric('test', 'test', None)
value = distribution.Distribution(bucketer=cd.bucketer)
with self.assertRaises(TypeError):
cd.set(value)
cd.dangerously_enable_cumulative_set()
cd.set(value)
def test_populate_buckets_last_zero(self):
pb = metrics_pb2.MetricsData()
m = metrics.DistributionMetric('test')
d = distribution.Distribution(
distribution.FixedWidthBucketer(10, num_finite_buckets=10))
d.add(5)
d.add(105)
m._populate_value(pb, d, 1234)
self.assertEquals([1], pb.distribution.bucket)
self.assertEquals(1, pb.distribution.overflow)
def test_populate_is_cumulative(self):
pb = metrics_pb2.MetricsData()
d = distribution.Distribution(
distribution.FixedWidthBucketer(10, num_finite_buckets=10))
m = metrics.CumulativeDistributionMetric('test')
m._populate_value(pb, d, 1234)
self.assertTrue(pb.distribution.is_cumulative)
m = metrics.NonCumulativeDistributionMetric('test2')
m._populate_value(pb, d, 1234)
self.assertFalse(pb.distribution.is_cumulative)
def test_populate_buckets_underflow(self):
pb = metrics_pb2.MetricsData()
m = metrics.DistributionMetric('test')
d = distribution.Distribution(
distribution.FixedWidthBucketer(10, num_finite_buckets=10))
d.add(-5)
d.add(-1000000)
m._populate_value(pb, d, 1234)
self.assertEquals([], pb.distribution.bucket)
self.assertEquals(2, pb.distribution.underflow)
self.assertEquals(0, pb.distribution.overflow)
self.assertEquals(-500002.5, pb.distribution.mean)
def test_populate_custom(self):
pb = metrics_pb2.MetricsData()
m = metrics.DistributionMetric('test')
m._populate_value(
pb, distribution.Distribution(distribution.GeometricBucketer(4)),
1234)
self.assertEquals(
pb.distribution.spec_type,
metrics_pb2.PrecomputedDistribution.CUSTOM_PARAMETERIZED)
self.assertEquals(0, pb.distribution.width)
self.assertEquals(4, pb.distribution.growth_factor)
self.assertEquals(100, pb.distribution.num_buckets)
m._populate_value(
pb, distribution.Distribution(distribution.FixedWidthBucketer(10)),
1234)
self.assertEquals(
pb.distribution.spec_type,
metrics_pb2.PrecomputedDistribution.CUSTOM_PARAMETERIZED)
self.assertEquals(10, pb.distribution.width)
self.assertEquals(0, pb.distribution.growth_factor)
self.assertEquals(100, pb.distribution.num_buckets)
def test_populate_canonical(self):
pb = metrics_pb2.MetricsData()
m = metrics.DistributionMetric('test')
m._populate_value(
pb, distribution.Distribution(distribution.GeometricBucketer()),
1234)
self.assertEquals(
pb.distribution.spec_type,
metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_10_P_0_2)
m._populate_value(
pb, distribution.Distribution(distribution.GeometricBucketer(2)),
1234)
self.assertEquals(
pb.distribution.spec_type,
metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_2)
m._populate_value(
pb, distribution.Distribution(distribution.GeometricBucketer(10)),
1234)
self.assertEquals(
pb.distribution.spec_type,
metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_10)
def test_overflow_bucket(self):
d = distribution.Distribution(
distribution.FixedWidthBucketer(width=10, num_finite_buckets=10))
d.add(100)
self.assertEqual(100, d.sum)
self.assertEqual(1, d.count)
self.assertEqual({11: 1}, d.buckets)
d.add(1000000)
self.assertEqual(1000100, d.sum)
self.assertEqual(2, d.count)
self.assertEqual({11: 2}, d.buckets)
def test_underflow_bucket(self):
d = distribution.Distribution(
distribution.FixedWidthBucketer(width=10))
d.add(-1)
self.assertEqual(-1, d.sum)
self.assertEqual(1, d.count)
self.assertEqual({0: 1}, d.buckets)
d.add(-1000000)
self.assertEqual(-1000001, d.sum)
self.assertEqual(2, d.count)
self.assertEqual({0: 2}, d.buckets)
def test_add_on_bucket_boundary(self):
d = distribution.Distribution(
distribution.FixedWidthBucketer(width=10))
d.add(10)
self.assertEqual(10, d.sum)
self.assertEqual(1, d.count)
self.assertEqual({2: 1}, d.buckets)
d.add(0)
self.assertEqual(10, d.sum)
self.assertEqual(2, d.count)
self.assertEqual({1: 1, 2: 1}, d.buckets)
def test_set(self):
d = distribution.Distribution(
distribution.FixedWidthBucketer(10, num_finite_buckets=10))
d.add(1)
d.add(10)
d.add(100)
m = metrics.CumulativeDistributionMetric('test')
with self.assertRaises(TypeError):
m.set(d)
m = metrics.NonCumulativeDistributionMetric('test2')
m.set(d)
self.assertEquals(d, m.get())
with self.assertRaises(errors.MonitoringInvalidValueTypeError):
m.set(1)
with self.assertRaises(errors.MonitoringInvalidValueTypeError):
m.set('foo')
def test_add(self):
d = distribution.Distribution(distribution.GeometricBucketer())
self.assertEqual(0, d.sum)
self.assertEqual(0, d.count)
self.assertEqual({}, d.buckets)
d.add(1)
d.add(10)
d.add(100)
self.assertEqual(111, d.sum)
self.assertEqual(3, d.count)
self.assertEqual({1: 1, 5: 1, 10: 1}, d.buckets)
d.add(50)
self.assertEqual(161, d.sum)
self.assertEqual(4, d.count)
self.assertEqual({1: 1, 5: 1, 9: 1, 10: 1}, d.buckets)
def modify_fn(dist, value):
if dist == 0:
dist = distribution.Distribution(self.bucketer)
dist.add(value)
return dist
def post(self):
"""POST expects a JSON body that's a dict which includes a key "metrics".
This key's value is an array of objects with schema:
{
"metrics": [{
"MetricInfo": {
"Name": "monorail/frontend/float_test",
"ValueType": 2
},
"Cells": [{
"value": 1,
"fields": {},
"start_time": 1538430628174
}]
}]
}
Important!
The user of this library is responsible for validating XSRF tokens via
implementing the method self.xsrf_is_valid.
"""
if not self._metrics:
self.response.set_status(400)
self.response.write('No metrics have been registered.')
logging.warning(
'gae_ts_mon error: No metrics have been registered.')
return
try:
body = json.loads(self.request.body)
except ValueError:
self.response.set_status(400)
self.response.write('Invalid JSON.')
logging.warning('gae_ts_mon error: Invalid JSON.')
return
if not self.xsrf_is_valid(body):
self.response.set_status(403)
self.response.write('XSRF token invalid.')
logging.warning('gae_ts_mon error: XSRF token invalid.')
return
if not isinstance(body, dict):
self.response.set_status(400)
self.response.write('Body must be a dictionary.')
logging.warning('gae_ts_mon error: Body must be a dictionary.')
return
if 'metrics' not in body:
self.response.set_status(400)
self.response.write('Key "metrics" must be in body.')
logging.warning('gae_ts_mon error: Key "metrics" must be in body.')
logging.warning('Request body: %s', body)
return
for metric_measurement in body.get('metrics', []):
name = metric_measurement['MetricInfo']['Name']
metric = self._metrics.get(name, None)
if not metric:
self.response.set_status(400)
self.response.write('Metric "%s" is not defined.' % name)
logging.warning(
'gae_ts_mon error: Metric "%s" is not defined.', name)
return
for cell in metric_measurement.get('Cells', []):
fields = cell.get('fields', {})
value = cell.get('value')
metric_field_keys = set(fs.name for fs in metric.field_spec)
if set(fields.keys()) != metric_field_keys:
self.response.set_status(400)
self.response.write(
'Supplied fields do not match metric "%s".' % name)
logging.warning(
'gae_ts_mon error: Supplied fields do not match metric "%s".',
name)
logging.warning('Supplied fields keys: %s', fields.keys())
logging.warning('Metric fields keys: %s',
metric_field_keys)
return
start_time = cell.get('start_time')
if metric.is_cumulative() and not start_time:
self.response.set_status(400)
self.response.write(
'Cumulative metrics must have start_time.')
logging.warning(
'gae_ts_mon error: Cumulative metrics must have start_time.'
)
logging.warning('Metric name: %s', name)
return
if metric.is_cumulative(
) and not self._start_time_is_valid(start_time):
self.response.set_status(400)
self.response.write('Invalid start_time: %s.' % start_time)
logging.warning(
'gae_ts_mon error: Invalid start_time: %s.',
start_time)
return
# Convert distribution metric values into Distribution classes.
if (isinstance(metric,
(metrics.CumulativeDistributionMetric,
metrics.NonCumulativeDistributionMetric))):
if not isinstance(value, dict):
self.response.set_status(400)
self.response.write(
'Distribution metric values must be a dict.')
logging.warning(
'gae_ts_mon error: Distribution metric values must be a dict.'
)
logging.warning('Metric value: %s', value)
return
dist_value = distribution.Distribution(
bucketer=metric.bucketer)
dist_value.sum = value.get('sum', 0)
dist_value.count = value.get('count', 0)
dist_value.buckets.update(value.get('buckets', {}))
metric.set(dist_value, fields=fields)
else:
metric.set(value, fields=fields)
if metric.is_cumulative():
metric.dangerously_set_start_time(start_time)
self.response.set_status(201)
self.response.write('Ok.')
