def test_populate_buckets(self): pb = metrics_pb2.MetricsData() m = metrics.DistributionMetric('test') d = distribution.Distribution(distribution.FixedWidthBucketer(10)) d.add(5) d.add(15) d.add(35) d.add(65) m._populate_value(pb, d, 1234) self.assertEquals([1, 1, -1, 1, -2, 1], pb.distribution.bucket) self.assertEquals(0, pb.distribution.underflow) self.assertEquals(0, pb.distribution.overflow) self.assertEquals(30, pb.distribution.mean) pb = metrics_pb2.MetricsData() d = distribution.Distribution( distribution.FixedWidthBucketer(10, num_finite_buckets=1)) d.add(5) d.add(15) d.add(25) m._populate_value(pb, d, 1234) self.assertEquals([1], pb.distribution.bucket) self.assertEquals(0, pb.distribution.underflow) self.assertEquals(2, pb.distribution.overflow) self.assertEquals(15, pb.distribution.mean)
def modify_fn(dist, delta): # This is the same as the modify_fn in _DistributionMetricBase's add(). if dist == 0: dist = distribution.Distribution( distribution.GeometricBucketer()) dist.add(delta) return dist
def test_dangerously_enable_cumulative_set(self): cd = metrics.CumulativeDistributionMetric('test', 'test', None) value = distribution.Distribution(bucketer=cd.bucketer) with self.assertRaises(TypeError): cd.set(value) cd.dangerously_enable_cumulative_set() cd.set(value)
def test_populate_buckets_last_zero(self): pb = metrics_pb2.MetricsData() m = metrics.DistributionMetric('test') d = distribution.Distribution( distribution.FixedWidthBucketer(10, num_finite_buckets=10)) d.add(5) d.add(105) m._populate_value(pb, d, 1234) self.assertEquals([1], pb.distribution.bucket) self.assertEquals(1, pb.distribution.overflow)
def test_populate_is_cumulative(self): pb = metrics_pb2.MetricsData() d = distribution.Distribution( distribution.FixedWidthBucketer(10, num_finite_buckets=10)) m = metrics.CumulativeDistributionMetric('test') m._populate_value(pb, d, 1234) self.assertTrue(pb.distribution.is_cumulative) m = metrics.NonCumulativeDistributionMetric('test2') m._populate_value(pb, d, 1234) self.assertFalse(pb.distribution.is_cumulative)
def test_populate_buckets_underflow(self): pb = metrics_pb2.MetricsData() m = metrics.DistributionMetric('test') d = distribution.Distribution( distribution.FixedWidthBucketer(10, num_finite_buckets=10)) d.add(-5) d.add(-1000000) m._populate_value(pb, d, 1234) self.assertEquals([], pb.distribution.bucket) self.assertEquals(2, pb.distribution.underflow) self.assertEquals(0, pb.distribution.overflow) self.assertEquals(-500002.5, pb.distribution.mean)
def test_populate_custom(self): pb = metrics_pb2.MetricsData() m = metrics.DistributionMetric('test') m._populate_value( pb, distribution.Distribution(distribution.GeometricBucketer(4)), 1234) self.assertEquals( pb.distribution.spec_type, metrics_pb2.PrecomputedDistribution.CUSTOM_PARAMETERIZED) self.assertEquals(0, pb.distribution.width) self.assertEquals(4, pb.distribution.growth_factor) self.assertEquals(100, pb.distribution.num_buckets) m._populate_value( pb, distribution.Distribution(distribution.FixedWidthBucketer(10)), 1234) self.assertEquals( pb.distribution.spec_type, metrics_pb2.PrecomputedDistribution.CUSTOM_PARAMETERIZED) self.assertEquals(10, pb.distribution.width) self.assertEquals(0, pb.distribution.growth_factor) self.assertEquals(100, pb.distribution.num_buckets)
def test_populate_canonical(self): pb = metrics_pb2.MetricsData() m = metrics.DistributionMetric('test') m._populate_value( pb, distribution.Distribution(distribution.GeometricBucketer()), 1234) self.assertEquals( pb.distribution.spec_type, metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_10_P_0_2) m._populate_value( pb, distribution.Distribution(distribution.GeometricBucketer(2)), 1234) self.assertEquals( pb.distribution.spec_type, metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_2) m._populate_value( pb, distribution.Distribution(distribution.GeometricBucketer(10)), 1234) self.assertEquals( pb.distribution.spec_type, metrics_pb2.PrecomputedDistribution.CANONICAL_POWERS_OF_10)
def test_overflow_bucket(self): d = distribution.Distribution( distribution.FixedWidthBucketer(width=10, num_finite_buckets=10)) d.add(100) self.assertEqual(100, d.sum) self.assertEqual(1, d.count) self.assertEqual({11: 1}, d.buckets) d.add(1000000) self.assertEqual(1000100, d.sum) self.assertEqual(2, d.count) self.assertEqual({11: 2}, d.buckets)
def test_underflow_bucket(self): d = distribution.Distribution( distribution.FixedWidthBucketer(width=10)) d.add(-1) self.assertEqual(-1, d.sum) self.assertEqual(1, d.count) self.assertEqual({0: 1}, d.buckets) d.add(-1000000) self.assertEqual(-1000001, d.sum) self.assertEqual(2, d.count) self.assertEqual({0: 2}, d.buckets)
def test_add_on_bucket_boundary(self): d = distribution.Distribution( distribution.FixedWidthBucketer(width=10)) d.add(10) self.assertEqual(10, d.sum) self.assertEqual(1, d.count) self.assertEqual({2: 1}, d.buckets) d.add(0) self.assertEqual(10, d.sum) self.assertEqual(2, d.count) self.assertEqual({1: 1, 2: 1}, d.buckets)
def test_set(self): d = distribution.Distribution( distribution.FixedWidthBucketer(10, num_finite_buckets=10)) d.add(1) d.add(10) d.add(100) m = metrics.CumulativeDistributionMetric('test') with self.assertRaises(TypeError): m.set(d) m = metrics.NonCumulativeDistributionMetric('test2') m.set(d) self.assertEquals(d, m.get()) with self.assertRaises(errors.MonitoringInvalidValueTypeError): m.set(1) with self.assertRaises(errors.MonitoringInvalidValueTypeError): m.set('foo')
def test_add(self): d = distribution.Distribution(distribution.GeometricBucketer()) self.assertEqual(0, d.sum) self.assertEqual(0, d.count) self.assertEqual({}, d.buckets) d.add(1) d.add(10) d.add(100) self.assertEqual(111, d.sum) self.assertEqual(3, d.count) self.assertEqual({1: 1, 5: 1, 10: 1}, d.buckets) d.add(50) self.assertEqual(161, d.sum) self.assertEqual(4, d.count) self.assertEqual({1: 1, 5: 1, 9: 1, 10: 1}, d.buckets)
def modify_fn(dist, value): if dist == 0: dist = distribution.Distribution(self.bucketer) dist.add(value) return dist
def post(self): """POST expects a JSON body that's a dict which includes a key "metrics". This key's value is an array of objects with schema: { "metrics": [{ "MetricInfo": { "Name": "monorail/frontend/float_test", "ValueType": 2 }, "Cells": [{ "value": 1, "fields": {}, "start_time": 1538430628174 }] }] } Important! The user of this library is responsible for validating XSRF tokens via implementing the method self.xsrf_is_valid. """ if not self._metrics: self.response.set_status(400) self.response.write('No metrics have been registered.') logging.warning( 'gae_ts_mon error: No metrics have been registered.') return try: body = json.loads(self.request.body) except ValueError: self.response.set_status(400) self.response.write('Invalid JSON.') logging.warning('gae_ts_mon error: Invalid JSON.') return if not self.xsrf_is_valid(body): self.response.set_status(403) self.response.write('XSRF token invalid.') logging.warning('gae_ts_mon error: XSRF token invalid.') return if not isinstance(body, dict): self.response.set_status(400) self.response.write('Body must be a dictionary.') logging.warning('gae_ts_mon error: Body must be a dictionary.') return if 'metrics' not in body: self.response.set_status(400) self.response.write('Key "metrics" must be in body.') logging.warning('gae_ts_mon error: Key "metrics" must be in body.') logging.warning('Request body: %s', body) return for metric_measurement in body.get('metrics', []): name = metric_measurement['MetricInfo']['Name'] metric = self._metrics.get(name, None) if not metric: self.response.set_status(400) self.response.write('Metric "%s" is not defined.' % name) logging.warning( 'gae_ts_mon error: Metric "%s" is not defined.', name) return for cell in metric_measurement.get('Cells', []): fields = cell.get('fields', {}) value = cell.get('value') metric_field_keys = set(fs.name for fs in metric.field_spec) if set(fields.keys()) != metric_field_keys: self.response.set_status(400) self.response.write( 'Supplied fields do not match metric "%s".' % name) logging.warning( 'gae_ts_mon error: Supplied fields do not match metric "%s".', name) logging.warning('Supplied fields keys: %s', fields.keys()) logging.warning('Metric fields keys: %s', metric_field_keys) return start_time = cell.get('start_time') if metric.is_cumulative() and not start_time: self.response.set_status(400) self.response.write( 'Cumulative metrics must have start_time.') logging.warning( 'gae_ts_mon error: Cumulative metrics must have start_time.' ) logging.warning('Metric name: %s', name) return if metric.is_cumulative( ) and not self._start_time_is_valid(start_time): self.response.set_status(400) self.response.write('Invalid start_time: %s.' % start_time) logging.warning( 'gae_ts_mon error: Invalid start_time: %s.', start_time) return # Convert distribution metric values into Distribution classes. if (isinstance(metric, (metrics.CumulativeDistributionMetric, metrics.NonCumulativeDistributionMetric))): if not isinstance(value, dict): self.response.set_status(400) self.response.write( 'Distribution metric values must be a dict.') logging.warning( 'gae_ts_mon error: Distribution metric values must be a dict.' ) logging.warning('Metric value: %s', value) return dist_value = distribution.Distribution( bucketer=metric.bucketer) dist_value.sum = value.get('sum', 0) dist_value.count = value.get('count', 0) dist_value.buckets.update(value.get('buckets', {})) metric.set(dist_value, fields=fields) else: metric.set(value, fields=fields) if metric.is_cumulative(): metric.dangerously_set_start_time(start_time) self.response.set_status(201) self.response.write('Ok.')