def test_populate_data(self):
m = metrics.CounterMetric('test', 'test', None)
data = metrics_pb2.MetricsData()
m.populate_data(data, 100.4, 1000.6, {}, 5)
self.assertEqual(100, data.start_timestamp.seconds)
self.assertEqual(1000, data.end_timestamp.seconds)
def test_decrement_raises(self):
m = metrics.CounterMetric('test')
m.set(1)
with self.assertRaises(errors.MonitoringDecreasingValueError):
m.set(0)
with self.assertRaises(errors.MonitoringDecreasingValueError):
m.increment_by(-1)
def test_start_timestamp(self):
t = targets.DeviceTarget('reg', 'role', 'net', 'host')
m = metrics.CounterMetric('test', fields={'foo': 'bar'})
m.increment()
p = metrics_pb2.MetricsCollection()
m.serialize_to(p, 1234, (), m.get(), t)
self.assertEquals(1234000000, p.data[0].start_timestamp_us)
def test_override_fields(self):
m = metrics.CounterMetric('test', fields={'foo': 'bar'})
m.increment()
m.increment({'foo': 'baz'})
self.assertEquals(1, m.get())
self.assertEquals(1, m.get({'foo': 'bar'}))
self.assertEquals(1, m.get({'foo': 'baz'}))
def test_send_modifies_metric_values(self):
interface.state.global_monitor = stubs.MockMonitor()
interface.state.target = stubs.MockTarget()
# pylint: disable=unused-argument
def serialize_to(pb, start_time, fields, value, target):
pb.data.add().name = 'foo'
fake_metric = mock.create_autospec(metrics.Metric, spec_set=True)
fake_metric.name = 'fake'
fake_metric.serialize_to.side_effect = serialize_to
interface.register(fake_metric)
# Setting this will modify store._values in the middle of iteration.
delayed_metric = metrics.CounterMetric('foo')
def send(proto):
delayed_metric.increment_by(1)
interface.state.global_monitor.send.side_effect = send
for i in xrange(1001):
interface.state.store.set('fake', ('field', i), None, 123)
# Shouldn't raise an exception.
interface.flush()
def test_flush_many_new(self):
interface.state.global_monitor = mock.create_autospec(monitors.Monitor)
interface.state.target = targets.TaskTarget('a', 'b', 'c', 'd', 1)
# We can't use the mock's call_args_list here because the same object is
# reused as the argument to both calls and cleared inbetween.
data_lengths = []
def send(proto):
count = 0
for coll in proto.metrics_collection:
for data_set in coll.metrics_data_set:
for _ in data_set.data:
count += 1
data_lengths.append(count)
interface.state.global_monitor.send.side_effect = send
counter = metrics.CounterMetric('counter', 'desc',
[metrics.IntegerField('field')])
interface.register(counter)
for i in range(interface.METRICS_DATA_LENGTH_LIMIT + 1):
counter.increment_by(i, {'field': i})
interface.flush()
self.assertEquals(2, interface.state.global_monitor.send.call_count)
self.assertListEqual([500, 1], data_lengths)
def test_send_modifies_metric_values(self):
interface.state.global_monitor = mock.create_autospec(monitors.Monitor)
interface.state.target = mock.create_autospec(targets.Target)
interface.state.target.__hash__.return_value = 42
# pylint: disable=unused-argument
def populate_data_set(pb):
pb.metric_name = 'foo'
fake_metric = mock.create_autospec(metrics.Metric, spec_set=True)
fake_metric.name = 'fake'
fake_metric.populate_data_set.side_effect = populate_data_set
interface.register(fake_metric)
# Setting this will modify store._values in the middle of iteration.
delayed_metric = metrics.CounterMetric('foo', 'desc', None)
def send(proto):
delayed_metric.increment_by(1)
interface.state.global_monitor.send.side_effect = send
for i in range(1001):
interface.state.store.set('fake', (i, ), None, 123)
# Shouldn't raise an exception.
interface.flush()
def test_generate_every_type_of_field(self):
counter = metrics.CounterMetric('counter', 'desc', [
metrics.IntegerField('a'),
metrics.BooleanField('b'),
metrics.StringField('c'),
])
interface.register(counter)
counter.increment({'a': 1, 'b': True, 'c': 'test'})
proto = list(interface._generate_proto())[0]
data_set = proto.metrics_collection[0].metrics_data_set[0]
field_type = metrics_pb2.MetricsDataSet.MetricFieldDescriptor
self.assertEqual('a', data_set.field_descriptor[0].name)
self.assertEqual(field_type.INT64,
data_set.field_descriptor[0].field_type)
self.assertEqual('b', data_set.field_descriptor[1].name)
self.assertEqual(field_type.BOOL,
data_set.field_descriptor[1].field_type)
self.assertEqual('c', data_set.field_descriptor[2].name)
self.assertEqual(field_type.STRING,
data_set.field_descriptor[2].field_type)
self.assertEqual(1, data_set.data[0].int64_value)
self.assertEqual('a', data_set.data[0].field[0].name)
self.assertEqual(1, data_set.data[0].field[0].int64_value)
self.assertEqual('b', data_set.data[0].field[1].name)
self.assertTrue(data_set.data[0].field[1].bool_value)
self.assertEqual('c', data_set.data[0].field[2].name)
self.assertEqual('test', data_set.data[0].field[2].string_value)
def test_reset_for_unittest(self):
metric = metrics.CounterMetric('foo', 'desc', None)
metric.increment()
self.assertEquals(1, metric.get())
interface.reset_for_unittest()
self.assertIsNone(metric.get())
def test_increment(self):
m = metrics.CounterMetric('test')
m.set(1)
self.assertEquals(m.get(), 1)
m.increment()
self.assertEquals(m.get(), 2)
m.increment_by(3)
self.assertAlmostEquals(m.get(), 5)
def test_get_all(self):
m = metrics.CounterMetric('test', 'test', [metrics.StringField('foo')])
m.increment({'foo': ''})
m.increment({'foo': 'bar'})
self.assertEqual([
(('', ), 1),
(('bar', ), 1),
], sorted(m.get_all()))
def test_non_int_raises(self):
m = metrics.CounterMetric('test')
with self.assertRaises(errors.MonitoringInvalidValueTypeError):
m.set(object())
with self.assertRaises(errors.MonitoringInvalidValueTypeError):
m.set(1.5)
with self.assertRaises(errors.MonitoringInvalidValueTypeError):
m.increment_by(1.5)
def test_multiple_field_values(self):
m = metrics.CounterMetric('test')
m.increment({'foo': 'bar'})
m.increment({'foo': 'baz'})
m.increment({'foo': 'bar'})
self.assertIsNone(None)
self.assertEquals(2, m.get({'foo': 'bar'}))
self.assertEquals(1, m.get({'foo': 'baz'}))
def test_with_target_type(self):
m = metrics.CounterMetric('test',
'test',
None,
target_type=my_target_pb2.MyTarget)
with self.assertRaises(AssertionError):
m._get_target_context()
with interface.target_context(my_target_pb2.MyTarget(s='x')):
self.assertEqual('x', m._get_target_context().s)
def test_grouping(self):
counter0 = metrics.CounterMetric('counter0', 'desc0',
[metrics.IntegerField('test')])
counter1 = metrics.CounterMetric('counter1', 'desc1', None)
counter2 = metrics.CounterMetric('counter2', 'desc2', None)
interface.register(counter0)
interface.register(counter1)
interface.register(counter2)
counter0.increment_by(3, {'test': 123})
counter0.increment_by(5, {'test': 999})
counter1.increment()
counter2.increment_by(4, target_fields={'task_num': 1})
protos = list(interface._generate_proto())
self.assertEqual(1, len(protos))
proto = protos[0]
self.assertEqual(2, len(proto.metrics_collection))
for coll in proto.metrics_collection:
self.assertEqual('service', coll.task.service_name)
self.assertEqual('job', coll.task.job_name)
self.assertEqual('region', coll.task.data_center)
self.assertEqual('hostname', coll.task.host_name)
first_coll = proto.metrics_collection[0]
second_coll = proto.metrics_collection[1]
self.assertEqual(0, first_coll.task.task_num)
self.assertEqual(1, second_coll.task.task_num)
self.assertEqual(2, len(first_coll.metrics_data_set))
self.assertEqual(1, len(second_coll.metrics_data_set))
data_sets = [
first_coll.metrics_data_set[0],
first_coll.metrics_data_set[1],
second_coll.metrics_data_set[0]
]
for i, data_set in enumerate(data_sets):
self.assertEqual('/infra/test/counter%d' % i, data_set.metric_name)
def test_get_all(self):
m = metrics.CounterMetric('test')
m.increment()
m.increment({'foo': 'bar'})
m.increment({'foo': 'baz', 'moo': 'wibble'})
self.assertEqual([
((), 1),
((('foo', 'bar'), ), 1),
((('foo', 'baz'), ('moo', 'wibble')), 1),
], sorted(m.get_all()))
def test_multiple_field_values(self):
m = metrics.CounterMetric('test', 'test', [metrics.StringField('foo')])
m.increment({'foo': 'bar'})
m.increment({'foo': 'baz'})
m.increment({'foo': 'bar'})
with self.assertRaises(errors.WrongFieldsError):
m.get()
self.assertIsNone(m.get({'foo': ''}))
self.assertEqual(2, m.get({'foo': 'bar'}))
self.assertEqual(1, m.get({'foo': 'baz'}))
def test_flush_new(self):
interface.state.metric_name_prefix = '/infra/test/'
interface.state.global_monitor = mock.create_autospec(monitors.Monitor)
interface.state.target = targets.TaskTarget('a', 'b', 'c', 'd', 1)
counter = metrics.CounterMetric('counter', 'desc', None)
interface.register(counter)
counter.increment_by(3)
interface.flush()
self.assertEqual(1, interface.state.global_monitor.send.call_count)
proto = interface.state.global_monitor.send.call_args[0][0]
self.assertEqual(1, len(proto.metrics_collection))
self.assertEqual(1, len(proto.metrics_collection[0].metrics_data_set))
data_set = proto.metrics_collection[0].metrics_data_set[0]
self.assertEqual('/infra/test/counter', data_set.metric_name)
def test_generate_proto_with_dangerously_set_start_time(self):
counter0 = metrics.CounterMetric('counter0', 'desc0',
[metrics.IntegerField('test')])
interface.register(counter0)
counter0.increment_by(3, {'test': 123})
counter0.increment_by(3, {'test': 456})
counter0.dangerously_set_start_time(1000)
proto = list(interface._generate_proto())[0]
data_set = proto.metrics_collection[0].metrics_data_set[0]
for d in data_set.data:
self.assertEqual(1000, d.start_timestamp.seconds)
counter0.dangerously_set_start_time(2000)
proto = list(interface._generate_proto())[0]
data_set = proto.metrics_collection[0].metrics_data_set[0]
for d in data_set.data:
self.assertEqual(2000, d.start_timestamp.seconds)
STATUS_ERROR = 901
STATUS_TIMEOUT = 902
STATUS_EXCEPTION = 909
request_bytes = metrics.CumulativeDistributionMetric(
'http/request_bytes',
description='Bytes sent per http request (body only).')
response_bytes = metrics.CumulativeDistributionMetric(
'http/response_bytes',
description='Bytes received per http request (content only).')
durations = metrics.CumulativeDistributionMetric(
'http/durations',
description='Time elapsed between sending a request and getting a'
' response (including parsing) in milliseconds.')
response_status = metrics.CounterMetric(
'http/response_status',
description='Number of responses received by HTTP status code.')
server_request_bytes = metrics.CumulativeDistributionMetric(
'http/server_request_bytes',
description='Bytes received per http request (body only).')
server_response_bytes = metrics.CumulativeDistributionMetric(
'http/server_response_bytes',
description='Bytes sent per http request (content only).')
server_durations = metrics.CumulativeDistributionMetric(
'http/server_durations',
description='Time elapsed between receiving a request and sending a'
' response (including parsing) in milliseconds.')
server_response_status = metrics.CounterMetric(
'http/server_response_status',
description='Number of responses sent by HTTP status code.')
from infra_libs.ts_mon.common import metrics
REGION = 'appengine'
PUBSUB_PROJECT = 'chrome-infra-mon-pubsub'
PUBSUB_TOPIC = 'monacq'
INSTANCE_NAMESPACE = 'ts_mon_instance_namespace'
# Duration of inactivity to consider an instance dead.
INSTANCE_EXPIRE_SEC = 30 * 60
INSTANCE_EXPECTED_TO_HAVE_TASK_NUM_SEC = 5 * 60
INTERNAL_CALLBACK_NAME = '__gae_ts_mon_callback'
appengine_default_version = metrics.StringMetric(
'appengine/default_version',
description='Name of the version currently marked as default.')
started_counter = metrics.CounterMetric(
'appengine/instances/started',
description='Count the number of GAE instance initializations.')
shutdown_counter = metrics.CounterMetric(
'appengine/instances/shutdown',
description='Count the number of GAE instance shutdowns.')
expired_counter = metrics.CounterMetric(
'appengine/instances/expired',
description=('Count the number of GAE instance expirations '
'due to inactivity.'))
global_metrics = {}
global_metrics_callbacks = {}
def reset_for_unittest():
global global_metrics
def test_generate_proto(self):
proto = self._test_proto(metrics.CounterMetric('c', 'test', None),
lambda m: m.increment_by(5),
metrics_pb2.INT64, metrics_pb2.CUMULATIVE)
self.assertEqual(5, proto.int64_value)
def test_is_cumulative(self):
m = metrics.CounterMetric('test', 'test', None)
self.assertTrue(m.is_cumulative())
def test_dangerously_set_start_time(self):
m = metrics.CounterMetric('test', 'test', None)
self.assertEqual(None, m.start_time)
m.dangerously_set_start_time(102.5)
self.assertEqual(102.5, m.start_time)
def test_set(self):
m = metrics.CounterMetric('test')
m.set(10)
self.assertEquals(m.get(), 10)
def test_populate_value(self):
pb = metrics_pb2.MetricsData()
m = metrics.CounterMetric('test')
m._populate_value(pb, 1, 1234)
self.assertEquals(pb.counter, 1)
PRODXMON_SERVICE_ACCOUNT_EMAIL = (
'app-engine-metric-publishers@'
'prodx-mon-chrome-infra.google.com.iam.gserviceaccount.com')
INSTANCE_NAMESPACE = 'ts_mon_instance_namespace'
# Duration of inactivity to consider an instance dead.
INSTANCE_EXPIRE_SEC = 30 * 60
INSTANCE_EXPECTED_TO_HAVE_TASK_NUM_SEC = 5 * 60
INTERNAL_CALLBACK_NAME = '__gae_ts_mon_callback'
appengine_default_version = metrics.StringMetric(
'appengine/default_version',
'Name of the version currently marked as default.',
None)
started_counter = metrics.CounterMetric(
'appengine/instances/started',
'Count the number of GAE instance initializations.',
None)
shutdown_counter = metrics.CounterMetric(
'appengine/instances/shutdown',
'Count the number of GAE instance shutdowns.',
None)
expired_counter = metrics.CounterMetric(
'appengine/instances/expired',
'Count the number of GAE instance expirations due to inactivity.',
None)
class Instance(ndb.Model):
"""Used to map instances to small integers.
Each instance "owns" an entity with the key <instance-id>.<version>.<module>.
def test_without_target_type(self):
m = metrics.CounterMetric('test', 'test', None)
self.assertIsNone(m._get_target_context())
with interface.target_context(my_target_pb2.MyTarget(s='x')):
self.assertIsNone(m._get_target_context())
])
response_bytes = metrics.CumulativeDistributionMetric(
'http/response_bytes', 'Bytes received per http request (content only).', [
metrics.StringField('name'),
metrics.StringField('client'),
])
durations = metrics.CumulativeDistributionMetric(
'http/durations', 'Time elapsed between sending a request and getting a'
' response (including parsing) in milliseconds.', [
metrics.StringField('name'),
metrics.StringField('client'),
])
response_status = metrics.CounterMetric(
'http/response_status',
'Number of responses received by HTTP status code.', [
metrics.IntegerField('status'),
metrics.StringField('name'),
metrics.StringField('client'),
])
server_request_bytes = metrics.CumulativeDistributionMetric(
'http/server_request_bytes',
'Bytes received per http request (body only).', [
metrics.IntegerField('status'),
metrics.StringField('name'),
metrics.BooleanField('is_robot'),
])
server_response_bytes = metrics.CumulativeDistributionMetric(
'http/server_response_bytes',
'Bytes sent per http request (content only).', [
metrics.IntegerField('status'),
STATUS_OK = 200
STATUS_ERROR = 901
STATUS_TIMEOUT = 902
STATUS_EXCEPTION = 909
request_bytes = metrics.CumulativeDistributionMetric(
'http/request_bytes',
description='Bytes sent per http request (body only).')
response_bytes = metrics.CumulativeDistributionMetric(
'http/response_bytes',
description='Bytes received per http request (content only).')
durations = metrics.CumulativeDistributionMetric(
'http/durations',
description='Time elapsed between sending a request and getting a'
' response (including parsing) in milliseconds.')
response_status = metrics.CounterMetric('http/response_status')
server_request_bytes = metrics.CumulativeDistributionMetric(
'http/server_request_bytes',
description='Bytes received per http request (body only).')
server_response_bytes = metrics.CumulativeDistributionMetric(
'http/server_response_bytes',
description='Bytes sent per http request (content only).')
server_durations = metrics.CumulativeDistributionMetric(
'http/server_durations',
description='Time elapsed between receiving a request and sending a'
' response (including parsing) in milliseconds.')
server_response_status = metrics.CounterMetric('http/server_response_status')