def _get_chrome_infra_event(timestamp_kind, service_name=None):
"""Compute a basic event.
Validates the inputs and returns a pre-filled ChromeInfraEvent or
None if any check failed.
The proto is filled using values provided in setup_monitoring() at
initialization time, and args.
Args:
timestamp_kind (string): any of ('POINT', 'BEGIN', 'END').
Returns:
event (chrome_infra_log_pb2.ChromeInfraEvent):
"""
if timestamp_kind not in TIMESTAMP_KINDS:
logging.error('Invalid value for timestamp_kind: %s' %
str(timestamp_kind))
return None
# We must accept unicode here.
if service_name is not None and not isinstance(service_name, basestring):
logging.error('Invalid type for service_name: %s' % type(service_name))
return None
event = ChromeInfraEvent()
event.CopyFrom(config.cache['default_event'])
if timestamp_kind:
event.timestamp_kind = getattr(ChromeInfraEvent, timestamp_kind)
if service_name:
event.event_source.service_name = service_name
return event
def _get_chrome_infra_event(timestamp_kind, service_name=None):
"""Compute a basic event.
Validates the inputs and returns a pre-filled ChromeInfraEvent or
None if any check failed.
The proto is filled using values provided in setup_monitoring() at
initialization time, and args.
Args:
timestamp_kind (string): any of ('POINT', 'BEGIN', 'END').
Returns:
event (chrome_infra_log_pb2.ChromeInfraEvent):
"""
if timestamp_kind not in TIMESTAMP_KINDS:
logging.error('Invalid value for timestamp_kind: %s' %
str(timestamp_kind))
return None
# We must accept unicode here.
if service_name is not None and not isinstance(service_name, basestring):
logging.error('Invalid type for service_name: %s' % type(service_name))
return None
event = ChromeInfraEvent()
event.CopyFrom(config.cache['default_event'])
if timestamp_kind:
event.timestamp_kind = getattr(ChromeInfraEvent, timestamp_kind)
if service_name:
event.event_source.service_name = service_name
return event
def test_get_build_event_test_result_mapping(self):
# Tests the hacky mapping between buildbot results and the proto values.
hostname = 'bot.host.name'
build_name = 'build_name'
build_number = 314159265
build_scheduling_time = 123456789
# WARNINGS -> WARNING
log_event = monitoring.get_build_event(
'BUILD',
hostname,
build_name,
build_number=build_number,
build_scheduling_time=build_scheduling_time,
result='WARNINGS') # with an S
self.assertIsInstance(log_event, LogRequestLite.LogEventLite)
# Check that source_extension deserializes to the right thing.
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('build_event'))
self.assertEquals(event.build_event.type, BuildEvent.BUILD)
self.assertEquals(event.build_event.host_name, hostname)
self.assertEquals(event.build_event.build_name, build_name)
self.assertEquals(event.build_event.build_number, build_number)
self.assertEquals(event.build_event.build_scheduling_time_ms,
build_scheduling_time)
self.assertEquals(event.build_event.result, BuildEvent.WARNING) # no S
# EXCEPTION -> INFRA_FAILURE
log_event = monitoring.get_build_event(
'BUILD',
hostname,
build_name,
build_number=build_number,
build_scheduling_time=build_scheduling_time,
result='EXCEPTION')
self.assertIsInstance(log_event, LogRequestLite.LogEventLite)
# Check that source_extension deserializes to the right thing.
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('build_event'))
self.assertEquals(event.build_event.type, BuildEvent.BUILD)
self.assertEquals(event.build_event.host_name, hostname)
self.assertEquals(event.build_event.build_name, build_name)
self.assertEquals(event.build_event.build_number, build_number)
self.assertEquals(event.build_event.build_scheduling_time_ms,
build_scheduling_time)
self.assertEquals(event.build_event.result, BuildEvent.INFRA_FAILURE)
def setup_monitoring(run_type='dry',
hostname=None,
service_name=None,
appengine_name=None,
service_account_creds=None,
service_accounts_creds_root=None):
"""Initializes event monitoring.
This function is mainly used to provide default global values which are
required for the module to work.
If you're implementing a command-line tool, use process_argparse_options
instead.
Args:
run_type (str): One of 'dry', 'test', or 'prod'. Do respectively nothing,
hit the testing endpoint and the production endpoint.
hostname (str): hostname as it should appear in the event. If not provided
a default value is computed.
service_name (str): logical name of the service that emits events. e.g.
"commit_queue".
appengine_name (str): name of the appengine app, if running on appengine.
service_account_creds (str): path to a json file containing a service
account's credentials obtained from a Google Cloud project. **Path is
relative to service_account_creds_root**, which is not the current path by
default. See infra_libs.authentication for details.
service_account_creds_root (str): path containing credentials files.
"""
global _router
logging.debug('event_mon: setting up monitoring.')
if not _router: # pragma: no cover
default_event = ChromeInfraEvent()
hostname = hostname or socket.getfqdn()
# hostname might be empty string or None on some systems, who knows.
if hostname: # pragma: no branch
default_event.event_source.host_name = hostname
else:
logging.warning('event_mon: unable to determine hostname.')
if service_name:
default_event.event_source.service_name = service_name
if appengine_name:
default_event.event_source.appengine_name = appengine_name
cache['default_event'] = default_event
cache['service_account_creds'] = service_account_creds
cache['service_accounts_creds_root'] = service_accounts_creds_root
if run_type not in ENDPOINTS:
logging.error('Unknown run_type (%s). Setting to "dry"', run_type)
endpoint = ENDPOINTS.get(run_type)
_router = _Router(cache, endpoint=endpoint)
def test_get_build_event_with_step_info_wrong_type(self):
# BUILD event with step info is invalid.
hostname = 'bot.host.name'
build_name = 'build_name'
build_number = 314159265
build_scheduling_time = 123456789
step_name = 'step_name'
step_number = 0 # valid step number
log_event = monitoring.get_build_event(
'BUILD',
hostname,
build_name,
build_number=build_number,
build_scheduling_time=build_scheduling_time,
step_name=step_name,
step_number=step_number)
self.assertIsInstance(log_event, LogRequestLite.LogEventLite)
# Check that source_extension deserializes to the right thing.
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('build_event'))
self.assertEquals(event.build_event.type, BuildEvent.BUILD)
self.assertEquals(event.build_event.host_name, hostname)
self.assertEquals(event.build_event.build_name, build_name)
self.assertEquals(event.build_event.build_number, build_number)
self.assertEquals(event.build_event.build_scheduling_time_ms,
build_scheduling_time)
self.assertEquals(event.build_event.step_name, step_name)
self.assertEquals(event.build_event.step_number, step_number)
def test_get_build_event_step_info_missing_build_info(self):
hostname = 'bot.host.name'
build_name = 'build_name'
step_name = 'step_name'
step_number = 0 # valid step number
log_event = monitoring.get_build_event(
'STEP',
hostname,
build_name,
step_name=step_name,
step_number=step_number)
self.assertIsInstance(log_event, LogRequestLite.LogEventLite)
# Check that source_extension deserializes to the right thing.
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('build_event'))
self.assertEquals(event.build_event.type, BuildEvent.STEP)
self.assertEquals(event.build_event.host_name, hostname)
self.assertEquals(event.build_event.build_name, build_name)
self.assertEquals(event.build_event.step_name, step_name)
self.assertEquals(event.build_event.step_number, step_number)
self.assertFalse(event.build_event.HasField('build_number'))
self.assertFalse(event.build_event.HasField('build_scheduling_time_ms'))
def test_get_service_event_crash_simple(self):
self.assertIsInstance(config._router, router._Router)
self.assertIsInstance(config.cache.get('default_event'), ChromeInfraEvent)
log_event = monitoring._get_service_event('CRASH')
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertEqual(event.service_event.type, ServiceEvent.CRASH)
def test_get_build_event_with_invalid_result(self):
hostname = 'bot.host.name'
build_name = 'build_name'
build_number = 314159265
build_scheduling_time = 123456789
result = '---INVALID---'
log_event = monitoring.get_build_event(
'BUILD',
hostname,
build_name,
build_number=build_number,
build_scheduling_time=build_scheduling_time,
result=result)
self.assertIsInstance(log_event, LogRequestLite.LogEventLite)
# Check that source_extension deserializes to the right thing.
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('build_event'))
self.assertEquals(event.build_event.type, BuildEvent.BUILD)
self.assertEquals(event.build_event.host_name, hostname)
self.assertEquals(event.build_event.build_name, build_name)
self.assertEquals(event.build_event.build_number, build_number)
self.assertEquals(event.build_event.build_scheduling_time_ms,
build_scheduling_time)
self.assertFalse(event.build_event.HasField('result'))
def test_get_build_event_valid_result_wrong_type(self):
# SCHEDULER can't have a result
hostname = 'bot.host.name'
build_name = 'build_name'
build_number = 314159265
build_scheduling_time = 123456789
result = 'SUCCESS'
log_event = monitoring.get_build_event(
'SCHEDULER',
hostname,
build_name,
build_number=build_number,
build_scheduling_time=build_scheduling_time,
result=result)
self.assertIsInstance(log_event, LogRequestLite.LogEventLite)
# Check that source_extension deserializes to the right thing.
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('build_event'))
self.assertEquals(event.build_event.type, BuildEvent.SCHEDULER)
self.assertEquals(event.build_event.host_name, hostname)
self.assertEquals(event.build_event.build_name, build_name)
self.assertEquals(event.build_event.build_number, build_number)
self.assertEquals(event.build_event.build_scheduling_time_ms,
build_scheduling_time)
self.assertEquals(event.build_event.result, BuildEvent.SUCCESS)
def test_get_service_event_crash_with_unicode_trace(self):
self.assertIsInstance(config._router, router._Router)
self.assertIsInstance(config.cache.get('default_event'), ChromeInfraEvent)
stack_trace = u"Soyez prêt à un étrange goût de Noël."
log_event = monitoring._get_service_event('CRASH',
stack_trace=stack_trace)
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertEqual(event.service_event.type, ServiceEvent.CRASH)
self.assertEqual(event.service_event.stack_trace, stack_trace)
def test_get_service_event_crash_with_ascii_trace(self):
self.assertIsInstance(config._router, router._Router)
self.assertIsInstance(config.cache.get('default_event'), ChromeInfraEvent)
stack_trace = 'A nice ascii string'
log_event = monitoring._get_service_event('CRASH',
stack_trace=stack_trace)
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertEqual(event.service_event.type, ServiceEvent.CRASH)
self.assertEqual(event.service_event.stack_trace, stack_trace)
def test_get_service_event_crash_with_big_trace(self):
self.assertIsInstance(config._router, router._Router)
self.assertIsInstance(config.cache.get('default_event'), ChromeInfraEvent)
stack_trace = "this is way too long" * 55
self.assertTrue(len(stack_trace) > monitoring.STACK_TRACE_MAX_SIZE)
log_event = monitoring._get_service_event('CRASH',
stack_trace=stack_trace)
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertEqual(event.service_event.type, ServiceEvent.CRASH)
self.assertEqual(len(event.service_event.stack_trace),
monitoring.STACK_TRACE_MAX_SIZE)
def test_get_service_event_trace_without_crash(self):
self.assertIsInstance(config._router, router._Router)
self.assertIsInstance(config.cache.get('default_event'), ChromeInfraEvent)
stack_trace = 'A nice ascii string'
log_event = monitoring._get_service_event('START',
stack_trace=stack_trace)
event = ChromeInfraEvent.FromString(log_event.source_extension)
# Make sure we send even invalid data.
self.assertEqual(event.service_event.type, ServiceEvent.START)
self.assertEqual(event.service_event.stack_trace, stack_trace)
def test_send_service_event_bad_versions(self):
# Check that an invalid version does not cause any exception.
self.assertIsInstance(config._router, router._Router)
self.assertIsInstance(config.cache.get('default_event'), ChromeInfraEvent)
code_version = [{}, {'revision': 'https://fake.url'}]
log_event = monitoring._get_service_event('START',
code_version=code_version)
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('service_event'))
self.assertTrue(event.service_event.HasField('type'))
self.assertEqual(len(event.service_event.code_version), 0)
def test_send_service_event_bad_type(self):
# Check that an invalid type for code_version does not raise
# any exception.
code_versions = [None, 123, 'string',
[None], [123], ['string'], [['list']]]
for code_version in code_versions:
log_event = monitoring._get_service_event('START',
code_version=code_version)
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('service_event'))
self.assertTrue(event.service_event.HasField('type'))
self.assertEqual(len(event.service_event.code_version), 0)
def test_get_build_event_invalid_hostname(self):
# an invalid hostname is not a critical error.
builder_name = 'builder_name'
log_event = monitoring.get_build_event('BUILD', None, builder_name)
self.assertIsInstance(log_event, LogRequestLite.LogEventLite)
# Check that source_extension deserializes to the right thing.
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('build_event'))
self.assertEquals(event.build_event.type, BuildEvent.BUILD)
self.assertEquals(event.build_event.build_name, builder_name)
self.assertFalse(event.build_event.HasField('host_name'))
def test_get_service_event_default(self):
self.assertIsInstance(config._router, router._Router)
self.assertIsInstance(config.cache.get('default_event'), ChromeInfraEvent)
log_event = monitoring._get_service_event('START')
self.assertIsInstance(log_event, LogRequestLite.LogEventLite)
self.assertTrue(log_event.HasField('event_time_ms'))
self.assertTrue(log_event.HasField('source_extension'))
# Check that source_extension deserializes to the right thing.
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('service_event'))
self.assertTrue(event.service_event.HasField('type'))
self.assertEquals(event.service_event.type, ServiceEvent.START)
def test_get_service_event_crash_invalid_trace(self):
self.assertIsInstance(config._router, router._Router)
self.assertIsInstance(config.cache.get('default_event'), ChromeInfraEvent)
# This is not a stacktrace
stack_trace = 123456
# Should not crash
log_event = monitoring._get_service_event('CRASH',
stack_trace=stack_trace)
event = ChromeInfraEvent.FromString(log_event.source_extension)
# Send only valid data this time
self.assertEqual(event.service_event.type, ServiceEvent.CRASH)
self.assertFalse(event.service_event.HasField('stack_trace'))
def test_get_build_event_default(self):
hostname = 'bot.host.name'
build_name = 'build_name'
log_event = monitoring.get_build_event('BUILD', hostname, build_name)
self.assertIsInstance(log_event, LogRequestLite.LogEventLite)
self.assertTrue(log_event.HasField('event_time_ms'))
self.assertTrue(log_event.HasField('source_extension'))
# Check that source_extension deserializes to the right thing.
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('build_event'))
self.assertEquals(event.build_event.type, BuildEvent.BUILD)
self.assertEquals(event.build_event.host_name, hostname)
self.assertEquals(event.build_event.build_name, build_name)
def _post_to_endpoint(self, events, try_num=3, retry_backoff=2.):
"""Post protobuf to endpoint.
Args:
events(LogRequestLite): the protobuf to post.
Keyword Args:
try_num(int): max number of http requests send to the endpoint.
retry_backoff(float): time in seconds before retrying posting to the
endpoint. Randomized exponential backoff is applied on subsequent
retries.
Returns:
success(bool): whether pushing to the endpoint succeeded or not.
"""
# Set this time at the very last moment
events.request_time_ms = time_ms()
if self.endpoint: # pragma: no cover
logging.info('event_mon: POSTing events to %s', self.endpoint)
for attempt in xrange(try_num - 1):
response, _ = self.http.request(
uri=self.endpoint,
method='POST',
headers={'Content-Type': 'application/octet-stream'},
body=events.SerializeToString()
)
if response.status == 200:
return True
logging.error('failed to POST data to %s (attempt %d)',
self.endpoint, attempt)
logging.error('data: %s', str(events)[:200])
time.sleep(backoff_time(attempt, retry_backoff=retry_backoff))
return False
else:
infra_events = [str(ChromeInfraEvent.FromString(
ev.source_extension)) for ev in events.log_event]
logging.info('Fake post request. Sending:\n%s',
'\n'.join(infra_events))
return True
def test_get_service_event_correct_versions(self):
self.assertIsInstance(config._router, router._Router)
self.assertIsInstance(config.cache.get('default_event'), ChromeInfraEvent)
code_version = [
{'source_url': 'https://fake.url/thing',
'revision': '708329c2aeece8aac33af6a5a772ffb14b55903f'},
{'source_url': 'svn://fake_svn.url/other_thing',
'revision': '123456'},
{'source_url': 'https://other_fake.url/yet_another_thing',
'version': 'v2.0'},
{'source_url': 'https://other_fake2.url/yet_another_thing2',
'dirty': True},
]
log_event = monitoring._get_service_event('START',
code_version=code_version)
event = ChromeInfraEvent.FromString(log_event.source_extension)
code_version_p = event.service_event.code_version
self.assertEquals(len(code_version_p), len(code_version))
self.assertEqual(code_version_p[0].source_url,
code_version[0]['source_url'])
self.assertEqual(code_version_p[0].git_hash, code_version[0]['revision'])
self.assertFalse(code_version_p[0].HasField('svn_revision'))
self.assertEqual(code_version_p[1].source_url,
code_version[1]['source_url'])
self.assertEqual(code_version_p[1].svn_revision,
int(code_version[1]['revision']))
self.assertFalse(code_version_p[1].HasField('git_hash'))
self.assertEqual(code_version_p[2].source_url,
code_version[2]['source_url'])
self.assertFalse(code_version_p[2].HasField('svn_revision'))
self.assertEqual(code_version_p[2].version,
code_version[2]['version'])
self.assertEqual(code_version_p[3].source_url,
code_version[3]['source_url'])
self.assertEqual(code_version_p[3].dirty, True)
def test_get_build_event_missing_build_number(self):
hostname = 'bot.host.name'
build_name = 'build_name'
build_scheduling_time = 123456789
log_event = monitoring.get_build_event(
'BUILD',
hostname,
build_name,
build_scheduling_time=build_scheduling_time)
self.assertIsInstance(log_event, LogRequestLite.LogEventLite)
# Check that source_extension deserializes to the right thing.
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('build_event'))
self.assertEquals(event.build_event.type, BuildEvent.BUILD)
self.assertEquals(event.build_event.host_name, hostname)
self.assertEquals(event.build_event.build_name, build_name)
self.assertEquals(event.build_event.build_scheduling_time_ms,
build_scheduling_time)
self.assertFalse(event.build_event.HasField('build_number'))
def test_get_build_event_invalid_scheduler(self):
# Providing a build number on a scheduler event is invalid.
hostname = 'bot.host.name'
build_name = 'build_name'
build_number = 314159265 # int32
log_event = monitoring.get_build_event(
'SCHEDULER',
hostname,
build_name,
build_number=build_number)
self.assertIsInstance(log_event, LogRequestLite.LogEventLite)
# Check that source_extension deserializes to the right thing.
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('build_event'))
self.assertEquals(event.build_event.type, BuildEvent.SCHEDULER)
self.assertEquals(event.build_event.host_name, hostname)
self.assertEquals(event.build_event.build_name, build_name)
self.assertEquals(event.build_event.build_number, build_number)
self.assertFalse(event.build_event.HasField('build_scheduling_time_ms'))
def test_get_build_event_with_build_zero(self):
# testing 0 is important because bool(0) == False
hostname = 'bot.host.name'
build_name = 'build_name'
build_number = 0
build_scheduling_time = 123456789
log_event = monitoring.get_build_event(
'BUILD',
hostname,
build_name,
build_number=build_number,
build_scheduling_time=build_scheduling_time)
self.assertIsInstance(log_event, LogRequestLite.LogEventLite)
# Check that source_extension deserializes to the right thing.
event = ChromeInfraEvent.FromString(log_event.source_extension)
self.assertTrue(event.HasField('build_event'))
self.assertEquals(event.build_event.type, BuildEvent.BUILD)
self.assertEquals(event.build_event.host_name, hostname)
self.assertEquals(event.build_event.build_name, build_name)
self.assertEquals(event.build_event.build_number, build_number)
self.assertEquals(event.build_event.build_scheduling_time_ms,
build_scheduling_time)