def test_send_build_event_with_goma_error_crashed(self):
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments([
'--event-mon-run-type', 'file', '--event-mon-output-file',
outfile, '--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD', '--build-event-hostname',
'foo.bar.dns', '--build-event-build-name', 'whatever',
'--build-event-goma-error', 'GOMA_ERROR_CRASHED',
'--build-event-goma-crash-report-id-path',
os.path.join(DATA_DIR, 'goma_error_report.txt')
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
self.assertTrue(common.send_build_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(
request.log_event[0].source_extension)
self.assertEqual(event.build_event.goma_error,
BuildEvent.GOMA_ERROR_CRASHED)
self.assertEqual(event.build_event.goma_crash_report_id,
'0123456789abcdef0')
self.assertEqual(event.build_event.host_name, 'foo.bar.dns')
def test_send_build_event_with_goma_stats(self):
# Write a file to avoid mocks
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD',
'--build-event-hostname', 'foo.bar.dns',
'--build-event-build-name', 'whatever',
'--build-event-goma-stats-path',
os.path.join(DATA_DIR, 'goma_stats.bin')])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
self.assertTrue(send_event.send_build_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(request.log_event[0].source_extension)
self.assertEqual(event.build_event.goma_stats.request_stats.total, 10)
self.assertEqual(event.build_event.goma_stats.request_stats.success, 9)
self.assertEqual(event.build_event.goma_stats.request_stats.failure, 1)
self.assertEqual(event.build_event.host_name, 'foo.bar.dns')
def test_logrequest_path_build_type_override(self):
# logrequest contains build event, overrid the type with an arg.
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments([
'--event-mon-run-type',
'file',
'--event-mon-output-file',
outfile,
'--event-mon-service-name',
'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'build-foo-builder.bin'),
'--build-event-build-number',
'3',
'--build-event-type',
'STEP',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
common.process_argparse_options(args)
self.assertTrue(common.send_build_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(
request.log_event[0].source_extension)
self.assertEqual(event.build_event.host_name, 'myhostname')
self.assertEqual(event.build_event.type, BuildEvent.STEP)
self.assertEqual(event.build_event.build_number, 3)
self.assertEqual(event.timestamp_kind, ChromeInfraEvent.BEGIN)
def test_send_build_event_with_goma_error_crashed(self):
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD',
'--build-event-hostname', 'foo.bar.dns',
'--build-event-build-name', 'whatever',
'--build-event-goma-error', 'GOMA_ERROR_CRASHED',
'--build-event-goma-crash-report-id-path',
os.path.join(DATA_DIR, 'goma_error_report.txt')])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
self.assertTrue(send_event.send_build_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(request.log_event[0].source_extension)
self.assertEqual(event.build_event.goma_error,
BuildEvent.GOMA_ERROR_CRASHED)
self.assertEqual(event.build_event.goma_crash_report_id,
'0123456789abcdef0')
self.assertEqual(event.build_event.host_name, 'foo.bar.dns')
def test_logrequest_path_build_type_override(self):
# logrequest contains build event, overrid the type with an arg.
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'build-foo-builder.bin'),
'--build-event-build-number', '3',
'--build-event-type', 'STEP',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
send_event._process_logrequest_path(args)
self.assertTrue(send_event.send_build_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(request.log_event[0].source_extension)
self.assertEqual(event.build_event.host_name, 'myhostname')
self.assertEqual(event.build_event.type, BuildEvent.STEP)
self.assertEqual(event.build_event.build_number, 3)
self.assertEqual(event.timestamp_kind, ChromeInfraEvent.BEGIN)
def test_logrequest_path_service_type_override(self):
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'service-bar-service.bin'),
'--service-event-type', 'STOP',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
send_event._process_logrequest_path(args)
self.assertTrue(send_event.send_service_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(request.log_event[0].source_extension)
self.assertEqual(event.event_source.host_name, 'myhostname')
self.assertEqual(event.service_event.type, ServiceEvent.STOP)
self.assertEqual(event.timestamp_kind, ChromeInfraEvent.END)
def test_send_build_event_with_goma_stats(self):
# Write a file to avoid mocks
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments([
'--event-mon-run-type', 'file', '--event-mon-output-file',
outfile, '--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD', '--build-event-hostname',
'foo.bar.dns', '--build-event-build-name', 'whatever',
'--build-event-goma-stats-path',
os.path.join(DATA_DIR, 'goma_stats.bin')
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
self.assertTrue(common.send_build_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(
request.log_event[0].source_extension)
self.assertEqual(event.build_event.goma_stats.request_stats.total, 10)
self.assertEqual(event.build_event.goma_stats.request_stats.success, 9)
self.assertEqual(event.build_event.goma_stats.request_stats.failure, 1)
self.assertEqual(event.build_event.host_name, 'foo.bar.dns')
def test_logrequest_path_service_type_override(self):
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments([
'--event-mon-run-type',
'file',
'--event-mon-output-file',
outfile,
'--event-mon-service-name',
'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'service-bar-service.bin'),
'--service-event-type',
'STOP',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
common.process_argparse_options(args)
self.assertTrue(common.send_service_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(
request.log_event[0].source_extension)
self.assertEqual(event.event_source.host_name, 'myhostname')
self.assertEqual(event.service_event.type, ServiceEvent.STOP)
self.assertEqual(event.timestamp_kind, ChromeInfraEvent.END)
def _set_up_args(self, args=None): # pragma: no cover parser = argparse.ArgumentParser() event_mon.add_argparse_options(parser) args = parser.parse_args((args or [])) self.assertEquals(args.event_mon_run_type, 'dry') event_mon.process_argparse_options(args) r = config._router self.assertIsInstance(r, router._Router) # Check that process_argparse_options is idempotent event_mon.process_argparse_options(args) self.assertIs(config._router, r)
def _set_up_args(self, args=None): # pragma: no cover
parser = argparse.ArgumentParser()
event_mon.add_argparse_options(parser)
args = parser.parse_args((args or []))
# Safety net. We really don't want to send something to a real endpoint.
self.assertTrue((args.event_mon_run_type not in ('test', 'prod'))
or args.dry_run)
event_mon.process_argparse_options(args)
r = config._router
self.assertIsInstance(r, router._Router)
# Check that process_argparse_options is idempotent
event_mon.process_argparse_options(args)
self.assertIs(config._router, r)
def _set_up_args(self, args=None): # pragma: no cover
parser = argparse.ArgumentParser()
event_mon.add_argparse_options(parser)
args = parser.parse_args((args or []))
# Safety net. We really don't want to send something to a real endpoint.
self.assertTrue((args.event_mon_run_type not in ('test', 'prod'))
or args.dry_run)
event_mon.process_argparse_options(args)
r = config._router
self.assertIsInstance(r, router._Router)
# Check that process_argparse_options is idempotent
event_mon.process_argparse_options(args)
self.assertIs(config._router, r)
def test_logrequest_path_service_build_conflict(self):
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'service-bar-service.bin'),
'--build-event-type', 'BUILD',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(ValueError):
common.process_argparse_options(args)
def test_logrequest_path_service_build_conflict(self):
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'service-bar-service.bin'),
'--build-event-type', 'BUILD',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(ValueError):
send_event._process_logrequest_path(args)
def test_send_build_event_with_invalid_goma_stats(self):
# Write a file to avoid mocks
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments([
'--event-mon-run-type', 'file', '--event-mon-output-file',
outfile, '--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD', '--build-event-hostname',
'foo.bar.dns', '--build-event-build-name', 'whatever',
'--build-event-goma-stats-path',
os.path.join(DATA_DIR, 'garbage')
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(google.protobuf.message.DecodeError):
common.send_build_event(args)
def test_logrequest_path_service_build_and_service(self):
# The logrequest provided contains both a service and a build type,
# which is invalid.
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'build-and-service-event.bin'),
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(ValueError):
send_event._process_logrequest_path(args)
def test_logrequest_path_service_build_and_service(self):
# The logrequest provided contains both a service and a build type,
# which is invalid.
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'build-and-service-event.bin'),
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(ValueError):
common.process_argparse_options(args)
def test_send_build_event_with_non_existing_goma_error_report(self):
# Write a file to avoid mocks
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments([
'--event-mon-run-type', 'file', '--event-mon-output-file',
outfile, '--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD', '--build-event-hostname',
'foo.bar.dns', '--build-event-build-name', 'whatever',
'--build-event-goma-error', 'GOMA_ERROR_CRASHED',
'--build-event-goma-crash-report-id-path',
os.path.join(DATA_DIR, 'this-file-does-not-exist')
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(IOError):
common.send_build_event(args)
def test_logrequest_path_build_service_conflicts(self):
# logrequest contains build event, provides service event as arg
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'build-foo-builder.bin'),
'--build-event-build-number', '3',
'--service-event-type', 'START',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(ValueError):
send_event._process_logrequest_path(args)
def test_logrequest_path_build_service_conflicts(self):
# logrequest contains build event, provides service event as arg
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'build-foo-builder.bin'),
'--build-event-build-number', '3',
'--service-event-type', 'START',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(ValueError):
common.process_argparse_options(args)
def test_send_build_event_with_invalid_goma_stats(self):
# Write a file to avoid mocks
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD',
'--build-event-hostname', 'foo.bar.dns',
'--build-event-build-name', 'whatever',
'--build-event-goma-stats-path',
os.path.join(DATA_DIR, 'garbage')])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(google.protobuf.message.DecodeError):
send_event.send_build_event(args)
def test_send_build_event_with_non_existing_goma_error_report(self):
# Write a file to avoid mocks
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD',
'--build-event-hostname', 'foo.bar.dns',
'--build-event-build-name', 'whatever',
'--build-event-goma-error', 'GOMA_ERROR_CRASHED',
'--build-event-goma-crash-report-id-path',
os.path.join(DATA_DIR, 'this-file-does-not-exist')])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(IOError):
send_event.send_build_event(args)
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--master-name', help='Buildbot master name')
parser.add_argument('--builder-name', help='Buildbot builder name')
parser.add_argument('--build-id', help='Build ID (buildnumber)')
parser.add_argument('--analyze-input',
help='JSON input passed to analyze',
type=argparse.FileType('r'))
parser.add_argument('--analyze-output',
help='JSON output from analyze',
type=argparse.FileType('r'))
event_mon.add_argparse_options(parser)
args = parser.parse_args(argv)
event_mon.process_argparse_options(args)
try:
return inner_main(args)
finally:
event_mon.close()
def process_argparse_options(args): # pragma: no cover infra_libs.logs.process_argparse_options(args) event_mon.process_argparse_options(args) # Put here because we want to send the presence metric as late as possible. ts_mon.process_argparse_options(args) _process_logrequest_path(args)
def process_argparse_options(args): # pragma: no cover
infra_libs.logs.process_argparse_options(args)
event_mon.process_argparse_options(args)
ts_mon.process_argparse_options(args)
def process_argparse_options(args): # pragma: no cover
infra_libs.logs.process_argparse_options(args)
event_mon.process_argparse_options(args)
ts_mon.process_argparse_options(args)
def process_argparse_options(args):
"""Sets the default event based on --event-logrequest-path.
This function raises exceptions because if the base event is wrong, then it's
not worth sending anything anyway.
"""
event_mon.process_argparse_options(args)
_extra_argument_checking(args)
if args.event_logrequest_path:
try:
with open(args.event_logrequest_path, 'rb') as f:
request = LogRequestLite.FromString(f.read())
if len(request.log_event) == 1:
default_event = ChromeInfraEvent.FromString(
request.log_event[0].source_extension)
# Assume that the content is sane because we don't want to duplicate
# any business logic here.
# TODO(pgervais): find a better solution.
event_mon.set_default_event(default_event)
else:
raise ValueError(
'Expected only one log_event in the LogRequestLite proto '
'pointed by --event-logrequest-path. Found %d in %s',
len(request.log_event), args.event_logrequest_path)
except Exception:
LOGGER.exception('Failure when reading/parsing file %s',
args.event_logrequest_path)
raise
default_event = event_mon.get_default_event()
# When the default event is set using --event-logrequest-path, passing
# --build-event-type or --service-event-type is optional. These options
# still takes precedence but they must keep the event type the same.
if (default_event.build_event.HasField('type')
and default_event.service_event.HasField('type')):
msg = (
'Default event contains both service_event_type and '
'build_event_type which is incorrect. Make sure you passed '
'a correct proto to --event-logrequest-path. Otherwise it\'s an '
'internal error. Aborting.')
LOGGER.error(msg)
raise ValueError(msg)
if default_event.build_event.HasField('type'):
if args.service_event_type:
msg = (
'The default event contains a type for build_event, but a '
'service_event type was provided on the command-line. At most '
'one of them can be specified. Aborting.')
LOGGER.error(msg)
raise ValueError(msg)
if not args.build_event_type:
args.build_event_type = event_mon.BuildEvent.BuildEventType.Name(
default_event.build_event.type)
if default_event.service_event.HasField('type'):
if args.build_event_type:
msg = (
'The default event contains a type for service_event, but a '
'build_event type was provided on the command-line. At most '
'one of them can be specified. Aborting.')
LOGGER.error(msg)
raise ValueError(msg)
if not args.service_event_type:
args.service_event_type = event_mon.ServiceEvent.ServiceEventType.Name(
default_event.service_event.type)
