def test_send_build_event_with_goma_stats(self):
# Write a file to avoid mocks
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD',
'--build-event-hostname', 'foo.bar.dns',
'--build-event-build-name', 'whatever',
'--build-event-goma-stats-path',
os.path.join(DATA_DIR, 'goma_stats.bin')])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
self.assertTrue(send_event.send_build_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(request.log_event[0].source_extension)
self.assertEqual(event.build_event.goma_stats.request_stats.total, 10)
self.assertEqual(event.build_event.goma_stats.request_stats.success, 9)
self.assertEqual(event.build_event.goma_stats.request_stats.failure, 1)
self.assertEqual(event.build_event.host_name, 'foo.bar.dns')
def test_send_build_event_with_goma_stats(self):
# Write a file to avoid mocks
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments([
'--event-mon-run-type', 'file', '--event-mon-output-file',
outfile, '--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD', '--build-event-hostname',
'foo.bar.dns', '--build-event-build-name', 'whatever',
'--build-event-goma-stats-path',
os.path.join(DATA_DIR, 'goma_stats.bin')
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
self.assertTrue(common.send_build_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(
request.log_event[0].source_extension)
self.assertEqual(event.build_event.goma_stats.request_stats.total, 10)
self.assertEqual(event.build_event.goma_stats.request_stats.success, 9)
self.assertEqual(event.build_event.goma_stats.request_stats.failure, 1)
self.assertEqual(event.build_event.host_name, 'foo.bar.dns')
def test_file_has_data(self, fake_add, fake_increment):
result1 = {'builder': 'b1', 'slave': 's1',
'result': 'r1', 'project_id': 'chromium',
'subproject_tag': 'unknown'}
result2 = {'builder': 'b1', 'slave': 's1',
'result': 'r1', 'project_id': 'unknown',
'subproject_tag': 'unknown'}
# Check that we've listed all the required metric fields.
self.assertEqual(set(result1), set(pollers.FilePoller.field_keys))
self.assertEqual(set(result2), set(pollers.FilePoller.field_keys))
data1 = result1.copy()
data2 = result2.copy()
data1['random'] = 'value' # Extra field, should be ignored.
del data2['project_id'] # Missing field, should become 'unknown'.
data2['duration_s'] = 5
data2['pending_s'] = 1
data2['total_s'] = data2['pending_s'] + data2['duration_s']
data2['pre_test_time_s'] = 2
with temporary_directory(prefix='poller-test-') as tempdir:
filename = self.create_data_file(tempdir, [data1, data2])
p = pollers.FilePoller(filename, {})
self.assertTrue(p.poll())
fake_increment.assert_any_call(result1)
fake_increment.assert_any_call(result2)
fake_add.assert_any_call(data2['duration_s'], result2)
fake_add.assert_any_call(data2['pending_s'], result2)
fake_add.assert_any_call(data2['total_s'], result2)
self.assertFalse(os.path.isfile(filename))
# Make sure the rotated file is still there - for debugging.
self.assertTrue(os.path.isfile(pollers.rotated_filename(filename)))
def test_logrequest_path_build_type_override(self):
# logrequest contains build event, overrid the type with an arg.
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments([
'--event-mon-run-type',
'file',
'--event-mon-output-file',
outfile,
'--event-mon-service-name',
'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'build-foo-builder.bin'),
'--build-event-build-number',
'3',
'--build-event-type',
'STEP',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
common.process_argparse_options(args)
self.assertTrue(common.send_build_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(
request.log_event[0].source_extension)
self.assertEqual(event.build_event.host_name, 'myhostname')
self.assertEqual(event.build_event.type, BuildEvent.STEP)
self.assertEqual(event.build_event.build_number, 3)
self.assertEqual(event.timestamp_kind, ChromeInfraEvent.BEGIN)
def test_send_build_event_with_goma_error_crashed(self):
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments([
'--event-mon-run-type', 'file', '--event-mon-output-file',
outfile, '--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD', '--build-event-hostname',
'foo.bar.dns', '--build-event-build-name', 'whatever',
'--build-event-goma-error', 'GOMA_ERROR_CRASHED',
'--build-event-goma-crash-report-id-path',
os.path.join(DATA_DIR, 'goma_error_report.txt')
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
self.assertTrue(common.send_build_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(
request.log_event[0].source_extension)
self.assertEqual(event.build_event.goma_error,
BuildEvent.GOMA_ERROR_CRASHED)
self.assertEqual(event.build_event.goma_crash_report_id,
'0123456789abcdef0')
self.assertEqual(event.build_event.host_name, 'foo.bar.dns')
def test_send_build_event_with_goma_error_crashed(self):
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD',
'--build-event-hostname', 'foo.bar.dns',
'--build-event-build-name', 'whatever',
'--build-event-goma-error', 'GOMA_ERROR_CRASHED',
'--build-event-goma-crash-report-id-path',
os.path.join(DATA_DIR, 'goma_error_report.txt')])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
self.assertTrue(send_event.send_build_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(request.log_event[0].source_extension)
self.assertEqual(event.build_event.goma_error,
BuildEvent.GOMA_ERROR_CRASHED)
self.assertEqual(event.build_event.goma_crash_report_id,
'0123456789abcdef0')
self.assertEqual(event.build_event.host_name, 'foo.bar.dns')
def test_logrequest_path_service_type_override(self):
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments([
'--event-mon-run-type',
'file',
'--event-mon-output-file',
outfile,
'--event-mon-service-name',
'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'service-bar-service.bin'),
'--service-event-type',
'STOP',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
common.process_argparse_options(args)
self.assertTrue(common.send_service_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(
request.log_event[0].source_extension)
self.assertEqual(event.event_source.host_name, 'myhostname')
self.assertEqual(event.service_event.type, ServiceEvent.STOP)
self.assertEqual(event.timestamp_kind, ChromeInfraEvent.END)
def test_load_machine_config(self):
with infra_libs.temporary_directory() as temp_dir:
filename = os.path.join(temp_dir, 'config')
with open(filename, 'w') as fh:
json.dump({'foo': 'bar'}, fh)
self.assertEquals({'foo': 'bar'}, config.load_machine_config(filename))
def test_rmtree_file(self):
with infra_libs.temporary_directory() as tempdir:
tmpfile = os.path.join(tempdir, 'foo')
with open(tmpfile, 'w') as f:
f.write('asdfasdf')
infra_libs.rmtree(tmpfile)
self.assertFalse(os.path.exists(tmpfile))
def test_load_machine_config(self):
with infra_libs.temporary_directory() as temp_dir:
filename = os.path.join(temp_dir, 'config')
with open(filename, 'w') as fh:
json.dump({'foo': 'bar'}, fh)
self.assertEquals({'foo': 'bar'}, config.load_machine_config(filename))
def test_logrequest_path_build_type_override(self):
# logrequest contains build event, overrid the type with an arg.
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'build-foo-builder.bin'),
'--build-event-build-number', '3',
'--build-event-type', 'STEP',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
send_event._process_logrequest_path(args)
self.assertTrue(send_event.send_build_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(request.log_event[0].source_extension)
self.assertEqual(event.build_event.host_name, 'myhostname')
self.assertEqual(event.build_event.type, BuildEvent.STEP)
self.assertEqual(event.build_event.build_number, 3)
self.assertEqual(event.timestamp_kind, ChromeInfraEvent.BEGIN)
def test_logrequest_path_service_type_override(self):
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'service-bar-service.bin'),
'--service-event-type', 'STOP',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
send_event._process_logrequest_path(args)
self.assertTrue(send_event.send_service_event(args))
# Now open the resulting file and check what was written
with open(outfile, 'rb') as f:
request = LogRequestLite.FromString(f.read())
self.assertEqual(len(request.log_event), 1)
event = ChromeInfraEvent.FromString(request.log_event[0].source_extension)
self.assertEqual(event.event_source.host_name, 'myhostname')
self.assertEqual(event.service_event.type, ServiceEvent.STOP)
self.assertEqual(event.timestamp_kind, ChromeInfraEvent.END)
def test_get_commits_by_user(self):
with infra_libs.temporary_directory(prefix='antibody-test') as dirname:
# tbr_no_lgtm: review_url, request_timestamp, subject,
# people_email_address, hash
mock_cc = mock.MagicMock();
mock_cc.fetchall.side_effect = (
(
('https://codereview.chromium.org/1175993003',
datetime.datetime(2015, 07, 13, 11, 11, 11), 'hello',
'pgervais', 'git_hash_1'),
('https://codereview.chromium.org/1175993003',
datetime.datetime(2015, 07, 13, 11, 11, 11), 'hello',
'hinoka', 'git_hash_1'),
('https://codereview.chromium.org/1171763002',
datetime.datetime(2015, 07, 13, 22, 22, 22), 'world',
'hinoka', 'git_hash_2'),
('https://codereview.chromium.org/1171763002',
datetime.datetime(2015, 07, 13, 22, 22, 22), 'world',
'keelerh', 'git_hash_2'),
),
(
('https://codereview.chromium.org/1175993001',
datetime.datetime(2015, 07, 13, 11, 11, 11), 'hi',
'pgervais', 'git_hash_3'),
('https://codereview.chromium.org/1171763000',
datetime.datetime(2015, 07, 13, 22, 22, 22), 'there',
'hinoka', 'git_hash_4'),
),
)
antibody.get_commits_by_user(mock_cc,
'https://chromium.googlesource.com/infra/infra/+/', dirname)
expected_out = {
"by_user" : {
"pgervais" : {
"tbr": [['hello', 'https://codereview.chromium.org/1175993003',
'2015-07-13 11:11:11', 'git_hash_1', 'TBR']],
"author": [['hi', 'https://codereview.chromium.org/1175993001',
'2015-07-13 11:11:11', 'git_hash_3', 'Author']],
},
"hinoka" : {
"tbr": [['world', 'https://codereview.chromium.org/1171763002',
'2015-07-13 22:22:22', 'git_hash_2', 'TBR'],
['hello', 'https://codereview.chromium.org/1175993003',
'2015-07-13 11:11:11', 'git_hash_1', 'TBR'],
],
"author": [['there', 'https://codereview.chromium.org/1171763000',
'2015-07-13 22:22:22', 'git_hash_4', 'Author']],
},
"keelerh" : {
"tbr": [['world', 'https://codereview.chromium.org/1171763002',
'2015-07-13 22:22:22', 'git_hash_2', 'TBR']],
"author": [],
},
},
"gitiles_prefix" : "https://chromium.googlesource.com/infra/infra/+/",
}
with open(os.path.join(dirname, 'search_by_user.json'), 'r') as f:
output = json.load(f)
print output
self.assertEqual(output, expected_out)
def test_tempdir_no_error(self):
with infra_libs.temporary_directory() as tempdir:
self.assertTrue(os.path.isdir(tempdir))
# This should work.
with open(os.path.join(tempdir, 'test_tempdir_no_error.txt'), 'w') as f:
f.write('nonsensical content')
# And everything should have been cleaned up afterward
self.assertFalse(os.path.isdir(tempdir))
def test_load_machine_config_bad(self):
with infra_libs.temporary_directory() as temp_dir:
filename = os.path.join(temp_dir, 'config')
with open(filename, 'w') as fh:
fh.write('not a json file')
with self.assertRaises(ValueError):
config.load_machine_config(filename)
def test_load_machine_config_bad(self):
with infra_libs.temporary_directory() as temp_dir:
filename = os.path.join(temp_dir, 'config')
with open(filename, 'w') as fh:
fh.write('not a json file')
with self.assertRaises(ValueError):
config.load_machine_config(filename)
def test_invalid_syntax(self):
with temporary_directory(prefix='metric_tool-') as tempdir:
tempfile = os.path.join(tempdir, 'invalid_syntax.py')
with open(tempfile, 'w') as f:
f.write('=1\n')
descriptions = metric_tool.extract_metrics_descriptions(tempfile)
self.assertEqual(descriptions, [])
def test_generate(self):
with infra_libs.temporary_directory(prefix="new-tool-test-") as tempdir:
toolname = "test_tool"
new_tool.generate_tool_files("test_tool", tempdir)
self.assertTrue(os.path.isdir(os.path.join(tempdir, toolname)))
self.assertTrue(os.path.isfile(os.path.join(tempdir, toolname, "__init__.py")))
self.assertTrue(os.path.isfile(os.path.join(tempdir, toolname, "__main__.py")))
self.assertTrue(os.path.isdir(os.path.join(tempdir, toolname, "test")))
self.assertTrue(os.path.isfile(os.path.join(tempdir, toolname, "test", "__init__.py")))
def test_send_file(self):
with infra_libs.temporary_directory() as temp_dir:
filename = os.path.join(temp_dir, 'out')
m = monitors.DebugMonitor(filename)
payload = metrics_pb2.MetricsPayload()
payload.metrics_collection.add().metrics_data_set.add().metric_name = 'm1'
m.send(payload)
with open(filename) as fh:
output = fh.read()
self.assertIn('metrics_data_set {\n metric_name: "m1"\n }', output)
def test_file_credentials_non_google(self, mock_storage):
with infra_libs.temporary_directory() as temp_dir:
path = os.path.join(temp_dir, 'foo')
with open(path, 'w') as fh:
json.dump({}, fh)
ret = monitors.FileCredentials(path).create(['bar'])
mock_storage.assert_called_once_with(path)
mock_storage.return_value.get.assert_called_once_with()
self.assertEqual(ret, mock_storage.return_value.get.return_value)
def test_ts_mon_file_polling_file_missing(self):
# Test that asking to poll a missing file works.
# Mostly a smoke test.
class MasterMonitor(monitor.MasterMonitor):
POLLER_CLASSES = []
with temporary_directory(prefix="monitor-test-") as tempdir:
ts_mon_filename = os.path.join(tempdir, "ts_mon.json")
self.assertFalse(os.path.isfile(ts_mon_filename))
MasterMonitor("http://example.com", name="foobar", results_file=ts_mon_filename)
self.assertFalse(os.path.isfile(ts_mon_filename))
def test_ts_mon_file_deletion(self):
class MasterMonitor(monitor.MasterMonitor):
POLLER_CLASSES = []
with temporary_directory(prefix="monitor-test-") as tempdir:
ts_mon_filename = os.path.join(tempdir, "ts_mon.json")
with open(ts_mon_filename, "w") as f:
f.write(" ")
self.assertTrue(os.path.isfile(ts_mon_filename))
MasterMonitor("http://example.com", name="foobar", results_file=ts_mon_filename)
self.assertFalse(os.path.isfile(ts_mon_filename))
def test_run_type_file_good(self):
try:
with infra_libs.temporary_directory(prefix='config_test-') as tempdir:
filename = os.path.join(tempdir, 'output.db')
self._set_up_args(args=['--event-mon-run-type', 'file',
'--event-mon-output-file', filename])
self.assertEqual(config._router.output_file, filename)
except Exception: # pragma: no cover
# help for debugging
traceback.print_exc()
raise
def test_file_has_bad_data(self):
"""Mostly a smoke test: don't crash on bad data."""
with temporary_directory(prefix='poller-test-') as tempdir:
filename = self.create_data_file(tempdir, [])
with open(filename, 'a') as f:
f.write('}')
p = pollers.FilePoller(filename, {})
self.assertTrue(p.poll())
self.assertFalse(os.path.isfile(filename))
# Make sure the rotated file is still there - for debugging.
self.assertTrue(os.path.isfile(pollers.rotated_filename(filename)))
def test_tempdir_with_exception(self):
with self.assertRaises(_UtilTestException):
with infra_libs.temporary_directory() as tempdir:
self.assertTrue(os.path.isdir(tempdir))
# Create a non-empty file to check that tempdir deletion works.
with open(os.path.join(tempdir, 'test_tempdir_no_error.txt'), 'w') as f:
f.write('nonsensical content')
raise _UtilTestException()
# And everything should have been cleaned up afterward
self.assertFalse(os.path.isdir(tempdir))
def test_file_credentials_google(self, mock_from_stream):
with infra_libs.temporary_directory() as temp_dir:
path = os.path.join(temp_dir, 'foo')
with open(path, 'w') as fh:
json.dump({'type': 'blah'}, fh)
ret = monitors.FileCredentials(path).create(['bar'])
mock_from_stream.assert_called_once_with(path)
creds = mock_from_stream.return_value
creds.create_scoped.assert_called_once_with(['bar'])
self.assertEqual(ret, creds.create_scoped.return_value)
def test_existing_directory_dry_run(self):
event = LogRequestLite.LogEventLite()
event.event_time_ms = router.time_ms()
event.event_code = 1
event.event_flow_id = 2
with infra_libs.temporary_directory(prefix='event-mon-') as tempdir:
filename = os.path.join(tempdir, 'output.db')
r = router._LocalFileRouter(filename, dry_run=True)
self.assertTrue(r.push_event(event))
# Check that the file has not been written
self.assertFalse(os.path.exists(filename))
def test_existing_directory_dry_run(self):
event = LogRequestLite.LogEventLite()
event.event_time_ms = router.time_ms()
event.event_code = 1
event.event_flow_id = 2
with infra_libs.temporary_directory(prefix='event-mon-') as tempdir:
filename = os.path.join(tempdir, 'output.db')
r = router._LocalFileRouter(filename, dry_run=True)
self.assertTrue(r.push_event(event))
# Check that the file has not been written
self.assertFalse(os.path.exists(filename))
def test_send_file(self):
with infra_libs.temporary_directory() as temp_dir:
filename = os.path.join(temp_dir, 'out')
m = monitors.DebugMonitor(filename)
metric1 = metrics_pb2.MetricsData(name='m1')
m.send(metric1)
metric2 = metrics_pb2.MetricsData(name='m2')
m.send([metric1, metric2])
collection = metrics_pb2.MetricsCollection(data=[metric1, metric2])
m.send(collection)
with open(filename) as fh:
output = fh.read()
self.assertEquals(output.count('data {\n name: "m1"\n}'), 3)
self.assertEquals(output.count('data {\n name: "m2"\n}'), 2)
def test_run_type_file_good(self):
try:
with infra_libs.temporary_directory(
prefix='config_test-') as tempdir:
filename = os.path.join(tempdir, 'output.db')
self._set_up_args(args=[
'--event-mon-run-type', 'file', '--event-mon-output-file',
filename
])
self.assertEqual(config._router.output_file, filename)
except Exception: # pragma: no cover
# help for debugging
traceback.print_exc()
raise
def test_send_file(self):
with infra_libs.temporary_directory() as temp_dir:
filename = os.path.join(temp_dir, 'out')
m = monitors.DebugMonitor(filename)
metric1 = metrics_pb2.MetricsData(name='m1')
m.send(metric1)
metric2 = metrics_pb2.MetricsData(name='m2')
m.send([metric1, metric2])
collection = metrics_pb2.MetricsCollection(data=[metric1, metric2])
m.send(collection)
with open(filename) as fh:
output = fh.read()
self.assertEquals(output.count('data {\n name: "m1"\n}'), 3)
self.assertEquals(output.count('data {\n name: "m2"\n}'), 2)
def DEACTIVATEtest_add_code_review_data_to_db(self, _,
dummy): # pragma: no cover
mock_cc = mock.MagicMock()
checkout = None
with infra_libs.temporary_directory(prefix='antibody-test') as dirname:
mock_cc.dirname.return_value = dirname
code_review_parse.get_code_review_data(mock_cc, checkout)
code_review_parse.get_code_review_data(mock_cc, checkout)
code_review_parse.get_code_review_data(mock_cc, checkout)
code_review_parse.get_code_review_data(mock_cc, checkout)
code_review_parse.get_code_review_data(mock_cc, checkout)
code_review_parse.get_code_review_data(mock_cc, checkout)
with open (os.path.join(dirname, 'output.txt'), 'r') as f:
return f.readlines()
def test_generate_file_from_template(self):
with infra_libs.temporary_directory(prefix="new-tool-test-") as tempdir:
filename = new_tool.generate_python_file(
tempdir, "test.py", "test_template", template_dir=DATA_DIR, value="Passed string."
)
self.assertTrue(os.path.isfile(filename))
# Read the content
with open(filename, "r") as f:
content = f.read()
expected_content = (
new_tool.COPYRIGHT_NOTICE + "This is a template used by the test suite.\n" "Passed string.\n"
)
self.assertEqual(content, expected_content)
def test_logrequest_path_service_build_conflict(self):
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'service-bar-service.bin'),
'--build-event-type', 'BUILD',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(ValueError):
send_event._process_logrequest_path(args)
def test_logrequest_path_service_build_conflict(self):
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'service-bar-service.bin'),
'--build-event-type', 'BUILD',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(ValueError):
common.process_argparse_options(args)
def test_send_build_event_with_invalid_goma_stats(self):
# Write a file to avoid mocks
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments([
'--event-mon-run-type', 'file', '--event-mon-output-file',
outfile, '--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD', '--build-event-hostname',
'foo.bar.dns', '--build-event-build-name', 'whatever',
'--build-event-goma-stats-path',
os.path.join(DATA_DIR, 'garbage')
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(google.protobuf.message.DecodeError):
common.send_build_event(args)
def test_logrequest_path_service_build_and_service(self):
# The logrequest provided contains both a service and a build type,
# which is invalid.
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'build-and-service-event.bin'),
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(ValueError):
common.process_argparse_options(args)
def test_logrequest_path_service_build_and_service(self):
# The logrequest provided contains both a service and a build type,
# which is invalid.
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'build-and-service-event.bin'),
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(ValueError):
send_event._process_logrequest_path(args)
def test_logrequest_path_build_service_conflicts(self):
# logrequest contains build event, provides service event as arg
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'build-foo-builder.bin'),
'--build-event-build-number', '3',
'--service-event-type', 'START',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(ValueError):
send_event._process_logrequest_path(args)
def test_logrequest_path_build_service_conflicts(self):
# logrequest contains build event, provides service event as arg
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--event-logrequest-path',
os.path.join(DATA_DIR, 'build-foo-builder.bin'),
'--build-event-build-number', '3',
'--service-event-type', 'START',
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(ValueError):
common.process_argparse_options(args)
def test_send_build_event_with_invalid_goma_stats(self):
# Write a file to avoid mocks
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD',
'--build-event-hostname', 'foo.bar.dns',
'--build-event-build-name', 'whatever',
'--build-event-goma-stats-path',
os.path.join(DATA_DIR, 'garbage')])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(google.protobuf.message.DecodeError):
send_event.send_build_event(args)
def test_send_build_event_with_non_existing_goma_error_report(self):
# Write a file to avoid mocks
with infra_libs.temporary_directory(prefix='common_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = get_arguments([
'--event-mon-run-type', 'file', '--event-mon-output-file',
outfile, '--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD', '--build-event-hostname',
'foo.bar.dns', '--build-event-build-name', 'whatever',
'--build-event-goma-error', 'GOMA_ERROR_CRASHED',
'--build-event-goma-crash-report-id-path',
os.path.join(DATA_DIR, 'this-file-does-not-exist')
])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(IOError):
common.send_build_event(args)
def test_send_build_event_with_non_existing_goma_error_report(self):
# Write a file to avoid mocks
with infra_libs.temporary_directory(prefix='send_event_test-') as tmpdir:
outfile = os.path.join(tmpdir, 'out.bin')
args = send_event.get_arguments(
['--event-mon-run-type', 'file',
'--event-mon-output-file', outfile,
'--event-mon-service-name', 'thing',
'--build-event-type', 'BUILD',
'--build-event-hostname', 'foo.bar.dns',
'--build-event-build-name', 'whatever',
'--build-event-goma-error', 'GOMA_ERROR_CRASHED',
'--build-event-goma-crash-report-id-path',
os.path.join(DATA_DIR, 'this-file-does-not-exist')])
self.assertEquals(args.event_mon_run_type, 'file')
event_mon.process_argparse_options(args)
with self.assertRaises(IOError):
send_event.send_build_event(args)
def test_get_project_name(self):
with infra_libs.temporary_directory(prefix='antibody-test') as dirname:
with open(os.path.join(dirname, 'codereview.settings'), 'w') as f:
f.writelines([
'This file is used by gcl to get repository specific information.\n',
'CODE_REVIEW_SERVER: https://codereview.chromium.org\n',
'VIEW_VC: https://chromium.googlesource.com/infra/infra/+/\n',
'CC_LIST: [email protected]\n',
'PROJECT: infra\n',
])
self.assertEqual(antibody.get_project_name(dirname), 'infra')
with open(os.path.join(dirname, 'codereview.settings'), 'w') as f:
f.writelines([
'CC_LIST: [email protected]\n',
'VIEW_VC: https://chromium.googlesource.com/infra/infra/+/\n',
])
self.assertEqual(antibody.get_project_name(dirname), None)
def test_generate_empty_file(self):
with infra_libs.temporary_directory(prefix="new-tool-test-") as tempdir:
filename = new_tool.generate_python_file(tempdir, "test.py", None)
self.assertTrue(os.path.isfile(filename))
# Read the content and do some basic check
with open(filename, "r") as f:
content = f.read()
self.assertTrue(content.startswith(new_tool.COPYRIGHT_NOTICE))
# Now make sure the file is not overwritten
new_content = "other content"
with open(filename, "w") as f:
f.write(new_content)
filename2 = new_tool.generate_python_file(tempdir, "test.py", None)
self.assertEqual(filename, filename2)
with open(filename, "r") as f:
content = f.read()
self.assertEqual(content, new_content)
def test_get_project_name(self):
with infra_libs.temporary_directory(prefix='antibody-test') as dirname:
with open(os.path.join(dirname, 'codereview.settings'), 'w') as f:
f.writelines([
'This file is used by gcl to get repository specific information.\n',
'CODE_REVIEW_SERVER: https://codereview.chromium.org\n',
'VIEW_VC: https://chromium.googlesource.com/infra/infra/+/\n',
'CC_LIST: [email protected]\n',
'PROJECT: infra\n',
])
self.assertEqual(antibody.get_project_name(dirname),
'infra')
with open(os.path.join(dirname, 'codereview.settings'), 'w') as f:
f.writelines([
'CC_LIST: [email protected]\n',
'VIEW_VC: https://chromium.googlesource.com/infra/infra/+/\n',
])
self.assertEqual(antibody.get_project_name(dirname), None)
def test_existing_directory(self):
event = LogRequestLite.LogEventLite()
event.event_time_ms = router.time_ms()
event.event_code = 1
event.event_flow_id = 2
with infra_libs.temporary_directory(prefix='event-mon-') as tempdir:
filename = os.path.join(tempdir, 'output.db')
r = router._LocalFileRouter(filename, dry_run=False)
self.assertTrue(r.push_event(event))
# Check that the file is readable and contains the right data.
with open(filename, 'rb') as f:
req_read = LogRequestLite.FromString(f.read())
self.assertEqual(len(req_read.log_event), 1)
event_read = req_read.log_event[0]
self.assertEqual(event_read.event_time_ms, event.event_time_ms)
self.assertEqual(event_read.event_code, event.event_code)
self.assertEqual(event_read.event_flow_id, event.event_flow_id)
def test_add_code_review_data_to_db(self, _, dummy):
mock_cc = mock.MagicMock()
checkout = None
with infra_libs.temporary_directory(prefix='antibody-test') as dirname:
mock_cc.dirname.return_value = dirname
code_review_parse.add_code_review_data_to_db(self.g_not_lgtm,
mock_cc, checkout)
code_review_parse.add_code_review_data_to_db(self.r_lgtm_not_lgtm_no_tbr,
mock_cc, checkout)
code_review_parse.add_code_review_data_to_db(self.g_lgtm,
mock_cc, checkout)
code_review_parse.add_code_review_data_to_db(self.r_lgtm_no_tbr,
mock_cc, checkout)
code_review_parse.add_code_review_data_to_db(self.r_lgtm_tbr,
mock_cc, checkout)
code_review_parse.add_code_review_data_to_db('fake url',
mock_cc, checkout)
with open (os.path.join(dirname, 'output.txt'), 'r') as f:
return f.readlines()
def test_add_code_review_data_to_db(self, _, dummy):
mock_cc = mock.MagicMock()
checkout = None
with infra_libs.temporary_directory(prefix='antibody-test') as dirname:
mock_cc.dirname.return_value = dirname
code_review_parse.add_code_review_data_to_db(
self.g_not_lgtm, mock_cc, checkout)
code_review_parse.add_code_review_data_to_db(
self.r_lgtm_not_lgtm_no_tbr, mock_cc, checkout)
code_review_parse.add_code_review_data_to_db(
self.g_lgtm, mock_cc, checkout)
code_review_parse.add_code_review_data_to_db(
self.r_lgtm_no_tbr, mock_cc, checkout)
code_review_parse.add_code_review_data_to_db(
self.r_lgtm_tbr, mock_cc, checkout)
code_review_parse.add_code_review_data_to_db(
'fake url', mock_cc, checkout)
with open(os.path.join(dirname, 'output.txt'), 'r') as f:
return f.readlines()
def test_existing_directory(self):
event = LogRequestLite.LogEventLite()
event.event_time_ms = router.time_ms()
event.event_code = 1
event.event_flow_id = 2
with infra_libs.temporary_directory(prefix='event-mon-') as tempdir:
filename = os.path.join(tempdir, 'output.db')
r = router._LocalFileRouter(filename, dry_run=False)
self.assertTrue(r.push_event(event))
# Check that the file is readable and contains the right data.
with open(filename, 'rb') as f:
req_read = LogRequestLite.FromString(f.read())
self.assertEqual(len(req_read.log_event), 1)
event_read = req_read.log_event[0]
self.assertEqual(event_read.event_time_ms, event.event_time_ms)
self.assertEqual(event_read.event_code, event.event_code)
self.assertEqual(event_read.event_flow_id, event.event_flow_id)
def test_send_events_from_file_delete_file_smoke(self):
# Create a temporary file because we don't want to risk deleting a
# checked-in file.
with infra_libs.temporary_directory(prefix='send-events-test-') as tempdir:
event_file = os.path.join(tempdir, 'events.log')
with open(event_file, 'w') as f:
f.write('{"build-event-type": "STEP", '
'"build-event-build-name": "infra-continuous-precise-64", '
'"event-mon-service-name": "buildbot/master/chromium.infra", '
'"build-event-step-number": 9, '
'"build-event-build-number": 5, '
'"event-mon-timestamp-kind": "END", '
'"build-event-step-name": "cipd - test packages integrity", '
'"build-event-build-scheduling-time": 1434665160000, '
'"build-event-hostname": "vm25-m1"}\n')
self.assertTrue(os.path.isfile(event_file))
args = send_event.get_arguments(['--events-from-file', event_file,
'--delete-file-when-sent'])
send_event.send_events_from_file(args)
self.assertFalse(os.path.isfile(event_file))
def test_send_events_from_file_delete_file_smoke(self):
# Create a temporary file because we don't want to risk deleting a
# checked-in file.
with infra_libs.temporary_directory(prefix='send-events-test-') as tempdir:
event_file = os.path.join(tempdir, 'events.log')
with open(event_file, 'w') as f:
f.write('{"build-event-type": "STEP", '
'"build-event-build-name": "infra-continuous-precise-64", '
'"event-mon-service-name": "buildbot/master/chromium.infra", '
'"build-event-step-number": 9, '
'"build-event-build-number": 5, '
'"event-mon-timestamp-kind": "END", '
'"build-event-step-name": "cipd - test packages integrity", '
'"build-event-build-scheduling-time": 1434665160000, '
'"build-event-hostname": "vm25-m1"}\n')
self.assertTrue(os.path.isfile(event_file))
args = get_arguments(['--events-from-file', event_file,
'--delete-file-when-sent'])
common.send_events_from_file(args)
self.assertFalse(os.path.isfile(event_file))
def test_generate_antibody_ui(self, mock_tbr_by_user, mock_gen_stats,
mock_tbr_no_lgtm):
#pylint: disable=W0613
fake_cc = None
with infra_libs.temporary_directory(prefix='antibody-test') as dirname:
commit_data = 'data/sample_suspicious_commits.txt'
with open(os.path.join(THIS_DIR, commit_data), 'r') as f:
suspicious_commits_data = [
line.rstrip('\n').split(',') for line in f
]
temp_data_gitiles = "https://chromium.googlesource.com/infra/infra/+/"
sample_monthly_stats = {
'7_days': {
'suspicious_to_total_ratio':
1,
'total_commits':
2,
'tbr_no_lgtm':
3,
'no_review_url':
4,
'blank_tbr':
5,
'tbr_no_lgtm_commits': [
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 1",
"123456789abcdefghijklmnop"
],
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 2",
"123456789abcdefghijklmnop"
],
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 3",
"123456789abcdefghijklmnop"
],
],
'no_review_url_commits': [
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 1",
"123456789abcdefghijklmnop"
],
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 3",
"123456789abcdefghijklmnop"
],
],
'blank_tbr_commits': [
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 3",
"123456789abcdefghijklmnop"
],
],
},
'30_days': {
'suspicious_to_total_ratio':
1,
'total_commits':
2,
'tbr_no_lgtm':
3,
'no_review_url':
4,
'blank_tbr':
5,
'tbr_no_lgtm_commits': [
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 1",
"123456789abcdefghijklmnop"
],
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 2",
"123456789abcdefghijklmnop"
],
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 3",
"123456789abcdefghijklmnop"
],
],
'no_review_url_commits': [
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 1",
"123456789abcdefghijklmnop"
],
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 3",
"123456789abcdefghijklmnop"
],
],
'blank_tbr_commits': [
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 3",
"123456789abcdefghijklmnop"
],
],
},
'all_time': {
'suspicious_to_total_ratio':
1,
'total_commits':
2,
'tbr_no_lgtm':
3,
'no_review_url':
4,
'blank_tbr':
5,
'tbr_no_lgtm_commits': [
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 1",
"123456789abcdefghijklmnop"
],
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 2",
"123456789abcdefghijklmnop"
],
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 3",
"123456789abcdefghijklmnop"
],
],
'no_review_url_commits': [
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 1",
"123456789abcdefghijklmnop"
],
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 3",
"123456789abcdefghijklmnop"
],
],
'blank_tbr_commits': [
[
"https://codereview.chromium.org/",
"2015-07-13 11:11:11", "Fake Commit Subject 3",
"123456789abcdefghijklmnop"
],
],
},
}
proj_dirname = os.path.join(dirname, 'proj')
os.makedirs(proj_dirname)
with open(os.path.join(proj_dirname, 'all_monthly_stats.json'),
'w') as f:
json.dump(sample_monthly_stats, f)
antibody.generate_antibody_ui(fake_cc, temp_data_gitiles, 'proj',
'2014', dirname,
suspicious_commits_data, ['db 1'])
with open(
os.path.join(proj_dirname, antibody.ANTIBODY_UI_MAIN_NAME),
'r') as f:
file_string = f.read()
self.assertTrue(file_string)
self.assertFalse('{{' in file_string)
self.assertFalse('}}' in file_string)
with open(os.path.join(proj_dirname, antibody.SEARCH_BY_USER_NAME),
'r') as f:
file_string = f.read()
self.assertTrue(file_string)
self.assertFalse('{{' in file_string)
self.assertFalse('}}' in file_string)
with open(os.path.join(proj_dirname, antibody.STATS_NAME),
'r') as f:
file_string = f.read()
self.assertTrue(file_string)
self.assertFalse('{{' in file_string)
self.assertFalse('}}' in file_string)
with open(os.path.join(proj_dirname, antibody.LEADERBOARD_NAME),
'r') as f:
file_string = f.read()
self.assertTrue(file_string)
self.assertFalse('{{' in file_string)
self.assertFalse('}}' in file_string)
with open(os.path.join(proj_dirname, antibody.STATS_7_NAME),
'r') as f:
file_string = f.read()
self.assertTrue(file_string)
self.assertFalse('{{' in file_string)
self.assertFalse('}}' in file_string)
with open(os.path.join(proj_dirname, antibody.STATS_30_NAME),
'r') as f:
file_string = f.read()
self.assertTrue(file_string)
self.assertFalse('{{' in file_string)
self.assertFalse('}}' in file_string)
with open(os.path.join(proj_dirname, antibody.STATS_ALL_TIME_NAME),
'r') as f:
file_string = f.read()
self.assertTrue(file_string)
self.assertFalse('{{' in file_string)
self.assertFalse('}}' in file_string)
self.assertTrue(os.path.exists(os.path.join(dirname, 'static')))
def test_run_main(self):
with infra_libs.temporary_directory(
prefix='new-tool-test-') as tempdir:
new_tool_main.main(['whatever_tool', '--base-dir', tempdir])
self.assertTrue(
os.path.isdir(os.path.join(tempdir, 'whatever_tool')))
def test_rmtree_directory(self):
with infra_libs.temporary_directory() as tempdir:
infra_libs.rmtree(tempdir)
self.assertFalse(os.path.exists(tempdir))
def test_get_commits_by_user(self):
with infra_libs.temporary_directory(prefix='antibody-test') as dirname:
# tbr_no_lgtm: review_url, request_timestamp, subject,
# people_email_address, hash
mock_cc = mock.MagicMock()
mock_cc.fetchall.side_effect = (
(
('https://codereview.chromium.org/1175993003',
datetime.datetime(2015, 07, 13, 11, 11,
11), 'hello', 'pgervais', 'git_hash_1'),
('https://codereview.chromium.org/1175993003',
datetime.datetime(2015, 07, 13, 11, 11,
11), 'hello', 'hinoka', 'git_hash_1'),
('https://codereview.chromium.org/1171763002',
datetime.datetime(2015, 07, 13, 22, 22,
22), 'world', 'hinoka', 'git_hash_2'),
('https://codereview.chromium.org/1171763002',
datetime.datetime(2015, 07, 13, 22, 22,
22), 'world', 'keelerh', 'git_hash_2'),
),
(
('https://codereview.chromium.org/1175993001',
datetime.datetime(2015, 07, 13, 11, 11,
11), 'hi', 'pgervais', 'git_hash_3'),
('https://codereview.chromium.org/1171763000',
datetime.datetime(2015, 07, 13, 22, 22,
22), 'there', 'hinoka', 'git_hash_4'),
),
)
antibody.get_commits_by_user(
mock_cc, 'https://chromium.googlesource.com/infra/infra/+/',
dirname)
expected_out = {
"by_user": {
"pgervais": {
"tbr": [[
'hello',
'https://codereview.chromium.org/1175993003',
'2015-07-13 11:11:11', 'git_hash_1', 'TBR'
]],
"author": [[
'hi', 'https://codereview.chromium.org/1175993001',
'2015-07-13 11:11:11', 'git_hash_3', 'Author'
]],
},
"hinoka": {
"tbr": [
[
'world',
'https://codereview.chromium.org/1171763002',
'2015-07-13 22:22:22', 'git_hash_2', 'TBR'
],
[
'hello',
'https://codereview.chromium.org/1175993003',
'2015-07-13 11:11:11', 'git_hash_1', 'TBR'
],
],
"author": [[
'there',
'https://codereview.chromium.org/1171763000',
'2015-07-13 22:22:22', 'git_hash_4', 'Author'
]],
},
"keelerh": {
"tbr": [[
'world',
'https://codereview.chromium.org/1171763002',
'2015-07-13 22:22:22', 'git_hash_2', 'TBR'
]],
"author": [],
},
},
"gitiles_prefix":
"https://chromium.googlesource.com/infra/infra/+/",
}
with open(os.path.join(dirname, 'search_by_user.json'), 'r') as f:
output = json.load(f)
print output
self.assertEqual(output, expected_out)
