def test_print_attempt_counts(self):
self.mock(cq_stats, 'output', self.print_mock)
stats = cq_stats.default_stats()
stats['patch_stats'] = {
(123, 1): {
'attempts': 1,
'false-rejections': 0,
'rejections': 1,
'committed': False,
'patchset-duration': 3600,
'patchset-duration-wallclock': 3600,
'failed-jobs-details': {
'builder_a': 1,
},
},
}
cq_stats._derive_stats_from_patch_stats(stats)
cq_stats.print_attempt_counts(stats,
'rejections',
'were unsuccessful',
item_name=None,
committed=False,
details=True)
cq_stats.print_attempt_counts(stats,
'rejections',
'failed jobs',
item_name=None,
committed=False)
return self.expectations
def test_print_attempt_counts(self):
self.mock(cq_stats, 'output', self.print_mock)
stats = cq_stats.default_stats()
stats['patch_stats'] = {
(123, 1): {
'attempts': 1,
'false-rejections': 0,
'rejections': 1,
'committed': False,
'patchset-duration': 3600,
'patchset-duration-wallclock': 3600,
'failed-jobs-details': {
'builder_a': 1,
},
},
}
cq_stats._derive_stats_from_patch_stats(stats)
cq_stats.print_attempt_counts(
stats, 'rejections', 'were unsuccessful',
item_name=None, committed=False, details=True)
cq_stats.print_attempt_counts(
stats, 'rejections', 'failed jobs',
item_name=None, committed=False)
return self.expectations
def test_organize_stats(self):
stats = {
'results': [{
'begin':
t,
'stats': [
{
'count': 3,
'type': 'count',
'name': 'attempt-count'
},
{
'count': 2,
'type': 'count',
'name': 'trybot-bot-false-reject-count'
},
{
'count': 1,
'type': 'count',
'name': 'trybot-bot-pass-count'
},
{
'description': 'Total time spent per CQ attempt.',
'max': 9999.99999,
'percentile_25': 2512.34567,
'percentile_75': 7512.34567,
'percentile_10': 1012.34567,
'unit': 'seconds',
'name': 'attempt-durations',
'percentile_50': 5012.34567,
'min': 0.00001,
'sample_size': 10000,
'percentile_90': 9012.34567,
'percentile_95': 9512.34567,
'percentile_99': 9912.34567,
'type': 'list',
'mean': 5555.555555
},
],
'interval_minutes':
15,
'project':
'chromium',
'key':
5976204561612800,
'end':
t + 900
} for t in [1415138400, 1415139300]]
}
result = cq_stats.organize_stats(stats)
# Test that the result stats have the minimal expected dict keys
# for print_stats().
expected_keys = set(cq_stats.default_stats().keys())
self.assertFalse(expected_keys - set(result.keys()))
self.assertIsNone(cq_stats.organize_stats({}))
return ensure_serializable(result)
def test_print_usage(self):
self.mock(cq_stats, 'output', self.print_mock)
stats = cq_stats.default_stats()
stats['usage'] = cq_stats.derive_log_stats([])
cq_stats.print_usage(stats)
return self.expectations
def test_acquire_stats(self):
self.mock(cq_stats, 'fetch_json', lambda _: 'json')
self.mock(cq_stats, 'organize_stats',
lambda *_args, **_kwargs: {
'latest': cq_stats.default_stats(),
'previous': cq_stats.default_stats()})
self.mock(cq_stats, 'derive_stats', lambda *_args, **_kwargs: {})
self.mock(cq_stats, 'derive_git_stats', lambda *_: {})
cq_stats.acquire_stats(Args(project='chromium', bots=[]),
add_tree_stats=False)
cq_stats.acquire_stats(Args(
project='chromium', bots=[], use_logs=True, range='week'))
cq_stats.acquire_stats(Args(
project='chromium', bots=[], use_logs=True, range='day'))
cq_stats.acquire_stats(Args(
project='chromium', bots=[], use_logs=True, range='hour'))
def test_main(self):
self.mock(cq_stats, 'output', self.print_mock)
self.mock(cq_stats, 'parse_args', lambda: Args(
project='chromium', log_level=logging.CRITICAL, logs_black_list=None,
logs_directory=tempfile.gettempdir()))
self.mock(cq_stats, 'acquire_stats', lambda _: cq_stats.default_stats())
cq_stats.main()
return self.expectations
def test_print_log_stats(self):
self.mock(cq_stats, 'output', self.print_mock)
args = Args(use_logs=True)
stats_set = cq_stats.default_stats()
stats_set['begin'] = args.date
stats_set['end'] = args.date + datetime.timedelta(days=7)
cq_stats.print_stats(args, stats_set)
return self.expectations
def test_print_log_stats(self):
self.mock(cq_stats, 'output', self.print_mock)
args = Args(use_logs=True)
stats_set = cq_stats.default_stats()
stats_set['begin'] = args.date
stats_set['end'] = args.date + datetime.timedelta(days=7)
cq_stats.print_stats(args, stats_set)
return self.expectations
def test_print_stats(self):
self.mock(cq_stats, 'output', self.print_mock)
args = Args()
stats_set = cq_stats.default_stats()
stats_set['begin'] = args.date
stats_set['end'] = args.date + datetime.timedelta(days=7)
stats_set['patch_stats'].update({
(123, 1): {
'attempts': 1,
'false-rejections': 0,
'rejections': 1,
'committed': False,
'patchset-duration': 3600,
'patchset-duration-wallclock': 3600,
'failed-jobs-details': {
'builder_a': 1,
},
},
(456, 1): {
'attempts': 1,
'false-rejections': 0,
'rejections': 1,
'committed': False,
'patchset-duration': 7200,
'patchset-duration-wallclock': 7200,
'failed-jobs-details': {
'builder_a': 1,
},
},
})
swapped_stats = copy.deepcopy(stats_set)
swapped_stats['begin'], swapped_stats['end'] = (
swapped_stats['end'], swapped_stats['begin'])
cq_stats.print_stats(args, None)
cq_stats.print_stats(args, stats_set)
cq_stats.print_stats(args, swapped_stats)
stats_set['per-day'] = [
copy.copy(stats_set),
copy.copy(stats_set),
copy.copy(stats_set),
]
args = Args()
args.path_filter_include = ['third_party/WebKit']
cq_stats.print_stats(args, stats_set)
args = Args()
args.path_filter_exclude = ['third_party/WebKit']
cq_stats.print_stats(args, stats_set)
return self.expectations
def test_print_flakiness_stats(self):
self.mock(cq_stats, "output", self.print_mock)
args = Args()
stats_set = cq_stats.default_stats()
stats_set["begin"] = args.date
stats_set["end"] = args.date + datetime.timedelta(days=7)
stats_set["patch_stats"].update(
{
(123, 1): {
"attempts": 1,
"false-rejections": 0,
"rejections": 1,
"committed": False,
"patchset-duration": 3600,
"patchset-duration-wallclock": 3600,
"failed-jobs-details": {"builder_a": 1},
}
}
)
self.mock(
cq_stats,
"fetch_json",
lambda _: [
{"master": "tryserver.chromium.linux", "builder": "chromium_presubmit", "result": 0},
{"master": "tryserver.chromium.linux", "builder": "chromium_presubmit", "result": -1},
{"master": "tryserver.chromium.linux", "builder": "chromium_presubmit", "result": 2},
{
"master": "tryserver.chromium.linux",
"builder": "linux_chromium_rel_ng",
"result": 2,
"build_properties": '{"failure_type": "COMPILE_FAILURE"}',
},
{
"master": "tryserver.chromium.linux",
"builder": "linux_chromium_rel_ng",
"result": 2,
"build_properties": '{"failure_type": "TEST_FAILURE"}',
},
{"master": "tryserver.chromium.linux", "builder": "chromium_presubmit", "result": 4},
],
)
cq_stats.print_flakiness_stats(args, stats_set)
args.seq = False
self.mock(
cq_stats,
"fetch_json",
lambda _: [{"master": "tryserver.chromium.linux", "builder": "chromium_presubmit", "result": 2}],
)
cq_stats.print_flakiness_stats(args, stats_set)
return self.expectations
def test_print_usage(self):
self.mock(cq_stats, "output", self.print_mock)
stats = cq_stats.default_stats()
stats["usage"] = cq_stats.derive_log_stats([], [])
cq_stats.print_usage(stats)
stats["usage"]["bot_manual_commits"] += 1
cq_stats.print_usage(stats)
return self.expectations
def test_print_usage(self):
self.mock(cq_stats, 'output', self.print_mock)
stats = cq_stats.default_stats()
stats['usage'] = cq_stats.derive_log_stats([], [])
cq_stats.print_usage(stats)
stats['usage']['bot_manual_commits'] += 1
cq_stats.print_usage(stats)
return self.expectations
def test_print_usage(self):
self.mock(cq_stats, 'output', self.print_mock)
stats = cq_stats.default_stats()
stats['usage'] = cq_stats.derive_log_stats([], [])
cq_stats.print_usage(stats)
stats['usage']['bot_manual_commits'] += 1
cq_stats.print_usage(stats)
return self.expectations
def test_main(self):
self.mock(cq_stats, 'output', self.print_mock)
self.mock(
cq_stats, 'parse_args',
lambda: Args(project='chromium',
log_level=logging.CRITICAL,
logs_black_list=None,
logs_directory=tempfile.gettempdir()))
self.mock(cq_stats, 'acquire_stats',
lambda _: cq_stats.default_stats())
cq_stats.main()
return self.expectations
def test_acquire_stats(self):
self.mock(cq_stats, 'fetch_json', lambda _: 'json')
self.mock(
cq_stats, 'organize_stats', lambda *_args, **_kwargs: {
'latest': cq_stats.default_stats(),
'previous': cq_stats.default_stats()
})
self.mock(cq_stats, 'derive_stats', lambda *_args, **_kwargs: {})
self.mock(cq_stats, 'derive_tree_stats', lambda *_: {
'open': 0.0,
'total': 3600.0
})
self.mock(cq_stats, 'derive_git_stats', lambda *_: {})
self.mock(cq_stats, 'derive_svn_stats', lambda *_: {})
cq_stats.acquire_stats(Args(project='blink', bots=[]))
cq_stats.acquire_stats(Args(project='chromium', bots=[]))
cq_stats.acquire_stats(
Args(project='chromium', bots=[], use_logs=True, range='week'))
cq_stats.acquire_stats(
Args(project='chromium', bots=[], use_logs=True, range='day'))
cq_stats.acquire_stats(
Args(project='chromium', bots=[], use_logs=True, range='hour'))
def test_organize_stats(self):
stats = {
"results": [
{
"begin": t,
"stats": [
{"count": 3, "type": "count", "name": "attempt-count"},
{"count": 2, "type": "count", "name": "trybot-bot-false-reject-count"},
{"count": 1, "type": "count", "name": "trybot-bot-pass-count"},
{
"description": "Total time spent per CQ attempt.",
"max": 9999.99999,
"percentile_25": 2512.34567,
"percentile_75": 7512.34567,
"percentile_10": 1012.34567,
"unit": "seconds",
"name": "attempt-durations",
"percentile_50": 5012.34567,
"min": 0.00001,
"sample_size": 10000,
"percentile_90": 9012.34567,
"percentile_95": 9512.34567,
"percentile_99": 9912.34567,
"type": "list",
"mean": 5555.555555,
},
],
"interval_minutes": 15,
"project": "chromium",
"key": 5976204561612800,
"end": t + 900,
}
for t in [1415138400, 1415139300]
]
}
result = cq_stats.organize_stats(stats)
# Test that the result stats have the minimal expected dict keys
# for print_stats().
expected_keys = set(cq_stats.default_stats().keys())
self.assertFalse(expected_keys - set(result.keys()))
self.assertIsNone(cq_stats.organize_stats({}))
return ensure_serializable(result)
def test_acquire_stats(self):
self.mock(cq_stats, "fetch_json", lambda _: "json")
self.mock(
cq_stats,
"organize_stats",
lambda *_args, **_kwargs: {"latest": cq_stats.default_stats(), "previous": cq_stats.default_stats()},
)
self.mock(cq_stats, "derive_stats", lambda *_args, **_kwargs: {})
self.mock(cq_stats, "derive_tree_stats", lambda *_: {"open": 0.0, "total": 3600.0})
self.mock(cq_stats, "derive_git_stats", lambda *_: {})
self.mock(cq_stats, "derive_svn_stats", lambda *_: {})
cq_stats.acquire_stats(Args(project="blink", bots=[]))
cq_stats.acquire_stats(Args(project="chromium", bots=[]))
cq_stats.acquire_stats(Args(project="chromium", bots=[], use_logs=True, range="week"))
cq_stats.acquire_stats(Args(project="chromium", bots=[], use_logs=True, range="day"))
cq_stats.acquire_stats(Args(project="chromium", bots=[], use_logs=True, range="hour"))
def test_organize_stats(self):
stats = {'results': [
{'begin': t,
'stats': [
{'count': 3, 'type': 'count',
'name': 'attempt-count'},
{'count': 2, 'type': 'count',
'name': 'trybot-bot-false-reject-count'},
{'count': 1, 'type': 'count',
'name': 'trybot-bot-pass-count'},
{'description': 'Total time spent per CQ attempt.',
'max': 9999.99999,
'percentile_25': 2512.34567,
'percentile_75': 7512.34567,
'percentile_10': 1012.34567,
'unit': 'seconds',
'name': 'attempt-durations',
'percentile_50': 5012.34567,
'min': 0.00001,
'sample_size': 10000,
'percentile_90': 9012.34567,
'percentile_95': 9512.34567,
'percentile_99': 9912.34567,
'type': 'list',
'mean': 5555.555555},
],
'interval_minutes': 15,
'project': 'chromium',
'key': 5976204561612800,
'end': t + 900} for t in [1415138400, 1415139300]]}
result = cq_stats.organize_stats(stats)
# Test that the result stats have the minimal expected dict keys
# for print_stats().
expected_keys = set(cq_stats.default_stats().keys())
self.assertFalse(expected_keys - set(result.keys()))
self.assertIsNone(cq_stats.organize_stats({}))
return ensure_serializable(result)
def test_print_stats(self):
self.mock(cq_stats, 'output', self.print_mock)
args = Args()
stats_set = cq_stats.default_stats()
stats_set['begin'] = args.date
stats_set['end'] = args.date + datetime.timedelta(days=7)
stats_set['patch_stats'].update({
(123, 1): {
'attempts': 1,
'false-rejections': 0,
'rejections': 1,
'committed': False,
'patchset-duration': 3600,
'patchset-duration-wallclock': 3600,
'failed-jobs-details': {
'builder_a': 1,
},
},
(456, 1): {
'attempts': 1,
'false-rejections': 0,
'rejections': 1,
'committed': False,
'patchset-duration': 7200,
'patchset-duration-wallclock': 7200,
'failed-jobs-details': {
'builder_a': 1,
},
},
})
swapped_stats = copy.deepcopy(stats_set)
swapped_stats['begin'], swapped_stats['end'] = (swapped_stats['end'],
swapped_stats['begin'])
cq_stats.print_stats(args, None)
cq_stats.print_stats(args, stats_set)
cq_stats.print_stats(args, swapped_stats)
return self.expectations
def test_print_stats(self):
self.mock(cq_stats, "output", self.print_mock)
args = Args()
stats_set = cq_stats.default_stats()
stats_set["begin"] = args.date
stats_set["end"] = args.date + datetime.timedelta(days=7)
stats_set["patch_stats"].update(
{
(123, 1): {
"attempts": 1,
"false-rejections": 0,
"rejections": 1,
"committed": False,
"patchset-duration": 3600,
"patchset-duration-wallclock": 3600,
"failed-jobs-details": {"builder_a": 1},
},
(456, 1): {
"attempts": 1,
"false-rejections": 0,
"rejections": 1,
"committed": False,
"patchset-duration": 7200,
"patchset-duration-wallclock": 7200,
"failed-jobs-details": {"builder_a": 1},
},
}
)
swapped_stats = copy.deepcopy(stats_set)
swapped_stats["begin"], swapped_stats["end"] = (swapped_stats["end"], swapped_stats["begin"])
cq_stats.print_stats(args, None)
cq_stats.print_stats(args, stats_set)
cq_stats.print_stats(args, swapped_stats)
return self.expectations
def test_print_attempt_counts(self):
self.mock(cq_stats, "output", self.print_mock)
stats = cq_stats.default_stats()
stats["patch_stats"] = {
(123, 1): {
"attempts": 1,
"false-rejections": 0,
"rejections": 1,
"committed": False,
"patchset-duration": 3600,
"patchset-duration-wallclock": 3600,
"failed-jobs-details": {"builder_a": 1},
}
}
cq_stats._derive_stats_from_patch_stats(stats)
cq_stats.print_attempt_counts(
stats, "rejections", "were unsuccessful", item_name=None, committed=False, details=True
)
cq_stats.print_attempt_counts(stats, "rejections", "failed jobs", item_name=None, committed=False)
return self.expectations
def test_print_duration(self):
self.mock(cq_stats, 'output', self.print_mock)
cq_stats.print_duration('mean', cq_stats.default_stats(), None)
return self.expectations
def test_print_flakiness_stats(self):
self.mock(cq_stats, 'output', self.print_mock)
args = Args()
stats_set = cq_stats.default_stats()
stats_set['begin'] = args.date
stats_set['end'] = args.date + datetime.timedelta(days=7)
stats_set['patch_stats'].update({
(123, 1): {
'attempts': 1,
'false-rejections': 0,
'rejections': 1,
'committed': False,
'patchset-duration': 3600,
'patchset-duration-wallclock': 3600,
'failed-jobs-details': {
'builder_a': 1,
},
},
})
self.mock(cq_stats, 'fetch_json', lambda _: [{
'master': 'tryserver.chromium.linux',
'builder': 'chromium_presubmit',
'result': 0,
'url': 'https://build.chromium.org/10',
}, {
'master': 'tryserver.chromium.linux',
'builder': 'chromium_presubmit',
'result': -1,
'url': 'https://build.chromium.org/20',
}, {
'master': 'tryserver.chromium.linux',
'builder': 'chromium_presubmit',
'result': 2,
'url': 'https://build.chromium.org/30',
}, {
'master': 'tryserver.chromium.linux',
'builder': 'linux_chromium_rel_ng',
'result': 2,
'build_properties': '{"failure_type": "COMPILE_FAILURE"}',
'url': 'https://build.chromium.org/40',
}, {
'master': 'tryserver.chromium.linux',
'builder': 'linux_chromium_rel_ng',
'result': 2,
'build_properties': '{"failure_type": "TEST_FAILURE"}',
'url': 'https://build.chromium.org/50',
}, {
'master': 'tryserver.chromium.linux',
'builder': 'linux_chromium_rel_ng',
'result': 2,
'build_properties': '{"failure_type": "INVALID_TEST_RESULTS"}',
'url': 'https://build.chromium.org/50',
}, {
'master': 'tryserver.chromium.linux',
'builder': 'linux_chromium_rel_ng',
'result': 2,
'build_properties': '{"failure_type": "PATCH_FAILURE"}',
'url': 'https://build.chromium.org/50',
}, {
'master': 'tryserver.chromium.linux',
'builder': 'chromium_presubmit',
'result': 4,
'url': 'https://build.chromium.org/60',
}])
cq_stats.print_flakiness_stats(args, stats_set)
args.seq = False
args.list_uncategorized_flakes = True
self.mock(cq_stats, 'fetch_json', lambda _: [{
'master': 'tryserver.chromium.linux',
'builder': 'chromium_presubmit',
'result': 0,
'url': 'https://build.chromium.org/70',
}, {
'master': 'tryserver.chromium.linux',
'builder': 'chromium_presubmit',
'result': 2,
'url': 'https://build.chromium.org/75',
}, {
'master': 'tryserver.chromium.linux',
'builder': 'linux_chromium_rel_ng',
'result': 2,
'build_properties': '{"failure_type": "COMPILE_FAILURE"}',
'url': 'https://build.chromium.org/80',
}])
cq_stats.print_flakiness_stats(args, stats_set)
return self.expectations
def test_print_flakiness_stats(self):
self.mock(cq_stats, 'output', self.print_mock)
args = Args()
stats_set = cq_stats.default_stats()
stats_set['begin'] = args.date
stats_set['end'] = args.date + datetime.timedelta(days=7)
stats_set['patch_stats'].update({
(123, 1): {
'attempts': 1,
'false-rejections': 0,
'rejections': 1,
'committed': False,
'patchset-duration': 3600,
'patchset-duration-wallclock': 3600,
'failed-jobs-details': {
'builder_a': 1,
},
},
})
self.mock(
cq_stats, 'fetch_json', lambda _:
[{
'master': 'tryserver.chromium.linux',
'builder': 'chromium_presubmit',
'result': 0,
}, {
'master': 'tryserver.chromium.linux',
'builder': 'chromium_presubmit',
'result': -1,
}, {
'master': 'tryserver.chromium.linux',
'builder': 'chromium_presubmit',
'result': 2,
}, {
'master': 'tryserver.chromium.linux',
'builder': 'linux_chromium_rel_ng',
'result': 2,
'build_properties': '{"failure_type": "COMPILE_FAILURE"}',
}, {
'master': 'tryserver.chromium.linux',
'builder': 'linux_chromium_rel_ng',
'result': 2,
'build_properties': '{"failure_type": "TEST_FAILURE"}',
}, {
'master': 'tryserver.chromium.linux',
'builder': 'chromium_presubmit',
'result': 4,
}])
cq_stats.print_flakiness_stats(args, stats_set)
args.seq = False
self.mock(
cq_stats, 'fetch_json', lambda _: [{
'master': 'tryserver.chromium.linux',
'builder': 'chromium_presubmit',
'result': 2,
}])
cq_stats.print_flakiness_stats(args, stats_set)
return self.expectations
def test_print_duration(self):
self.mock(cq_stats, 'output', self.print_mock)
cq_stats.print_duration('mean', cq_stats.default_stats(), None)
return self.expectations
def test_print_duration(self):
self.mock(cq_stats, "output", self.print_mock)
cq_stats.print_duration("mean", cq_stats.default_stats(), None)
return self.expectations
