def test_print_attempt_counts(self): self.mock(cq_stats, 'output', self.print_mock) stats = cq_stats.default_stats() stats['patch_stats'] = { (123, 1): { 'attempts': 1, 'false-rejections': 0, 'rejections': 1, 'committed': False, 'patchset-duration': 3600, 'patchset-duration-wallclock': 3600, 'failed-jobs-details': { 'builder_a': 1, }, }, } cq_stats._derive_stats_from_patch_stats(stats) cq_stats.print_attempt_counts(stats, 'rejections', 'were unsuccessful', item_name=None, committed=False, details=True) cq_stats.print_attempt_counts(stats, 'rejections', 'failed jobs', item_name=None, committed=False) return self.expectations
def test_print_attempt_counts(self): self.mock(cq_stats, 'output', self.print_mock) stats = cq_stats.default_stats() stats['patch_stats'] = { (123, 1): { 'attempts': 1, 'false-rejections': 0, 'rejections': 1, 'committed': False, 'patchset-duration': 3600, 'patchset-duration-wallclock': 3600, 'failed-jobs-details': { 'builder_a': 1, }, }, } cq_stats._derive_stats_from_patch_stats(stats) cq_stats.print_attempt_counts( stats, 'rejections', 'were unsuccessful', item_name=None, committed=False, details=True) cq_stats.print_attempt_counts( stats, 'rejections', 'failed jobs', item_name=None, committed=False) return self.expectations
def test_organize_stats(self): stats = { 'results': [{ 'begin': t, 'stats': [ { 'count': 3, 'type': 'count', 'name': 'attempt-count' }, { 'count': 2, 'type': 'count', 'name': 'trybot-bot-false-reject-count' }, { 'count': 1, 'type': 'count', 'name': 'trybot-bot-pass-count' }, { 'description': 'Total time spent per CQ attempt.', 'max': 9999.99999, 'percentile_25': 2512.34567, 'percentile_75': 7512.34567, 'percentile_10': 1012.34567, 'unit': 'seconds', 'name': 'attempt-durations', 'percentile_50': 5012.34567, 'min': 0.00001, 'sample_size': 10000, 'percentile_90': 9012.34567, 'percentile_95': 9512.34567, 'percentile_99': 9912.34567, 'type': 'list', 'mean': 5555.555555 }, ], 'interval_minutes': 15, 'project': 'chromium', 'key': 5976204561612800, 'end': t + 900 } for t in [1415138400, 1415139300]] } result = cq_stats.organize_stats(stats) # Test that the result stats have the minimal expected dict keys # for print_stats(). expected_keys = set(cq_stats.default_stats().keys()) self.assertFalse(expected_keys - set(result.keys())) self.assertIsNone(cq_stats.organize_stats({})) return ensure_serializable(result)
def test_print_usage(self): self.mock(cq_stats, 'output', self.print_mock) stats = cq_stats.default_stats() stats['usage'] = cq_stats.derive_log_stats([]) cq_stats.print_usage(stats) return self.expectations
def test_acquire_stats(self): self.mock(cq_stats, 'fetch_json', lambda _: 'json') self.mock(cq_stats, 'organize_stats', lambda *_args, **_kwargs: { 'latest': cq_stats.default_stats(), 'previous': cq_stats.default_stats()}) self.mock(cq_stats, 'derive_stats', lambda *_args, **_kwargs: {}) self.mock(cq_stats, 'derive_git_stats', lambda *_: {}) cq_stats.acquire_stats(Args(project='chromium', bots=[]), add_tree_stats=False) cq_stats.acquire_stats(Args( project='chromium', bots=[], use_logs=True, range='week')) cq_stats.acquire_stats(Args( project='chromium', bots=[], use_logs=True, range='day')) cq_stats.acquire_stats(Args( project='chromium', bots=[], use_logs=True, range='hour'))
def test_main(self): self.mock(cq_stats, 'output', self.print_mock) self.mock(cq_stats, 'parse_args', lambda: Args( project='chromium', log_level=logging.CRITICAL, logs_black_list=None, logs_directory=tempfile.gettempdir())) self.mock(cq_stats, 'acquire_stats', lambda _: cq_stats.default_stats()) cq_stats.main() return self.expectations
def test_print_log_stats(self): self.mock(cq_stats, 'output', self.print_mock) args = Args(use_logs=True) stats_set = cq_stats.default_stats() stats_set['begin'] = args.date stats_set['end'] = args.date + datetime.timedelta(days=7) cq_stats.print_stats(args, stats_set) return self.expectations
def test_print_stats(self): self.mock(cq_stats, 'output', self.print_mock) args = Args() stats_set = cq_stats.default_stats() stats_set['begin'] = args.date stats_set['end'] = args.date + datetime.timedelta(days=7) stats_set['patch_stats'].update({ (123, 1): { 'attempts': 1, 'false-rejections': 0, 'rejections': 1, 'committed': False, 'patchset-duration': 3600, 'patchset-duration-wallclock': 3600, 'failed-jobs-details': { 'builder_a': 1, }, }, (456, 1): { 'attempts': 1, 'false-rejections': 0, 'rejections': 1, 'committed': False, 'patchset-duration': 7200, 'patchset-duration-wallclock': 7200, 'failed-jobs-details': { 'builder_a': 1, }, }, }) swapped_stats = copy.deepcopy(stats_set) swapped_stats['begin'], swapped_stats['end'] = ( swapped_stats['end'], swapped_stats['begin']) cq_stats.print_stats(args, None) cq_stats.print_stats(args, stats_set) cq_stats.print_stats(args, swapped_stats) stats_set['per-day'] = [ copy.copy(stats_set), copy.copy(stats_set), copy.copy(stats_set), ] args = Args() args.path_filter_include = ['third_party/WebKit'] cq_stats.print_stats(args, stats_set) args = Args() args.path_filter_exclude = ['third_party/WebKit'] cq_stats.print_stats(args, stats_set) return self.expectations
def test_print_flakiness_stats(self): self.mock(cq_stats, "output", self.print_mock) args = Args() stats_set = cq_stats.default_stats() stats_set["begin"] = args.date stats_set["end"] = args.date + datetime.timedelta(days=7) stats_set["patch_stats"].update( { (123, 1): { "attempts": 1, "false-rejections": 0, "rejections": 1, "committed": False, "patchset-duration": 3600, "patchset-duration-wallclock": 3600, "failed-jobs-details": {"builder_a": 1}, } } ) self.mock( cq_stats, "fetch_json", lambda _: [ {"master": "tryserver.chromium.linux", "builder": "chromium_presubmit", "result": 0}, {"master": "tryserver.chromium.linux", "builder": "chromium_presubmit", "result": -1}, {"master": "tryserver.chromium.linux", "builder": "chromium_presubmit", "result": 2}, { "master": "tryserver.chromium.linux", "builder": "linux_chromium_rel_ng", "result": 2, "build_properties": '{"failure_type": "COMPILE_FAILURE"}', }, { "master": "tryserver.chromium.linux", "builder": "linux_chromium_rel_ng", "result": 2, "build_properties": '{"failure_type": "TEST_FAILURE"}', }, {"master": "tryserver.chromium.linux", "builder": "chromium_presubmit", "result": 4}, ], ) cq_stats.print_flakiness_stats(args, stats_set) args.seq = False self.mock( cq_stats, "fetch_json", lambda _: [{"master": "tryserver.chromium.linux", "builder": "chromium_presubmit", "result": 2}], ) cq_stats.print_flakiness_stats(args, stats_set) return self.expectations
def test_print_usage(self): self.mock(cq_stats, "output", self.print_mock) stats = cq_stats.default_stats() stats["usage"] = cq_stats.derive_log_stats([], []) cq_stats.print_usage(stats) stats["usage"]["bot_manual_commits"] += 1 cq_stats.print_usage(stats) return self.expectations
def test_print_usage(self): self.mock(cq_stats, 'output', self.print_mock) stats = cq_stats.default_stats() stats['usage'] = cq_stats.derive_log_stats([], []) cq_stats.print_usage(stats) stats['usage']['bot_manual_commits'] += 1 cq_stats.print_usage(stats) return self.expectations
def test_main(self): self.mock(cq_stats, 'output', self.print_mock) self.mock( cq_stats, 'parse_args', lambda: Args(project='chromium', log_level=logging.CRITICAL, logs_black_list=None, logs_directory=tempfile.gettempdir())) self.mock(cq_stats, 'acquire_stats', lambda _: cq_stats.default_stats()) cq_stats.main() return self.expectations
def test_acquire_stats(self): self.mock(cq_stats, 'fetch_json', lambda _: 'json') self.mock( cq_stats, 'organize_stats', lambda *_args, **_kwargs: { 'latest': cq_stats.default_stats(), 'previous': cq_stats.default_stats() }) self.mock(cq_stats, 'derive_stats', lambda *_args, **_kwargs: {}) self.mock(cq_stats, 'derive_tree_stats', lambda *_: { 'open': 0.0, 'total': 3600.0 }) self.mock(cq_stats, 'derive_git_stats', lambda *_: {}) self.mock(cq_stats, 'derive_svn_stats', lambda *_: {}) cq_stats.acquire_stats(Args(project='blink', bots=[])) cq_stats.acquire_stats(Args(project='chromium', bots=[])) cq_stats.acquire_stats( Args(project='chromium', bots=[], use_logs=True, range='week')) cq_stats.acquire_stats( Args(project='chromium', bots=[], use_logs=True, range='day')) cq_stats.acquire_stats( Args(project='chromium', bots=[], use_logs=True, range='hour'))
def test_organize_stats(self): stats = { "results": [ { "begin": t, "stats": [ {"count": 3, "type": "count", "name": "attempt-count"}, {"count": 2, "type": "count", "name": "trybot-bot-false-reject-count"}, {"count": 1, "type": "count", "name": "trybot-bot-pass-count"}, { "description": "Total time spent per CQ attempt.", "max": 9999.99999, "percentile_25": 2512.34567, "percentile_75": 7512.34567, "percentile_10": 1012.34567, "unit": "seconds", "name": "attempt-durations", "percentile_50": 5012.34567, "min": 0.00001, "sample_size": 10000, "percentile_90": 9012.34567, "percentile_95": 9512.34567, "percentile_99": 9912.34567, "type": "list", "mean": 5555.555555, }, ], "interval_minutes": 15, "project": "chromium", "key": 5976204561612800, "end": t + 900, } for t in [1415138400, 1415139300] ] } result = cq_stats.organize_stats(stats) # Test that the result stats have the minimal expected dict keys # for print_stats(). expected_keys = set(cq_stats.default_stats().keys()) self.assertFalse(expected_keys - set(result.keys())) self.assertIsNone(cq_stats.organize_stats({})) return ensure_serializable(result)
def test_acquire_stats(self): self.mock(cq_stats, "fetch_json", lambda _: "json") self.mock( cq_stats, "organize_stats", lambda *_args, **_kwargs: {"latest": cq_stats.default_stats(), "previous": cq_stats.default_stats()}, ) self.mock(cq_stats, "derive_stats", lambda *_args, **_kwargs: {}) self.mock(cq_stats, "derive_tree_stats", lambda *_: {"open": 0.0, "total": 3600.0}) self.mock(cq_stats, "derive_git_stats", lambda *_: {}) self.mock(cq_stats, "derive_svn_stats", lambda *_: {}) cq_stats.acquire_stats(Args(project="blink", bots=[])) cq_stats.acquire_stats(Args(project="chromium", bots=[])) cq_stats.acquire_stats(Args(project="chromium", bots=[], use_logs=True, range="week")) cq_stats.acquire_stats(Args(project="chromium", bots=[], use_logs=True, range="day")) cq_stats.acquire_stats(Args(project="chromium", bots=[], use_logs=True, range="hour"))
def test_organize_stats(self): stats = {'results': [ {'begin': t, 'stats': [ {'count': 3, 'type': 'count', 'name': 'attempt-count'}, {'count': 2, 'type': 'count', 'name': 'trybot-bot-false-reject-count'}, {'count': 1, 'type': 'count', 'name': 'trybot-bot-pass-count'}, {'description': 'Total time spent per CQ attempt.', 'max': 9999.99999, 'percentile_25': 2512.34567, 'percentile_75': 7512.34567, 'percentile_10': 1012.34567, 'unit': 'seconds', 'name': 'attempt-durations', 'percentile_50': 5012.34567, 'min': 0.00001, 'sample_size': 10000, 'percentile_90': 9012.34567, 'percentile_95': 9512.34567, 'percentile_99': 9912.34567, 'type': 'list', 'mean': 5555.555555}, ], 'interval_minutes': 15, 'project': 'chromium', 'key': 5976204561612800, 'end': t + 900} for t in [1415138400, 1415139300]]} result = cq_stats.organize_stats(stats) # Test that the result stats have the minimal expected dict keys # for print_stats(). expected_keys = set(cq_stats.default_stats().keys()) self.assertFalse(expected_keys - set(result.keys())) self.assertIsNone(cq_stats.organize_stats({})) return ensure_serializable(result)
def test_print_stats(self): self.mock(cq_stats, 'output', self.print_mock) args = Args() stats_set = cq_stats.default_stats() stats_set['begin'] = args.date stats_set['end'] = args.date + datetime.timedelta(days=7) stats_set['patch_stats'].update({ (123, 1): { 'attempts': 1, 'false-rejections': 0, 'rejections': 1, 'committed': False, 'patchset-duration': 3600, 'patchset-duration-wallclock': 3600, 'failed-jobs-details': { 'builder_a': 1, }, }, (456, 1): { 'attempts': 1, 'false-rejections': 0, 'rejections': 1, 'committed': False, 'patchset-duration': 7200, 'patchset-duration-wallclock': 7200, 'failed-jobs-details': { 'builder_a': 1, }, }, }) swapped_stats = copy.deepcopy(stats_set) swapped_stats['begin'], swapped_stats['end'] = (swapped_stats['end'], swapped_stats['begin']) cq_stats.print_stats(args, None) cq_stats.print_stats(args, stats_set) cq_stats.print_stats(args, swapped_stats) return self.expectations
def test_print_stats(self): self.mock(cq_stats, "output", self.print_mock) args = Args() stats_set = cq_stats.default_stats() stats_set["begin"] = args.date stats_set["end"] = args.date + datetime.timedelta(days=7) stats_set["patch_stats"].update( { (123, 1): { "attempts": 1, "false-rejections": 0, "rejections": 1, "committed": False, "patchset-duration": 3600, "patchset-duration-wallclock": 3600, "failed-jobs-details": {"builder_a": 1}, }, (456, 1): { "attempts": 1, "false-rejections": 0, "rejections": 1, "committed": False, "patchset-duration": 7200, "patchset-duration-wallclock": 7200, "failed-jobs-details": {"builder_a": 1}, }, } ) swapped_stats = copy.deepcopy(stats_set) swapped_stats["begin"], swapped_stats["end"] = (swapped_stats["end"], swapped_stats["begin"]) cq_stats.print_stats(args, None) cq_stats.print_stats(args, stats_set) cq_stats.print_stats(args, swapped_stats) return self.expectations
def test_print_attempt_counts(self): self.mock(cq_stats, "output", self.print_mock) stats = cq_stats.default_stats() stats["patch_stats"] = { (123, 1): { "attempts": 1, "false-rejections": 0, "rejections": 1, "committed": False, "patchset-duration": 3600, "patchset-duration-wallclock": 3600, "failed-jobs-details": {"builder_a": 1}, } } cq_stats._derive_stats_from_patch_stats(stats) cq_stats.print_attempt_counts( stats, "rejections", "were unsuccessful", item_name=None, committed=False, details=True ) cq_stats.print_attempt_counts(stats, "rejections", "failed jobs", item_name=None, committed=False) return self.expectations
def test_print_duration(self): self.mock(cq_stats, 'output', self.print_mock) cq_stats.print_duration('mean', cq_stats.default_stats(), None) return self.expectations
def test_print_flakiness_stats(self): self.mock(cq_stats, 'output', self.print_mock) args = Args() stats_set = cq_stats.default_stats() stats_set['begin'] = args.date stats_set['end'] = args.date + datetime.timedelta(days=7) stats_set['patch_stats'].update({ (123, 1): { 'attempts': 1, 'false-rejections': 0, 'rejections': 1, 'committed': False, 'patchset-duration': 3600, 'patchset-duration-wallclock': 3600, 'failed-jobs-details': { 'builder_a': 1, }, }, }) self.mock(cq_stats, 'fetch_json', lambda _: [{ 'master': 'tryserver.chromium.linux', 'builder': 'chromium_presubmit', 'result': 0, 'url': 'https://build.chromium.org/10', }, { 'master': 'tryserver.chromium.linux', 'builder': 'chromium_presubmit', 'result': -1, 'url': 'https://build.chromium.org/20', }, { 'master': 'tryserver.chromium.linux', 'builder': 'chromium_presubmit', 'result': 2, 'url': 'https://build.chromium.org/30', }, { 'master': 'tryserver.chromium.linux', 'builder': 'linux_chromium_rel_ng', 'result': 2, 'build_properties': '{"failure_type": "COMPILE_FAILURE"}', 'url': 'https://build.chromium.org/40', }, { 'master': 'tryserver.chromium.linux', 'builder': 'linux_chromium_rel_ng', 'result': 2, 'build_properties': '{"failure_type": "TEST_FAILURE"}', 'url': 'https://build.chromium.org/50', }, { 'master': 'tryserver.chromium.linux', 'builder': 'linux_chromium_rel_ng', 'result': 2, 'build_properties': '{"failure_type": "INVALID_TEST_RESULTS"}', 'url': 'https://build.chromium.org/50', }, { 'master': 'tryserver.chromium.linux', 'builder': 'linux_chromium_rel_ng', 'result': 2, 'build_properties': '{"failure_type": "PATCH_FAILURE"}', 'url': 'https://build.chromium.org/50', }, { 'master': 'tryserver.chromium.linux', 'builder': 'chromium_presubmit', 'result': 4, 'url': 'https://build.chromium.org/60', }]) cq_stats.print_flakiness_stats(args, stats_set) args.seq = False args.list_uncategorized_flakes = True self.mock(cq_stats, 'fetch_json', lambda _: [{ 'master': 'tryserver.chromium.linux', 'builder': 'chromium_presubmit', 'result': 0, 'url': 'https://build.chromium.org/70', }, { 'master': 'tryserver.chromium.linux', 'builder': 'chromium_presubmit', 'result': 2, 'url': 'https://build.chromium.org/75', }, { 'master': 'tryserver.chromium.linux', 'builder': 'linux_chromium_rel_ng', 'result': 2, 'build_properties': '{"failure_type": "COMPILE_FAILURE"}', 'url': 'https://build.chromium.org/80', }]) cq_stats.print_flakiness_stats(args, stats_set) return self.expectations
def test_print_flakiness_stats(self): self.mock(cq_stats, 'output', self.print_mock) args = Args() stats_set = cq_stats.default_stats() stats_set['begin'] = args.date stats_set['end'] = args.date + datetime.timedelta(days=7) stats_set['patch_stats'].update({ (123, 1): { 'attempts': 1, 'false-rejections': 0, 'rejections': 1, 'committed': False, 'patchset-duration': 3600, 'patchset-duration-wallclock': 3600, 'failed-jobs-details': { 'builder_a': 1, }, }, }) self.mock( cq_stats, 'fetch_json', lambda _: [{ 'master': 'tryserver.chromium.linux', 'builder': 'chromium_presubmit', 'result': 0, }, { 'master': 'tryserver.chromium.linux', 'builder': 'chromium_presubmit', 'result': -1, }, { 'master': 'tryserver.chromium.linux', 'builder': 'chromium_presubmit', 'result': 2, }, { 'master': 'tryserver.chromium.linux', 'builder': 'linux_chromium_rel_ng', 'result': 2, 'build_properties': '{"failure_type": "COMPILE_FAILURE"}', }, { 'master': 'tryserver.chromium.linux', 'builder': 'linux_chromium_rel_ng', 'result': 2, 'build_properties': '{"failure_type": "TEST_FAILURE"}', }, { 'master': 'tryserver.chromium.linux', 'builder': 'chromium_presubmit', 'result': 4, }]) cq_stats.print_flakiness_stats(args, stats_set) args.seq = False self.mock( cq_stats, 'fetch_json', lambda _: [{ 'master': 'tryserver.chromium.linux', 'builder': 'chromium_presubmit', 'result': 2, }]) cq_stats.print_flakiness_stats(args, stats_set) return self.expectations
def test_print_duration(self): self.mock(cq_stats, "output", self.print_mock) cq_stats.print_duration("mean", cq_stats.default_stats(), None) return self.expectations