def test_issue_status_can_be_queried_from_bugrefs(): args = bugrefs_test_args_factory() args.verbose_test = 1 args.query_issue_status = True args.include_softfails = True args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tags_labels') args.arch = 'i586' report = str(openqa_review.generate_report(args)) compare_report(report, os.path.join(args.load_dir, 'report25_bugrefs_query_issues.md')) # report generated when no todo items are left and some bugref is not accessible args.builds = '1508,1500' args.include_softfails = False report = str(openqa_review.generate_report(args)) compare_report(report, os.path.join(args.load_dir, 'report25_bugrefs_build1508.md'))
def test_custom_reports_based_on_issue_status(): args = bugrefs_test_args_factory() args.verbose_test = 1 args.query_issue_status = True args.include_softfails = True # now, try filtering: unassigned report = openqa_review.generate_report(args) openqa_review.filter_report(report, openqa_review.ie_filters["unassigned"]) compare_report(str(report), os.path.join(args.load_dir, 'report25_bugrefs_query_issues_filter_unassigned.md')) # 2nd filter: closed report = openqa_review.generate_report(args) openqa_review.filter_report(report, openqa_review.ie_filters["closed"]) compare_report(str(report), os.path.join(args.load_dir, 'report25_bugrefs_query_issues_filter_closed.md'))
def test_single_job_group_pages_can_be_cached_from_cache(): args = cache_test_args_factory() with TemporaryDirectory() as tmp_dir: args.save_dir = tmp_dir args.save = True report = str(openqa_review.generate_report(args)) assert "**Common issues:**" in report
def test_arch_distinguish(): args = cache_test_args_factory() args.arch = None args.job_group_urls = args.host + '/group_overview/4' report = str(openqa_review.generate_report(args)) assert 'ppc64le' in report
def test_generate_report_with_progress_notification_does_not_fail(): args = cache_test_args_factory() # Not easy to test automatically but at least we can call it and assume it also gives valid results args.no_progress = False args.job_groups_url = None report = str(openqa_review.generate_report(args)) assert "**Common issues:**" in report
def test_single_job_group_pages_can_be_cached_from_cache(): args = cache_test_args_factory() with TemporaryDirectory() as tmp_dir: args.save_dir = tmp_dir args.save = True report = str(openqa_review.generate_report(args)) assert '**Common issues:**' in report
def test_ha_tests_yields_valid_report_with_valid_build_nr(): args = cache_test_args_factory() args.arch = None # let this test check architectures by itself to reach good test coverage args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "live") args.job_group_urls = args.host + "/group_overview/27" report = str(openqa_review.generate_report(args)) assert "0104@0351" in report
def test_previously_loaded_cache_file_is_generated_into_valid_terse_report_by_default(): args = cache_test_args_factory() args.verbose_test = 1 report = str(openqa_review.generate_report(args)) assert '**Common issues:**' in report ref_report = open(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'report25_terse.md')).read() compare_report(report, ref_report)
def test_previously_loaded_cache_file_is_generated_into_valid_verbose_report_if_configured(): args = cache_test_args_factory() report = str(openqa_review.generate_report(args)) assert '**Common issues:**' in report # Missing architecture is reported assert re.search("Missing arch.*i586", report) compare_report(report, os.path.join(os.path.dirname(os.path.realpath(__file__)), 'report25_TTT.md'))
def test_generate_report_with_progress_notification_does_not_fail(): args = cache_test_args_factory() # Not easy to test automatically but at least we can call it and assume it also gives valid results args.no_progress = False args.job_groups_url = None report = str(openqa_review.generate_report(args)) assert '**Common issues:**' in report
def test_previously_loaded_cache_file_is_generated_into_valid_terse_report_by_default(): args = cache_test_args_factory() args.verbose_test = 1 report = str(openqa_review.generate_report(args)) assert "**Common issues:**" in report ref_report = open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "report25_terse.md")).read() compare_report(report, ref_report)
def test_arch_distinguish(): args = cache_test_args_factory() args.arch = None args.job_group_urls = args.host + "/group_overview/4" report = str(openqa_review.generate_report(args)) assert "ppc64le" in report
def test_openqa_45_bootstrap_4_can_parse_failed_modules(): args = cache_test_args_factory() args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'openqa_4.5_dashboard') args.job_group_urls = 'https://openqa.opensuse.org/group_overview/41' report = str(openqa_review.generate_report(args)) assert '20180424' in report assert 'installation' in report
def test_new_job_group_json_syntax_after_openqa_9b50b22(): args = cache_test_args_factory() args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'job_group_after_openqa_9b50b22') args.job_group_urls = 'http://openqa.opensuse.org/group_overview/70' report = str(openqa_review.generate_report(args)) assert '0211' in report assert 'Green' in report
def test_bugrefs_with_abbreviated_format_can_be_used(): args = bugrefs_test_args_factory() args.verbose_test = 1 args.short_failure_str = True args.abbreviate_test_issues = True report = str(openqa_review.generate_report(args)) compare_report(report, os.path.join(args.load_dir, 'report25_bugrefs_abbreviated.md'))
def test_ha_tests_yields_valid_report_with_valid_build_nr(): args = cache_test_args_factory() args.arch = None # let this test check architectures by itself to reach good test coverage args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'live') args.job_group_urls = args.host + '/group_overview/27' report = str(openqa_review.generate_report(args)) assert '0104@0351' in report
def test_specified_job_group_yields_single_product_report(): args = cache_test_args_factory() args.job_group_urls = None args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "single_job_group") args.job_groups = "openSUSE Argon" report = str(openqa_review.generate_report(args)) assert args.job_groups in report # There must be only one job group tag assert len([line for line in report.splitlines() if line.startswith("#")]) == 1 # Invalid name should yield assertion with helpful message args.job_groups = "openSUSE Tumbleweed FOO" with pytest.raises(AssertionError) as e: report = openqa_review.generate_report(args) assert "No job group" in str(e.value) # Multiple job groups can be specified args.job_groups = "openSUSE Argon,openSUSE Leap 42.2 Updates" # we don't actually need to check the parsing just make sure # openqa_review tries to parse all and as there is no cache page # for 'openSUSE Tumbleweed 2.KDE' saved, we assume its corresponding # page can not be retrieved. # Unfortunately we can not easily be more specific about the # exception as python2 raises IOError, python3 FileNotFoundError # but we can check the content anyway. with pytest.raises(Exception) as e: report = str(openqa_review.generate_report(args)) assert "group_overview:26" in str(e.value) # job groups can also be used as an incomplete search tags or regex args.job_groups = "(42.2 Updates|Argon)" # To increase statement and branch coverage we enable progress report here. # It will be invisible but executed. args.no_progress = False # see above with pytest.raises(Exception) as e: report = str(openqa_review.generate_report(args)) assert "group_overview:26" in str(e.value) # job group with only a single recent build yields empty report args.job_groups = "openSUSE Leap 42.2 AArch64" report = str(openqa_review.generate_report(args)) assert args.job_groups in report # There must be only one job group tag assert "Not enough finished builds" in report
def test_single_job_group_pages_can_be_cached_from_web(): args = args_factory() args.job_group_urls = args.host + '/group_overview/25' with TemporaryDirectory() as tmp_dir: args.save_dir = tmp_dir report = openqa_review.generate_report(args) assert '**Common issues:**' in report
def test_previously_loaded_cache_file_is_generated_into_valid_verbose_report_if_configured(): args = cache_test_args_factory() report = str(openqa_review.generate_report(args)) assert "**Common issues:**" in report # Missing architecture is reported assert re.search("Missing arch.*i586", report) ref_report = open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "report25_TTT.md")).read() compare_report(report, ref_report)
def test_bugrefs_with_report_links_new_issue(): args = bugrefs_test_args_factory() args.report_links = True args.include_softfails = False args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tags_labels/report_link_new_issue') args.arch = 'arm' report = str(openqa_review.generate_report(args)) compare_report(report, os.path.join(args.load_dir, 'report25_bugrefs_bug_link_new_issue.md'))
def test_previously_loaded_cache_file_is_generated_into_ref_report_l3(): args = cache_test_args_factory() args.verbose_test = 3 report = openqa_review.generate_report(args) ref_report = open( os.path.join(os.path.dirname(os.path.realpath(__file__)), 'report25_TT.md')).read() compare_report(report, ref_report)
def test_previously_loaded_cache_file_is_generated_into_ref_report_l2(): args = cache_test_args_factory() args.verbose_test = 2 report = str(openqa_review.generate_report(args)) compare_report( report, os.path.join(os.path.dirname(os.path.realpath(__file__)), "report25_T.md"))
def test_single_job_group_pages_can_be_cached_from_web(): args = args_factory() args.job_group_urls = args.host + "/group_overview/25" with TemporaryDirectory() as tmp_dir: args.save_dir = tmp_dir report = openqa_review.generate_report(args) assert "**Common issues:**" in report
def test_specified_job_group_yields_single_product_report(): args = cache_test_args_factory() args.job_group_urls = None args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "single_job_group") args.job_groups = "openSUSE Argon" report = str(openqa_review.generate_report(args)) assert args.job_groups in report # There must be only one job group tag assert len([l for l in report.splitlines() if l.startswith("#")]) == 1 # Invalid name should yield assertion with helpful message args.job_groups = "openSUSE Tumbleweed FOO" with pytest.raises(AssertionError) as e: report = openqa_review.generate_report(args) assert "No job group" in str(e.value) # Multiple job groups can be specified args.job_groups = "openSUSE Argon,openSUSE Leap 42.2 Updates" # we don't actually need to check the parsing just make sure # openqa_review tries to parse all and as there is no cache page # for 'openSUSE Tumbleweed 2.KDE' saved, we assume its corresponding # page can not be retrieved. # Unfortunately we can not easily be more specific about the # exception as python2 raises IOError, python3 FileNotFoundError # but we can check the content anyway. with pytest.raises(Exception) as e: report = str(openqa_review.generate_report(args)) assert "group_overview:26" in str(e.value) # job groups can also be used as an incomplete search tags or regex args.job_groups = "(42.2 Updates|Argon)" # To increase statement and branch coverage we enable progress report here. # It will be invisible but executed. args.no_progress = False # see above with pytest.raises(Exception) as e: report = str(openqa_review.generate_report(args)) assert "group_overview:26" in str(e.value) # job group with only a single recent build yields empty report args.job_groups = "openSUSE Leap 42.2 AArch64" report = str(openqa_review.generate_report(args)) assert args.job_groups in report # There must be only one job group tag assert "Not enough finished builds" in report
def test_new_tests_appearing_in_builds_are_supported(): args = cache_test_args_factory() args.job_groups = None args.builds = "0405,0389" args.arch = "i586" args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "differing_tests") report = str(openqa_review.generate_report(args)) # There should be one new test which is failing and has not been there in before assert '* [btrfs@zkvm](https://openqa.opensuse.org/tests/181148 "Failed modules: livecdreboot")' in report
def test_bugrefs_are_used_for_triaging(): args = bugrefs_test_args_factory() args.verbose_test = 1 report = str(openqa_review.generate_report(args)) # report should feature bug references assert 'bsc#' in report # and references to 'test issues' assert 'poo#' in report compare_report(report, os.path.join(args.load_dir, 'report25_bugrefs.md'))
def test_new_tests_appearing_in_builds_are_supported(): args = cache_test_args_factory() args.job_groups = None args.builds = '0405,0389' args.arch = 'i586' args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'differing_tests') report = openqa_review.generate_report(args) # There should be one new test which is failing and has not been there in before assert '* ***btrfs@zkvm***: https://openqa.opensuse.org/tests/181148 (reference NONE )' in report
def test_new_tests_appearing_in_builds_are_supported(): args = cache_test_args_factory() args.job_groups = None args.builds = '0405,0389' args.arch = 'i586' args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'differing_tests') report = str(openqa_review.generate_report(args)) # There should be one new test which is failing and has not been there in before assert '* [btrfs@zkvm](https://openqa.opensuse.org/tests/181148 "Failed modules: livecdreboot")' in report
def test_skip_passed(): args = cache_test_args_factory() args.skip_passed = True args.show_empty = False args.job_groups = [ "openSUSE Tumbleweed WSL", "openSUSE Tumbleweed PowerPC" ] args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "skip-passed") args.job_group_urls = "https://openqa.opensuse.org/group_overview/68,https://openqa.opensuse.org/group_overview/4" args.arch = None report = str(openqa_review.generate_report(args)) compare_report(report, os.path.join(args.load_dir, "report-68-4.md"))
def test_reminder_comments_on_referenced_bugs_are_posted(): args = bugrefs_test_args_factory() args.verbose_test = 1 args.query_issue_status = True args.dry_run = True report = openqa_review.generate_report(args) # test double comment prevention code p, pr = list(iteritems(report.report))[0] report.report[p + 237] = pr openqa_review.reminder_comment_on_issues(report) args.dry_run = False
def test_bugrefs_are_used_for_triaging(): # python openqa_review/openqa_review.py --load-dir tests/tags_labels --host https://openqa.opensuse.org -J https://openqa.opensuse.org/group_overview/25 -b # 1507,1500 --load -n > tests/tags_labels/report25_bugrefs.md args = cache_test_args_factory() args.job_groups = None args.bugrefs = True args.builds = '1507,1500' args.arch = 'i586' args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tags_labels') args.show_empty = False args.include_softfails = False report = openqa_review.generate_report(args) # report should feature bug references assert 'bsc#' in report # and references to 'test issues' assert 'poo#' in report ref_report = open(os.path.join(args.load_dir, 'report25_bugrefs.md')).read() compare_report(report, ref_report) # now, with query issues args.query_issue_status = True report = openqa_review.generate_report(args) ref_report = open( os.path.join(args.load_dir, 'report25_bugrefs_query_issues.md')).read() compare_report(report, ref_report) # report generated when no todo items are left args.builds = '1508,1500' args.query_issue_status = False report = openqa_review.generate_report(args) ref_report = open( os.path.join(args.load_dir, 'report25_bugrefs_build1508.md')).read() compare_report(report, ref_report)
def test_bugrefs_are_used_for_triaging(): # python openqa_review/openqa_review.py --load-dir tests/tags_labels --host https://openqa.opensuse.org -J https://openqa.opensuse.org/group_overview/25 -b # 1507,1500 --load -n > tests/tags_labels/report25_bugrefs.md args = cache_test_args_factory() args.job_groups = None args.bugrefs = True args.builds = '1507,1500' args.arch = 'i586' args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tags_labels') args.show_empty = False args.include_softfails = False args.verbose_test = 1 openqa_review.config = ConfigParser() openqa_review.config.add_section('product_issues') openqa_review.config.set('product_issues', 'base_url', 'https://%(username)s:%(password)[email protected]') openqa_review.config.set('product_issues', 'username', 'user') openqa_review.config.set('product_issues', 'password', 'pass') openqa_review.config.set('product_issues', 'report_url', 'https://bugzilla.opensuse.org') openqa_review.config.add_section('product_issues:https://openqa.opensuse.org:product_mapping') openqa_review.config.set('product_issues:https://openqa.opensuse.org:product_mapping', '25', 'openSUSE Tumbleweed') openqa_review.config.add_section('product_issues:https://openqa.opensuse.org:component_mapping') openqa_review.config.set('product_issues:https://openqa.opensuse.org:component_mapping', 'installation-bootloader', 'Bootloader') openqa_review.config.add_section('test_issues') openqa_review.config.set('test_issues', 'report_url', 'https://progress.opensuse.org/projects/openqatests/issues/new') report = str(openqa_review.generate_report(args)) # report should feature bug references assert 'bsc#' in report # and references to 'test issues' assert 'poo#' in report ref_report = open(os.path.join(args.load_dir, 'report25_bugrefs.md')).read() compare_report(report, ref_report) args.verbose_test = 2 args.report_links = True report = str(openqa_review.generate_report(args)) ref_report = open(os.path.join(args.load_dir, 'report25_T_bugrefs.md')).read() compare_report(report, ref_report) # report bug link(s) with 'new issue' args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tags_labels/report_link_new_issue') args.arch = 'arm' report = str(openqa_review.generate_report(args)) ref_report = open(os.path.join(args.load_dir, 'report25_bugrefs_bug_link_new_issue.md')).read() compare_report(report, ref_report) # now, with query issues args.verbose_test = 1 args.report_links = False args.query_issue_status = True args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'tags_labels') args.arch = 'i586' report = openqa_review.generate_report(args) ref_report = open(os.path.join(args.load_dir, 'report25_bugrefs_query_issues.md')).read() compare_report(report, ref_report) # now, try filtering: unassigned openqa_review.filter_report(report, openqa_review.ie_filters["unassigned"]) ref_report = open(os.path.join(args.load_dir, 'report25_bugrefs_query_issues_filter_unassigned.md')).read() compare_report(report, ref_report) # 2nd filter: closed report = openqa_review.generate_report(args) openqa_review.filter_report(report, openqa_review.ie_filters["closed"]) ref_report = open(os.path.join(args.load_dir, 'report25_bugrefs_query_issues_filter_closed.md')).read() compare_report(report, ref_report) # report generated when no todo items are left args.builds = '1508,1500' args.query_issue_status = False report = openqa_review.generate_report(args) ref_report = open(os.path.join(args.load_dir, 'report25_bugrefs_build1508.md')).read() compare_report(report, ref_report)
def test_bugrefs_including_softfails(): args = bugrefs_test_args_factory() args.include_softfails = True report = str(openqa_review.generate_report(args)) compare_report(report, os.path.join(args.load_dir, 'report25_T_bugrefs_softfails.md'))
def test_default_returns_valid_markdown_document(): args = args_factory() report = openqa_review.generate_report(args) assert '**Common issues:**' in report
def test_single_job_group_with_extended_test_output_returns_valid_markdown_document( ): args = args_factory() args.job_group_urls = args.host + '/group_overview/25' report = openqa_review.generate_report(args) assert '**Common issues:**' in report
def test_bugrefs_with_report_links(): args = bugrefs_test_args_factory() args.report_links = True report = str(openqa_review.generate_report(args)) compare_report(report, os.path.join(args.load_dir, 'report25_T_bugrefs.md'))
def test_builds_can_be_specified_and_appear_in_report(): args = cache_test_args_factory() args.builds = "0313,0308" report = str(openqa_review.generate_report(args)) assert "**Build:** {} (reference {})".format(*args.builds.split(",")) in report
def test_previously_loaded_cache_file_is_generated_into_ref_report_l3(): args = cache_test_args_factory() args.verbose_test = 3 report = str(openqa_review.generate_report(args)) ref_report = open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "report25_TT.md")).read() compare_report(report, ref_report)
def test_too_high_verbosity_selection_yields_still_valid_selection(): args = cache_test_args_factory() args.verbose_test = 5 report = str(openqa_review.generate_report(args)) assert report != ""
def test_state_report_does_not_break_generation(): args = cache_test_args_factory() args.output_state_results = True report = openqa_review.generate_report(args) assert report
def test_default_returns_valid_markdown_document(): args = args_factory() report = openqa_review.generate_report(args) assert "**Common issues:**" in report
def test_single_job_group_with_extended_test_output_returns_valid_markdown_document(): args = args_factory() args.job_group_urls = args.host + "/group_overview/25" report = openqa_review.generate_report(args) assert "**Common issues:**" in report
def test_too_high_verbosity_selection_yields_still_valid_selection(): args = cache_test_args_factory() args.verbose_test = 5 report = str(openqa_review.generate_report(args)) assert report != ''
def test_builds_can_be_specified_and_appear_in_report(): args = cache_test_args_factory() args.builds = '0313,0308' report = str(openqa_review.generate_report(args)) assert '**Build:** {} (reference {})'.format(*args.builds.split(',')) in report
def test_bugrefs_are_used_for_triaging(): # python openqa_review/openqa_review.py --load-dir tests/tags_labels --host https://openqa.opensuse.org -J https://openqa.opensuse.org/group_overview/25 -b # 1507,1500 --load -n > tests/tags_labels/report25_bugrefs.md args = cache_test_args_factory() args.job_groups = None args.bugrefs = True args.builds = "1507,1500" args.arch = "i586" args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tags_labels") args.show_empty = False args.include_softfails = False args.verbose_test = 1 openqa_review.config = ConfigParser() openqa_review.config.add_section("product_issues") openqa_review.config.set("product_issues", "base_url", "https://%(username)s:%(password)[email protected]") openqa_review.config.set("product_issues", "username", "user") openqa_review.config.set("product_issues", "password", "pass") openqa_review.config.set("product_issues", "report_url", "https://bugzilla.opensuse.org") openqa_review.config.add_section("product_issues:https://openqa.opensuse.org:product_mapping") openqa_review.config.set("product_issues:https://openqa.opensuse.org:product_mapping", "25", "openSUSE Tumbleweed") openqa_review.config.add_section("product_issues:https://openqa.opensuse.org:component_mapping") openqa_review.config.set( "product_issues:https://openqa.opensuse.org:component_mapping", "installation-bootloader", "Bootloader" ) openqa_review.config.add_section("test_issues") openqa_review.config.set( "test_issues", "report_url", "https://progress.opensuse.org/projects/openqatests/issues/new" ) report = str(openqa_review.generate_report(args)) # report should feature bug references assert "bsc#" in report # and references to 'test issues' assert "poo#" in report ref_report = open(os.path.join(args.load_dir, "report25_bugrefs.md")).read() compare_report(report, ref_report) args.verbose_test = 2 args.report_links = True report = str(openqa_review.generate_report(args)) ref_report = open(os.path.join(args.load_dir, "report25_T_bugrefs.md")).read() compare_report(report, ref_report) # report bug link(s) with 'new issue' args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tags_labels/report_link_new_issue") args.arch = "arm" report = str(openqa_review.generate_report(args)) ref_report = open(os.path.join(args.load_dir, "report25_bugrefs_bug_link_new_issue.md")).read() compare_report(report, ref_report) # now, with query issues args.verbose_test = 1 args.report_links = False args.query_issue_status = True args.load_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tags_labels") args.arch = "i586" report = openqa_review.generate_report(args) ref_report = open(os.path.join(args.load_dir, "report25_bugrefs_query_issues.md")).read() compare_report(report, ref_report) # now, try filtering: unassigned openqa_review.filter_report(report, openqa_review.ie_filters["unassigned"]) ref_report = open(os.path.join(args.load_dir, "report25_bugrefs_query_issues_filter_unassigned.md")).read() compare_report(report, ref_report) # 2nd filter: closed report = openqa_review.generate_report(args) openqa_review.filter_report(report, openqa_review.ie_filters["closed"]) ref_report = open(os.path.join(args.load_dir, "report25_bugrefs_query_issues_filter_closed.md")).read() compare_report(report, ref_report) # report generated when no todo items are left and some bugref is not accessible args.builds = "1508,1500" args.query_issue_status = True report = openqa_review.generate_report(args) ref_report = open(os.path.join(args.load_dir, "report25_bugrefs_build1508.md")).read() compare_report(report, ref_report)
def test_builds_can_be_specified_and_appear_in_report(): args = cache_test_args_factory() args.builds = '0313,0308' report = openqa_review.generate_report(args) assert '**Build: {}** (reference {})'.format(*args.builds.split(',')) in report