def report_view(document, view): jobs = get_jobs_for_view(view) system_test_jobs = [name for name in jobs if '.system_test.' in name] failures = [] for name in system_test_jobs: try: report_build(document, name, 'latest') except Exception as e: logger.debug("Failed to report {0} with error: {1}".format( name, e)) failures.append(name) # Retry failed for name in failures: logger.debug("Retry failed {0}".format(name)) try: report_build(document, name, 'latest') except Exception as e: logger.debug("Failed again to report {0}: {1}".format(name, e)) else: failures.remove(name) logger.debug("Failures: {0}".format(",".join(failures))) return system_test_jobs, failures
def report_view(document, view): jobs = get_jobs_for_view(view) system_test_jobs = [name for name in jobs if ".system_test." in name] failures = [] for name in system_test_jobs: try: report_build(document, name, "latest") except Exception as e: logger.debug("Failed to report {0} with error: {1}".format(name, e)) failures.append(name) # Retry failed for name in failures: logger.debug("Retry failed {0}".format(name)) try: report_build(document, name, "latest") except Exception as e: logger.debug("Failed again to report {0}: {1}".format(name, e)) else: failures.remove(name) logger.debug("Failures: {0}".format(",".join(failures))) return system_test_jobs, failures
def main(): parser = OptionParser( description="Publish results of system tests from Jenkins build to " "TestRail. See settings.py for configuration.") parser.add_option('-j', '--job-name', dest='job_name', default=None, help='Jenkins swarm runner job name') parser.add_option('-N', '--build-number', dest='build_number', default='latest', help='Jenkins swarm runner build number') parser.add_option('-o', '--one-job', dest='one_job_name', default=None, help=('Process only one job name from the specified ' 'parent job or view')) parser.add_option("-w", "--view", dest="jenkins_view", default=False, help="Get system tests jobs from Jenkins view") parser.add_option("-l", "--live", dest="live_report", action="store_true", help="Get tests results from running swarm") parser.add_option("-m", "--manual", dest="manual_run", action="store_true", help="Manually add tests cases to TestRun (tested only)") parser.add_option("-s", "--statistics", action="store_true", dest="bug_statistics", default=False, help="Make a statistics for bugs linked to TestRail for " "the test run") parser.add_option('-c', '--create-plan-only', action="store_true", dest="create_plan_only", default=False, help='Jenkins swarm runner job name') parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Enable debug output") (options, args) = parser.parse_args() if options.verbose: logger.setLevel(DEBUG) if options.live_report and options.build_number == 'latest': options.build_number = 'latest_started' # STEP #1 # Initialize TestRail Project and define configuration logger.info('Initializing TestRail Project configuration...') project = TestRailProject(url=TestRailSettings.url, user=TestRailSettings.user, password=TestRailSettings.password, project=TestRailSettings.project) tests_suite = project.get_suite_by_name(TestRailSettings.tests_suite) operation_systems = [ { 'name': config['name'], 'id': config['id'], 'distro': config['name'].split()[0].lower() } for config in project.get_config_by_name('Operation System')['configs'] if config['name'] in TestRailSettings.operation_systems ] tests_results = {os['distro']: [] for os in operation_systems} # STEP #2 # Get tests results from Jenkins logger.info('Getting tests results from Jenkins...') if options.jenkins_view: jobs = get_jobs_for_view(options.jenkins_view) tests_jobs = [{'name': j, 'number': 'latest'} for j in jobs if 'system_test' in j] if \ not options.create_plan_only else [] runner_job = [j for j in jobs if 'runner' in j][0] runner_build = Build(runner_job, 'latest') elif options.job_name: runner_build = Build(options.job_name, options.build_number) tests_jobs = get_downstream_builds(runner_build.build_data) if \ not options.create_plan_only else [] else: logger.error("Please specify either Jenkins swarm runner job name (-j)" " or Jenkins view with system tests jobs (-w). Exiting..") return is_running_builds = False for systest_build in tests_jobs: if (options.one_job_name and options.one_job_name != systest_build['name']): logger.debug( "Skipping '{0}' because --one-job is specified".format( systest_build['name'])) continue if options.job_name: if 'result' not in systest_build.keys(): logger.debug("Skipping '{0}' job because it does't run tests " "(build #{1} contains no results)".format( systest_build['name'], systest_build['number'])) continue if systest_build['result'] is None: logger.debug("Skipping '{0}' job (build #{1}) because it's sti" "ll running...".format( systest_build['name'], systest_build['number'], )) is_running_builds = True continue for os in tests_results.keys(): if os in systest_build['name'].lower(): tests_results[os].extend(get_tests_results(systest_build, os)) # STEP #3 # Create new TestPlan in TestRail (or get existing) and add TestRuns milestone, iso_number, prefix = get_version(runner_build.build_data) milestone = project.get_milestone_by_name(name=milestone) test_plan_name = ' '.join( filter(lambda x: bool(x), (milestone['name'], prefix, 'iso', '#' + str(iso_number)))) test_plan = project.get_plan_by_name(test_plan_name) if not test_plan: test_plan = project.add_plan(test_plan_name, description='/'.join([ JENKINS['url'], 'job', '{0}.all'.format(milestone['name']), str(iso_number) ]), milestone_id=milestone['id'], entries=[]) logger.info('Created new TestPlan "{0}".'.format(test_plan_name)) else: logger.info('Found existing TestPlan "{0}".'.format(test_plan_name)) if options.create_plan_only: return plan_entries = [] all_cases = project.get_cases(suite_id=tests_suite['id']) for os in operation_systems: cases_ids = [] if options.manual_run: all_results_groups = [r.group for r in tests_results[os['distro']]] for case in all_cases: if case['custom_test_group'] in all_results_groups: cases_ids.append(case['id']) plan_entries.append( project.test_run_struct( name='{suite_name}'.format(suite_name=tests_suite['name']), suite_id=tests_suite['id'], milestone_id=milestone['id'], description='Results of system tests ({tests_suite}) on is' 'o #"{iso_number}"'.format(tests_suite=tests_suite['name'], iso_number=iso_number), config_ids=[os['id']], include_all=True, case_ids=cases_ids)) if not any(entry['suite_id'] == tests_suite['id'] for entry in test_plan['entries']): if project.add_plan_entry( plan_id=test_plan['id'], suite_id=tests_suite['id'], config_ids=[os['id'] for os in operation_systems], runs=plan_entries): test_plan = project.get_plan(test_plan['id']) # STEP #4 # Upload tests results to TestRail logger.info('Uploading tests results to TestRail...') for os in operation_systems: logger.info('Checking tests results for "{0}"...'.format(os['name'])) results_to_publish = publish_results( project=project, milestone_id=milestone['id'], test_plan=test_plan, suite_id=tests_suite['id'], config_id=os['id'], results=tests_results[os['distro']]) logger.debug('Added new results for tests ({os}): {tests}'.format( os=os['name'], tests=[r.group for r in results_to_publish])) logger.info('Report URL: {0}'.format(test_plan['url'])) # STEP #5 # Provide the bugs linked in TestRail for current run as a short statistics if options.bug_statistics: if is_running_builds: logger.info("Some jobs are still running. " "Skipping bug statistics report, please try later.") else: logger.info("Generating a bug statistics report...") make_bug_statistics(tests_results, operation_systems)
def main(): parser = OptionParser( description="Publish results of system tests from Jenkins build to " "TestRail. See settings.py for configuration." ) parser.add_option('-j', '--job-name', dest='job_name', default=None, help='Jenkins swarm runner job name') parser.add_option('-N', '--build-number', dest='build_number', default='latest', help='Jenkins swarm runner build number') parser.add_option('-o', '--one-job', dest='one_job_name', default=None, help=('Process only one job name from the specified ' 'parent job or view')) parser.add_option("-w", "--view", dest="jenkins_view", default=False, help="Get system tests jobs from Jenkins view") parser.add_option("-l", "--live", dest="live_report", action="store_true", help="Get tests results from running swarm") parser.add_option("-m", "--manual", dest="manual_run", action="store_true", help="Manually add tests cases to TestRun (tested only)") parser.add_option("-s", "--statistics", action="store_true", dest="bug_statistics", default=False, help="Make a statistics for bugs linked to TestRail for " "the test run") parser.add_option('-c', '--create-plan-only', action="store_true", dest="create_plan_only", default=False, help='Jenkins swarm runner job name') parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Enable debug output") (options, args) = parser.parse_args() if options.verbose: logger.setLevel(DEBUG) if options.live_report and options.build_number == 'latest': options.build_number = 'latest_started' # STEP #1 # Initialize TestRail Project and define configuration logger.info('Initializing TestRail Project configuration...') project = TestRailProject(url=TestRailSettings.url, user=TestRailSettings.user, password=TestRailSettings.password, project=TestRailSettings.project) tests_suite = project.get_suite_by_name(TestRailSettings.tests_suite) operation_systems = [{'name': config['name'], 'id': config['id'], 'distro': config['name'].split()[0].lower()} for config in project.get_config_by_name( 'Operation System')['configs'] if config['name'] in TestRailSettings.operation_systems] tests_results = {os['distro']: [] for os in operation_systems} # STEP #2 # Get tests results from Jenkins logger.info('Getting tests results from Jenkins...') if options.jenkins_view: jobs = get_jobs_for_view(options.jenkins_view) tests_jobs = [{'name': j, 'number': 'latest'} for j in jobs if 'system_test' in j] if \ not options.create_plan_only else [] runner_job = [j for j in jobs if 'runner' in j][0] runner_build = Build(runner_job, 'latest') elif options.job_name: runner_build = Build(options.job_name, options.build_number) tests_jobs = get_downstream_builds(runner_build.build_data) if \ not options.create_plan_only else [] else: logger.error("Please specify either Jenkins swarm runner job name (-j)" " or Jenkins view with system tests jobs (-w). Exiting..") return is_running_builds = False for systest_build in tests_jobs: if (options.one_job_name and options.one_job_name != systest_build['name']): logger.debug("Skipping '{0}' because --one-job is specified" .format(systest_build['name'])) continue if options.job_name: if 'result' not in systest_build.keys(): logger.debug("Skipping '{0}' job because it does't run tests " "(build #{1} contains no results)".format( systest_build['name'], systest_build['number'])) continue if systest_build['result'] is None: logger.debug("Skipping '{0}' job (build #{1}) because it's sti" "ll running...".format(systest_build['name'], systest_build['number'],)) is_running_builds = True continue for os in tests_results.keys(): if os in systest_build['name'].lower(): tests_results[os].extend(get_tests_results(systest_build, os)) # STEP #3 # Create new TestPlan in TestRail (or get existing) and add TestRuns milestone, iso_number, prefix = get_version(runner_build.build_data) milestone = project.get_milestone_by_name(name=milestone) test_plan_name = ' '.join( filter(lambda x: bool(x), (milestone['name'], prefix, 'iso', '#' + str(iso_number)))) test_plan = project.get_plan_by_name(test_plan_name) iso_link = '/'.join([JENKINS['url'], 'job', '{0}.all'.format(milestone['name']), str(iso_number)]) if not test_plan: test_plan = project.add_plan(test_plan_name, description=iso_link, milestone_id=milestone['id'], entries=[] ) logger.info('Created new TestPlan "{0}".'.format(test_plan_name)) else: logger.info('Found existing TestPlan "{0}".'.format(test_plan_name)) if options.create_plan_only: return plan_entries = [] all_cases = project.get_cases(suite_id=tests_suite['id']) for os in operation_systems: cases_ids = [] if options.manual_run: all_results_groups = [r.group for r in tests_results[os['distro']]] for case in all_cases: if case['custom_test_group'] in all_results_groups: cases_ids.append(case['id']) plan_entries.append( project.test_run_struct( name='{suite_name}'.format(suite_name=tests_suite['name']), suite_id=tests_suite['id'], milestone_id=milestone['id'], description='Results of system tests ({tests_suite}) on is' 'o #"{iso_number}"'.format(tests_suite=tests_suite['name'], iso_number=iso_number), config_ids=[os['id']], include_all=True, case_ids=cases_ids ) ) if not any(entry['suite_id'] == tests_suite['id'] for entry in test_plan['entries']): if project.add_plan_entry(plan_id=test_plan['id'], suite_id=tests_suite['id'], config_ids=[os['id'] for os in operation_systems], runs=plan_entries): test_plan = project.get_plan(test_plan['id']) # STEP #4 # Upload tests results to TestRail logger.info('Uploading tests results to TestRail...') for os in operation_systems: logger.info('Checking tests results for "{0}"...'.format(os['name'])) results_to_publish = publish_results( project=project, milestone_id=milestone['id'], test_plan=test_plan, suite_id=tests_suite['id'], config_id=os['id'], results=tests_results[os['distro']] ) logger.debug('Added new results for tests ({os}): {tests}'.format( os=os['name'], tests=[r.group for r in results_to_publish] )) logger.info('Report URL: {0}'.format(test_plan['url'])) # STEP #5 # Provide the bugs linked in TestRail for current run as a short statistics if options.bug_statistics: if is_running_builds: logger.info("Some jobs are still running. " "Skipping bug statistics report, please try later.") else: logger.info("Generating a bug statistics report...") bug_results = make_bug_statistics(tests_results, test_plan, tests_suite, project, operation_systems) project.update_plan(plan_id=test_plan['id'], description=test_plan['description'] + '\n' + bug_results)
def main(): parser = OptionParser( description="Publish results of system tests from Jenkins build to " "TestRail. See settings.py for configuration." ) parser.add_option("-j", "--job-name", dest="job_name", default=None, help="Jenkins swarm runner job name") parser.add_option( "-N", "--build-number", dest="build_number", default="latest", help="Jenkins swarm runner build number" ) parser.add_option( "-o", "--one-job", dest="one_job_name", default=None, help=("Process only one job name from the specified " "parent job or view"), ) parser.add_option( "-w", "--view", dest="jenkins_view", default=False, help="Get system tests jobs from Jenkins view" ) parser.add_option( "-l", "--live", dest="live_report", action="store_true", help="Get tests results from running swarm" ) parser.add_option( "-m", "--manual", dest="manual_run", action="store_true", help="Manually add tests cases to TestRun (tested only)", ) parser.add_option( "-s", "--statistics", action="store_true", dest="bug_statistics", default=False, help="Make a statistics for bugs linked to TestRail for " "the test run", ) parser.add_option( "-c", "--create-plan-only", action="store_true", dest="create_plan_only", default=False, help="Jenkins swarm runner job name", ) parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Enable debug output") (options, args) = parser.parse_args() if options.verbose: logger.setLevel(DEBUG) if options.live_report and options.build_number == "latest": options.build_number = "latest_started" # STEP #1 # Initialize TestRail Project and define configuration logger.info("Initializing TestRail Project configuration...") project = TestRailProject( url=TestRailSettings.url, user=TestRailSettings.user, password=TestRailSettings.password, project=TestRailSettings.project, ) tests_suite = project.get_suite_by_name(TestRailSettings.tests_suite) operation_systems = [ {"name": config["name"], "id": config["id"], "distro": config["name"].split()[0].lower()} for config in project.get_config_by_name("Operation System")["configs"] if config["name"] in TestRailSettings.operation_systems ] tests_results = {os["distro"]: [] for os in operation_systems} # STEP #2 # Get tests results from Jenkins logger.info("Getting tests results from Jenkins...") if options.jenkins_view: jobs = get_jobs_for_view(options.jenkins_view) tests_jobs = ( [{"name": j, "number": "latest"} for j in jobs if "system_test" in j] if not options.create_plan_only else [] ) runner_job = [j for j in jobs if "runner" in j][0] runner_build = Build(runner_job, "latest") elif options.job_name: runner_build = Build(options.job_name, options.build_number) tests_jobs = get_downstream_builds(runner_build.build_data) if not options.create_plan_only else [] else: logger.error( "Please specify either Jenkins swarm runner job name (-j)" " or Jenkins view with system tests jobs (-w). Exiting.." ) return is_running_builds = False for systest_build in tests_jobs: if options.one_job_name and options.one_job_name != systest_build["name"]: logger.debug("Skipping '{0}' because --one-job is specified".format(systest_build["name"])) continue if options.job_name: if "result" not in systest_build.keys(): logger.debug( "Skipping '{0}' job because it does't run tests " "(build #{1} contains no results)".format(systest_build["name"], systest_build["number"]) ) continue if systest_build["result"] is None: logger.debug( "Skipping '{0}' job (build #{1}) because it's sti" "ll running...".format(systest_build["name"], systest_build["number"]) ) is_running_builds = True continue for os in tests_results.keys(): if os in systest_build["name"].lower(): tests_results[os].extend(get_tests_results(systest_build, os)) # STEP #3 # Create new TestPlan in TestRail (or get existing) and add TestRuns milestone, iso_number, prefix = get_version(runner_build.build_data) milestone = project.get_milestone_by_name(name=milestone) test_plan_name = " ".join(filter(lambda x: bool(x), (milestone["name"], prefix, "iso", "#" + str(iso_number)))) test_plan = project.get_plan_by_name(test_plan_name) iso_link = "/".join([JENKINS["url"], "job", "{0}.all".format(milestone["name"]), str(iso_number)]) if not test_plan: test_plan = project.add_plan(test_plan_name, description=iso_link, milestone_id=milestone["id"], entries=[]) logger.info('Created new TestPlan "{0}".'.format(test_plan_name)) else: logger.info('Found existing TestPlan "{0}".'.format(test_plan_name)) if options.create_plan_only: return plan_entries = [] all_cases = project.get_cases(suite_id=tests_suite["id"]) for os in operation_systems: cases_ids = [] if options.manual_run: all_results_groups = [r.group for r in tests_results[os["distro"]]] for case in all_cases: if case["custom_test_group"] in all_results_groups: cases_ids.append(case["id"]) plan_entries.append( project.test_run_struct( name="{suite_name}".format(suite_name=tests_suite["name"]), suite_id=tests_suite["id"], milestone_id=milestone["id"], description="Results of system tests ({tests_suite}) on is" 'o #"{iso_number}"'.format(tests_suite=tests_suite["name"], iso_number=iso_number), config_ids=[os["id"]], include_all=True, case_ids=cases_ids, ) ) if not any(entry["suite_id"] == tests_suite["id"] for entry in test_plan["entries"]): if project.add_plan_entry( plan_id=test_plan["id"], suite_id=tests_suite["id"], config_ids=[os["id"] for os in operation_systems], runs=plan_entries, ): test_plan = project.get_plan(test_plan["id"]) # STEP #4 # Upload tests results to TestRail logger.info("Uploading tests results to TestRail...") for os in operation_systems: logger.info('Checking tests results for "{0}"...'.format(os["name"])) results_to_publish = publish_results( project=project, milestone_id=milestone["id"], test_plan=test_plan, suite_id=tests_suite["id"], config_id=os["id"], results=tests_results[os["distro"]], ) logger.debug( "Added new results for tests ({os}): {tests}".format( os=os["name"], tests=[r.group for r in results_to_publish] ) ) logger.info("Report URL: {0}".format(test_plan["url"])) # STEP #5 # Provide the bugs linked in TestRail for current run as a short statistics if options.bug_statistics: if is_running_builds: logger.info("Some jobs are still running. " "Skipping bug statistics report, please try later.") else: logger.info("Generating a bug statistics report...") bug_results = make_bug_statistics(tests_results, test_plan, tests_suite, project, operation_systems) project.update_plan(plan_id=test_plan["id"], description=test_plan["description"] + "\n" + bug_results)