コード例 #1
0
ファイル: report.py プロジェクト: naveenzhang/fuel-main_5.1.2
def report_view(document, view):
    jobs = get_jobs_for_view(view)
    system_test_jobs = [name for name in jobs if '.system_test.' in name]

    failures = []

    for name in system_test_jobs:
        try:
            report_build(document, name, 'latest')
        except Exception as e:
            logger.debug("Failed to report {0} with error: {1}".format(
                name, e))
            failures.append(name)

    # Retry failed
    for name in failures:
        logger.debug("Retry failed {0}".format(name))
        try:
            report_build(document, name, 'latest')
        except Exception as e:
            logger.debug("Failed again to report {0}: {1}".format(name, e))
        else:
            failures.remove(name)

    logger.debug("Failures: {0}".format(",".join(failures)))

    return system_test_jobs, failures
コード例 #2
0
ファイル: report.py プロジェクト: Fiware/ops.Fuel-main-dev
def report_view(document, view):
    jobs = get_jobs_for_view(view)
    system_test_jobs = [name for name in jobs if ".system_test." in name]

    failures = []

    for name in system_test_jobs:
        try:
            report_build(document, name, "latest")
        except Exception as e:
            logger.debug("Failed to report {0} with error: {1}".format(name, e))
            failures.append(name)

    # Retry failed
    for name in failures:
        logger.debug("Retry failed {0}".format(name))
        try:
            report_build(document, name, "latest")
        except Exception as e:
            logger.debug("Failed again to report {0}: {1}".format(name, e))
        else:
            failures.remove(name)

    logger.debug("Failures: {0}".format(",".join(failures)))

    return system_test_jobs, failures
コード例 #3
0
def main():

    parser = OptionParser(
        description="Publish results of system tests from Jenkins build to "
        "TestRail. See settings.py for configuration.")
    parser.add_option('-j',
                      '--job-name',
                      dest='job_name',
                      default=None,
                      help='Jenkins swarm runner job name')
    parser.add_option('-N',
                      '--build-number',
                      dest='build_number',
                      default='latest',
                      help='Jenkins swarm runner build number')
    parser.add_option('-o',
                      '--one-job',
                      dest='one_job_name',
                      default=None,
                      help=('Process only one job name from the specified '
                            'parent job or view'))
    parser.add_option("-w",
                      "--view",
                      dest="jenkins_view",
                      default=False,
                      help="Get system tests jobs from Jenkins view")
    parser.add_option("-l",
                      "--live",
                      dest="live_report",
                      action="store_true",
                      help="Get tests results from running swarm")
    parser.add_option("-m",
                      "--manual",
                      dest="manual_run",
                      action="store_true",
                      help="Manually add tests cases to TestRun (tested only)")
    parser.add_option("-s",
                      "--statistics",
                      action="store_true",
                      dest="bug_statistics",
                      default=False,
                      help="Make a statistics for bugs linked to TestRail for "
                      "the test run")
    parser.add_option('-c',
                      '--create-plan-only',
                      action="store_true",
                      dest="create_plan_only",
                      default=False,
                      help='Jenkins swarm runner job name')
    parser.add_option("-v",
                      "--verbose",
                      action="store_true",
                      dest="verbose",
                      default=False,
                      help="Enable debug output")

    (options, args) = parser.parse_args()

    if options.verbose:
        logger.setLevel(DEBUG)

    if options.live_report and options.build_number == 'latest':
        options.build_number = 'latest_started'

    # STEP #1
    # Initialize TestRail Project and define configuration
    logger.info('Initializing TestRail Project configuration...')
    project = TestRailProject(url=TestRailSettings.url,
                              user=TestRailSettings.user,
                              password=TestRailSettings.password,
                              project=TestRailSettings.project)

    tests_suite = project.get_suite_by_name(TestRailSettings.tests_suite)
    operation_systems = [
        {
            'name': config['name'],
            'id': config['id'],
            'distro': config['name'].split()[0].lower()
        }
        for config in project.get_config_by_name('Operation System')['configs']
        if config['name'] in TestRailSettings.operation_systems
    ]
    tests_results = {os['distro']: [] for os in operation_systems}

    # STEP #2
    # Get tests results from Jenkins
    logger.info('Getting tests results from Jenkins...')
    if options.jenkins_view:
        jobs = get_jobs_for_view(options.jenkins_view)
        tests_jobs = [{'name': j, 'number': 'latest'}
                      for j in jobs if 'system_test' in j] if \
            not options.create_plan_only else []
        runner_job = [j for j in jobs if 'runner' in j][0]
        runner_build = Build(runner_job, 'latest')
    elif options.job_name:
        runner_build = Build(options.job_name, options.build_number)
        tests_jobs = get_downstream_builds(runner_build.build_data) if \
            not options.create_plan_only else []
    else:
        logger.error("Please specify either Jenkins swarm runner job name (-j)"
                     " or Jenkins view with system tests jobs (-w). Exiting..")
        return

    is_running_builds = False

    for systest_build in tests_jobs:
        if (options.one_job_name
                and options.one_job_name != systest_build['name']):
            logger.debug(
                "Skipping '{0}' because --one-job is specified".format(
                    systest_build['name']))
            continue
        if options.job_name:
            if 'result' not in systest_build.keys():
                logger.debug("Skipping '{0}' job because it does't run tests "
                             "(build #{1} contains no results)".format(
                                 systest_build['name'],
                                 systest_build['number']))
                continue
            if systest_build['result'] is None:
                logger.debug("Skipping '{0}' job (build #{1}) because it's sti"
                             "ll running...".format(
                                 systest_build['name'],
                                 systest_build['number'],
                             ))
                is_running_builds = True
                continue
        for os in tests_results.keys():
            if os in systest_build['name'].lower():
                tests_results[os].extend(get_tests_results(systest_build, os))

    # STEP #3
    # Create new TestPlan in TestRail (or get existing) and add TestRuns
    milestone, iso_number, prefix = get_version(runner_build.build_data)
    milestone = project.get_milestone_by_name(name=milestone)

    test_plan_name = ' '.join(
        filter(lambda x: bool(x),
               (milestone['name'], prefix, 'iso', '#' + str(iso_number))))
    test_plan = project.get_plan_by_name(test_plan_name)
    if not test_plan:
        test_plan = project.add_plan(test_plan_name,
                                     description='/'.join([
                                         JENKINS['url'], 'job',
                                         '{0}.all'.format(milestone['name']),
                                         str(iso_number)
                                     ]),
                                     milestone_id=milestone['id'],
                                     entries=[])
        logger.info('Created new TestPlan "{0}".'.format(test_plan_name))
    else:
        logger.info('Found existing TestPlan "{0}".'.format(test_plan_name))

    if options.create_plan_only:
        return

    plan_entries = []
    all_cases = project.get_cases(suite_id=tests_suite['id'])
    for os in operation_systems:
        cases_ids = []
        if options.manual_run:
            all_results_groups = [r.group for r in tests_results[os['distro']]]
            for case in all_cases:
                if case['custom_test_group'] in all_results_groups:
                    cases_ids.append(case['id'])
        plan_entries.append(
            project.test_run_struct(
                name='{suite_name}'.format(suite_name=tests_suite['name']),
                suite_id=tests_suite['id'],
                milestone_id=milestone['id'],
                description='Results of system tests ({tests_suite}) on is'
                'o #"{iso_number}"'.format(tests_suite=tests_suite['name'],
                                           iso_number=iso_number),
                config_ids=[os['id']],
                include_all=True,
                case_ids=cases_ids))

    if not any(entry['suite_id'] == tests_suite['id']
               for entry in test_plan['entries']):
        if project.add_plan_entry(
                plan_id=test_plan['id'],
                suite_id=tests_suite['id'],
                config_ids=[os['id'] for os in operation_systems],
                runs=plan_entries):
            test_plan = project.get_plan(test_plan['id'])

    # STEP #4
    # Upload tests results to TestRail
    logger.info('Uploading tests results to TestRail...')
    for os in operation_systems:
        logger.info('Checking tests results for "{0}"...'.format(os['name']))
        results_to_publish = publish_results(
            project=project,
            milestone_id=milestone['id'],
            test_plan=test_plan,
            suite_id=tests_suite['id'],
            config_id=os['id'],
            results=tests_results[os['distro']])
        logger.debug('Added new results for tests ({os}): {tests}'.format(
            os=os['name'], tests=[r.group for r in results_to_publish]))

    logger.info('Report URL: {0}'.format(test_plan['url']))

    # STEP #5
    # Provide the bugs linked in TestRail for current run as a short statistics
    if options.bug_statistics:
        if is_running_builds:
            logger.info("Some jobs are still running. "
                        "Skipping bug statistics report, please try later.")
        else:
            logger.info("Generating a bug statistics report...")
            make_bug_statistics(tests_results, operation_systems)
コード例 #4
0
ファイル: report.py プロジェクト: izadorozhna/fuel-qa
def main():

    parser = OptionParser(
        description="Publish results of system tests from Jenkins build to "
                    "TestRail. See settings.py for configuration."
    )
    parser.add_option('-j', '--job-name', dest='job_name', default=None,
                      help='Jenkins swarm runner job name')
    parser.add_option('-N', '--build-number', dest='build_number',
                      default='latest',
                      help='Jenkins swarm runner build number')
    parser.add_option('-o', '--one-job', dest='one_job_name',
                      default=None,
                      help=('Process only one job name from the specified '
                            'parent job or view'))
    parser.add_option("-w", "--view", dest="jenkins_view", default=False,
                      help="Get system tests jobs from Jenkins view")
    parser.add_option("-l", "--live", dest="live_report", action="store_true",
                      help="Get tests results from running swarm")
    parser.add_option("-m", "--manual", dest="manual_run", action="store_true",
                      help="Manually add tests cases to TestRun (tested only)")
    parser.add_option("-s", "--statistics", action="store_true",
                      dest="bug_statistics", default=False,
                      help="Make a statistics for bugs linked to TestRail for "
                      "the test run")
    parser.add_option('-c', '--create-plan-only', action="store_true",
                      dest="create_plan_only", default=False,
                      help='Jenkins swarm runner job name')
    parser.add_option("-v", "--verbose",
                      action="store_true", dest="verbose", default=False,
                      help="Enable debug output")

    (options, args) = parser.parse_args()

    if options.verbose:
        logger.setLevel(DEBUG)

    if options.live_report and options.build_number == 'latest':
        options.build_number = 'latest_started'

    # STEP #1
    # Initialize TestRail Project and define configuration
    logger.info('Initializing TestRail Project configuration...')
    project = TestRailProject(url=TestRailSettings.url,
                              user=TestRailSettings.user,
                              password=TestRailSettings.password,
                              project=TestRailSettings.project)

    tests_suite = project.get_suite_by_name(TestRailSettings.tests_suite)
    operation_systems = [{'name': config['name'], 'id': config['id'],
                         'distro': config['name'].split()[0].lower()}
                         for config in project.get_config_by_name(
                             'Operation System')['configs'] if
                         config['name'] in TestRailSettings.operation_systems]
    tests_results = {os['distro']: [] for os in operation_systems}

    # STEP #2
    # Get tests results from Jenkins
    logger.info('Getting tests results from Jenkins...')
    if options.jenkins_view:
        jobs = get_jobs_for_view(options.jenkins_view)
        tests_jobs = [{'name': j, 'number': 'latest'}
                      for j in jobs if 'system_test' in j] if \
            not options.create_plan_only else []
        runner_job = [j for j in jobs if 'runner' in j][0]
        runner_build = Build(runner_job, 'latest')
    elif options.job_name:
        runner_build = Build(options.job_name, options.build_number)
        tests_jobs = get_downstream_builds(runner_build.build_data) if \
            not options.create_plan_only else []
    else:
        logger.error("Please specify either Jenkins swarm runner job name (-j)"
                     " or Jenkins view with system tests jobs (-w). Exiting..")
        return

    is_running_builds = False

    for systest_build in tests_jobs:
        if (options.one_job_name and
                options.one_job_name != systest_build['name']):
            logger.debug("Skipping '{0}' because --one-job is specified"
                         .format(systest_build['name']))
            continue
        if options.job_name:
            if 'result' not in systest_build.keys():
                logger.debug("Skipping '{0}' job because it does't run tests "
                             "(build #{1} contains no results)".format(
                                 systest_build['name'],
                                 systest_build['number']))
                continue
            if systest_build['result'] is None:
                logger.debug("Skipping '{0}' job (build #{1}) because it's sti"
                             "ll running...".format(systest_build['name'],
                                                    systest_build['number'],))
                is_running_builds = True
                continue
        for os in tests_results.keys():
            if os in systest_build['name'].lower():
                tests_results[os].extend(get_tests_results(systest_build, os))

    # STEP #3
    # Create new TestPlan in TestRail (or get existing) and add TestRuns
    milestone, iso_number, prefix = get_version(runner_build.build_data)
    milestone = project.get_milestone_by_name(name=milestone)

    test_plan_name = ' '.join(
        filter(lambda x: bool(x),
               (milestone['name'], prefix, 'iso', '#' + str(iso_number))))

    test_plan = project.get_plan_by_name(test_plan_name)
    iso_link = '/'.join([JENKINS['url'], 'job',
                         '{0}.all'.format(milestone['name']), str(iso_number)])
    if not test_plan:
        test_plan = project.add_plan(test_plan_name,
                                     description=iso_link,
                                     milestone_id=milestone['id'],
                                     entries=[]
                                     )
        logger.info('Created new TestPlan "{0}".'.format(test_plan_name))
    else:
        logger.info('Found existing TestPlan "{0}".'.format(test_plan_name))

    if options.create_plan_only:
        return

    plan_entries = []
    all_cases = project.get_cases(suite_id=tests_suite['id'])
    for os in operation_systems:
        cases_ids = []
        if options.manual_run:
            all_results_groups = [r.group for r in tests_results[os['distro']]]
            for case in all_cases:
                if case['custom_test_group'] in all_results_groups:
                    cases_ids.append(case['id'])
        plan_entries.append(
            project.test_run_struct(
                name='{suite_name}'.format(suite_name=tests_suite['name']),
                suite_id=tests_suite['id'],
                milestone_id=milestone['id'],
                description='Results of system tests ({tests_suite}) on is'
                'o #"{iso_number}"'.format(tests_suite=tests_suite['name'],
                                           iso_number=iso_number),
                config_ids=[os['id']],
                include_all=True,
                case_ids=cases_ids
            )
        )

    if not any(entry['suite_id'] == tests_suite['id']
               for entry in test_plan['entries']):
        if project.add_plan_entry(plan_id=test_plan['id'],
                                  suite_id=tests_suite['id'],
                                  config_ids=[os['id'] for os
                                              in operation_systems],
                                  runs=plan_entries):
            test_plan = project.get_plan(test_plan['id'])

    # STEP #4
    # Upload tests results to TestRail
    logger.info('Uploading tests results to TestRail...')
    for os in operation_systems:
        logger.info('Checking tests results for "{0}"...'.format(os['name']))
        results_to_publish = publish_results(
            project=project,
            milestone_id=milestone['id'],
            test_plan=test_plan,
            suite_id=tests_suite['id'],
            config_id=os['id'],
            results=tests_results[os['distro']]
        )
        logger.debug('Added new results for tests ({os}): {tests}'.format(
            os=os['name'], tests=[r.group for r in results_to_publish]
        ))

    logger.info('Report URL: {0}'.format(test_plan['url']))

    # STEP #5
    # Provide the bugs linked in TestRail for current run as a short statistics
    if options.bug_statistics:
        if is_running_builds:
            logger.info("Some jobs are still running. "
                        "Skipping bug statistics report, please try later.")
        else:
            logger.info("Generating a bug statistics report...")
            bug_results = make_bug_statistics(tests_results, test_plan,
                                              tests_suite, project,
                                              operation_systems)
            project.update_plan(plan_id=test_plan['id'],
                                description=test_plan['description'] + '\n' +
                                bug_results)
コード例 #5
0
ファイル: report.py プロジェクト: dkulykov/fuel-qa-1
def main():

    parser = OptionParser(
        description="Publish results of system tests from Jenkins build to "
        "TestRail. See settings.py for configuration."
    )
    parser.add_option("-j", "--job-name", dest="job_name", default=None, help="Jenkins swarm runner job name")
    parser.add_option(
        "-N", "--build-number", dest="build_number", default="latest", help="Jenkins swarm runner build number"
    )
    parser.add_option(
        "-o",
        "--one-job",
        dest="one_job_name",
        default=None,
        help=("Process only one job name from the specified " "parent job or view"),
    )
    parser.add_option(
        "-w", "--view", dest="jenkins_view", default=False, help="Get system tests jobs from Jenkins view"
    )
    parser.add_option(
        "-l", "--live", dest="live_report", action="store_true", help="Get tests results from running swarm"
    )
    parser.add_option(
        "-m",
        "--manual",
        dest="manual_run",
        action="store_true",
        help="Manually add tests cases to TestRun (tested only)",
    )
    parser.add_option(
        "-s",
        "--statistics",
        action="store_true",
        dest="bug_statistics",
        default=False,
        help="Make a statistics for bugs linked to TestRail for " "the test run",
    )
    parser.add_option(
        "-c",
        "--create-plan-only",
        action="store_true",
        dest="create_plan_only",
        default=False,
        help="Jenkins swarm runner job name",
    )
    parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Enable debug output")

    (options, args) = parser.parse_args()

    if options.verbose:
        logger.setLevel(DEBUG)

    if options.live_report and options.build_number == "latest":
        options.build_number = "latest_started"

    # STEP #1
    # Initialize TestRail Project and define configuration
    logger.info("Initializing TestRail Project configuration...")
    project = TestRailProject(
        url=TestRailSettings.url,
        user=TestRailSettings.user,
        password=TestRailSettings.password,
        project=TestRailSettings.project,
    )

    tests_suite = project.get_suite_by_name(TestRailSettings.tests_suite)
    operation_systems = [
        {"name": config["name"], "id": config["id"], "distro": config["name"].split()[0].lower()}
        for config in project.get_config_by_name("Operation System")["configs"]
        if config["name"] in TestRailSettings.operation_systems
    ]
    tests_results = {os["distro"]: [] for os in operation_systems}

    # STEP #2
    # Get tests results from Jenkins
    logger.info("Getting tests results from Jenkins...")
    if options.jenkins_view:
        jobs = get_jobs_for_view(options.jenkins_view)
        tests_jobs = (
            [{"name": j, "number": "latest"} for j in jobs if "system_test" in j]
            if not options.create_plan_only
            else []
        )
        runner_job = [j for j in jobs if "runner" in j][0]
        runner_build = Build(runner_job, "latest")
    elif options.job_name:
        runner_build = Build(options.job_name, options.build_number)
        tests_jobs = get_downstream_builds(runner_build.build_data) if not options.create_plan_only else []
    else:
        logger.error(
            "Please specify either Jenkins swarm runner job name (-j)"
            " or Jenkins view with system tests jobs (-w). Exiting.."
        )
        return

    is_running_builds = False

    for systest_build in tests_jobs:
        if options.one_job_name and options.one_job_name != systest_build["name"]:
            logger.debug("Skipping '{0}' because --one-job is specified".format(systest_build["name"]))
            continue
        if options.job_name:
            if "result" not in systest_build.keys():
                logger.debug(
                    "Skipping '{0}' job because it does't run tests "
                    "(build #{1} contains no results)".format(systest_build["name"], systest_build["number"])
                )
                continue
            if systest_build["result"] is None:
                logger.debug(
                    "Skipping '{0}' job (build #{1}) because it's sti"
                    "ll running...".format(systest_build["name"], systest_build["number"])
                )
                is_running_builds = True
                continue
        for os in tests_results.keys():
            if os in systest_build["name"].lower():
                tests_results[os].extend(get_tests_results(systest_build, os))

    # STEP #3
    # Create new TestPlan in TestRail (or get existing) and add TestRuns
    milestone, iso_number, prefix = get_version(runner_build.build_data)
    milestone = project.get_milestone_by_name(name=milestone)

    test_plan_name = " ".join(filter(lambda x: bool(x), (milestone["name"], prefix, "iso", "#" + str(iso_number))))

    test_plan = project.get_plan_by_name(test_plan_name)
    iso_link = "/".join([JENKINS["url"], "job", "{0}.all".format(milestone["name"]), str(iso_number)])
    if not test_plan:
        test_plan = project.add_plan(test_plan_name, description=iso_link, milestone_id=milestone["id"], entries=[])
        logger.info('Created new TestPlan "{0}".'.format(test_plan_name))
    else:
        logger.info('Found existing TestPlan "{0}".'.format(test_plan_name))

    if options.create_plan_only:
        return

    plan_entries = []
    all_cases = project.get_cases(suite_id=tests_suite["id"])
    for os in operation_systems:
        cases_ids = []
        if options.manual_run:
            all_results_groups = [r.group for r in tests_results[os["distro"]]]
            for case in all_cases:
                if case["custom_test_group"] in all_results_groups:
                    cases_ids.append(case["id"])
        plan_entries.append(
            project.test_run_struct(
                name="{suite_name}".format(suite_name=tests_suite["name"]),
                suite_id=tests_suite["id"],
                milestone_id=milestone["id"],
                description="Results of system tests ({tests_suite}) on is"
                'o #"{iso_number}"'.format(tests_suite=tests_suite["name"], iso_number=iso_number),
                config_ids=[os["id"]],
                include_all=True,
                case_ids=cases_ids,
            )
        )

    if not any(entry["suite_id"] == tests_suite["id"] for entry in test_plan["entries"]):
        if project.add_plan_entry(
            plan_id=test_plan["id"],
            suite_id=tests_suite["id"],
            config_ids=[os["id"] for os in operation_systems],
            runs=plan_entries,
        ):
            test_plan = project.get_plan(test_plan["id"])

    # STEP #4
    # Upload tests results to TestRail
    logger.info("Uploading tests results to TestRail...")
    for os in operation_systems:
        logger.info('Checking tests results for "{0}"...'.format(os["name"]))
        results_to_publish = publish_results(
            project=project,
            milestone_id=milestone["id"],
            test_plan=test_plan,
            suite_id=tests_suite["id"],
            config_id=os["id"],
            results=tests_results[os["distro"]],
        )
        logger.debug(
            "Added new results for tests ({os}): {tests}".format(
                os=os["name"], tests=[r.group for r in results_to_publish]
            )
        )

    logger.info("Report URL: {0}".format(test_plan["url"]))

    # STEP #5
    # Provide the bugs linked in TestRail for current run as a short statistics
    if options.bug_statistics:
        if is_running_builds:
            logger.info("Some jobs are still running. " "Skipping bug statistics report, please try later.")
        else:
            logger.info("Generating a bug statistics report...")
            bug_results = make_bug_statistics(tests_results, test_plan, tests_suite, project, operation_systems)
            project.update_plan(plan_id=test_plan["id"], description=test_plan["description"] + "\n" + bug_results)