def main(argv=sys.argv[1:]):
    parser = argparse.ArgumentParser(
        description="Generate the 'doc' management jobs on Jenkins")
    add_argument_config_url(parser)
    add_argument_rosdistro_name(parser)
    add_argument_build_name(parser, 'doc')
    args = parser.parse_args(argv)

    config = get_index(args.config_url)
    build_files = get_doc_build_files(config, args.rosdistro_name)
    build_file = build_files[args.doc_build_name]

    if build_file.documentation_type != DOC_TYPE_ROSDOC:
        print(("The doc build file '%s' has the wrong documentation type to " +
               "be used with this script") % args.doc_build_name,
              file=sys.stderr)
        return 1

    jenkins = connect(config.jenkins_url)
    configure_management_view(jenkins)
    group_name = get_doc_view_name(
        args.rosdistro_name, args.doc_build_name)

    configure_reconfigure_jobs_job(
        jenkins, group_name, args, config, build_file)
    configure_trigger_jobs_job(jenkins, group_name, build_file)
Exemple #2
0
def main(argv=sys.argv[1:]):
    parser = argparse.ArgumentParser(
        description="Generate the 'doc' management jobs on Jenkins")
    add_argument_config_url(parser)
    add_argument_rosdistro_name(parser)
    add_argument_build_name(parser, 'doc')
    add_argument_dry_run(parser)
    args = parser.parse_args(argv)

    config = get_index(args.config_url)
    build_files = get_doc_build_files(config, args.rosdistro_name)
    build_file = build_files[args.doc_build_name]

    if build_file.documentation_type != DOC_TYPE_ROSDOC:
        print(("The doc build file '%s' has the wrong documentation type to " +
               "be used with this script") % args.doc_build_name,
              file=sys.stderr)
        return 1

    jenkins = connect(config.jenkins_url)
    configure_management_view(jenkins, dry_run=args.dry_run)
    group_name = get_doc_view_name(args.rosdistro_name, args.doc_build_name)

    configure_reconfigure_jobs_job(jenkins,
                                   group_name,
                                   args,
                                   config,
                                   build_file,
                                   dry_run=args.dry_run)
    configure_trigger_jobs_job(jenkins,
                               group_name,
                               build_file,
                               dry_run=args.dry_run)
Exemple #3
0
def get_job_config(args, config):
    template_name = 'misc/rosdistro_cache_job.xml.em'

    repository_args, script_generating_key_files = \
        get_repositories_and_script_generating_key_files(config=config)

    reconfigure_job_names = []
    build_files = get_release_build_files(config, args.rosdistro_name)
    for release_build_name in sorted(build_files.keys()):
        group_name = get_release_job_prefix(args.rosdistro_name,
                                            release_build_name)
        job_name = '%s_%s' % (group_name, 'reconfigure-jobs')
        reconfigure_job_names.append(job_name)

    reconfigure_doc_job_names = []
    build_files = get_doc_build_files(config, args.rosdistro_name)
    for doc_build_name in sorted(build_files.keys()):
        group_name = get_doc_view_name(args.rosdistro_name, doc_build_name)
        job_name = '%s_%s' % (group_name, 'reconfigure-jobs')
        reconfigure_doc_job_names.append(job_name)

    reconfigure_source_job_names = []
    build_files = get_source_build_files(config, args.rosdistro_name)
    for source_build_name in sorted(build_files.keys()):
        group_name = get_devel_view_name(args.rosdistro_name,
                                         source_build_name)
        job_name = '%s_%s' % (group_name, 'reconfigure-jobs')
        reconfigure_source_job_names.append(job_name)

    job_data = copy.deepcopy(args.__dict__)
    job_data.update({
        'ros_buildfarm_repository':
        get_repository(),
        'script_generating_key_files':
        script_generating_key_files,
        'rosdistro_index_url':
        config.rosdistro_index_url,
        'repository_args':
        repository_args,
        'reconfigure_job_names':
        reconfigure_job_names,
        'reconfigure_doc_job_names':
        reconfigure_doc_job_names,
        'reconfigure_source_job_names':
        reconfigure_source_job_names,
        'notification_emails':
        config.distributions[args.rosdistro_name]['notification_emails'],
        'git_ssh_credential_id':
        config.git_ssh_credential_id,
    })
    job_config = expand_template(template_name, job_data)
    return job_config
def get_job_config(args, config):
    template_name = 'misc/rosdistro_cache_job.xml.em'

    repository_args, script_generating_key_files = \
        get_repositories_and_script_generating_key_files(config=config)

    reconfigure_job_names = []
    build_files = get_release_build_files(config, args.rosdistro_name)
    for release_build_name in sorted(build_files.keys()):
        group_name = get_release_job_prefix(
            args.rosdistro_name, release_build_name)
        job_name = '%s_%s' % (group_name, 'reconfigure-jobs')
        reconfigure_job_names.append(job_name)

    reconfigure_doc_job_names = []
    build_files = get_doc_build_files(config, args.rosdistro_name)
    for doc_build_name in sorted(build_files.keys()):
        group_name = get_doc_view_name(
            args.rosdistro_name, doc_build_name)
        job_name = '%s_%s' % (group_name, 'reconfigure-jobs')
        reconfigure_doc_job_names.append(job_name)

    reconfigure_source_job_names = []
    build_files = get_source_build_files(config, args.rosdistro_name)
    for source_build_name in sorted(build_files.keys()):
        group_name = get_devel_view_name(
            args.rosdistro_name, source_build_name)
        job_name = '%s_%s' % (group_name, 'reconfigure-jobs')
        reconfigure_source_job_names.append(job_name)

    job_data = copy.deepcopy(args.__dict__)
    job_data.update({
        'ros_buildfarm_repository': get_repository(),

        'script_generating_key_files': script_generating_key_files,

        'rosdistro_index_url': config.rosdistro_index_url,

        'repository_args': repository_args,

        'reconfigure_job_names': reconfigure_job_names,
        'reconfigure_doc_job_names': reconfigure_doc_job_names,
        'reconfigure_source_job_names': reconfigure_source_job_names,

        'notification_emails':
        config.distributions[args.rosdistro_name]['notification_emails'],

        'git_ssh_credential_id': config.git_ssh_credential_id,
    })
    job_config = expand_template(template_name, job_data)
    return job_config
def configure_doc_metadata_job(
        config_url, rosdistro_name, doc_build_name,
        config=None, build_file=None):
    if config is None:
        config = get_config_index(config_url)
    if build_file is None:
        build_files = get_doc_build_files(config, rosdistro_name)
        build_file = build_files[doc_build_name]

    from ros_buildfarm.jenkins import connect
    jenkins = connect(config.jenkins_url)

    job_name = get_doc_view_name(rosdistro_name, doc_build_name)

    job_config = _get_doc_metadata_job_config(
        config, config_url, rosdistro_name, doc_build_name, build_file)
    # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
    if isinstance(jenkins, object) and jenkins is not False:
        from ros_buildfarm.jenkins import configure_job
        configure_job(jenkins, job_name, job_config)
def configure_doc_metadata_job(
        config_url, rosdistro_name, doc_build_name,
        config=None, build_file=None, dry_run=False):
    if config is None:
        config = get_config_index(config_url)
    if build_file is None:
        build_files = get_doc_build_files(config, rosdistro_name)
        build_file = build_files[doc_build_name]

    from ros_buildfarm.jenkins import connect
    jenkins = connect(config.jenkins_url)

    job_name = get_doc_view_name(rosdistro_name, doc_build_name)

    job_config = _get_doc_metadata_job_config(
        config, config_url, rosdistro_name, doc_build_name, build_file)
    # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
    if isinstance(jenkins, object) and jenkins is not False:
        from ros_buildfarm.jenkins import configure_job
        configure_job(jenkins, job_name, job_config, dry_run=dry_run)
def configure_doc_jobs(
        config_url, rosdistro_name, doc_build_name, groovy_script=None):
    """
    Configure all Jenkins doc jobs.

    L{configure_doc_job} will be invoked for doc repository and target
    which matches the build file criteria.
    """
    config = get_config_index(config_url)
    build_files = get_doc_build_files(config, rosdistro_name)
    build_file = build_files[doc_build_name]

    index = get_index(config.rosdistro_index_url)

    dist_cache = None
    if build_file.notify_maintainers:
        dist_cache = get_distribution_cache(index, rosdistro_name)

    # get targets
    targets = []
    for os_name in build_file.targets.keys():
        for os_code_name in build_file.targets[os_name].keys():
            for arch in build_file.targets[os_name][os_code_name]:
                targets.append((os_name, os_code_name, arch))
    print('The build file contains the following targets:')
    for os_name, os_code_name, arch in targets:
        print('  -', os_name, os_code_name, arch)

    dist_file = get_distribution_file(index, rosdistro_name, build_file)
    if not dist_file:
        print('No distribution file matches the build file')
        return

    doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name)

    from ros_buildfarm.jenkins import connect
    jenkins = connect(config.jenkins_url)

    views = []
    views.append(configure_doc_view(jenkins, doc_view_name))

    if groovy_script is not None:
        # all further configuration will be handled by the groovy script
        jenkins = False

    repo_names = dist_file.repositories.keys()
    filtered_repo_names = build_file.filter_repositories(repo_names)

    job_names = []
    job_configs = {}
    for repo_name in sorted(repo_names):
        is_disabled = repo_name not in filtered_repo_names
        if is_disabled and build_file.skip_ignored_repositories:
            print("Skipping ignored repository '%s'" % repo_name,
                  file=sys.stderr)
            continue

        repo = dist_file.repositories[repo_name]
        if not repo.doc_repository:
            print("Skipping repository '%s': no doc section" % repo_name)
            continue
        if not repo.doc_repository.version:
            print("Skipping repository '%s': no doc version" % repo_name)
            continue

        for os_name, os_code_name, arch in targets:
            try:
                job_name, job_config = configure_doc_job(
                    config_url, rosdistro_name, doc_build_name,
                    repo_name, os_name, os_code_name, arch,
                    config=config, build_file=build_file,
                    index=index, dist_file=dist_file,
                    dist_cache=dist_cache, jenkins=jenkins, views=views,
                    is_disabled=is_disabled,
                    groovy_script=groovy_script)
                job_names.append(job_name)
                if groovy_script is not None:
                    print("Configuration for job '%s'" % job_name)
                    job_configs[job_name] = job_config
            except JobValidationError as e:
                print(e.message, file=sys.stderr)

    job_prefix = '%s__' % doc_view_name
    if groovy_script is None:
        # delete obsolete jobs in this view
        from ros_buildfarm.jenkins import remove_jobs
        print('Removing obsolete doc jobs')
        remove_jobs(jenkins, job_prefix, job_names)
    else:
        print("Writing groovy script '%s' to reconfigure %d jobs" %
              (groovy_script, len(job_configs)))
        data = {
            'expected_num_jobs': len(job_configs),
            'job_prefixes_and_names': {
                'doc': (job_prefix, job_names),
            }
        }
        content = expand_template('snippet/reconfigure_jobs.groovy.em', data)
        write_groovy_script_and_configs(
            groovy_script, content, job_configs)
def configure_doc_job(
        config_url, rosdistro_name, doc_build_name,
        repo_name, os_name, os_code_name, arch,
        config=None, build_file=None,
        index=None, dist_file=None, dist_cache=None,
        jenkins=None, views=None,
        is_disabled=False,
        groovy_script=None,
        doc_repository=None):
    """
    Configure a single Jenkins doc job.

    This includes the following steps:
    - clone the doc repository to use
    - clone the ros_buildfarm repository
    - write the distribution repository keys into files
    - invoke the run_doc_job.py script
    """
    if config is None:
        config = get_config_index(config_url)
    if build_file is None:
        build_files = get_doc_build_files(config, rosdistro_name)
        build_file = build_files[doc_build_name]

    if index is None:
        index = get_index(config.rosdistro_index_url)
    if dist_file is None:
        dist_file = get_distribution_file(index, rosdistro_name, build_file)
        if not dist_file:
            raise JobValidationError(
                'No distribution file matches the build file')

    repo_names = dist_file.repositories.keys()

    if repo_name is not None:
        if repo_name not in repo_names:
            raise JobValidationError(
                "Invalid repository name '%s' " % repo_name +
                'choose one of the following: %s' %
                ', '.join(sorted(repo_names)))

        repo = dist_file.repositories[repo_name]
        if not repo.doc_repository:
            raise JobValidationError(
                "Repository '%s' has no doc section" % repo_name)
        if not repo.doc_repository.version:
            raise JobValidationError(
                "Repository '%s' has no doc version" % repo_name)
        doc_repository = repo.doc_repository

    if os_name not in build_file.targets.keys():
        raise JobValidationError(
            "Invalid OS name '%s' " % os_name +
            'choose one of the following: ' +
            ', '.join(sorted(build_file.targets.keys())))
    if os_code_name not in build_file.targets[os_name].keys():
        raise JobValidationError(
            "Invalid OS code name '%s' " % os_code_name +
            'choose one of the following: ' +
            ', '.join(sorted(build_file.targets[os_name].keys())))
    if arch not in build_file.targets[os_name][os_code_name]:
        raise JobValidationError(
            "Invalid architecture '%s' " % arch +
            'choose one of the following: %s' % ', '.join(sorted(
                build_file.targets[os_name][os_code_name])))

    if dist_cache is None and build_file.notify_maintainers:
        dist_cache = get_distribution_cache(index, rosdistro_name)
    if jenkins is None:
        from ros_buildfarm.jenkins import connect
        jenkins = connect(config.jenkins_url)
    if views is None:
        view_name = get_doc_view_name(
            rosdistro_name, doc_build_name)
        configure_doc_view(jenkins, view_name)

    job_name = get_doc_job_name(
        rosdistro_name, doc_build_name,
        repo_name, os_name, os_code_name, arch)

    job_config = _get_doc_job_config(
        config, config_url, rosdistro_name, doc_build_name,
        build_file, os_name, os_code_name, arch, doc_repository,
        repo_name, dist_cache=dist_cache, is_disabled=is_disabled)
    # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
    if isinstance(jenkins, object) and jenkins is not False:
        from ros_buildfarm.jenkins import configure_job
        configure_job(jenkins, job_name, job_config)

    return job_name, job_config
def configure_doc_jobs(config_url,
                       rosdistro_name,
                       doc_build_name,
                       groovy_script=None,
                       dry_run=False,
                       whitelist_repository_names=None):
    """
    Configure all Jenkins doc jobs.

    L{configure_doc_job} will be invoked for doc repository and target
    which matches the build file criteria.
    """
    config = get_config_index(config_url)
    build_files = get_doc_build_files(config, rosdistro_name)
    build_file = build_files[doc_build_name]

    index = get_index(config.rosdistro_index_url)

    dist_cache = None
    if build_file.notify_maintainers:
        dist_cache = get_distribution_cache(index, rosdistro_name)

    # get targets
    targets = []
    for os_name in build_file.targets.keys():
        for os_code_name in build_file.targets[os_name].keys():
            for arch in build_file.targets[os_name][os_code_name]:
                targets.append((os_name, os_code_name, arch))
    print('The build file contains the following targets:')
    for os_name, os_code_name, arch in targets:
        print('  -', os_name, os_code_name, arch)

    dist_file = get_distribution_file(index, rosdistro_name, build_file)
    if not dist_file:
        print('No distribution file matches the build file')
        return

    doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name)

    # all further configuration will be handled by either the Jenkins API
    # or by a generated groovy script
    from ros_buildfarm.jenkins import connect
    jenkins = connect(config.jenkins_url) if groovy_script is None else False

    view_configs = {}
    views = {}
    views[doc_view_name] = configure_doc_view(jenkins,
                                              doc_view_name,
                                              dry_run=dry_run)
    if not jenkins:
        view_configs.update(views)
    groovy_data = {
        'dry_run': dry_run,
        'expected_num_views': len(view_configs),
    }

    repo_names = dist_file.repositories.keys()
    filtered_repo_names = build_file.filter_repositories(repo_names)

    job_names = []
    job_configs = OrderedDict()
    for repo_name in sorted(repo_names):
        if whitelist_repository_names:
            if repo_name not in whitelist_repository_names:
                print(
                    "Skipping repository '%s' not in explicitly passed list" %
                    repo_name,
                    file=sys.stderr)
                continue
        is_disabled = repo_name not in filtered_repo_names
        if is_disabled and build_file.skip_ignored_repositories:
            print("Skipping ignored repository '%s'" % repo_name,
                  file=sys.stderr)
            continue

        repo = dist_file.repositories[repo_name]
        if not repo.doc_repository:
            print("Skipping repository '%s': no doc section" % repo_name)
            continue
        if not repo.doc_repository.version:
            print("Skipping repository '%s': no doc version" % repo_name)
            continue

        for os_name, os_code_name, arch in targets:
            try:
                job_name, job_config = configure_doc_job(
                    config_url,
                    rosdistro_name,
                    doc_build_name,
                    repo_name,
                    os_name,
                    os_code_name,
                    arch,
                    config=config,
                    build_file=build_file,
                    index=index,
                    dist_file=dist_file,
                    dist_cache=dist_cache,
                    jenkins=jenkins,
                    views=views,
                    is_disabled=is_disabled,
                    groovy_script=groovy_script,
                    dry_run=dry_run)
                job_names.append(job_name)
                if groovy_script is not None:
                    print("Configuration for job '%s'" % job_name)
                    job_configs[job_name] = job_config
            except JobValidationError as e:
                print(e.message, file=sys.stderr)

    groovy_data['expected_num_jobs'] = len(job_configs)
    groovy_data['job_prefixes_and_names'] = {}

    job_prefix = '%s__' % doc_view_name
    if not whitelist_repository_names:
        groovy_data['job_prefixes_and_names']['doc'] = (job_prefix, job_names)

        if groovy_script is None:
            # delete obsolete jobs in this view
            from ros_buildfarm.jenkins import remove_jobs
            print('Removing obsolete doc jobs')
            remove_jobs(jenkins, job_prefix, job_names, dry_run=dry_run)
    if groovy_script is not None:
        print(
            "Writing groovy script '%s' to reconfigure %d views and %d jobs" %
            (groovy_script, len(view_configs), len(job_configs)))
        content = expand_template('snippet/reconfigure_jobs.groovy.em',
                                  groovy_data)
        write_groovy_script_and_configs(groovy_script,
                                        content,
                                        job_configs,
                                        view_configs=view_configs)
def configure_doc_job(config_url,
                      rosdistro_name,
                      doc_build_name,
                      repo_name,
                      os_name,
                      os_code_name,
                      arch,
                      config=None,
                      build_file=None,
                      index=None,
                      dist_file=None,
                      dist_cache=None,
                      jenkins=None,
                      views=None,
                      is_disabled=False,
                      groovy_script=None,
                      doc_repository=None,
                      dry_run=False):
    """
    Configure a single Jenkins doc job.

    This includes the following steps:
    - clone the doc repository to use
    - clone the ros_buildfarm repository
    - write the distribution repository keys into files
    - invoke the run_doc_job.py script
    """
    if config is None:
        config = get_config_index(config_url)
    if build_file is None:
        build_files = get_doc_build_files(config, rosdistro_name)
        build_file = build_files[doc_build_name]

    if index is None:
        index = get_index(config.rosdistro_index_url)
    if dist_file is None:
        dist_file = get_distribution_file(index, rosdistro_name, build_file)
        if not dist_file:
            raise JobValidationError(
                'No distribution file matches the build file')

    repo_names = dist_file.repositories.keys()

    if repo_name is not None:
        if repo_name not in repo_names:
            raise JobValidationError("Invalid repository name '%s' " %
                                     repo_name +
                                     'choose one of the following: %s' %
                                     ', '.join(sorted(repo_names)))

        repo = dist_file.repositories[repo_name]
        if not repo.doc_repository:
            raise JobValidationError("Repository '%s' has no doc section" %
                                     repo_name)
        if not repo.doc_repository.version:
            raise JobValidationError("Repository '%s' has no doc version" %
                                     repo_name)
        doc_repository = repo.doc_repository

    if os_name not in build_file.targets.keys():
        raise JobValidationError("Invalid OS name '%s' " % os_name +
                                 'choose one of the following: ' +
                                 ', '.join(sorted(build_file.targets.keys())))
    if os_code_name not in build_file.targets[os_name].keys():
        raise JobValidationError(
            "Invalid OS code name '%s' " % os_code_name +
            'choose one of the following: ' +
            ', '.join(sorted(build_file.targets[os_name].keys())))
    if arch not in build_file.targets[os_name][os_code_name]:
        raise JobValidationError(
            "Invalid architecture '%s' " % arch +
            'choose one of the following: %s' %
            ', '.join(sorted(build_file.targets[os_name][os_code_name])))

    if dist_cache is None and build_file.notify_maintainers:
        dist_cache = get_distribution_cache(index, rosdistro_name)
    if jenkins is None:
        from ros_buildfarm.jenkins import connect
        jenkins = connect(config.jenkins_url)
    if views is None:
        view_name = get_doc_view_name(rosdistro_name, doc_build_name)
        configure_doc_view(jenkins, view_name, dry_run=dry_run)

    job_name = get_doc_job_name(rosdistro_name, doc_build_name, repo_name,
                                os_name, os_code_name, arch)

    job_config = _get_doc_job_config(config,
                                     config_url,
                                     rosdistro_name,
                                     doc_build_name,
                                     build_file,
                                     os_name,
                                     os_code_name,
                                     arch,
                                     doc_repository,
                                     repo_name,
                                     dist_cache=dist_cache,
                                     is_disabled=is_disabled)
    # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
    if isinstance(jenkins, object) and jenkins is not False:
        from ros_buildfarm.jenkins import configure_job
        configure_job(jenkins, job_name, job_config, dry_run=dry_run)

    return job_name, job_config