def configure_devel_job(config_url, rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, pull_request=False, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, views=None, is_disabled=False, groovy_script=None, source_repository=None, build_targets=None, dry_run=False): """ Configure a single Jenkins devel job. This includes the following steps: - clone the source repository to use - clone the ros_buildfarm repository - write the distribution repository keys into files - invoke the release/run_devel_job.py script """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_source_build_files(config, rosdistro_name) build_file = build_files[source_build_name] # Overwrite build_file.targets if build_targets is specified if build_targets is not None: build_file.targets = build_targets if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: raise JobValidationError( 'No distribution file matches the build file') repo_names = dist_file.repositories.keys() if repo_name is not None: if repo_name not in repo_names: raise JobValidationError("Invalid repository name '%s' " % repo_name + 'choose one of the following: %s' % ', '.join(sorted(repo_names))) repo = dist_file.repositories[repo_name] if not repo.source_repository: raise JobValidationError("Repository '%s' has no source section" % repo_name) if not repo.source_repository.version: raise JobValidationError("Repository '%s' has no source version" % repo_name) source_repository = repo.source_repository if os_name not in build_file.targets.keys(): raise JobValidationError("Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if arch not in build_file.targets[os_name][os_code_name]: raise JobValidationError( "Invalid architecture '%s' " % arch + 'choose one of the following: %s' % ', '.join(sorted(build_file.targets[os_name][os_code_name]))) if dist_cache is None and build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if views is None: view_name = get_devel_view_name(rosdistro_name, source_build_name, pull_request=pull_request) configure_devel_view(jenkins, view_name, dry_run=dry_run) job_name = get_devel_job_name(rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, pull_request) job_config = _get_devel_job_config(config, rosdistro_name, source_build_name, build_file, os_name, os_code_name, arch, source_repository, repo_name, pull_request, job_name, dist_cache=dist_cache, is_disabled=is_disabled) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: from ros_buildfarm.jenkins import configure_job configure_job(jenkins, job_name, job_config, dry_run=dry_run) return job_name, job_config
def configure_devel_jobs(config_url, rosdistro_name, source_build_name, groovy_script=None, dry_run=False, whitelist_repository_names=None): """ Configure all Jenkins devel jobs. L{configure_release_job} will be invoked for source repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_source_build_files(config, rosdistro_name) build_file = build_files[source_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return devel_view_name = get_devel_view_name(rosdistro_name, source_build_name, pull_request=False) pull_request_view_name = get_devel_view_name(rosdistro_name, source_build_name, pull_request=True) # all further configuration will be handled by either the Jenkins API # or by a generated groovy script from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if groovy_script is None else False view_configs = {} views = {} if build_file.test_commits_force is not False: views[devel_view_name] = configure_devel_view(jenkins, devel_view_name, dry_run=dry_run) if build_file.test_pull_requests_force is not False: views[pull_request_view_name] = configure_devel_view( jenkins, pull_request_view_name, dry_run=dry_run) if not jenkins: view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(view_configs), } repo_names = dist_file.repositories.keys() filtered_repo_names = build_file.filter_repositories(repo_names) devel_job_names = [] pull_request_job_names = [] job_configs = OrderedDict() for repo_name in sorted(repo_names): if whitelist_repository_names: if repo_name not in whitelist_repository_names: print( "Skipping repository '%s' not in explicitly passed list" % repo_name, file=sys.stderr) continue is_disabled = repo_name not in filtered_repo_names if is_disabled and build_file.skip_ignored_repositories: print("Skipping ignored repository '%s'" % repo_name, file=sys.stderr) continue repo = dist_file.repositories[repo_name] if not repo.source_repository: print("Skipping repository '%s': no source section" % repo_name) continue if not repo.source_repository.version: print("Skipping repository '%s': no source version" % repo_name) continue job_types = [] # check for testing commits if build_file.test_commits_force is False: print(("Skipping repository '%s': 'test_commits' is forced to " + "false in the build file") % repo_name) elif repo.source_repository.test_commits is False: print(("Skipping repository '%s': 'test_commits' of the " + "repository set to false") % repo_name) elif repo.source_repository.test_commits is None and \ not build_file.test_commits_default: print(("Skipping repository '%s': 'test_commits' defaults to " + "false in the build file") % repo_name) else: job_types.append('commit') if not is_disabled: # check for testing pull requests if build_file.test_pull_requests_force is False: # print(("Skipping repository '%s': 'test_pull_requests' " + # "is forced to false in the build file") % repo_name) pass elif repo.source_repository.test_pull_requests is False: # print(("Skipping repository '%s': 'test_pull_requests' of " + # "the repository set to false") % repo_name) pass elif repo.source_repository.test_pull_requests is None and \ not build_file.test_pull_requests_default: # print(("Skipping repository '%s': 'test_pull_requests' " + # "defaults to false in the build file") % repo_name) pass else: print("Pull request job for repository '%s'" % repo_name) job_types.append('pull_request') for job_type in job_types: pull_request = job_type == 'pull_request' for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_devel_job( config_url, rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, pull_request, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script, dry_run=dry_run) if not pull_request: devel_job_names.append(job_name) else: pull_request_job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(job_configs) groovy_data['job_prefixes_and_names'] = {} devel_job_prefix = '%s__' % devel_view_name pull_request_job_prefix = '%s__' % pull_request_view_name if not whitelist_repository_names: groovy_data['job_prefixes_and_names']['devel'] = \ (devel_job_prefix, devel_job_names) groovy_data['job_prefixes_and_names']['pull_request'] = \ (pull_request_job_prefix, pull_request_job_names) if groovy_script is None: # delete obsolete jobs in these views from ros_buildfarm.jenkins import remove_jobs print('Removing obsolete devel jobs') remove_jobs(jenkins, devel_job_prefix, devel_job_names, dry_run=dry_run) print('Removing obsolete pull request jobs') remove_jobs(jenkins, pull_request_job_prefix, pull_request_job_names, dry_run=dry_run) if groovy_script is not None: print( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % (groovy_script, len(view_configs), len(job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, job_configs, view_configs=view_configs)
def configure_doc_job( config_url, rosdistro_name, doc_build_name, repo_name, os_name, os_code_name, arch, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, views=None, is_disabled=False, groovy_script=None, doc_repository=None): """ Configure a single Jenkins doc job. This includes the following steps: - clone the doc repository to use - clone the ros_buildfarm repository - write the distribution repository keys into files - invoke the run_doc_job.py script """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_doc_build_files(config, rosdistro_name) build_file = build_files[doc_build_name] if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: raise JobValidationError( 'No distribution file matches the build file') repo_names = dist_file.repositories.keys() if repo_name is not None: if repo_name not in repo_names: raise JobValidationError( "Invalid repository name '%s' " % repo_name + 'choose one of the following: %s' % ', '.join(sorted(repo_names))) repo = dist_file.repositories[repo_name] if not repo.doc_repository: raise JobValidationError( "Repository '%s' has no doc section" % repo_name) if not repo.doc_repository.version: raise JobValidationError( "Repository '%s' has no doc version" % repo_name) doc_repository = repo.doc_repository if os_name not in build_file.targets.keys(): raise JobValidationError( "Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if arch not in build_file.targets[os_name][os_code_name]: raise JobValidationError( "Invalid architecture '%s' " % arch + 'choose one of the following: %s' % ', '.join(sorted( build_file.targets[os_name][os_code_name]))) if dist_cache is None and build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if views is None: view_name = get_doc_view_name( rosdistro_name, doc_build_name) configure_doc_view(jenkins, view_name) job_name = get_doc_job_name( rosdistro_name, doc_build_name, repo_name, os_name, os_code_name, arch) job_config = _get_doc_job_config( config, config_url, rosdistro_name, doc_build_name, build_file, os_name, os_code_name, arch, doc_repository, repo_name, dist_cache=dist_cache, is_disabled=is_disabled) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: from ros_buildfarm.jenkins import configure_job configure_job(jenkins, job_name, job_config) return job_name, job_config
def configure_doc_jobs( config_url, rosdistro_name, doc_build_name, groovy_script=None): """ Configure all Jenkins doc jobs. L{configure_doc_job} will be invoked for doc repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_doc_build_files(config, rosdistro_name) build_file = build_files[doc_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name) from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) views = [] views.append(configure_doc_view(jenkins, doc_view_name)) if groovy_script is not None: # all further configuration will be handled by the groovy script jenkins = False repo_names = dist_file.repositories.keys() filtered_repo_names = build_file.filter_repositories(repo_names) job_names = [] job_configs = {} for repo_name in sorted(repo_names): is_disabled = repo_name not in filtered_repo_names if is_disabled and build_file.skip_ignored_repositories: print("Skipping ignored repository '%s'" % repo_name, file=sys.stderr) continue repo = dist_file.repositories[repo_name] if not repo.doc_repository: print("Skipping repository '%s': no doc section" % repo_name) continue if not repo.doc_repository.version: print("Skipping repository '%s': no doc version" % repo_name) continue for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_doc_job( config_url, rosdistro_name, doc_build_name, repo_name, os_name, os_code_name, arch, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script) job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) job_prefix = '%s__' % doc_view_name if groovy_script is None: # delete obsolete jobs in this view from ros_buildfarm.jenkins import remove_jobs print('Removing obsolete doc jobs') remove_jobs(jenkins, job_prefix, job_names) else: print("Writing groovy script '%s' to reconfigure %d jobs" % (groovy_script, len(job_configs))) data = { 'expected_num_jobs': len(job_configs), 'job_prefixes_and_names': { 'doc': (job_prefix, job_names), } } content = expand_template('snippet/reconfigure_jobs.groovy.em', data) write_groovy_script_and_configs( groovy_script, content, job_configs)
def configure_release_job(config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, views=None, generate_import_package_job=True, generate_sync_packages_jobs=True, is_disabled=False, other_build_files_same_platform=None, groovy_script=None, filter_arches=None, dry_run=False): """ Configure a Jenkins release job. The following jobs are created for each package: - M source jobs, one for each OS node name - M * N binary jobs, one for each combination of OS code name and arch """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: raise JobValidationError( 'No distribution file matches the build file') pkg_names = dist_file.release_packages.keys() if pkg_name not in pkg_names: raise JobValidationError("Invalid package name '%s' " % pkg_name + 'choose one of the following: ' + ', '.join(sorted(pkg_names))) pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] if not repo.release_repository: raise JobValidationError("Repository '%s' has no release section" % repo_name) if not repo.release_repository.version: raise JobValidationError("Repository '%s' has no release version" % repo_name) if os_name not in build_file.targets.keys(): raise JobValidationError("Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if dist_cache is None and \ (build_file.notify_maintainers or build_file.abi_incompatibility_assumed): dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if views is None: targets = [] targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) configure_release_views(jenkins, rosdistro_name, release_build_name, targets, dry_run=dry_run) if generate_import_package_job: configure_import_package_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if generate_sync_packages_jobs: configure_sync_packages_to_main_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) for arch in build_file.targets[os_name][os_code_name]: configure_sync_packages_to_testing_job(config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) source_job_names = [] binary_job_names = [] job_configs = {} # sourcedeb job # since sourcedeb jobs are potentially being shared across multiple build # files the configuration has to take all of them into account in order to # generate a job which all build files agree on source_job_name = get_sourcedeb_job_name(rosdistro_name, release_build_name, pkg_name, os_name, os_code_name) # while the package is disabled in the current build file # it might be used by sibling build files is_source_disabled = is_disabled if is_source_disabled and other_build_files_same_platform: # check if sourcedeb job is used by any other build file with the same platform for other_build_file in other_build_files_same_platform: if other_build_file.filter_packages([pkg_name]): is_source_disabled = False break job_config = _get_sourcedeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache, is_disabled=is_source_disabled, other_build_files_same_platform=other_build_files_same_platform) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: from ros_buildfarm.jenkins import configure_job configure_job(jenkins, source_job_name, job_config, dry_run=dry_run) source_job_names.append(source_job_name) job_configs[source_job_name] = job_config dependency_names = [] if build_file.abi_incompatibility_assumed: dependency_names = _get_direct_dependencies(pkg_name, dist_cache, pkg_names) # if dependencies are not yet available in rosdistro cache # skip binary jobs if dependency_names is None: print(("Skipping binary jobs for package '%s' because it is not " + "yet in the rosdistro cache") % pkg_name, file=sys.stderr) return source_job_names, binary_job_names, job_configs # binarydeb jobs for arch in build_file.targets[os_name][os_code_name]: if filter_arches and arch not in filter_arches: continue job_name = get_binarydeb_job_name(rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, arch) upstream_job_names = [source_job_name] + [ get_binarydeb_job_name( rosdistro_name, release_build_name, dependency_name, os_name, os_code_name, arch) for dependency_name in dependency_names ] job_config = _get_binarydeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, arch, pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache, upstream_job_names=upstream_job_names, is_disabled=is_disabled) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, job_name, job_config, dry_run=dry_run) binary_job_names.append(job_name) job_configs[job_name] = job_config return source_job_names, binary_job_names, job_configs
def configure_release_jobs(config_url, rosdistro_name, release_build_name, groovy_script=None, dry_run=False, whitelist_package_names=None): """ Configure all Jenkins release jobs. L{configure_release_job} will be invoked for every released package and target which matches the build file criteria. Additionally a job to import Debian packages into the Debian repository is created. """ config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) # get targets platforms = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): platforms.append((os_name, os_code_name)) print('The build file contains the following targets:') for os_name, os_code_name in platforms: print(' - %s %s: %s' % (os_name, os_code_name, ', '.join( build_file.targets[os_name][os_code_name]))) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return pkg_names = dist_file.release_packages.keys() filtered_pkg_names = build_file.filter_packages(pkg_names) explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names) if explicitly_ignored_pkg_names: print(('The following packages are being %s because of ' + 'white-/blacklisting:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(explicitly_ignored_pkg_names): print(' -', pkg_name) dist_cache = get_distribution_cache(index, rosdistro_name) if explicitly_ignored_pkg_names: # get direct dependencies from distro cache for each package direct_dependencies = {} for pkg_name in pkg_names: direct_dependencies[pkg_name] = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) or set([]) # find recursive downstream deps for all explicitly ignored packages ignored_pkg_names = set(explicitly_ignored_pkg_names) while True: implicitly_ignored_pkg_names = _get_downstream_package_names( ignored_pkg_names, direct_dependencies) if implicitly_ignored_pkg_names - ignored_pkg_names: ignored_pkg_names |= implicitly_ignored_pkg_names continue break implicitly_ignored_pkg_names = \ ignored_pkg_names - explicitly_ignored_pkg_names if implicitly_ignored_pkg_names: print(('The following packages are being %s because their ' + 'dependencies are being ignored:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(implicitly_ignored_pkg_names): print(' -', pkg_name) filtered_pkg_names = \ set(filtered_pkg_names) - implicitly_ignored_pkg_names # all further configuration will be handled by either the Jenkins API # or by a generated groovy script jenkins = False if groovy_script is None: from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) all_view_configs = {} all_job_configs = OrderedDict() job_name, job_config = configure_import_package_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config job_name, job_config = configure_sync_packages_to_main_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config for os_name, os_code_name in platforms: for arch in sorted(build_file.targets[os_name][os_code_name]): job_name, job_config = configure_sync_packages_to_testing_job( config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config targets = [] for os_name, os_code_name in platforms: targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) views = configure_release_views(jenkins, rosdistro_name, release_build_name, targets, dry_run=dry_run) if not jenkins: all_view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(views), } # binary jobs must be generated in topological order from catkin_pkg.package import parse_package_string from ros_buildfarm.common import topological_order_packages pkgs = {} for pkg_name in pkg_names: if pkg_name not in dist_cache.release_package_xmls: print("Skipping package '%s': no released package.xml in cache" % (pkg_name), file=sys.stderr) continue pkg_xml = dist_cache.release_package_xmls[pkg_name] pkg = parse_package_string(pkg_xml) pkgs[pkg_name] = pkg ordered_pkg_tuples = topological_order_packages(pkgs) other_build_files = [ v for k, v in build_files.items() if k != release_build_name ] all_source_job_names = [] all_binary_job_names = [] for pkg_name in [p.name for _, p in ordered_pkg_tuples]: if whitelist_package_names: if pkg_name not in whitelist_package_names: print( "Skipping package '%s' not in the explicitly passed list" % pkg_name, file=sys.stderr) continue pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] is_disabled = pkg_name not in filtered_pkg_names if is_disabled and build_file.skip_ignored_packages: print("Skipping ignored package '%s' in repository '%s'" % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository: print(("Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository.version: print(("Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name), file=sys.stderr) continue for os_name, os_code_name in platforms: other_build_files_same_platform = [] for other_build_file in other_build_files: if os_name not in other_build_file.targets: continue if os_code_name not in other_build_file.targets[os_name]: continue other_build_files_same_platform.append(other_build_file) try: source_job_names, binary_job_names, job_configs = \ configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, generate_import_package_job=False, generate_sync_packages_jobs=False, is_disabled=is_disabled, other_build_files_same_platform=other_build_files_same_platform, groovy_script=groovy_script, dry_run=dry_run) all_source_job_names += source_job_names all_binary_job_names += binary_job_names if groovy_script is not None: print('Configuration for jobs: ' + ', '.join(source_job_names + binary_job_names)) for source_job_name in source_job_names: all_job_configs[source_job_name] = job_configs[ source_job_name] for binary_job_name in binary_job_names: all_job_configs[binary_job_name] = job_configs[ binary_job_name] except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(all_job_configs) groovy_data['job_prefixes_and_names'] = {} # with an explicit list of packages we don't delete obsolete jobs if not whitelist_package_names: # delete obsolete binary jobs for os_name, os_code_name in platforms: for arch in build_file.targets[os_name][os_code_name]: binary_view = get_release_binary_view_name( rosdistro_name, release_build_name, os_name, os_code_name, arch) binary_job_prefix = '%s__' % binary_view excluded_job_names = set([ j for j in all_binary_job_names if j.startswith(binary_job_prefix) ]) if groovy_script is None: print("Removing obsolete binary jobs with prefix '%s'" % binary_job_prefix) from ros_buildfarm.jenkins import remove_jobs remove_jobs(jenkins, binary_job_prefix, excluded_job_names, dry_run=dry_run) else: binary_key = 'binary_%s_%s_%s' % \ (os_name, os_code_name, arch) groovy_data['job_prefixes_and_names'][binary_key] = \ (binary_job_prefix, excluded_job_names) # delete obsolete source jobs # requires knowledge about all other release build files for os_name, os_code_name in platforms: other_source_job_names = [] # get source job names for all other release build files for other_release_build_name in [ k for k in build_files.keys() if k != release_build_name ]: other_build_file = build_files[other_release_build_name] other_dist_file = get_distribution_file( index, rosdistro_name, other_build_file) if not other_dist_file: continue if os_name not in other_build_file.targets or \ os_code_name not in other_build_file.targets[os_name]: continue if other_build_file.skip_ignored_packages: filtered_pkg_names = other_build_file.filter_packages( pkg_names) else: filtered_pkg_names = pkg_names for pkg_name in sorted(filtered_pkg_names): pkg = other_dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = other_dist_file.repositories[repo_name] if not repo.release_repository: continue if not repo.release_repository.version: continue other_job_name = get_sourcedeb_job_name( rosdistro_name, other_release_build_name, pkg_name, os_name, os_code_name) other_source_job_names.append(other_job_name) source_view_prefix = get_release_source_view_name( rosdistro_name, os_name, os_code_name) source_job_prefix = '%s__' % source_view_prefix excluded_job_names = set([ j for j in (all_source_job_names + other_source_job_names) if j.startswith(source_job_prefix) ]) if groovy_script is None: print("Removing obsolete source jobs with prefix '%s'" % source_job_prefix) from ros_buildfarm.jenkins import remove_jobs remove_jobs(jenkins, source_job_prefix, excluded_job_names, dry_run=dry_run) else: source_key = 'source_%s_%s' % (os_name, os_code_name) groovy_data['job_prefixes_and_names'][source_key] = ( source_job_prefix, excluded_job_names) if groovy_script is not None: print( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % (groovy_script, len(all_view_configs), len(all_job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, all_job_configs, view_configs=all_view_configs)
def _configure_ci_jobs(config, build_files, config_url, rosdistro_name, ci_build_name, groovy_script=None, dry_run=False): """Configure all Jenkins CI jobs for a specific CI build name.""" build_file = build_files[ci_build_name] index = get_index(config.rosdistro_index_url) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print("The build file '%s' contains the following targets:" % ci_build_name) for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return ci_view_name = get_ci_view_name(rosdistro_name) # all further configuration will be handled by either the Jenkins API # or by a generated groovy script from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if groovy_script is None else False view_configs = {} views = { ci_view_name: configure_ci_view(jenkins, ci_view_name, dry_run=dry_run) } if not jenkins: view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(view_configs), } ci_job_names = [] job_configs = OrderedDict() is_disabled = False for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_ci_job( config_url, rosdistro_name, ci_build_name, os_name, os_code_name, arch, config=config, build_file=build_file, index=index, dist_file=dist_file, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script, dry_run=dry_run, trigger_timer=build_file.jenkins_job_schedule) ci_job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(job_configs) groovy_data['job_prefixes_and_names'] = {} if groovy_script is not None: print("Writing groovy script '%s' to reconfigure %d jobs" % (groovy_script, len(job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, job_configs, view_configs)
def configure_devel_jobs( config_url, rosdistro_name, source_build_name, groovy_script=None): """ Configure all Jenkins devel jobs. L{configure_release_job} will be invoked for source repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_source_build_files(config, rosdistro_name) build_file = build_files[source_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return devel_view_name = get_devel_view_name( rosdistro_name, source_build_name, pull_request=False) pull_request_view_name = get_devel_view_name( rosdistro_name, source_build_name, pull_request=True) from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) views = [] if build_file.test_commits_force is not False: views.append(configure_devel_view(jenkins, devel_view_name)) if build_file.test_pull_requests_force is not False: views.append(configure_devel_view(jenkins, pull_request_view_name)) if groovy_script is not None: # all further configuration will be handled by the groovy script jenkins = False repo_names = dist_file.repositories.keys() filtered_repo_names = build_file.filter_repositories(repo_names) devel_job_names = [] pull_request_job_names = [] job_configs = {} for repo_name in sorted(repo_names): is_disabled = repo_name not in filtered_repo_names if is_disabled and build_file.skip_ignored_repositories: print("Skipping ignored repository '%s'" % repo_name, file=sys.stderr) continue repo = dist_file.repositories[repo_name] if not repo.source_repository: print("Skipping repository '%s': no source section" % repo_name) continue if not repo.source_repository.version: print("Skipping repository '%s': no source version" % repo_name) continue job_types = [] # check for testing commits if build_file.test_commits_force is False: print(("Skipping repository '%s': 'test_commits' is forced to " + "false in the build file") % repo_name) elif repo.source_repository.test_commits is False: print(("Skipping repository '%s': 'test_commits' of the " + "repository set to false") % repo_name) elif repo.source_repository.test_commits is None and \ not build_file.test_commits_default: print(("Skipping repository '%s': 'test_commits' defaults to " + "false in the build file") % repo_name) else: job_types.append('commit') if not is_disabled: # check for testing pull requests if build_file.test_pull_requests_force is False: # print(("Skipping repository '%s': 'test_pull_requests' " + # "is forced to false in the build file") % repo_name) pass elif repo.source_repository.test_pull_requests is False: # print(("Skipping repository '%s': 'test_pull_requests' of " + # "the repository set to false") % repo_name) pass elif repo.source_repository.test_pull_requests is None and \ not build_file.test_pull_requests_default: # print(("Skipping repository '%s': 'test_pull_requests' " + # "defaults to false in the build file") % repo_name) pass else: print("Pull request job for repository '%s'" % repo_name) job_types.append('pull_request') for job_type in job_types: pull_request = job_type == 'pull_request' for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_devel_job( config_url, rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, pull_request, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script) if not pull_request: devel_job_names.append(job_name) else: pull_request_job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) devel_job_prefix = '%s__' % devel_view_name pull_request_job_prefix = '%s__' % pull_request_view_name if groovy_script is None: # delete obsolete jobs in these views from ros_buildfarm.jenkins import remove_jobs print('Removing obsolete devel jobs') remove_jobs(jenkins, devel_job_prefix, devel_job_names) print('Removing obsolete pull request jobs') remove_jobs( jenkins, pull_request_job_prefix, pull_request_job_names) else: print("Writing groovy script '%s' to reconfigure %d jobs" % (groovy_script, len(job_configs))) data = { 'expected_num_jobs': len(job_configs), 'job_prefixes_and_names': { 'devel': (devel_job_prefix, devel_job_names), 'pull_request': ( pull_request_job_prefix, pull_request_job_names), } } content = expand_template('snippet/reconfigure_jobs.groovy.em', data) write_groovy_script_and_configs( groovy_script, content, job_configs)
def configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, views=None, generate_import_package_job=True, generate_sync_packages_jobs=True, is_disabled=False, other_build_files_same_platform=None, groovy_script=None, filter_arches=None, dry_run=False): """ Configure a Jenkins release job. The following jobs are created for each package: - M source jobs, one for each OS node name - M * N binary jobs, one for each combination of OS code name and arch """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: raise JobValidationError( 'No distribution file matches the build file') pkg_names = dist_file.release_packages.keys() if pkg_name not in pkg_names: raise JobValidationError( "Invalid package name '%s' " % pkg_name + 'choose one of the following: ' + ', '.join(sorted(pkg_names))) pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] if not repo.release_repository: raise JobValidationError( "Repository '%s' has no release section" % repo_name) if not repo.release_repository.version: raise JobValidationError( "Repository '%s' has no release version" % repo_name) if os_name not in build_file.targets.keys(): raise JobValidationError( "Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if dist_cache is None and \ (build_file.notify_maintainers or build_file.abi_incompatibility_assumed): dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: jenkins = connect(config.jenkins_url) if views is None: targets = [] targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) configure_release_views( jenkins, rosdistro_name, release_build_name, targets, dry_run=dry_run) if generate_import_package_job: configure_import_package_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if generate_sync_packages_jobs: configure_sync_packages_to_main_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) for arch in build_file.targets[os_name][os_code_name]: configure_sync_packages_to_testing_job( config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) source_job_names = [] binary_job_names = [] job_configs = {} # sourcedeb job # since sourcedeb jobs are potentially being shared across multiple build # files the configuration has to take all of them into account in order to # generate a job which all build files agree on source_job_name = get_sourcedeb_job_name( rosdistro_name, release_build_name, pkg_name, os_name, os_code_name) # while the package is disabled in the current build file # it might be used by sibling build files is_source_disabled = is_disabled if is_source_disabled and other_build_files_same_platform: # check if sourcedeb job is used by any other build file with the same platform for other_build_file in other_build_files_same_platform: if other_build_file.filter_packages([pkg_name]): is_source_disabled = False break job_config = _get_sourcedeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache, is_disabled=is_source_disabled, other_build_files_same_platform=other_build_files_same_platform) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, source_job_name, job_config, dry_run=dry_run) source_job_names.append(source_job_name) job_configs[source_job_name] = job_config dependency_names = [] if build_file.abi_incompatibility_assumed: dependency_names = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) # if dependencies are not yet available in rosdistro cache # skip binary jobs if dependency_names is None: print(("Skipping binary jobs for package '%s' because it is not " + "yet in the rosdistro cache") % pkg_name, file=sys.stderr) return source_job_names, binary_job_names, job_configs # binarydeb jobs for arch in build_file.targets[os_name][os_code_name]: if filter_arches and arch not in filter_arches: continue job_name = get_binarydeb_job_name( rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, arch) upstream_job_names = [source_job_name] + [ get_binarydeb_job_name( rosdistro_name, release_build_name, dependency_name, os_name, os_code_name, arch) for dependency_name in dependency_names] job_config = _get_binarydeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, arch, pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache, upstream_job_names=upstream_job_names, is_disabled=is_disabled) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, job_name, job_config, dry_run=dry_run) binary_job_names.append(job_name) job_configs[job_name] = job_config return source_job_names, binary_job_names, job_configs
def configure_ci_job(config_url, rosdistro_name, ci_build_name, os_name, os_code_name, arch, config=None, build_file=None, index=None, dist_file=None, jenkins=None, views=None, is_disabled=False, groovy_script=None, build_targets=None, dry_run=False, underlay_source_paths=None, trigger_timer=None): """ Configure a single Jenkins CI job. This includes the following steps: - clone the ros_buildfarm repository - write the distribution repository keys into files - invoke the ci/run_ci_job.py script """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_ci_build_files(config, rosdistro_name) build_file = build_files[ci_build_name] # Overwrite build_file.targets if build_targets is specified if build_targets is not None: build_file.targets = build_targets if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: raise JobValidationError( 'No distribution file matches the build file') if os_name not in build_file.targets.keys(): raise JobValidationError("Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if arch not in build_file.targets[os_name][os_code_name]: raise JobValidationError( "Invalid architecture '%s' " % arch + 'choose one of the following: %s' % ', '.join(sorted(build_file.targets[os_name][os_code_name]))) underlay_source_jobs = [ get_ci_job_name(rosdistro_name, os_name, os_code_name, arch, underlay_job) for underlay_job in build_file.underlay_from_ci_jobs ] underlay_source_paths = (underlay_source_paths or []) + \ ['$UNDERLAY%d_JOB_SPACE' % (index + 1) for index in range(len(underlay_source_jobs))] trigger_jobs = [ get_ci_job_name(rosdistro_name, os_name, os_code_name, arch, trigger_job) for trigger_job in build_file.jenkins_job_upstream_triggers ] if jenkins is None: from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if views is None: view_name = get_ci_view_name(rosdistro_name) configure_ci_view(jenkins, view_name, dry_run=dry_run) job_name = get_ci_job_name(rosdistro_name, os_name, os_code_name, arch, ci_build_name) job_config = _get_ci_job_config(index, rosdistro_name, build_file, os_name, os_code_name, arch, build_file.repos_files, build_file.repository_names, underlay_source_jobs, underlay_source_paths, trigger_timer, trigger_jobs, is_disabled=is_disabled) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: from ros_buildfarm.jenkins import configure_job configure_job(jenkins, job_name, job_config, dry_run=dry_run) return job_name, job_config
def configure_release_jobs( config_url, rosdistro_name, release_build_name, groovy_script=None, dry_run=False, whitelist_package_names=None): """ Configure all Jenkins release jobs. L{configure_release_job} will be invoked for every released package and target which matches the build file criteria. Additionally a job to import Debian packages into the Debian repository is created. """ config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) # get targets platforms = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): platforms.append((os_name, os_code_name)) print('The build file contains the following targets:') for os_name, os_code_name in platforms: print(' - %s %s: %s' % (os_name, os_code_name, ', '.join( build_file.targets[os_name][os_code_name]))) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return pkg_names = dist_file.release_packages.keys() filtered_pkg_names = build_file.filter_packages(pkg_names) explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names) if explicitly_ignored_pkg_names: print(('The following packages are being %s because of ' + 'white-/blacklisting:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(explicitly_ignored_pkg_names): print(' -', pkg_name) dist_cache = None if build_file.notify_maintainers or \ build_file.abi_incompatibility_assumed or \ explicitly_ignored_pkg_names: dist_cache = get_distribution_cache(index, rosdistro_name) if explicitly_ignored_pkg_names: # get direct dependencies from distro cache for each package direct_dependencies = {} for pkg_name in pkg_names: direct_dependencies[pkg_name] = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) or set([]) # find recursive downstream deps for all explicitly ignored packages ignored_pkg_names = set(explicitly_ignored_pkg_names) while True: implicitly_ignored_pkg_names = _get_downstream_package_names( ignored_pkg_names, direct_dependencies) if implicitly_ignored_pkg_names - ignored_pkg_names: ignored_pkg_names |= implicitly_ignored_pkg_names continue break implicitly_ignored_pkg_names = \ ignored_pkg_names - explicitly_ignored_pkg_names if implicitly_ignored_pkg_names: print(('The following packages are being %s because their ' + 'dependencies are being ignored:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(implicitly_ignored_pkg_names): print(' -', pkg_name) filtered_pkg_names = \ set(filtered_pkg_names) - implicitly_ignored_pkg_names # all further configuration will be handled by either the Jenkins API # or by a generated groovy script jenkins = connect(config.jenkins_url) if groovy_script is None else False all_view_configs = {} all_job_configs = {} job_name, job_config = configure_import_package_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config job_name, job_config = configure_sync_packages_to_main_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config for os_name, os_code_name in platforms: for arch in sorted(build_file.targets[os_name][os_code_name]): job_name, job_config = configure_sync_packages_to_testing_job( config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config targets = [] for os_name, os_code_name in platforms: targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) views = configure_release_views( jenkins, rosdistro_name, release_build_name, targets, dry_run=dry_run) if not jenkins: all_view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(views), } other_build_files = [v for k, v in build_files.items() if k != release_build_name] all_source_job_names = [] all_binary_job_names = [] for pkg_name in sorted(pkg_names): if whitelist_package_names: if pkg_name not in whitelist_package_names: print("Skipping package '%s' not in the explicitly passed list" % pkg_name, file=sys.stderr) continue pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] is_disabled = pkg_name not in filtered_pkg_names if is_disabled and build_file.skip_ignored_packages: print("Skipping ignored package '%s' in repository '%s'" % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository: print(("Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository.version: print(("Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name), file=sys.stderr) continue for os_name, os_code_name in platforms: other_build_files_same_platform = [] for other_build_file in other_build_files: if os_name not in other_build_file.targets: continue if os_code_name not in other_build_file.targets[os_name]: continue other_build_files_same_platform.append(other_build_file) try: source_job_names, binary_job_names, job_configs = \ configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, generate_import_package_job=False, generate_sync_packages_jobs=False, is_disabled=is_disabled, other_build_files_same_platform=other_build_files_same_platform, groovy_script=groovy_script, dry_run=dry_run) all_source_job_names += source_job_names all_binary_job_names += binary_job_names if groovy_script is not None: print('Configuration for jobs: ' + ', '.join(source_job_names + binary_job_names)) all_job_configs.update(job_configs) except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(all_job_configs) groovy_data['job_prefixes_and_names'] = {} # with an explicit list of packages we don't delete obsolete jobs if not whitelist_package_names: # delete obsolete binary jobs for os_name, os_code_name in platforms: for arch in build_file.targets[os_name][os_code_name]: binary_view = get_release_binary_view_name( rosdistro_name, release_build_name, os_name, os_code_name, arch) binary_job_prefix = '%s__' % binary_view excluded_job_names = set([ j for j in all_binary_job_names if j.startswith(binary_job_prefix)]) if groovy_script is None: print("Removing obsolete binary jobs with prefix '%s'" % binary_job_prefix) remove_jobs( jenkins, binary_job_prefix, excluded_job_names, dry_run=dry_run) else: binary_key = 'binary_%s_%s_%s' % \ (os_name, os_code_name, arch) groovy_data['job_prefixes_and_names'][binary_key] = \ (binary_job_prefix, excluded_job_names) # delete obsolete source jobs # requires knowledge about all other release build files for os_name, os_code_name in platforms: other_source_job_names = [] # get source job names for all other release build files for other_release_build_name in [ k for k in build_files.keys() if k != release_build_name]: other_build_file = build_files[other_release_build_name] other_dist_file = get_distribution_file( index, rosdistro_name, other_build_file) if not other_dist_file: continue if os_name not in other_build_file.targets or \ os_code_name not in other_build_file.targets[os_name]: continue if other_build_file.skip_ignored_packages: filtered_pkg_names = other_build_file.filter_packages( pkg_names) else: filtered_pkg_names = pkg_names for pkg_name in sorted(filtered_pkg_names): pkg = other_dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = other_dist_file.repositories[repo_name] if not repo.release_repository: continue if not repo.release_repository.version: continue other_job_name = get_sourcedeb_job_name( rosdistro_name, other_release_build_name, pkg_name, os_name, os_code_name) other_source_job_names.append(other_job_name) source_view_prefix = get_release_source_view_name( rosdistro_name, os_name, os_code_name) source_job_prefix = '%s__' % source_view_prefix excluded_job_names = set([ j for j in (all_source_job_names + other_source_job_names) if j.startswith(source_job_prefix)]) if groovy_script is None: print("Removing obsolete source jobs with prefix '%s'" % source_job_prefix) remove_jobs( jenkins, source_job_prefix, excluded_job_names, dry_run=dry_run) else: source_key = 'source_%s_%s' % (os_name, os_code_name) groovy_data['job_prefixes_and_names'][source_key] = ( source_job_prefix, excluded_job_names) if groovy_script is not None: print( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % (groovy_script, len(all_view_configs), len(all_job_configs))) content = expand_template( 'snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs( groovy_script, content, all_job_configs, view_configs=all_view_configs)
def configure_doc_jobs(config_url, rosdistro_name, doc_build_name, groovy_script=None, dry_run=False, whitelist_repository_names=None): """ Configure all Jenkins doc jobs. L{configure_doc_job} will be invoked for doc repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_doc_build_files(config, rosdistro_name) build_file = build_files[doc_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name) # all further configuration will be handled by either the Jenkins API # or by a generated groovy script from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if groovy_script is None else False view_configs = {} views = {} views[doc_view_name] = configure_doc_view(jenkins, doc_view_name, dry_run=dry_run) if not jenkins: view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(view_configs), } repo_names = dist_file.repositories.keys() filtered_repo_names = build_file.filter_repositories(repo_names) job_names = [] job_configs = OrderedDict() for repo_name in sorted(repo_names): if whitelist_repository_names: if repo_name not in whitelist_repository_names: print( "Skipping repository '%s' not in explicitly passed list" % repo_name, file=sys.stderr) continue is_disabled = repo_name not in filtered_repo_names if is_disabled and build_file.skip_ignored_repositories: print("Skipping ignored repository '%s'" % repo_name, file=sys.stderr) continue repo = dist_file.repositories[repo_name] if not repo.doc_repository: print("Skipping repository '%s': no doc section" % repo_name) continue if not repo.doc_repository.version: print("Skipping repository '%s': no doc version" % repo_name) continue for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_doc_job( config_url, rosdistro_name, doc_build_name, repo_name, os_name, os_code_name, arch, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script, dry_run=dry_run) job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(job_configs) groovy_data['job_prefixes_and_names'] = {} job_prefix = '%s__' % doc_view_name if not whitelist_repository_names: groovy_data['job_prefixes_and_names']['doc'] = (job_prefix, job_names) if groovy_script is None: # delete obsolete jobs in this view from ros_buildfarm.jenkins import remove_jobs print('Removing obsolete doc jobs') remove_jobs(jenkins, job_prefix, job_names, dry_run=dry_run) if groovy_script is not None: print( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % (groovy_script, len(view_configs), len(job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, job_configs, view_configs=view_configs)
def trigger_release_jobs(config_url, rosdistro_name, release_build_name, missing_only, source_only, cache_dir, cause=None, groovy_script=None): config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) # get targets targets = [] for os_name in sorted(build_file.targets.keys()): for os_code_name in sorted(build_file.targets[os_name].keys()): targets.append(Target(os_name, os_code_name, 'source')) if source_only: continue for arch in sorted( build_file.targets[os_name][os_code_name].keys()): targets.append(Target(os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' - %s %s %s' % (os_name, os_code_name, arch)) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return repo_data = None if missing_only: repo_data = get_debian_repo_data(build_file.target_repository, targets, cache_dir) if groovy_script is None: jenkins = connect(config.jenkins_url) pkg_names = dist_file.release_packages.keys() pkg_names = build_file.filter_packages(pkg_names) triggered_jobs = [] skipped_jobs = [] for pkg_name in sorted(pkg_names): pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] if not repo.release_repository: print((" Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name)) continue if not repo.release_repository.version: print((" Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name)) continue pkg_version = repo.release_repository.version debian_package_name = get_debian_package_name(rosdistro_name, pkg_name) for target in targets: job_name = get_sourcedeb_job_name(rosdistro_name, release_build_name, pkg_name, target.os_name, target.os_code_name) if target.arch != 'source': # binary job can be skipped if source job was triggered if job_name in triggered_jobs: print((" Skipping binary jobs of '%s' since the source " + "job was triggered") % job_name) continue job_name = get_binarydeb_job_name(rosdistro_name, release_build_name, pkg_name, target.os_name, target.os_code_name, target.arch) if repo_data: # check if artifact is missing repo_index = repo_data[target] if debian_package_name in repo_index: version = repo_index[debian_package_name] version = _strip_version_suffix(version) if version == pkg_version: print((" Skipping job '%s' since the artifact is " + "already up-to-date") % job_name) continue if groovy_script is None: success = invoke_job(jenkins, job_name, cause=cause) else: success = True if success: triggered_jobs.append(job_name) else: skipped_jobs.append(job_name) if groovy_script is None: print('Triggered %d jobs, skipped %d jobs.' % (len(triggered_jobs), len(skipped_jobs))) else: print("Writing groovy script '%s' to trigger %d jobs" % (groovy_script, len(triggered_jobs))) data = { 'job_names': triggered_jobs, } content = expand_template('release/trigger_jobs.groovy.em', data) with open(groovy_script, 'w') as h: h.write(content)
def configure_release_jobs(config_url, rosdistro_name, release_build_name, groovy_script=None): """ Configure all Jenkins release jobs. L{configure_release_job} will be invoked for every released package and target which matches the build file criteria. Additionally a job to import Debian packages into the Debian repository is created. """ config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) # get targets platforms = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): platforms.append((os_name, os_code_name)) print('The build file contains the following targets:') for os_name, os_code_name in platforms: print(' - %s %s: %s' % (os_name, os_code_name, ', '.join( build_file.targets[os_name][os_code_name]))) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return pkg_names = dist_file.release_packages.keys() filtered_pkg_names = build_file.filter_packages(pkg_names) explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names) if explicitly_ignored_pkg_names: print(('The following packages are being %s because of ' + 'white-/blacklisting:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(explicitly_ignored_pkg_names): print(' -', pkg_name) dist_cache = None if build_file.notify_maintainers or \ build_file.abi_incompatibility_assumed or \ explicitly_ignored_pkg_names: dist_cache = get_distribution_cache(index, rosdistro_name) if explicitly_ignored_pkg_names: # get direct dependencies from distro cache for each package direct_dependencies = {} for pkg_name in pkg_names: direct_dependencies[pkg_name] = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) or set([]) # find recursive downstream deps for all explicitly ignored packages ignored_pkg_names = set(explicitly_ignored_pkg_names) while True: implicitly_ignored_pkg_names = _get_downstream_package_names( ignored_pkg_names, direct_dependencies) if implicitly_ignored_pkg_names - ignored_pkg_names: ignored_pkg_names |= implicitly_ignored_pkg_names continue break implicitly_ignored_pkg_names = \ ignored_pkg_names - explicitly_ignored_pkg_names if implicitly_ignored_pkg_names: print(('The following packages are being %s because their ' + 'dependencies are being ignored:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(implicitly_ignored_pkg_names): print(' -', pkg_name) filtered_pkg_names = \ set(filtered_pkg_names) - implicitly_ignored_pkg_names jenkins = connect(config.jenkins_url) configure_import_package_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins) configure_sync_packages_to_main_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins) for os_name, os_code_name in platforms: for arch in sorted(build_file.targets[os_name][os_code_name]): configure_sync_packages_to_testing_job(config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins) targets = [] for os_name, os_code_name in platforms: targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) views = configure_release_views(jenkins, rosdistro_name, release_build_name, targets) if groovy_script is not None: # all further configuration will be handled by the groovy script jenkins = False all_source_job_names = [] all_binary_job_names = [] all_job_configs = {} for pkg_name in sorted(pkg_names): pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] is_disabled = pkg_name not in filtered_pkg_names if is_disabled and build_file.skip_ignored_packages: print("Skipping ignored package '%s' in repository '%s'" % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository: print(("Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository.version: print(("Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name), file=sys.stderr) continue for os_name, os_code_name in platforms: try: source_job_names, binary_job_names, job_configs = \ configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, generate_import_package_job=False, generate_sync_packages_jobs=False, is_disabled=is_disabled, groovy_script=groovy_script) all_source_job_names += source_job_names all_binary_job_names += binary_job_names if groovy_script is not None: print('Configuration for jobs: ' + ', '.join(source_job_names + binary_job_names)) all_job_configs.update(job_configs) except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data = { 'expected_num_jobs': len(all_job_configs), 'job_prefixes_and_names': {}, } # delete obsolete binary jobs for os_name, os_code_name in platforms: for arch in build_file.targets[os_name][os_code_name]: binary_view = get_release_binary_view_name(rosdistro_name, release_build_name, os_name, os_code_name, arch) binary_job_prefix = '%s__' % binary_view excluded_job_names = set([ j for j in all_binary_job_names if j.startswith(binary_job_prefix) ]) if groovy_script is None: print("Removing obsolete binary jobs with prefix '%s'" % binary_job_prefix) remove_jobs(jenkins, binary_job_prefix, excluded_job_names) else: binary_key = 'binary_%s_%s_%s' % (os_name, os_code_name, arch) groovy_data['job_prefixes_and_names'][binary_key] = \ (binary_job_prefix, excluded_job_names) # delete obsolete source jobs # requires knowledge about all other release build files for os_name, os_code_name in platforms: other_source_job_names = [] # get source job names for all other release build files for other_release_build_name in [ k for k in build_files.keys() if k != release_build_name ]: other_build_file = build_files[other_release_build_name] other_dist_file = get_distribution_file(index, rosdistro_name, other_build_file) if not other_dist_file: continue if os_name not in other_build_file.targets or \ os_code_name not in other_build_file.targets[os_name]: continue if other_build_file.skip_ignored_packages: filtered_pkg_names = other_build_file.filter_packages( pkg_names) else: filtered_pkg_names = pkg_names for pkg_name in sorted(filtered_pkg_names): pkg = other_dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = other_dist_file.repositories[repo_name] if not repo.release_repository: continue if not repo.release_repository.version: continue other_job_name = get_sourcedeb_job_name( rosdistro_name, other_release_build_name, pkg_name, os_name, os_code_name) other_source_job_names.append(other_job_name) source_view_prefix = get_release_source_view_name( rosdistro_name, os_name, os_code_name) source_job_prefix = '%s__' % source_view_prefix excluded_job_names = set([ j for j in (all_source_job_names + other_source_job_names) if j.startswith(source_job_prefix) ]) if groovy_script is None: print("Removing obsolete source jobs with prefix '%s'" % source_job_prefix) remove_jobs(jenkins, source_job_prefix, excluded_job_names) else: source_key = 'source_%s_%s' % (os_name, os_code_name) groovy_data['job_prefixes_and_names'][source_key] = ( source_job_prefix, excluded_job_names) if groovy_script is not None: print("Writing groovy script '%s' to reconfigure %d jobs" % (groovy_script, len(all_job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, all_job_configs)
def trigger_release_jobs( config_url, rosdistro_name, release_build_name, missing_only, source_only, cache_dir, cause=None, groovy_script=None): config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) # get targets targets = [] for os_name in sorted(build_file.targets.keys()): for os_code_name in sorted(build_file.targets[os_name].keys()): targets.append(Target(os_name, os_code_name, 'source')) if source_only: continue for arch in sorted( build_file.targets[os_name][os_code_name].keys()): targets.append(Target(os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' - %s %s %s' % (os_name, os_code_name, arch)) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return repo_data = None if missing_only: repo_data = get_debian_repo_data( build_file.target_repository, targets, cache_dir) if groovy_script is None: jenkins = connect(config.jenkins_url) pkg_names = dist_file.release_packages.keys() pkg_names = build_file.filter_packages(pkg_names) triggered_jobs = [] skipped_jobs = [] for pkg_name in sorted(pkg_names): pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] if not repo.release_repository: print((" Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name)) continue if not repo.release_repository.version: print((" Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name)) continue pkg_version = repo.release_repository.version debian_package_name = get_debian_package_name(rosdistro_name, pkg_name) for target in targets: job_name = get_sourcedeb_job_name( rosdistro_name, release_build_name, pkg_name, target.os_name, target.os_code_name) if target.arch != 'source': # binary job can be skipped if source job was triggered if job_name in triggered_jobs: print((" Skipping binary jobs of '%s' since the source " + "job was triggered") % job_name) continue job_name = get_binarydeb_job_name( rosdistro_name, release_build_name, pkg_name, target.os_name, target.os_code_name, target.arch) if repo_data: # check if artifact is missing repo_index = repo_data[target] if debian_package_name in repo_index: version = repo_index[debian_package_name] version = _strip_version_suffix(version) if version == pkg_version: print((" Skipping job '%s' since the artifact is " + "already up-to-date") % job_name) continue if groovy_script is None: success = invoke_job(jenkins, job_name, cause=cause) else: success = True if success: triggered_jobs.append(job_name) else: skipped_jobs.append(job_name) if groovy_script is None: print('Triggered %d jobs, skipped %d jobs.' % (len(triggered_jobs), len(skipped_jobs))) else: print("Writing groovy script '%s' to trigger %d jobs" % (groovy_script, len(triggered_jobs))) data = { 'job_names': triggered_jobs, } content = expand_template('release/trigger_jobs.groovy.em', data) with open(groovy_script, 'w') as h: h.write(content)