def get_sourcerpm(rosdistro_index_url, rosdistro_name, package_name, sourcepkg_dir, skip_download_sourcepkg=False): # ensure that no source subfolder exists rpm_package_name = get_os_package_name(rosdistro_name, package_name) if not skip_download_sourcepkg: # get expected package version from rosdistro from rosdistro import get_distribution_cache from rosdistro import get_index index = get_index(rosdistro_index_url) dist_cache = get_distribution_cache(index, rosdistro_name) dist_file = dist_cache.distribution_file assert package_name in dist_file.release_packages pkg = dist_file.release_packages[package_name] repo = dist_file.repositories[pkg.repository_name] package_version = repo.release_repository.version cmd = [ 'mock', '--resultdir', '%s' % sourcepkg_dir, '--no-cleanup-after', '--verbose', '--root', 'ros_buildfarm', '--dnf-cmd', '--', 'download', '--source', '--disablerepo', '*', '--enablerepo', 'ros-buildfarm-target-source', '%s-%s.*' % (rpm_package_name, package_version) ] print("Invoking '%s'" % ' '.join(cmd)) subprocess.check_call(cmd, cwd=sourcepkg_dir)
def _get_and_parse_distribution_cache(index, rosdistro_name, pkg_names): from catkin_pkg.package import parse_package_string from catkin_pkg.package import Dependency dist_cache = get_distribution_cache(index, rosdistro_name) pkg_names = set(['ros_workspace']).union(pkg_names) cached_pkgs = { pkg_name: parse_package_string(pkg_xml) for pkg_name, pkg_xml in dist_cache.release_package_xmls.items() if pkg_name in pkg_names } condition_context = get_package_condition_context(index, rosdistro_name) for pkg in cached_pkgs.values(): pkg.evaluate_conditions(condition_context) for pkg in cached_pkgs.values(): for group_depend in pkg.group_depends: if group_depend.evaluated_condition is not False: group_depend.extract_group_members(cached_pkgs.values()) # for ROS 2 distributions bloom injects a dependency on ros_workspace # into almost all packages (except its dependencies) # therefore the same dependency needs to to be injected here distribution_type = index.distributions[rosdistro_name].get( 'distribution_type') if distribution_type == 'ros2' and 'ros_workspace' in cached_pkgs: no_ros_workspace_dep = set(['ros_workspace']).union( get_direct_dependencies('ros_workspace', cached_pkgs, pkg_names)) for pkg_name, pkg in cached_pkgs.items(): if pkg_name not in no_ros_workspace_dep: pkg.exec_depends.append(Dependency('ros_workspace')) return cached_pkgs
def __init__(self, name, ros_distro): SrcAptBase.__init__(self, name) self.ros_distro = self.detect_ros_distribution(ros_distro) self.rosdistro_index = rosdistro.get_index(rosdistro.get_index_url()) self.cache = rosdistro.get_distribution_cache(self.rosdistro_index, self.ros_distro) self.distro_file = self.cache.distribution_file # More logic could be needed with new ros distributions # ROS1 - https://www.ros.org/reps/rep-0003.html # ROS2 - http://design.ros2.org/articles/changes.html if self.ros_distro == 'melodic': self.compilation_flags.append('--std=c++14') else: self.compilation_flags.append('--std=c++17') # needed for gazebo_ros_pkgs self.compilation_flags.append('-DBOOST_HAS_PTHREADS=1') # gtest-vendor is ROS2 self.compilation_flags.append('-I' + join('/opt/ros/', self.ros_distro, 'src', 'gtest_vendor', 'include')) # flag to avoid problems in rcutils # https://github.com/osrf/auto-abi-checker/issues/17 self.compilation_flags.append('-DRCUTILS__STDATOMIC_HELPER_H_') # flags for rmw_connext packages self.compilation_flags.append('-DRTI_UNIX') for rti_path in glob.glob('/opt/rti.com/rti_connext_dds-*'): self.compilation_flags.append('-I' + rti_path + '/include/') self.compilation_flags.append('-I' + rti_path + '/include/ndds') # Needs to add /opt/ros includes to compile ROS software self.compilation_flags.append('-I' + join('/opt/ros/', self.ros_distro, 'include'))
def configure_release_jobs( config_url, rosdistro_name, release_build_name, append_timestamp=False): config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers or build_file.abi_incompatibility_assumed: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): targets.append((os_name, os_code_name)) print('The build file contains the following targets:') for os_name, os_code_name in targets: print(' - %s %s: %s' % (os_name, os_code_name, ', '.join( build_file.targets[os_name][os_code_name]))) dist_file = get_distribution_file(index, rosdistro_name) jenkins = connect(config.jenkins_url) configure_import_package_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins) view_name = get_release_view_name(rosdistro_name, release_build_name) view = configure_release_view(jenkins, view_name) pkg_names = dist_file.release_packages.keys() pkg_names = build_file.filter_packages(pkg_names) for pkg_name in sorted(pkg_names): pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] if not repo.release_repository: print(("Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name)) continue if not repo.release_repository.version: print(("Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name)) continue for os_name, os_code_name in targets: configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, append_timestamp=append_timestamp, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, view=view, generate_import_package_job=False)
def __init__(self, distro): self.distro = distro try: index = get_index(get_index_url()) self._distribution_file = get_distribution_cache(index, distro).distribution_file except: logger.error("Could not load rosdistro distribution cache") self._distribution_file = None
def configure_devel_jobs( config_url, rosdistro_name, source_build_name): config = get_config_index(config_url) build_files = get_source_build_files(config, rosdistro_name) build_file = build_files[source_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name) jenkins = connect(config.jenkins_url) view_name = get_devel_view_name(rosdistro_name, source_build_name) view = configure_devel_view(jenkins, view_name) repo_names = dist_file.repositories.keys() repo_names = build_file.filter_repositories(repo_names) for repo_name in sorted(repo_names): repo = dist_file.repositories[repo_name] if not repo.source_repository: print("Skipping repository '%s': no source section" % repo_name) continue if not repo.source_repository.version: print("Skipping repository '%s': no source version" % repo_name) continue for os_name, os_code_name, arch in targets: configure_devel_job( config_url, rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, view=view)
def get_proposed_packages_repo_names(pkg_name_list): index = get_index( 'https://raw.githubusercontent.com/ros/rosdistro/master/index.yaml') dist_cache = get_distribution_cache(index, 'foxy') dist_file = dist_cache.distribution_file repositories = [] for pkg_name in pkg_name_list: if not pkg_name: continue try: repo_name = dist_file.release_packages[pkg_name].repository_name release_repo = dist_file.repositories[ repo_name].source_repository.get_url_parts()[1] except KeyError: if pkg_name.startswith('rosbag2_'): release_repo = 'ros2/rosbag2' elif pkg_name.startswith('test_launch_ros'): release_repo = 'ros2/launch_ros' elif pkg_name.startswith('test_'): release_repo = 'ros2/system_tests' elif pkg_name.startswith('tf2_'): release_repo = 'ros2/geometry2' elif pkg_name.startswith('libstatistics_collector' ) or pkg_name.startswith('statistics_'): release_repo = 'ros-tooling/libstatistics_collector' elif pkg_name.startswith('rmw_dds_common'): release_repo = 'ros2/rmw_dds_common' elif pkg_name.startswith('rosidl_runtime_c'): release_repo = 'ros2/rosidl' elif pkg_name.startswith('rmw_cyclonedds'): release_repo = 'ros2/rmw_cyclonedds' elif pkg_name.startswith('rmw_connext'): release_repo = 'ros2/rmw_connext' elif pkg_name.startswith('rmw_fastcdr'): release_repo = 'eProsima/Fast-CDR' else: raise Exception( "package " + pkg_name + " not found in foxy release. Might need to add code manually" ) repositories.append(release_repo) return list(dict.fromkeys(repositories))
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description="Generate a 'prerelease' script") add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_build_name(parser, 'source') add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_argument_build_tool(parser) add_argument_output_dir(parser, required=True) group = parser.add_argument_group( 'Repositories in underlay workspace', description='The repositories in the underlay workspace will be ' + 'built and installed as well as built and tested. ' + 'Dependencies will be provided by binary packages.') group.add_argument( 'source_repos', nargs='*', default=[], metavar='REPO_NAME', help="A name of a 'repository' from the distribution file") group.add_argument( '--custom-branch', nargs='*', type=_repository_name_and_branch, default=[], metavar='REPO_NAME:BRANCH_OR_TAG_NAME', help="A name of a 'repository' from the distribution file followed " + 'by a colon and a branch / tag name') group.add_argument( '--custom-repo', nargs='*', type=_repository_name_and_type_and_url_and_branch, default=[], metavar='REPO_NAME:REPO_TYPE:REPO_URL:BRANCH_OR_TAG_NAME', help='The name, type, url and branch / tag name of a repository, ' 'e.g. "common_tutorials:git:https://github.com/ros/common_tutorials:pullrequest-1"') add_overlay_arguments(parser) args = parser.parse_args(argv) print('Fetching buildfarm configuration...') config = get_config_index(args.config_url) build_files = get_source_build_files(config, args.rosdistro_name) build_file = build_files[args.source_build_name] print('Fetching rosdistro cache...') # Targets defined by source build file are subset of targets # defined by release build files. To increase the number of supported # pre-release targets, we combine all targets defined by all release # build files and use that when configuring the devel job. release_build_files = get_release_build_files(config, args.rosdistro_name) release_targets_combined = {} if release_build_files: release_targets_combined[args.os_name] = {} for build_name, rel_obj in release_build_files.items(): if args.os_name not in rel_obj.targets: continue for dist_name, targets in rel_obj.targets[args.os_name].items(): if dist_name not in release_targets_combined[args.os_name]: release_targets_combined[args.os_name][dist_name] = {} release_targets_combined[args.os_name][dist_name].update(targets) index = get_index(config.rosdistro_index_url) dist_cache = get_distribution_cache(index, args.rosdistro_name) dist_file = dist_cache.distribution_file # determine source repositories for underlay workspace repositories = {} for repo_name in args.source_repos: if repo_name in repositories: print("The repository '%s' appears multiple times" % repo_name, file=sys.stderr) return 1 try: repositories[repo_name] = \ dist_file.repositories[repo_name].source_repository except KeyError: print(("The repository '%s' was not found in the distribution " + "file") % repo_name, file=sys.stderr) return 1 if not repositories[repo_name]: print(("The repository '%s' has no source entry in the " + "distribution file") % repo_name, file=sys.stderr) return 1 for repo_name, custom_version in args.custom_branch: if repo_name in repositories: print("The repository '%s' appears multiple times" % repo_name, file=sys.stderr) return 1 try: source_repo = dist_file.repositories[repo_name].source_repository except KeyError: print(("The repository '%s' was not found in the distribution " + "file") % repo_name, file=sys.stderr) return 1 if not source_repo: print(("The repository '%s' has no source entry in the " + "distribution file") % repo_name, file=sys.stderr) return 1 source_repo = deepcopy(source_repo) source_repo.version = custom_version repositories[repo_name] = source_repo for repo_name, repo_type, repo_url, version in args.custom_repo: if repo_name in repositories and repositories[repo_name]: print("custom_repos option overriding '%s' to pull via '%s' " "from '%s' with version '%s'. " % (repo_name, repo_type, repo_url, version), file=sys.stderr) source_repo = RepositorySpecification( repo_name, { 'type': repo_type, 'url': repo_url, 'version': version, }) repositories[repo_name] = source_repo scms = [(repositories[k], 'ws/src/%s' % k) for k in sorted(repositories.keys())] # collect all template snippets of specific types class IncludeHook(Hook): def __init__(self): Hook.__init__(self) self.scripts = [] def beforeInclude(self, *_, **kwargs): template_path = kwargs['file'].name if template_path.endswith('/snippet/builder_shell.xml.em'): script = kwargs['locals']['script'] # reuse existing ros_buildfarm folder if it exists if 'Clone ros_buildfarm' in script: lines = script.splitlines() lines.insert(0, 'if [ ! -d "ros_buildfarm" ]; then') lines += [ 'else', 'echo "Using existing ros_buildfarm folder"', 'fi', ] script = '\n'.join(lines) if args.build_tool and ' --build-tool ' in script: script = script.replace( ' --build-tool catkin_make_isolated', ' --build-tool ' + args.build_tool) self.scripts.append(script) hook = IncludeHook() from ros_buildfarm import templates templates.template_hooks = [hook] # use any source repo to pass to devel job template if index.distributions[args.rosdistro_name].get('distribution_type', 'ros1') == 'ros1': package_name = 'catkin' elif index.distributions[args.rosdistro_name].get('distribution_type', 'ros1') == 'ros2': package_name = 'ros_workspace' else: assert False, 'Unsupported ROS version ' + \ str(index.distributions[args.rosdistro_name].get('distribution_type', None)) source_repository = deepcopy( dist_file.repositories[package_name].source_repository) if not source_repository: print(("The repository '%s' does not have a source entry in the distribution " + 'file. We cannot generate a prerelease without a source entry.') % package_name, file=sys.stderr) return 1 source_repository.name = 'prerelease' print('Evaluating job templates...') configure_devel_job( args.config_url, args.rosdistro_name, args.source_build_name, None, args.os_name, args.os_code_name, args.arch, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=False, views=False, source_repository=source_repository, build_targets=release_targets_combined) templates.template_hooks = None # derive scripts for overlay workspace from underlay overlay_scripts = [] for script in hook.scripts: # skip cloning of ros_buildfarm repository if 'git clone' in script and '.git ros_buildfarm' in script: continue # skip build-and-install step if 'build and install' in script: continue # add prerelease overlay flag run_devel_job = '/run_devel_job.py' if run_devel_job in script: script = script.replace( run_devel_job, run_devel_job + ' --prerelease-overlay') # replace mounted workspace volume with overlay and underlay # used by: # - create_devel_task_generator.py needs to find packages in both # the underlay as well as the overlay workspace # - build_and_test.py needs to source the environment of # the underlay before building the overlay mount_volume = '-v $WORKSPACE/ws:/tmp/ws' if mount_volume in script: script = script.replace( mount_volume, mount_volume + ':ro ' + '-v $WORKSPACE/' + 'ws_overlay:/tmp/ws_overlay') # relocate all docker files docker_path = '$WORKSPACE/docker_' if docker_path in script: script = script.replace( docker_path, docker_path + 'overlay_') # rename all docker images name_suffix = '_prerelease' if name_suffix in script: script = script.replace( name_suffix, name_suffix + '_overlay') overlay_scripts.append(script) from ros_buildfarm import __file__ as ros_buildfarm_file data = deepcopy(args.__dict__) data.update({ 'scms': scms, 'scripts': hook.scripts, 'overlay_scripts': overlay_scripts, 'ros_buildfarm_python_path': os.path.dirname( os.path.dirname(os.path.abspath(ros_buildfarm_file))), 'python_executable': sys.executable, 'prerelease_script_path': os.path.dirname(os.path.abspath(__file__)), 'build_tool': args.build_tool or build_file.build_tool}) if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) # generate multiple scripts for script_name in [ 'prerelease', 'prerelease_build_overlay', 'prerelease_build_underlay', 'prerelease_clone_overlay', 'prerelease_clone_underlay']: content = expand_template( 'prerelease/%s_script.sh.em' % script_name, data, options={BANGPATH_OPT: False}) script_file = os.path.join(args.output_dir, script_name + '.sh') with open(script_file, 'w') as h: h.write(content) os.chmod(script_file, os.stat(script_file).st_mode | stat.S_IEXEC) print('') print('Generated prerelease script - to execute it run:') if os.path.abspath(args.output_dir) != os.path.abspath(os.curdir): print(' cd %s' % args.output_dir) print(' ./prerelease.sh')
def configure_doc_job( config_url, rosdistro_name, doc_build_name, repo_name, os_name, os_code_name, arch, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, views=None, is_disabled=False, groovy_script=None, doc_repository=None): """ Configure a single Jenkins doc job. This includes the following steps: - clone the doc repository to use - clone the ros_buildfarm repository - write the distribution repository keys into files - invoke the run_doc_job.py script """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_doc_build_files(config, rosdistro_name) build_file = build_files[doc_build_name] if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: raise JobValidationError( 'No distribution file matches the build file') repo_names = dist_file.repositories.keys() if repo_name is not None: if repo_name not in repo_names: raise JobValidationError( "Invalid repository name '%s' " % repo_name + 'choose one of the following: %s' % ', '.join(sorted(repo_names))) repo = dist_file.repositories[repo_name] if not repo.doc_repository: raise JobValidationError( "Repository '%s' has no doc section" % repo_name) if not repo.doc_repository.version: raise JobValidationError( "Repository '%s' has no doc version" % repo_name) doc_repository = repo.doc_repository if os_name not in build_file.targets.keys(): raise JobValidationError( "Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if arch not in build_file.targets[os_name][os_code_name]: raise JobValidationError( "Invalid architecture '%s' " % arch + 'choose one of the following: %s' % ', '.join(sorted( build_file.targets[os_name][os_code_name]))) if dist_cache is None and build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if views is None: view_name = get_doc_view_name( rosdistro_name, doc_build_name) configure_doc_view(jenkins, view_name) job_name = get_doc_job_name( rosdistro_name, doc_build_name, repo_name, os_name, os_code_name, arch) job_config = _get_doc_job_config( config, config_url, rosdistro_name, doc_build_name, build_file, os_name, os_code_name, arch, doc_repository, repo_name, dist_cache=dist_cache, is_disabled=is_disabled) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: from ros_buildfarm.jenkins import configure_job configure_job(jenkins, job_name, job_config) return job_name, job_config
def configure_devel_job( config_url, rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, view=None): """ Configure a single Jenkins devel job. This includes the following steps: - clone the source repository to use - clone the ros_buildfarm repository - write the distribution repository keys into files - invoke the release/run_devel_job.py script """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_source_build_files(config, rosdistro_name) build_file = build_files[source_build_name] if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name) repo_names = dist_file.repositories.keys() repo_names = build_file.filter_repositories(repo_names) if repo_name not in repo_names: raise JobValidationError( "Invalid repository name '%s' " % repo_name + 'choose one of the following: %s' % ', '.join(sorted(repo_names))) repo = dist_file.repositories[repo_name] if not repo.source_repository: raise JobValidationError( "Repository '%s' has no source section" % repo_name) if not repo.source_repository.version: raise JobValidationError( "Repository '%s' has no source version" % repo_name) if os_name not in build_file.targets.keys(): raise JobValidationError( "Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if arch not in build_file.targets[os_name][os_code_name]: raise JobValidationError( "Invalid architecture '%s' " % arch + 'choose one of the following: %s' % ', '.join(sorted( build_file.targets[os_name][os_code_name]))) if dist_cache is None and build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: jenkins = connect(config.jenkins_url) if view is None: view_name = get_devel_view_name(rosdistro_name, source_build_name) configure_devel_view(jenkins, view_name) job_name = get_devel_job_name( rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch) job_config = _get_devel_job_config( config, rosdistro_name, source_build_name, build_file, os_name, os_code_name, arch, repo.source_repository, repo_name, dist_cache=dist_cache) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, job_name, job_config) return job_name
def configure_release_job(config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, views=None, generate_import_package_job=True, generate_sync_packages_jobs=True, is_disabled=False, other_build_files_same_platform=None, groovy_script=None, filter_arches=None, dry_run=False): """ Configure a Jenkins release job. The following jobs are created for each package: - M source jobs, one for each OS node name - M * N binary jobs, one for each combination of OS code name and arch """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: raise JobValidationError( 'No distribution file matches the build file') pkg_names = dist_file.release_packages.keys() if pkg_name not in pkg_names: raise JobValidationError("Invalid package name '%s' " % pkg_name + 'choose one of the following: ' + ', '.join(sorted(pkg_names))) pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] if not repo.release_repository: raise JobValidationError("Repository '%s' has no release section" % repo_name) if not repo.release_repository.version: raise JobValidationError("Repository '%s' has no release version" % repo_name) if os_name not in build_file.targets.keys(): raise JobValidationError("Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if dist_cache is None and \ (build_file.notify_maintainers or build_file.abi_incompatibility_assumed): dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if views is None: targets = [] targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) configure_release_views(jenkins, rosdistro_name, release_build_name, targets, dry_run=dry_run) if generate_import_package_job: configure_import_package_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if generate_sync_packages_jobs: configure_sync_packages_to_main_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) for arch in build_file.targets[os_name][os_code_name]: configure_sync_packages_to_testing_job(config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) source_job_names = [] binary_job_names = [] job_configs = {} # sourcedeb job # since sourcedeb jobs are potentially being shared across multiple build # files the configuration has to take all of them into account in order to # generate a job which all build files agree on source_job_name = get_sourcedeb_job_name(rosdistro_name, release_build_name, pkg_name, os_name, os_code_name) # while the package is disabled in the current build file # it might be used by sibling build files is_source_disabled = is_disabled if is_source_disabled and other_build_files_same_platform: # check if sourcedeb job is used by any other build file with the same platform for other_build_file in other_build_files_same_platform: if other_build_file.filter_packages([pkg_name]): is_source_disabled = False break job_config = _get_sourcedeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache, is_disabled=is_source_disabled, other_build_files_same_platform=other_build_files_same_platform) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: from ros_buildfarm.jenkins import configure_job configure_job(jenkins, source_job_name, job_config, dry_run=dry_run) source_job_names.append(source_job_name) job_configs[source_job_name] = job_config dependency_names = [] if build_file.abi_incompatibility_assumed: dependency_names = _get_direct_dependencies(pkg_name, dist_cache, pkg_names) # if dependencies are not yet available in rosdistro cache # skip binary jobs if dependency_names is None: print(("Skipping binary jobs for package '%s' because it is not " + "yet in the rosdistro cache") % pkg_name, file=sys.stderr) return source_job_names, binary_job_names, job_configs # binarydeb jobs for arch in build_file.targets[os_name][os_code_name]: if filter_arches and arch not in filter_arches: continue job_name = get_binarydeb_job_name(rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, arch) upstream_job_names = [source_job_name] + [ get_binarydeb_job_name( rosdistro_name, release_build_name, dependency_name, os_name, os_code_name, arch) for dependency_name in dependency_names ] job_config = _get_binarydeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, arch, pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache, upstream_job_names=upstream_job_names, is_disabled=is_disabled) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, job_name, job_config, dry_run=dry_run) binary_job_names.append(job_name) job_configs[job_name] = job_config return source_job_names, binary_job_names, job_configs
print('No previous distribution found.', file=sys.stderr) exit(-1) if args.comparison: valid_comparison_keys = valid_distro_keys[:] valid_comparison_keys.remove(distro_key) if args.comparison not in valid_comparison_keys: print('Invalid rosdistro [%s] selected for comparison to [%s].' % (args.comparison, distro_key), file=sys.stderr) print('Valid rosdistros are %s.' % valid_comparison_keys, file=sys.stderr) exit(-1) prev_distro_key = args.comparison else: prev_distro_key = valid_distro_keys[i - 1] cache = rosdistro.get_distribution_cache(index, distro_key) distro_file = cache.distribution_file prev_cache = rosdistro.get_distribution_cache(index, prev_distro_key) prev_distribution = rosdistro.get_cached_distribution( index, prev_distro_key, cache=prev_cache) prev_distro_file = prev_cache.distribution_file dependency_walker = DependencyWalker(prev_distribution) if repo_names_argument is None: # Check missing dependencies for packages that were in the previous # distribution that have not yet been released in the current distribution # Filter repos without a version or a release repository repo_names_argument = prev_distro_file.repositories.keys()
def configure_release_jobs( config_url, rosdistro_name, release_build_name, groovy_script=None, dry_run=False, whitelist_package_names=None): """ Configure all Jenkins release jobs. L{configure_release_job} will be invoked for every released package and target which matches the build file criteria. Additionally a job to import Debian packages into the Debian repository is created. """ config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) # get targets platforms = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): platforms.append((os_name, os_code_name)) print('The build file contains the following targets:') for os_name, os_code_name in platforms: print(' - %s %s: %s' % (os_name, os_code_name, ', '.join( build_file.targets[os_name][os_code_name]))) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return pkg_names = dist_file.release_packages.keys() filtered_pkg_names = build_file.filter_packages(pkg_names) explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names) if explicitly_ignored_pkg_names: print(('The following packages are being %s because of ' + 'white-/blacklisting:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(explicitly_ignored_pkg_names): print(' -', pkg_name) dist_cache = None if build_file.notify_maintainers or \ build_file.abi_incompatibility_assumed or \ explicitly_ignored_pkg_names: dist_cache = get_distribution_cache(index, rosdistro_name) if explicitly_ignored_pkg_names: # get direct dependencies from distro cache for each package direct_dependencies = {} for pkg_name in pkg_names: direct_dependencies[pkg_name] = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) or set([]) # find recursive downstream deps for all explicitly ignored packages ignored_pkg_names = set(explicitly_ignored_pkg_names) while True: implicitly_ignored_pkg_names = _get_downstream_package_names( ignored_pkg_names, direct_dependencies) if implicitly_ignored_pkg_names - ignored_pkg_names: ignored_pkg_names |= implicitly_ignored_pkg_names continue break implicitly_ignored_pkg_names = \ ignored_pkg_names - explicitly_ignored_pkg_names if implicitly_ignored_pkg_names: print(('The following packages are being %s because their ' + 'dependencies are being ignored:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(implicitly_ignored_pkg_names): print(' -', pkg_name) filtered_pkg_names = \ set(filtered_pkg_names) - implicitly_ignored_pkg_names # all further configuration will be handled by either the Jenkins API # or by a generated groovy script jenkins = connect(config.jenkins_url) if groovy_script is None else False all_view_configs = {} all_job_configs = {} job_name, job_config = configure_import_package_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config job_name, job_config = configure_sync_packages_to_main_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config for os_name, os_code_name in platforms: for arch in sorted(build_file.targets[os_name][os_code_name]): job_name, job_config = configure_sync_packages_to_testing_job( config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config targets = [] for os_name, os_code_name in platforms: targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) views = configure_release_views( jenkins, rosdistro_name, release_build_name, targets, dry_run=dry_run) if not jenkins: all_view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(views), } other_build_files = [v for k, v in build_files.items() if k != release_build_name] all_source_job_names = [] all_binary_job_names = [] for pkg_name in sorted(pkg_names): if whitelist_package_names: if pkg_name not in whitelist_package_names: print("Skipping package '%s' not in the explicitly passed list" % pkg_name, file=sys.stderr) continue pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] is_disabled = pkg_name not in filtered_pkg_names if is_disabled and build_file.skip_ignored_packages: print("Skipping ignored package '%s' in repository '%s'" % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository: print(("Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository.version: print(("Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name), file=sys.stderr) continue for os_name, os_code_name in platforms: other_build_files_same_platform = [] for other_build_file in other_build_files: if os_name not in other_build_file.targets: continue if os_code_name not in other_build_file.targets[os_name]: continue other_build_files_same_platform.append(other_build_file) try: source_job_names, binary_job_names, job_configs = \ configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, generate_import_package_job=False, generate_sync_packages_jobs=False, is_disabled=is_disabled, other_build_files_same_platform=other_build_files_same_platform, groovy_script=groovy_script, dry_run=dry_run) all_source_job_names += source_job_names all_binary_job_names += binary_job_names if groovy_script is not None: print('Configuration for jobs: ' + ', '.join(source_job_names + binary_job_names)) all_job_configs.update(job_configs) except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(all_job_configs) groovy_data['job_prefixes_and_names'] = {} # with an explicit list of packages we don't delete obsolete jobs if not whitelist_package_names: # delete obsolete binary jobs for os_name, os_code_name in platforms: for arch in build_file.targets[os_name][os_code_name]: binary_view = get_release_binary_view_name( rosdistro_name, release_build_name, os_name, os_code_name, arch) binary_job_prefix = '%s__' % binary_view excluded_job_names = set([ j for j in all_binary_job_names if j.startswith(binary_job_prefix)]) if groovy_script is None: print("Removing obsolete binary jobs with prefix '%s'" % binary_job_prefix) remove_jobs( jenkins, binary_job_prefix, excluded_job_names, dry_run=dry_run) else: binary_key = 'binary_%s_%s_%s' % \ (os_name, os_code_name, arch) groovy_data['job_prefixes_and_names'][binary_key] = \ (binary_job_prefix, excluded_job_names) # delete obsolete source jobs # requires knowledge about all other release build files for os_name, os_code_name in platforms: other_source_job_names = [] # get source job names for all other release build files for other_release_build_name in [ k for k in build_files.keys() if k != release_build_name]: other_build_file = build_files[other_release_build_name] other_dist_file = get_distribution_file( index, rosdistro_name, other_build_file) if not other_dist_file: continue if os_name not in other_build_file.targets or \ os_code_name not in other_build_file.targets[os_name]: continue if other_build_file.skip_ignored_packages: filtered_pkg_names = other_build_file.filter_packages( pkg_names) else: filtered_pkg_names = pkg_names for pkg_name in sorted(filtered_pkg_names): pkg = other_dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = other_dist_file.repositories[repo_name] if not repo.release_repository: continue if not repo.release_repository.version: continue other_job_name = get_sourcedeb_job_name( rosdistro_name, other_release_build_name, pkg_name, os_name, os_code_name) other_source_job_names.append(other_job_name) source_view_prefix = get_release_source_view_name( rosdistro_name, os_name, os_code_name) source_job_prefix = '%s__' % source_view_prefix excluded_job_names = set([ j for j in (all_source_job_names + other_source_job_names) if j.startswith(source_job_prefix)]) if groovy_script is None: print("Removing obsolete source jobs with prefix '%s'" % source_job_prefix) remove_jobs( jenkins, source_job_prefix, excluded_job_names, dry_run=dry_run) else: source_key = 'source_%s_%s' % (os_name, os_code_name) groovy_data['job_prefixes_and_names'][source_key] = ( source_job_prefix, excluded_job_names) if groovy_script is not None: print( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % (groovy_script, len(all_view_configs), len(all_job_configs))) content = expand_template( 'snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs( groovy_script, content, all_job_configs, view_configs=all_view_configs)
def configure_devel_jobs( config_url, rosdistro_name, source_build_name, groovy_script=None): """ Configure all Jenkins devel jobs. L{configure_release_job} will be invoked for source repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_source_build_files(config, rosdistro_name) build_file = build_files[source_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return devel_view_name = get_devel_view_name( rosdistro_name, source_build_name, pull_request=False) pull_request_view_name = get_devel_view_name( rosdistro_name, source_build_name, pull_request=True) from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) views = [] if build_file.test_commits_force is not False: views.append(configure_devel_view(jenkins, devel_view_name)) if build_file.test_pull_requests_force is not False: views.append(configure_devel_view(jenkins, pull_request_view_name)) if groovy_script is not None: # all further configuration will be handled by the groovy script jenkins = False repo_names = dist_file.repositories.keys() filtered_repo_names = build_file.filter_repositories(repo_names) devel_job_names = [] pull_request_job_names = [] job_configs = {} for repo_name in sorted(repo_names): is_disabled = repo_name not in filtered_repo_names if is_disabled and build_file.skip_ignored_repositories: print("Skipping ignored repository '%s'" % repo_name, file=sys.stderr) continue repo = dist_file.repositories[repo_name] if not repo.source_repository: print("Skipping repository '%s': no source section" % repo_name) continue if not repo.source_repository.version: print("Skipping repository '%s': no source version" % repo_name) continue job_types = [] # check for testing commits if build_file.test_commits_force is False: print(("Skipping repository '%s': 'test_commits' is forced to " + "false in the build file") % repo_name) elif repo.source_repository.test_commits is False: print(("Skipping repository '%s': 'test_commits' of the " + "repository set to false") % repo_name) elif repo.source_repository.test_commits is None and \ not build_file.test_commits_default: print(("Skipping repository '%s': 'test_commits' defaults to " + "false in the build file") % repo_name) else: job_types.append('commit') if not is_disabled: # check for testing pull requests if build_file.test_pull_requests_force is False: # print(("Skipping repository '%s': 'test_pull_requests' " + # "is forced to false in the build file") % repo_name) pass elif repo.source_repository.test_pull_requests is False: # print(("Skipping repository '%s': 'test_pull_requests' of " + # "the repository set to false") % repo_name) pass elif repo.source_repository.test_pull_requests is None and \ not build_file.test_pull_requests_default: # print(("Skipping repository '%s': 'test_pull_requests' " + # "defaults to false in the build file") % repo_name) pass else: print("Pull request job for repository '%s'" % repo_name) job_types.append('pull_request') for job_type in job_types: pull_request = job_type == 'pull_request' for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_devel_job( config_url, rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, pull_request, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script) if not pull_request: devel_job_names.append(job_name) else: pull_request_job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) devel_job_prefix = '%s__' % devel_view_name pull_request_job_prefix = '%s__' % pull_request_view_name if groovy_script is None: # delete obsolete jobs in these views from ros_buildfarm.jenkins import remove_jobs print('Removing obsolete devel jobs') remove_jobs(jenkins, devel_job_prefix, devel_job_names) print('Removing obsolete pull request jobs') remove_jobs( jenkins, pull_request_job_prefix, pull_request_job_names) else: print("Writing groovy script '%s' to reconfigure %d jobs" % (groovy_script, len(job_configs))) data = { 'expected_num_jobs': len(job_configs), 'job_prefixes_and_names': { 'devel': (devel_job_prefix, devel_job_names), 'pull_request': ( pull_request_job_prefix, pull_request_job_names), } } content = expand_template('snippet/reconfigure_jobs.groovy.em', data) write_groovy_script_and_configs( groovy_script, content, job_configs)
def configure_release_jobs(config_url, rosdistro_name, release_build_name, groovy_script=None): """ Configure all Jenkins release jobs. L{configure_release_job} will be invoked for every released package and target which matches the build file criteria. Additionally a job to import Debian packages into the Debian repository is created. """ config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) # get targets platforms = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): platforms.append((os_name, os_code_name)) print('The build file contains the following targets:') for os_name, os_code_name in platforms: print(' - %s %s: %s' % (os_name, os_code_name, ', '.join( build_file.targets[os_name][os_code_name]))) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return pkg_names = dist_file.release_packages.keys() filtered_pkg_names = build_file.filter_packages(pkg_names) explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names) if explicitly_ignored_pkg_names: print(('The following packages are being %s because of ' + 'white-/blacklisting:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(explicitly_ignored_pkg_names): print(' -', pkg_name) dist_cache = None if build_file.notify_maintainers or \ build_file.abi_incompatibility_assumed or \ explicitly_ignored_pkg_names: dist_cache = get_distribution_cache(index, rosdistro_name) if explicitly_ignored_pkg_names: # get direct dependencies from distro cache for each package direct_dependencies = {} for pkg_name in pkg_names: direct_dependencies[pkg_name] = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) or set([]) # find recursive downstream deps for all explicitly ignored packages ignored_pkg_names = set(explicitly_ignored_pkg_names) while True: implicitly_ignored_pkg_names = _get_downstream_package_names( ignored_pkg_names, direct_dependencies) if implicitly_ignored_pkg_names - ignored_pkg_names: ignored_pkg_names |= implicitly_ignored_pkg_names continue break implicitly_ignored_pkg_names = \ ignored_pkg_names - explicitly_ignored_pkg_names if implicitly_ignored_pkg_names: print(('The following packages are being %s because their ' + 'dependencies are being ignored:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(implicitly_ignored_pkg_names): print(' -', pkg_name) filtered_pkg_names = \ set(filtered_pkg_names) - implicitly_ignored_pkg_names jenkins = connect(config.jenkins_url) configure_import_package_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins) configure_sync_packages_to_main_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins) for os_name, os_code_name in platforms: for arch in sorted(build_file.targets[os_name][os_code_name]): configure_sync_packages_to_testing_job(config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins) targets = [] for os_name, os_code_name in platforms: targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) views = configure_release_views(jenkins, rosdistro_name, release_build_name, targets) if groovy_script is not None: # all further configuration will be handled by the groovy script jenkins = False all_source_job_names = [] all_binary_job_names = [] all_job_configs = {} for pkg_name in sorted(pkg_names): pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] is_disabled = pkg_name not in filtered_pkg_names if is_disabled and build_file.skip_ignored_packages: print("Skipping ignored package '%s' in repository '%s'" % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository: print(("Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository.version: print(("Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name), file=sys.stderr) continue for os_name, os_code_name in platforms: try: source_job_names, binary_job_names, job_configs = \ configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, generate_import_package_job=False, generate_sync_packages_jobs=False, is_disabled=is_disabled, groovy_script=groovy_script) all_source_job_names += source_job_names all_binary_job_names += binary_job_names if groovy_script is not None: print('Configuration for jobs: ' + ', '.join(source_job_names + binary_job_names)) all_job_configs.update(job_configs) except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data = { 'expected_num_jobs': len(all_job_configs), 'job_prefixes_and_names': {}, } # delete obsolete binary jobs for os_name, os_code_name in platforms: for arch in build_file.targets[os_name][os_code_name]: binary_view = get_release_binary_view_name(rosdistro_name, release_build_name, os_name, os_code_name, arch) binary_job_prefix = '%s__' % binary_view excluded_job_names = set([ j for j in all_binary_job_names if j.startswith(binary_job_prefix) ]) if groovy_script is None: print("Removing obsolete binary jobs with prefix '%s'" % binary_job_prefix) remove_jobs(jenkins, binary_job_prefix, excluded_job_names) else: binary_key = 'binary_%s_%s_%s' % (os_name, os_code_name, arch) groovy_data['job_prefixes_and_names'][binary_key] = \ (binary_job_prefix, excluded_job_names) # delete obsolete source jobs # requires knowledge about all other release build files for os_name, os_code_name in platforms: other_source_job_names = [] # get source job names for all other release build files for other_release_build_name in [ k for k in build_files.keys() if k != release_build_name ]: other_build_file = build_files[other_release_build_name] other_dist_file = get_distribution_file(index, rosdistro_name, other_build_file) if not other_dist_file: continue if os_name not in other_build_file.targets or \ os_code_name not in other_build_file.targets[os_name]: continue if other_build_file.skip_ignored_packages: filtered_pkg_names = other_build_file.filter_packages( pkg_names) else: filtered_pkg_names = pkg_names for pkg_name in sorted(filtered_pkg_names): pkg = other_dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = other_dist_file.repositories[repo_name] if not repo.release_repository: continue if not repo.release_repository.version: continue other_job_name = get_sourcedeb_job_name( rosdistro_name, other_release_build_name, pkg_name, os_name, os_code_name) other_source_job_names.append(other_job_name) source_view_prefix = get_release_source_view_name( rosdistro_name, os_name, os_code_name) source_job_prefix = '%s__' % source_view_prefix excluded_job_names = set([ j for j in (all_source_job_names + other_source_job_names) if j.startswith(source_job_prefix) ]) if groovy_script is None: print("Removing obsolete source jobs with prefix '%s'" % source_job_prefix) remove_jobs(jenkins, source_job_prefix, excluded_job_names) else: source_key = 'source_%s_%s' % (os_name, os_code_name) groovy_data['job_prefixes_and_names'][source_key] = ( source_job_prefix, excluded_job_names) if groovy_script is not None: print("Writing groovy script '%s' to reconfigure %d jobs" % (groovy_script, len(all_job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, all_job_configs)
def configure_release_job_with_validation( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, append_timestamp=False, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, view=None, generate_import_package_job=True, filter_arches=None): if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name) pkg_names = dist_file.release_packages.keys() pkg_names = build_file.filter_packages(pkg_names) if pkg_name not in pkg_names: raise JobValidationError( "Invalid package name '%s' " % pkg_name + 'choose one of the following: ' + ', '.join(sorted(pkg_names))) pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] if not repo.release_repository: raise JobValidationError( "Repository '%s' has no release section" % repo_name) if not repo.release_repository.version: raise JobValidationError( "Repository '%s' has no release version" % repo_name) if os_name not in build_file.targets.keys(): raise JobValidationError( "Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if dist_cache is None and \ (build_file.notify_maintainers or build_file.abi_incompatibility_assumed): dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: jenkins = connect(config.jenkins_url) if view is None: view_name = get_release_view_name(rosdistro_name, release_build_name) configure_release_view(jenkins, view_name) if generate_import_package_job: configure_import_package_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins) # sourcedeb job job_name = get_sourcedeb_job_name( rosdistro_name, release_build_name, pkg_name, os_name, os_code_name) job_config = _get_sourcedeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, _get_target_arches( build_file, os_name, os_code_name, print_skipped=False), repo.release_repository, pkg_name, repo_name, dist_cache=dist_cache) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, job_name, job_config) dependency_names = [] if build_file.abi_incompatibility_assumed: dependency_names = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) if dependency_names is None: return # binarydeb jobs for arch in _get_target_arches(build_file, os_name, os_code_name): if filter_arches and arch not in filter_arches: continue job_name = get_binarydeb_job_name( rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, arch) upstream_job_names = [ get_binarydeb_job_name( rosdistro_name, release_build_name, dependency_name, os_name, os_code_name, arch) for dependency_name in dependency_names] job_config = _get_binarydeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, arch, repo.release_repository, pkg_name, append_timestamp, repo_name, dist_cache=dist_cache, upstream_job_names=upstream_job_names) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, job_name, job_config)
def test_get_release_cache(): url = 'file://' + FILES_DIR + '/index_v2.yaml' i = get_index(url) get_distribution_cache(i, 'foo')
def get_sourcedeb( rosdistro_index_url, rosdistro_name, package_name, sourcedeb_dir, skip_download_sourcedeb=False): # ensure that no source subfolder exists debian_package_name = get_debian_package_name(rosdistro_name, package_name) subfolders = _get_package_subfolders(sourcedeb_dir, debian_package_name) assert not subfolders, \ ("Sourcedeb directory '%s' must not have any " + "subfolders starting with '%s-'") % (sourcedeb_dir, package_name) debian_package_name = get_debian_package_name(rosdistro_name, package_name) if not skip_download_sourcedeb: # get expected package version from rosdistro from rosdistro import get_distribution_cache from rosdistro import get_index index = get_index(rosdistro_index_url) dist_cache = get_distribution_cache(index, rosdistro_name) dist_file = dist_cache.distribution_file assert package_name in dist_file.release_packages pkg = dist_file.release_packages[package_name] repo = dist_file.repositories[pkg.repository_name] package_version = repo.release_repository.version # get the exact sourcedeb version showsrc_output = subprocess.check_output([ 'apt-cache', 'showsrc', debian_package_name]).decode() line_prefix = 'Version: ' debian_package_versions = [ l[len(line_prefix):] for l in showsrc_output.splitlines() if l.startswith(line_prefix + package_version)] assert len(debian_package_versions) == 1, \ "Failed to find sourcedeb with version '%s', only found: %s" % \ (package_version, ', '.join(debian_package_versions)) # download sourcedeb apt_script = os.path.join( os.path.dirname(__file__), 'wrapper', 'apt.py') cmd = [ sys.executable, apt_script, 'source', '--download-only', '--only-source', debian_package_name + '=' + debian_package_versions[0]] print("Invoking '%s'" % ' '.join(cmd)) subprocess.check_call(cmd, cwd=sourcedeb_dir) # extract sourcedeb filenames = _get_package_dsc_filename(sourcedeb_dir, debian_package_name) assert len(filenames) == 1, filenames dsc_filename = filenames[0] cmd = ['dpkg-source', '-x', dsc_filename] print("Invoking '%s'" % ' '.join(cmd)) subprocess.check_call(cmd, cwd=sourcedeb_dir) # ensure that one source subfolder exists subfolders = _get_package_subfolders(sourcedeb_dir, debian_package_name) assert len(subfolders) == 1, subfolders source_dir = subfolders[0] # output package maintainers for job notification from catkin_pkg.package import parse_package pkg = parse_package(source_dir) maintainer_emails = set([]) for m in pkg.maintainers: maintainer_emails.add(m.email) if maintainer_emails: print('Package maintainer emails: %s' % ' '.join(sorted(maintainer_emails)))
def _get_blocked_releases_info(config_url, rosdistro_name, repo_names=None): import rosdistro from rosdistro.dependency_walker import DependencyWalker from catkin_pkg.package import InvalidPackage, parse_package_string prev_rosdistro_name = None config = get_config_index(config_url) index = rosdistro.get_index(config.rosdistro_index_url) valid_rosdistro_names = list(index.distributions.keys()) valid_rosdistro_names.sort() if rosdistro_name is None: rosdistro_name = valid_rosdistro_names[-1] print('Checking packages for "%s" distribution' % rosdistro_name) # Find the previous distribution to the current one try: i = valid_rosdistro_names.index(rosdistro_name) except ValueError: print('Distribution key not found in list of valid distributions.', file=sys.stderr) exit(-1) if i == 0: print('No previous distribution found.', file=sys.stderr) exit(-1) prev_rosdistro_name = valid_rosdistro_names[i - 1] cache = rosdistro.get_distribution_cache(index, rosdistro_name) distro_file = cache.distribution_file prev_cache = rosdistro.get_distribution_cache(index, prev_rosdistro_name) prev_distribution = rosdistro.get_cached_distribution(index, prev_rosdistro_name, cache=prev_cache) prev_distro_file = prev_cache.distribution_file dependency_walker = DependencyWalker(prev_distribution) if repo_names is None: # Check missing dependencies for packages that were in the previous # distribution that have not yet been released in the current distribution # Filter repos without a version or a release repository keys = prev_distro_file.repositories.keys() prev_repo_names = set(repo for repo in keys if _is_released(repo, prev_distro_file)) repo_names = prev_repo_names ignored_inputs = [] else: prev_repo_names = set(repo for repo in repo_names if _is_released(repo, prev_distro_file)) ignored_inputs = list(set(repo_names).difference(prev_repo_names)) if len(ignored_inputs) > 0: print( 'Ignoring inputs for which repository info not found in previous distribution ' '(did you list a package instead of a repository?):') print('\n'.join( sorted('\t{0}'.format(repo) for repo in ignored_inputs))) keys = distro_file.repositories.keys() current_repo_names = set(repo for repo in keys if _is_released(repo, distro_file)) # Get a list of currently released packages current_package_names = set( pkg for repo in current_repo_names for pkg in distro_file.repositories[repo].release_repository.package_names) released_repos = prev_repo_names.intersection(current_repo_names) unreleased_repos = list(prev_repo_names.difference(current_repo_names)) if len(unreleased_repos) == 0: print('All inputs already released in {0}.'.format(rosdistro_name)) repos_info = defaultdict(dict) unprocessed_repos = prev_repo_names while unprocessed_repos: print('Processing repos:\n%s' % '\n'.join(['- %s' % r for r in sorted(unprocessed_repos)])) new_repos_to_process = set( ) # set containing repos that come up while processing others for repo_name in unprocessed_repos: repos_info[repo_name]['released'] = repo_name in released_repos if repo_name in released_repos: repo = distro_file.repositories[repo_name] version = repo.release_repository.version repos_info[repo_name]['version'] = version else: # Gather info on which required repos have not been released yet # Assume dependencies will be the same as in the previous distribution and find # which ones have been released repo = prev_distro_file.repositories[repo_name] release_repo = repo.release_repository package_dependencies = set() packages = release_repo.package_names # Accumulate all dependencies for those packages for package in packages: try: package_dependencies |= dependency_walker.get_recursive_depends( package, ['build', 'buildtool', 'run', 'test'], ros_packages_only=True, limit_depth=1) except AssertionError as e: print(e, file=sys.stderr) # For all package dependencies, check if they are released yet unreleased_pkgs = package_dependencies.difference( current_package_names) # Remove the packages which this repo provides unreleased_pkgs = unreleased_pkgs.difference(packages) # Get maintainer info and repo of unreleased packages maintainers = {} repos_blocked_by = set() for pkg_name in unreleased_pkgs: unreleased_repo_name = \ prev_distro_file.release_packages[pkg_name].repository_name repos_blocked_by.add(unreleased_repo_name) pkg_xml = prev_distribution.get_release_package_xml( pkg_name) if pkg_xml is not None: try: pkg = parse_package_string(pkg_xml) except InvalidPackage: pass else: pkg_maintainers = { m.name: m.email for m in pkg.maintainers } if unreleased_repo_name not in maintainers: maintainers[unreleased_repo_name] = {} maintainers[unreleased_repo_name].update( pkg_maintainers) if maintainers: repos_info[repo_name]['maintainers'] = maintainers repos_info[repo_name]['repos_blocked_by'] = {} for blocking_repo_name in repos_blocked_by: # Get url of blocking repos repo_url = None blocking_repo = prev_distro_file.repositories[ blocking_repo_name] if blocking_repo.source_repository: repo_url = blocking_repo.source_repository.url elif blocking_repo.doc_repository: repo_url = blocking_repo.doc_repository.url repos_info[repo_name]['repos_blocked_by'].update( {blocking_repo_name: repo_url}) # Mark blocking relationship in other direction if blocking_repo_name not in repos_info: new_repos_to_process.add(blocking_repo_name) repos_info[blocking_repo_name] = {} if 'repos_blocking' not in repos_info[blocking_repo_name]: repos_info[blocking_repo_name]['repos_blocking'] = set( []) repos_info[blocking_repo_name]['repos_blocking'].add( repo_name) # Get url of repo repo_url = None if repo.source_repository: repo_url = repo.source_repository.url elif repo.doc_repository: repo_url = repo.doc_repository.url if repo_url: repos_info[repo_name]['url'] = repo_url new_repos_to_process.discard( repo_name) # this repo has been fully processed now for repo_name in repos_info.keys(): # Recursively get all repos being blocked by this repo recursive_blocks = set([]) repos_to_check = set([repo_name]) while repos_to_check: next_repo_to_check = repos_to_check.pop() blocks = repos_info[next_repo_to_check].get( 'repos_blocking', set([])) new_blocks = blocks - recursive_blocks repos_to_check |= new_blocks recursive_blocks |= new_blocks if recursive_blocks: repos_info[repo_name][ 'recursive_repos_blocking'] = recursive_blocks unprocessed_repos = new_repos_to_process return repos_info
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description="Generate a 'prerelease' script") add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_build_name(parser, 'source') add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_argument_output_dir(parser, required=True) group = parser.add_argument_group( 'Repositories in underlay workspace', description='The repositories in the underlay workspace will be ' + 'built and installed as well as built and tested. ' + 'Dependencies will be provided by binary packages.') group.add_argument( 'source_repos', nargs='*', default=[], metavar='REPO_NAME', help="A name of a 'repository' from the distribution file") group.add_argument( '--custom-branch', nargs='*', type=_repository_name_and_branch, default=[], metavar='REPO_NAME:BRANCH_OR_TAG_NAME', help="A name of a 'repository' from the distribution file followed " + 'by a colon and a branch / tag name') group.add_argument( '--custom-repo', nargs='*', type=_repository_name_and_type_and_url_and_branch, default=[], metavar='REPO_NAME:REPO_TYPE:REPO_URL:BRANCH_OR_TAG_NAME', help='The name, type, url and branch / tag name of a repository') add_overlay_arguments(parser) args = parser.parse_args(argv) print('Fetching buildfarm configuration...') config = get_config_index(args.config_url) build_files = get_source_build_files(config, args.rosdistro_name) build_file = build_files[args.source_build_name] print('Fetching rosdistro cache...') # Targets defined by source build file are subset of targets # defined by release build files. To increase the number of supported # pre-release targets, we combine all targets defined by all release # build files and use that when configuring the devel job. release_build_files = get_release_build_files(config, args.rosdistro_name) release_targets_combined = {} if release_build_files: release_targets_combined[args.os_name] = {} for build_name, rel_obj in release_build_files.items(): if args.os_name not in rel_obj.targets: continue for dist_name, targets in rel_obj.targets[args.os_name].items(): if dist_name not in release_targets_combined[args.os_name]: release_targets_combined[args.os_name][dist_name] = {} release_targets_combined[args.os_name][dist_name].update(targets) index = get_index(config.rosdistro_index_url) dist_cache = get_distribution_cache(index, args.rosdistro_name) dist_file = dist_cache.distribution_file # determine source repositories for underlay workspace repositories = {} for repo_name in args.source_repos: if repo_name in repositories: print("The repository '%s' appears multiple times" % repo_name, file=sys.stderr) return 1 try: repositories[repo_name] = \ dist_file.repositories[repo_name].source_repository except KeyError: print(("The repository '%s' was not found in the distribution " + "file") % repo_name, file=sys.stderr) return 1 for repo_name, custom_version in args.custom_branch: if repo_name in repositories: print("The repository '%s' appears multiple times" % repo_name, file=sys.stderr) return 1 try: source_repo = dist_file.repositories[repo_name].source_repository except KeyError: print(("The repository '%s' was not found in the distribution " + "file") % repo_name, file=sys.stderr) return 1 source_repo = deepcopy(source_repo) source_repo.version = custom_version repositories[repo_name] = source_repo for repo_name, repo_type, repo_url, version in args.custom_repo: if repo_name in repositories and repositories[repo_name]: print("custom_repos option overriding '%s' to pull via '%s' " "from '%s' with version '%s'. " % (repo_name, repo_type, repo_url, version), file=sys.stderr) source_repo = RepositorySpecification( repo_name, { 'type': repo_type, 'url': repo_url, 'version': version, }) repositories[repo_name] = source_repo scms = [(repositories[k], 'catkin_workspace/src/%s' % k) for k in sorted(repositories.keys())] # collect all template snippets of specific types class IncludeHook(Hook): def __init__(self): Hook.__init__(self) self.scripts = [] def beforeInclude(self, *args, **kwargs): template_path = kwargs['file'].name if template_path.endswith('/snippet/builder_shell.xml.em'): self.scripts.append(kwargs['locals']['script']) hook = IncludeHook() from ros_buildfarm import templates templates.template_hooks = [hook] # use random source repo to pass to devel job template source_repository = deepcopy(list(repositories.values())[0]) if not source_repository: print(("The repository '%s' does not have a source entry in the distribution " + 'file. We cannot generate a prerelease without a source entry.') % repo_name, file=sys.stderr) return 1 source_repository.name = 'prerelease' print('Evaluating job templates...') configure_devel_job( args.config_url, args.rosdistro_name, args.source_build_name, None, args.os_name, args.os_code_name, args.arch, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=False, views=False, source_repository=source_repository, build_targets=release_targets_combined) templates.template_hooks = None # derive scripts for overlay workspace from underlay overlay_scripts = [] for script in hook.scripts: # skip cloning of ros_buildfarm repository if 'git clone' in script and '.git ros_buildfarm' in script: continue # skip build-and-install step if 'build and install' in script: continue # add prerelease overlay flag run_devel_job = '/run_devel_job.py' if run_devel_job in script: script = script.replace( run_devel_job, run_devel_job + ' --prerelease-overlay') # replace mounted workspace volume with overlay and underlay # used by: # - create_devel_task_generator.py needs to find packages in both # the underlay as well as the overlay workspace # - catkin_make_isolated_and_test.py needs to source the environment of # the underlay before building the overlay mount_volume = '-v $WORKSPACE/catkin_workspace:/tmp/catkin_workspace' if mount_volume in script: script = script.replace( mount_volume, mount_volume + ':ro ' + '-v $WORKSPACE/' + 'catkin_workspace_overlay:/tmp/catkin_workspace_overlay') # relocate all docker files docker_path = '$WORKSPACE/docker_' if docker_path in script: script = script.replace( docker_path, docker_path + 'overlay_') # rename all docker images name_suffix = '_prerelease' if name_suffix in script: script = script.replace( name_suffix, name_suffix + '_overlay') overlay_scripts.append(script) from ros_buildfarm import __file__ as ros_buildfarm_file data = deepcopy(args.__dict__) data.update({ 'scms': scms, 'scripts': hook.scripts, 'overlay_scripts': overlay_scripts, 'ros_buildfarm_python_path': os.path.dirname( os.path.dirname(os.path.abspath(ros_buildfarm_file))), 'python_executable': sys.executable, 'prerelease_script_path': os.path.dirname(os.path.abspath(__file__))}) if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) # generate multiple scripts for script_name in [ 'prerelease', 'prerelease_build_overlay', 'prerelease_build_underlay', 'prerelease_clone_overlay', 'prerelease_clone_underlay']: content = expand_template( 'prerelease/%s_script.sh.em' % script_name, data, options={BANGPATH_OPT: False}) script_file = os.path.join(args.output_dir, script_name + '.sh') with open(script_file, 'w') as h: h.write(content) os.chmod(script_file, os.stat(script_file).st_mode | stat.S_IEXEC) print('') print('Generated prerelease script - to execute it run:') if os.path.abspath(args.output_dir) != os.path.abspath(os.curdir): print(' cd %s' % args.output_dir) print(' ./prerelease.sh')
def main(argv=sys.argv[1:]): global templates parser = argparse.ArgumentParser( description="Generate a 'prerelease overlay' script") add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_overlay_arguments(parser) parser.add_argument( '--underlay-packages', nargs='+', help='Names of packages on which the overlay builds ' '(by default package names come from packages found in ' "'catkin_workspace/src')" ) parser.add_argument( '--json', action='store_true', help='Output overlay information as JSON instead of a shell script' ) args = parser.parse_args(argv) config = get_config_index(args.config_url) index = get_index(config.rosdistro_index_url) dist_cache = get_distribution_cache(index, args.rosdistro_name) dist_file = dist_cache.distribution_file # determine source repositories for overlay workspace underlay_package_names = args.underlay_packages if underlay_package_names is None: packages = find_packages('catkin_workspace/src') underlay_package_names = [pkg.name for pkg in packages.values()] print('Underlay workspace contains %d packages:%s' % (len(underlay_package_names), ''.join(['\n- %s' % pkg_name for pkg_name in sorted(underlay_package_names)])), file=sys.stderr) overlay_package_names = get_overlay_package_names( args.pkg, args.exclude_pkg, args.level, underlay_package_names, dist_cache.release_package_xmls, output=True) print('Overlay workspace will contain %d packages:%s' % (len(overlay_package_names), ''.join(['\n- %s' % pkg_name for pkg_name in sorted(overlay_package_names)])), file=sys.stderr) repositories = {} for pkg_name in overlay_package_names: repositories[pkg_name] = \ get_repository_specification_for_released_package( dist_file, pkg_name) scms = [ (repositories[k], 'catkin_workspace_overlay/src/%s' % k) for k in sorted(repositories.keys())] if not args.json: value = expand_template( 'prerelease/prerelease_overlay_script.sh.em', { 'scms': scms}, options={BANGPATH_OPT: False}) print(value) else: print(json.dumps([vars(r) for r, p in scms], sort_keys=True, indent=2))
if args.comparison: valid_comparison_keys = valid_distro_keys[:] valid_comparison_keys.remove(distro_key) if args.comparison not in valid_comparison_keys: print('Invalid rosdistro [%s] selected for comparison to [%s].' % (args.comparison, distro_key), file=sys.stderr) print('Valid rosdistros are %s.' % valid_comparison_keys, file=sys.stderr) exit(-1) prev_distro_key = args.comparison else: prev_distro_key = valid_distro_keys[i - 1] cache = rosdistro.get_distribution_cache(index, distro_key) distro_file = cache.distribution_file prev_cache = rosdistro.get_distribution_cache(index, prev_distro_key) prev_distribution = rosdistro.get_cached_distribution(index, prev_distro_key, cache=prev_cache) prev_distro_file = prev_cache.distribution_file dependency_walker = DependencyWalker(prev_distribution) if repo_names_argument is None: # Check missing dependencies for packages that were in the previous # distribution that have not yet been released in the current distribution # Filter repos without a version or a release repository
def get_blocking_info(distro_key, repo_names, depth): prev_distro_key = None index = rosdistro.get_index(rosdistro.get_index_url()) valid_distro_keys = index.distributions.keys() valid_distro_keys.sort() if distro_key is None: distro_key = valid_distro_keys[-1] print('Checking packages for "%s" distribution' % distro_key) # Find the previous distribution to the current one try: i = valid_distro_keys.index(distro_key) except ValueError: print('Distribution key not found in list of valid distributions.') exit(-1) if i == 0: print('No previous distribution found.') exit(-1) prev_distro_key = valid_distro_keys[i - 1] cache = rosdistro.get_distribution_cache(index, distro_key) distro_file = cache.distribution_file prev_cache = rosdistro.get_distribution_cache(index, prev_distro_key) prev_distribution = rosdistro.get_cached_distribution( index, prev_distro_key, cache=prev_cache) prev_distro_file = prev_cache.distribution_file dependency_walker = DependencyWalker(prev_distribution) if repo_names is None: # Check missing dependencies for packages that were in the previous # distribution that have not yet been released in the current distribution # Filter repos without a version or a release repository keys = prev_distro_file.repositories.keys() prev_repo_names = set( repo for repo in keys if is_released(repo, prev_distro_file)) repo_names = prev_repo_names ignored_inputs = [] else: prev_repo_names = set( repo for repo in repo_names if is_released(repo, prev_distro_file)) ignored_inputs = list(set(repo_names).difference(prev_repo_names)) if len(ignored_inputs) > 0: print('Ignoring inputs for which repository info not found in previous distribution' + ' (did you list a package instead of a repository?):') print('\n'.join( sorted('\t{0}'.format(repo) for repo in ignored_inputs))) keys = distro_file.repositories.keys() current_repo_names = set( repo for repo in keys if is_released(repo, distro_file)) released_repos = prev_repo_names.intersection( current_repo_names) unreleased_repos = list(prev_repo_names.difference( current_repo_names)) # Get a list of currently released packages current_package_names = set( pkg for repo in current_repo_names for pkg in distro_file.repositories[repo].release_repository.package_names) # Construct a dictionary where keys are repository names and values are a list # of the repos blocking/blocked by that repo blocked_repos = {} blocking_repos = {} unblocked_blocking_repos = set() if len(unreleased_repos) == 0: print('All inputs already released in {0}.'.format( distro_key)) # Process repo dependencies unblocked_repos = set() total_blocking_repos = set() for repository_name in unreleased_repos: repo = prev_distro_file.repositories[repository_name] release_repo = repo.release_repository package_dependencies = set() packages = release_repo.package_names # Accumulate all dependencies for those packages for package in packages: recursive_dependencies = dependency_walker.get_recursive_depends( package, ['build', 'run', 'buildtool'], ros_packages_only=True, limit_depth=depth) package_dependencies = package_dependencies.union( recursive_dependencies) # For all package dependencies, check if they are released yet unreleased_pkgs = package_dependencies.difference( current_package_names) # remove the packages which this repo provides. unreleased_pkgs = unreleased_pkgs.difference(packages) # Now get the repositories for these packages. blocking_repos_for_this_repo = set(prev_distro_file.release_packages[pkg].repository_name for pkg in unreleased_pkgs) if len(blocking_repos_for_this_repo) == 0: unblocked_repos.add(repository_name) else: # Get the repository for the unreleased packages blocked_repos[repository_name] = blocking_repos_for_this_repo total_blocking_repos |= blocking_repos_for_this_repo for blocking_repo in blocking_repos_for_this_repo: try: blocking_repos[blocking_repo] |= set([repository_name]) except KeyError: blocking_repos[blocking_repo] = set([repository_name]) unblocked_blocking_repos_names = total_blocking_repos.intersection(unblocked_repos) unblocked_blocking_repos = { repo:blocking for repo, blocking in blocking_repos.iteritems() if repo in unblocked_blocking_repos_names } unblocked_leaf_repos = unblocked_repos.difference(unblocked_blocking_repos_names) # Double-check repositories that we think are leaf repos for repo in unblocked_leaf_repos: # Check only one level of depends_on depends_on = dependency_walker.get_depends_on(package, 'build') | \ dependency_walker.get_depends_on(package, 'run') | \ dependency_walker.get_depends_on(package, 'buildtool') if len(depends_on) != 0: # There are packages that depend on this "leaf", but we didn't find # them initially because they weren't related to our inputs for package in depends_on: depends_on_repo = prev_distro_file.release_packages[package].repository_name try: unblocked_blocking_repos[repo] |= set([depends_on_repo]) except KeyError: unblocked_blocking_repos[repo] = set([depends_on_repo]) unblocked_unblocking_repos = unblocked_leaf_repos.difference( unblocked_blocking_repos.keys()) if not len(repo_names) == (len(ignored_inputs) + len(released_repos) + len(blocked_repos.keys()) + len(unblocked_blocking_repos.keys()) + len(unblocked_unblocking_repos)): raise Exception('Somewhere a repo has not been accounted for') return released_repos, blocked_repos, unblocked_blocking_repos, unblocked_unblocking_repos
# needs pygithub, yaml, rosdistro from github import Github import yaml import os import catkin_pkg.package import rosdistro maintainers = set() ind = rosdistro.get_index(rosdistro.get_index_url()) for d in ind.distributions: distribution_cache = rosdistro.get_distribution_cache(ind, d) for p, x in distribution_cache.release_package_xmls.items(): pkg = catkin_pkg.package.parse_package_string(x) for p in pkg.maintainers: #print p.email, p.name maintainers.add(p) print "Found %s maintainers in %s" % (len(maintainers), ind.distributions) g = Github() # Can add Oauth Token for higher rate limits limit = g.get_rate_limit() print "remaining API queries", limit.rate.remaining, "until", limit.rate.reset
def get_sourcedeb(rosdistro_index_url, rosdistro_name, package_name, sourcedeb_dir, skip_download_sourcedeb=False): # ensure that no source subfolder exists debian_package_name = get_debian_package_name(rosdistro_name, package_name) subfolders = _get_package_subfolders(sourcedeb_dir, debian_package_name) assert not subfolders, \ ("Sourcedeb directory '%s' must not have any " + "subfolders starting with '%s-'") % (sourcedeb_dir, package_name) debian_package_name = get_debian_package_name(rosdistro_name, package_name) if not skip_download_sourcedeb: # get expected package version from rosdistro from rosdistro import get_distribution_cache from rosdistro import get_index index = get_index(rosdistro_index_url) dist_cache = get_distribution_cache(index, rosdistro_name) dist_file = dist_cache.distribution_file assert package_name in dist_file.release_packages pkg = dist_file.release_packages[package_name] repo = dist_file.repositories[pkg.repository_name] package_version = repo.release_repository.version # get the exact sourcedeb version showsrc_output = subprocess.check_output( ['apt-cache', 'showsrc', debian_package_name]).decode() line_prefix = 'Version: ' debian_package_versions = [ l[len(line_prefix):] for l in showsrc_output.splitlines() if l.startswith(line_prefix + package_version) ] assert len(debian_package_versions) == 1, \ "Failed to find sourcedeb with version '%s', only found: %s" % \ (package_version, ', '.join(debian_package_versions)) # download sourcedeb apt_script = os.path.join(os.path.dirname(__file__), 'wrapper', 'apt.py') cmd = [ sys.executable, apt_script, 'source', '--download-only', '--only-source', debian_package_name + '=' + debian_package_versions[0] ] print("Invoking '%s'" % ' '.join(cmd)) subprocess.check_call(cmd, cwd=sourcedeb_dir) # extract sourcedeb filenames = _get_package_dsc_filename(sourcedeb_dir, debian_package_name) assert len(filenames) == 1, filenames dsc_filename = filenames[0] cmd = ['dpkg-source', '-x', dsc_filename] print("Invoking '%s'" % ' '.join(cmd)) subprocess.check_call(cmd, cwd=sourcedeb_dir) # ensure that one source subfolder exists subfolders = _get_package_subfolders(sourcedeb_dir, debian_package_name) assert len(subfolders) == 1, subfolders source_dir = subfolders[0] # output package maintainers for job notification from catkin_pkg.package import parse_package pkg = parse_package(source_dir) maintainer_emails = set([]) for m in pkg.maintainers: maintainer_emails.add(m.email) if maintainer_emails: print('Package maintainer emails: %s' % ' '.join(sorted(maintainer_emails)))
def configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, views=None, generate_import_package_job=True, generate_sync_packages_jobs=True, is_disabled=False, other_build_files_same_platform=None, groovy_script=None, filter_arches=None, dry_run=False): """ Configure a Jenkins release job. The following jobs are created for each package: - M source jobs, one for each OS node name - M * N binary jobs, one for each combination of OS code name and arch """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: raise JobValidationError( 'No distribution file matches the build file') pkg_names = dist_file.release_packages.keys() if pkg_name not in pkg_names: raise JobValidationError( "Invalid package name '%s' " % pkg_name + 'choose one of the following: ' + ', '.join(sorted(pkg_names))) pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] if not repo.release_repository: raise JobValidationError( "Repository '%s' has no release section" % repo_name) if not repo.release_repository.version: raise JobValidationError( "Repository '%s' has no release version" % repo_name) if os_name not in build_file.targets.keys(): raise JobValidationError( "Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if dist_cache is None and \ (build_file.notify_maintainers or build_file.abi_incompatibility_assumed): dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: jenkins = connect(config.jenkins_url) if views is None: targets = [] targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) configure_release_views( jenkins, rosdistro_name, release_build_name, targets, dry_run=dry_run) if generate_import_package_job: configure_import_package_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if generate_sync_packages_jobs: configure_sync_packages_to_main_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) for arch in build_file.targets[os_name][os_code_name]: configure_sync_packages_to_testing_job( config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) source_job_names = [] binary_job_names = [] job_configs = {} # sourcedeb job # since sourcedeb jobs are potentially being shared across multiple build # files the configuration has to take all of them into account in order to # generate a job which all build files agree on source_job_name = get_sourcedeb_job_name( rosdistro_name, release_build_name, pkg_name, os_name, os_code_name) # while the package is disabled in the current build file # it might be used by sibling build files is_source_disabled = is_disabled if is_source_disabled and other_build_files_same_platform: # check if sourcedeb job is used by any other build file with the same platform for other_build_file in other_build_files_same_platform: if other_build_file.filter_packages([pkg_name]): is_source_disabled = False break job_config = _get_sourcedeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache, is_disabled=is_source_disabled, other_build_files_same_platform=other_build_files_same_platform) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, source_job_name, job_config, dry_run=dry_run) source_job_names.append(source_job_name) job_configs[source_job_name] = job_config dependency_names = [] if build_file.abi_incompatibility_assumed: dependency_names = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) # if dependencies are not yet available in rosdistro cache # skip binary jobs if dependency_names is None: print(("Skipping binary jobs for package '%s' because it is not " + "yet in the rosdistro cache") % pkg_name, file=sys.stderr) return source_job_names, binary_job_names, job_configs # binarydeb jobs for arch in build_file.targets[os_name][os_code_name]: if filter_arches and arch not in filter_arches: continue job_name = get_binarydeb_job_name( rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, arch) upstream_job_names = [source_job_name] + [ get_binarydeb_job_name( rosdistro_name, release_build_name, dependency_name, os_name, os_code_name, arch) for dependency_name in dependency_names] job_config = _get_binarydeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, arch, pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache, upstream_job_names=upstream_job_names, is_disabled=is_disabled) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, job_name, job_config, dry_run=dry_run) binary_job_names.append(job_name) job_configs[job_name] = job_config return source_job_names, binary_job_names, job_configs
def configure_devel_job(config_url, rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, pull_request=False, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, views=None, is_disabled=False, groovy_script=None, source_repository=None, build_targets=None, dry_run=False): """ Configure a single Jenkins devel job. This includes the following steps: - clone the source repository to use - clone the ros_buildfarm repository - write the distribution repository keys into files - invoke the release/run_devel_job.py script """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_source_build_files(config, rosdistro_name) build_file = build_files[source_build_name] # Overwrite build_file.targets if build_targets is specified if build_targets is not None: build_file.targets = build_targets if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: raise JobValidationError( 'No distribution file matches the build file') repo_names = dist_file.repositories.keys() if repo_name is not None: if repo_name not in repo_names: raise JobValidationError("Invalid repository name '%s' " % repo_name + 'choose one of the following: %s' % ', '.join(sorted(repo_names))) repo = dist_file.repositories[repo_name] if not repo.source_repository: raise JobValidationError("Repository '%s' has no source section" % repo_name) if not repo.source_repository.version: raise JobValidationError("Repository '%s' has no source version" % repo_name) source_repository = repo.source_repository if os_name not in build_file.targets.keys(): raise JobValidationError("Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if arch not in build_file.targets[os_name][os_code_name]: raise JobValidationError( "Invalid architecture '%s' " % arch + 'choose one of the following: %s' % ', '.join(sorted(build_file.targets[os_name][os_code_name]))) if dist_cache is None and build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if views is None: view_name = get_devel_view_name(rosdistro_name, source_build_name, pull_request=pull_request) configure_devel_view(jenkins, view_name, dry_run=dry_run) job_name = get_devel_job_name(rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, pull_request) job_config = _get_devel_job_config(config, rosdistro_name, source_build_name, build_file, os_name, os_code_name, arch, source_repository, repo_name, pull_request, job_name, dist_cache=dist_cache, is_disabled=is_disabled) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: from ros_buildfarm.jenkins import configure_job configure_job(jenkins, job_name, job_config, dry_run=dry_run) return job_name, job_config
def main(argv=sys.argv[1:]): global templates parser = argparse.ArgumentParser( description="Generate a 'prerelease overlay' script") add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_overlay_arguments(parser) parser.add_argument( '--underlay-packages', nargs='+', help='Names of packages on which the overlay builds ' '(by default package names come from packages found in ' "'ws/src')") group = parser.add_mutually_exclusive_group() group.add_argument( '--json', action='store_true', help='Output overlay information as JSON instead of a shell script') group.add_argument('--vcstool', action='store_true', help='Output overlay information as vcstool repos file') args = parser.parse_args(argv) config = get_config_index(args.config_url) index = get_index(config.rosdistro_index_url) dist_cache = get_distribution_cache(index, args.rosdistro_name) dist_file = dist_cache.distribution_file # determine source repositories for overlay workspace underlay_package_names = args.underlay_packages if underlay_package_names is None: packages = find_packages('ws/src') underlay_package_names = [pkg.name for pkg in packages.values()] print('Underlay workspace contains %d packages:%s' % (len(underlay_package_names), ''.join([ '\n- %s' % pkg_name for pkg_name in sorted(underlay_package_names) ])), file=sys.stderr) overlay_package_names = get_overlay_package_names( args.pkg, args.exclude_pkg, args.level, underlay_package_names, dist_cache.release_package_xmls, output=True) print('Overlay workspace will contain %d packages:%s' % (len(overlay_package_names), ''.join([ '\n- %s' % pkg_name for pkg_name in sorted(overlay_package_names) ])), file=sys.stderr) repositories = {} for pkg_name in overlay_package_names: repositories[pkg_name] = \ get_repository_specification_for_released_package( dist_file, pkg_name) scms = [(repositories[k], 'ws_overlay/src/%s' % k) for k in sorted(repositories.keys())] if args.json: print(json.dumps([vars(r) for r, p in scms], sort_keys=True, indent=2)) elif args.vcstool: print('repositories:') for r, p in scms: print(' %s:' % p) print(' type: ' + r.type) print(' url: ' + r.url) print(' version: ' + r.version) else: value = expand_template('prerelease/prerelease_overlay_script.sh.em', {'scms': scms}, options={BANGPATH_OPT: False}) print(value)
def configure_release_jobs( config_url, rosdistro_name, release_build_name, append_timestamp=False): """ Configure all Jenkins release jobs. L{configure_release_job} will be invoked for every released package and target which matches the build file criteria. Additionally a job to import Debian packages into the Debian repository is created. """ config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers or build_file.abi_incompatibility_assumed: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): targets.append((os_name, os_code_name)) print('The build file contains the following targets:') for os_name, os_code_name in targets: print(' - %s %s: %s' % (os_name, os_code_name, ', '.join( build_file.targets[os_name][os_code_name]))) dist_file = get_distribution_file(index, rosdistro_name) jenkins = connect(config.jenkins_url) configure_import_package_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins) for os_name, os_code_name in targets: if os_name != 'ubuntu': continue for arch in sorted(build_file.targets[os_name][os_code_name]): configure_sync_packages_to_testing_job( config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins) view_name = get_release_view_name(rosdistro_name, release_build_name) view = configure_release_view(jenkins, view_name) pkg_names = dist_file.release_packages.keys() pkg_names = build_file.filter_packages(pkg_names) all_job_names = [] for pkg_name in sorted(pkg_names): pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] if not repo.release_repository: print(("Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository.version: print(("Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name), file=sys.stderr) continue for os_name, os_code_name in targets: try: job_names = configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, append_timestamp=append_timestamp, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, view=view, generate_import_package_job=False, generate_sync_packages_to_testing_job=False) all_job_names += job_names except JobValidationError as e: print(e.message, file=sys.stderr) # delete obsolete jobs in this view remove_jobs(jenkins, '%s__' % view_name, all_job_names)
def configure_release_jobs(config_url, rosdistro_name, release_build_name, groovy_script=None, dry_run=False, whitelist_package_names=None): """ Configure all Jenkins release jobs. L{configure_release_job} will be invoked for every released package and target which matches the build file criteria. Additionally a job to import Debian packages into the Debian repository is created. """ config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) # get targets platforms = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): platforms.append((os_name, os_code_name)) print('The build file contains the following targets:') for os_name, os_code_name in platforms: print(' - %s %s: %s' % (os_name, os_code_name, ', '.join( build_file.targets[os_name][os_code_name]))) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return pkg_names = dist_file.release_packages.keys() filtered_pkg_names = build_file.filter_packages(pkg_names) explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names) if explicitly_ignored_pkg_names: print(('The following packages are being %s because of ' + 'white-/blacklisting:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(explicitly_ignored_pkg_names): print(' -', pkg_name) dist_cache = get_distribution_cache(index, rosdistro_name) if explicitly_ignored_pkg_names: # get direct dependencies from distro cache for each package direct_dependencies = {} for pkg_name in pkg_names: direct_dependencies[pkg_name] = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) or set([]) # find recursive downstream deps for all explicitly ignored packages ignored_pkg_names = set(explicitly_ignored_pkg_names) while True: implicitly_ignored_pkg_names = _get_downstream_package_names( ignored_pkg_names, direct_dependencies) if implicitly_ignored_pkg_names - ignored_pkg_names: ignored_pkg_names |= implicitly_ignored_pkg_names continue break implicitly_ignored_pkg_names = \ ignored_pkg_names - explicitly_ignored_pkg_names if implicitly_ignored_pkg_names: print(('The following packages are being %s because their ' + 'dependencies are being ignored:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(implicitly_ignored_pkg_names): print(' -', pkg_name) filtered_pkg_names = \ set(filtered_pkg_names) - implicitly_ignored_pkg_names # all further configuration will be handled by either the Jenkins API # or by a generated groovy script jenkins = False if groovy_script is None: from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) all_view_configs = {} all_job_configs = OrderedDict() job_name, job_config = configure_import_package_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config job_name, job_config = configure_sync_packages_to_main_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config for os_name, os_code_name in platforms: for arch in sorted(build_file.targets[os_name][os_code_name]): job_name, job_config = configure_sync_packages_to_testing_job( config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config targets = [] for os_name, os_code_name in platforms: targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) views = configure_release_views(jenkins, rosdistro_name, release_build_name, targets, dry_run=dry_run) if not jenkins: all_view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(views), } # binary jobs must be generated in topological order from catkin_pkg.package import parse_package_string from ros_buildfarm.common import topological_order_packages pkgs = {} for pkg_name in pkg_names: if pkg_name not in dist_cache.release_package_xmls: print("Skipping package '%s': no released package.xml in cache" % (pkg_name), file=sys.stderr) continue pkg_xml = dist_cache.release_package_xmls[pkg_name] pkg = parse_package_string(pkg_xml) pkgs[pkg_name] = pkg ordered_pkg_tuples = topological_order_packages(pkgs) other_build_files = [ v for k, v in build_files.items() if k != release_build_name ] all_source_job_names = [] all_binary_job_names = [] for pkg_name in [p.name for _, p in ordered_pkg_tuples]: if whitelist_package_names: if pkg_name not in whitelist_package_names: print( "Skipping package '%s' not in the explicitly passed list" % pkg_name, file=sys.stderr) continue pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] is_disabled = pkg_name not in filtered_pkg_names if is_disabled and build_file.skip_ignored_packages: print("Skipping ignored package '%s' in repository '%s'" % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository: print(("Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository.version: print(("Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name), file=sys.stderr) continue for os_name, os_code_name in platforms: other_build_files_same_platform = [] for other_build_file in other_build_files: if os_name not in other_build_file.targets: continue if os_code_name not in other_build_file.targets[os_name]: continue other_build_files_same_platform.append(other_build_file) try: source_job_names, binary_job_names, job_configs = \ configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, generate_import_package_job=False, generate_sync_packages_jobs=False, is_disabled=is_disabled, other_build_files_same_platform=other_build_files_same_platform, groovy_script=groovy_script, dry_run=dry_run) all_source_job_names += source_job_names all_binary_job_names += binary_job_names if groovy_script is not None: print('Configuration for jobs: ' + ', '.join(source_job_names + binary_job_names)) for source_job_name in source_job_names: all_job_configs[source_job_name] = job_configs[ source_job_name] for binary_job_name in binary_job_names: all_job_configs[binary_job_name] = job_configs[ binary_job_name] except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(all_job_configs) groovy_data['job_prefixes_and_names'] = {} # with an explicit list of packages we don't delete obsolete jobs if not whitelist_package_names: # delete obsolete binary jobs for os_name, os_code_name in platforms: for arch in build_file.targets[os_name][os_code_name]: binary_view = get_release_binary_view_name( rosdistro_name, release_build_name, os_name, os_code_name, arch) binary_job_prefix = '%s__' % binary_view excluded_job_names = set([ j for j in all_binary_job_names if j.startswith(binary_job_prefix) ]) if groovy_script is None: print("Removing obsolete binary jobs with prefix '%s'" % binary_job_prefix) from ros_buildfarm.jenkins import remove_jobs remove_jobs(jenkins, binary_job_prefix, excluded_job_names, dry_run=dry_run) else: binary_key = 'binary_%s_%s_%s' % \ (os_name, os_code_name, arch) groovy_data['job_prefixes_and_names'][binary_key] = \ (binary_job_prefix, excluded_job_names) # delete obsolete source jobs # requires knowledge about all other release build files for os_name, os_code_name in platforms: other_source_job_names = [] # get source job names for all other release build files for other_release_build_name in [ k for k in build_files.keys() if k != release_build_name ]: other_build_file = build_files[other_release_build_name] other_dist_file = get_distribution_file( index, rosdistro_name, other_build_file) if not other_dist_file: continue if os_name not in other_build_file.targets or \ os_code_name not in other_build_file.targets[os_name]: continue if other_build_file.skip_ignored_packages: filtered_pkg_names = other_build_file.filter_packages( pkg_names) else: filtered_pkg_names = pkg_names for pkg_name in sorted(filtered_pkg_names): pkg = other_dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = other_dist_file.repositories[repo_name] if not repo.release_repository: continue if not repo.release_repository.version: continue other_job_name = get_sourcedeb_job_name( rosdistro_name, other_release_build_name, pkg_name, os_name, os_code_name) other_source_job_names.append(other_job_name) source_view_prefix = get_release_source_view_name( rosdistro_name, os_name, os_code_name) source_job_prefix = '%s__' % source_view_prefix excluded_job_names = set([ j for j in (all_source_job_names + other_source_job_names) if j.startswith(source_job_prefix) ]) if groovy_script is None: print("Removing obsolete source jobs with prefix '%s'" % source_job_prefix) from ros_buildfarm.jenkins import remove_jobs remove_jobs(jenkins, source_job_prefix, excluded_job_names, dry_run=dry_run) else: source_key = 'source_%s_%s' % (os_name, os_code_name) groovy_data['job_prefixes_and_names'][source_key] = ( source_job_prefix, excluded_job_names) if groovy_script is not None: print( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % (groovy_script, len(all_view_configs), len(all_job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, all_job_configs, view_configs=all_view_configs)
def configure_doc_jobs(config_url, rosdistro_name, doc_build_name, groovy_script=None, dry_run=False, whitelist_repository_names=None): """ Configure all Jenkins doc jobs. L{configure_doc_job} will be invoked for doc repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_doc_build_files(config, rosdistro_name) build_file = build_files[doc_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name) # all further configuration will be handled by either the Jenkins API # or by a generated groovy script from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if groovy_script is None else False view_configs = {} views = {} views[doc_view_name] = configure_doc_view(jenkins, doc_view_name, dry_run=dry_run) if not jenkins: view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(view_configs), } repo_names = dist_file.repositories.keys() filtered_repo_names = build_file.filter_repositories(repo_names) job_names = [] job_configs = OrderedDict() for repo_name in sorted(repo_names): if whitelist_repository_names: if repo_name not in whitelist_repository_names: print( "Skipping repository '%s' not in explicitly passed list" % repo_name, file=sys.stderr) continue is_disabled = repo_name not in filtered_repo_names if is_disabled and build_file.skip_ignored_repositories: print("Skipping ignored repository '%s'" % repo_name, file=sys.stderr) continue repo = dist_file.repositories[repo_name] if not repo.doc_repository: print("Skipping repository '%s': no doc section" % repo_name) continue if not repo.doc_repository.version: print("Skipping repository '%s': no doc version" % repo_name) continue for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_doc_job( config_url, rosdistro_name, doc_build_name, repo_name, os_name, os_code_name, arch, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script, dry_run=dry_run) job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(job_configs) groovy_data['job_prefixes_and_names'] = {} job_prefix = '%s__' % doc_view_name if not whitelist_repository_names: groovy_data['job_prefixes_and_names']['doc'] = (job_prefix, job_names) if groovy_script is None: # delete obsolete jobs in this view from ros_buildfarm.jenkins import remove_jobs print('Removing obsolete doc jobs') remove_jobs(jenkins, job_prefix, job_names, dry_run=dry_run) if groovy_script is not None: print( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % (groovy_script, len(view_configs), len(job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, job_configs, view_configs=view_configs)
def _get_blocked_releases_info(config_url, rosdistro_name, repo_names=None): import rosdistro from rosdistro.dependency_walker import DependencyWalker config = get_config_index(config_url) index = rosdistro.get_index(config.rosdistro_index_url) print('Checking packages for "%s" distribution' % rosdistro_name) # Find the previous distribution to the current one try: prev_rosdistro_name = _prev_rosdistro(index, rosdistro_name) except ValueError as e: print(e.args[0], file=sys.stderr) exit(-1) cache = rosdistro.get_distribution_cache(index, rosdistro_name) distro_file = cache.distribution_file prev_cache = rosdistro.get_distribution_cache(index, prev_rosdistro_name) prev_distribution = rosdistro.get_cached_distribution(index, prev_rosdistro_name, cache=prev_cache) prev_distro_file = prev_cache.distribution_file dependency_walker = DependencyWalker(prev_distribution) # Check missing dependencies for packages that were in the previous # distribution that have not yet been released in the current distribution # Filter repos without a version or a release repository prev_repo_names = set(_released_repos(prev_distro_file)) if repo_names is not None: ignored_inputs = prev_repo_names.difference(repo_names) prev_repo_names.intersection_update(repo_names) repo_names = prev_repo_names if len(ignored_inputs) > 0: print( 'Ignoring inputs for which repository info not found in previous distribution ' '(did you list a package instead of a repository?):') print('\n'.join( sorted('\t{0}'.format(repo) for repo in ignored_inputs))) current_repo_names = set(_released_repos(distro_file)) # Get a list of currently released packages current_package_names = set( _released_packages(distro_file, current_repo_names)) released_repos = prev_repo_names.intersection(current_repo_names) if prev_repo_names.issubset(current_repo_names): print('All inputs already released in {0}.'.format(rosdistro_name)) repos_info = defaultdict(dict) unprocessed_repos = prev_repo_names while unprocessed_repos: print('Processing repos:\n%s' % '\n'.join(['- %s' % r for r in sorted(unprocessed_repos)])) new_repos_to_process = set( ) # set containing repos that come up while processing others for repo_name in unprocessed_repos: repos_info[repo_name]['released'] = repo_name in released_repos if repo_name in released_repos: repo = distro_file.repositories[repo_name] version = repo.release_repository.version repos_info[repo_name]['version'] = version else: # Gather info on which required repos have not been released yet # Assume dependencies will be the same as in the previous distribution and find # which ones have been released repo = prev_distro_file.repositories[repo_name] release_repo = repo.release_repository package_dependencies = set() packages = release_repo.package_names # Accumulate all dependencies for those packages for package in packages: package_dependencies.update( _package_dependencies(dependency_walker, package)) # For all package dependencies, check if they are released yet unreleased_pkgs = package_dependencies.difference( current_package_names) # Remove the packages which this repo provides unreleased_pkgs = unreleased_pkgs.difference(packages) # Get maintainer info and repo of unreleased packages maintainers = defaultdict(dict) repos_blocked_by = set() for pkg_name in unreleased_pkgs: unreleased_repo_name = \ prev_distro_file.release_packages[pkg_name].repository_name repos_blocked_by.add(unreleased_repo_name) maintainers[unreleased_repo_name].update( dict(_maintainers(prev_distribution, pkg_name))) if maintainers: repos_info[repo_name]['maintainers'] = maintainers repos_info[repo_name]['repos_blocked_by'] = {} for blocking_repo_name in repos_blocked_by: # Get url of blocking repos repo_url = _repo_url(prev_distribution, blocking_repo_name) repos_info[repo_name]['repos_blocked_by'].update( {blocking_repo_name: repo_url}) # Mark blocking relationship in other direction if blocking_repo_name not in repos_info: new_repos_to_process.add(blocking_repo_name) repos_info[blocking_repo_name] = {} if 'repos_blocking' not in repos_info[blocking_repo_name]: repos_info[blocking_repo_name]['repos_blocking'] = set( []) repos_info[blocking_repo_name]['repos_blocking'].add( repo_name) # Get url of repo repo_url = _repo_url(prev_distribution, repo_name) if repo_url: repos_info[repo_name]['url'] = repo_url new_repos_to_process.discard( repo_name) # this repo has been fully processed now for repo_name in repos_info.keys(): # Recursively get all repos being blocked by this repo recursive_blocks = set([]) repos_to_check = set([repo_name]) while repos_to_check: next_repo_to_check = repos_to_check.pop() blocks = repos_info[next_repo_to_check].get( 'repos_blocking', set([])) new_blocks = blocks - recursive_blocks repos_to_check |= new_blocks recursive_blocks |= new_blocks if recursive_blocks: repos_info[repo_name][ 'recursive_repos_blocking'] = recursive_blocks unprocessed_repos = new_repos_to_process return repos_info
def run_audit(config_url, rosdistro_name, cache_dir): config = get_config_index(config_url) index = get_index(config.rosdistro_index_url) dist_file = get_distribution_file(index, rosdistro_name) dist_cache = get_distribution_cache(index, rosdistro_name) build_files = get_release_build_files(config, rosdistro_name) missing_packages = {} for bf_name, bf_value in build_files.items(): missing_packages[bf_name] = copy.deepcopy(bf_value.targets) for target in bf_value.get_targets_list(): all_pkgs, missing_pkgs = partition_packages( config_url, rosdistro_name, bf_name, target, cache_dir, deduplicate_dependencies=True, dist_cache=dist_cache) missing_packages[bf_name][target] = missing_pkgs if 'all' in missing_packages[bf_name]: missing_packages[bf_name]['all'] &= missing_pkgs else: missing_packages[bf_name]['all'] = missing_pkgs if 'all' in missing_packages: missing_packages['all'] &= missing_pkgs else: missing_packages['all'] = missing_pkgs recommended_actions = len(missing_packages['all']) print('# Sync preparation report for %s' % rosdistro_name) print('Prepared for configuration: %s' % config_url) print('Prepared for rosdistro index: %s' % config.rosdistro_index_url) print('\n\n') if missing_packages['all']: print('## Packages failing on all platforms\n\n' 'These releases are recommended to be rolled back:\n') for mp in sorted(missing_packages['all']): print(' - %s ' % mp) print('\n\n') else: print('## No packages detected failing on all platforms\n\n') def get_package_repository_link(dist_file, pkg_name): """Return the best guess of the url for filing a ticket against the package.""" pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] if repo.source_repository and repo.source_repository.url: return repo.source_repository.url if repo.release_repository and repo.release_repository.url: return repo.release_repository.url return None for bf_name in build_files.keys(): print('## Audit of buildfile %s\n\n' % bf_name) # TODO(tfoote) use rosdistro API to print the release build config for editing recommended_blacklists = sorted(missing_packages[bf_name]['all'] - missing_packages['all']) recommended_actions += len(recommended_blacklists) if not recommended_blacklists: print( 'Congratulations! ' 'No packages are failing to build on all targets for this buildfile.\n\n' ) continue print( 'Attention! ' 'The following packages are failing to build on all targets for this buildfile. ' 'It is recommended to blacklist them in the buildfile.\n\n') for rb in recommended_blacklists: print(' - %s:' % rb) jenkins_urls = get_jenkins_job_urls( rosdistro_name, config.jenkins_url, bf_name, build_files[bf_name].get_targets_list()) url = get_package_repository_link(dist_file, rb) print(' - Suggested ticket location [%s](%s)' % (url, url)) print('') print(' Title:') print('') print(' %s in %s fails to build on %s targets' % (rb, rosdistro_name, bf_name)) print('') print(' Body:') print('') print( ' The package %s in %s has been detected as not building' % (rb, rosdistro_name) + ' on all platforms in the buildfile %s.' % (bf_name) + ' The release manager for %s will consider disabling' % (rosdistro_name) + ' this build if it continues to fail to build.') print(' - jenkins_urls:') for target, ju in jenkins_urls.items(): target_str = ' '.join([x for x in target]) url = ju.format(pkg=rb) print(' - [%s](%s)' % (target_str, url)) # TODO(tfoote) embed build status when buildfarm has https # print(' - %s [![Build Status](%s)](%s)' % (' '.join([x for x in target]), # ju.format(pkg = rb) + '/badge/icon', ju.format(pkg = rb))) print( ' This is being filed because this package is about to be blacklisted.' ' If this ticket is resolved please review whether it can be removed from' ' the blacklist that should cross reference here.') print('') return recommended_actions
def configure_doc_jobs( config_url, rosdistro_name, doc_build_name, groovy_script=None): """ Configure all Jenkins doc jobs. L{configure_doc_job} will be invoked for doc repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_doc_build_files(config, rosdistro_name) build_file = build_files[doc_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name) from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) views = [] views.append(configure_doc_view(jenkins, doc_view_name)) if groovy_script is not None: # all further configuration will be handled by the groovy script jenkins = False repo_names = dist_file.repositories.keys() filtered_repo_names = build_file.filter_repositories(repo_names) job_names = [] job_configs = {} for repo_name in sorted(repo_names): is_disabled = repo_name not in filtered_repo_names if is_disabled and build_file.skip_ignored_repositories: print("Skipping ignored repository '%s'" % repo_name, file=sys.stderr) continue repo = dist_file.repositories[repo_name] if not repo.doc_repository: print("Skipping repository '%s': no doc section" % repo_name) continue if not repo.doc_repository.version: print("Skipping repository '%s': no doc version" % repo_name) continue for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_doc_job( config_url, rosdistro_name, doc_build_name, repo_name, os_name, os_code_name, arch, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script) job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) job_prefix = '%s__' % doc_view_name if groovy_script is None: # delete obsolete jobs in this view from ros_buildfarm.jenkins import remove_jobs print('Removing obsolete doc jobs') remove_jobs(jenkins, job_prefix, job_names) else: print("Writing groovy script '%s' to reconfigure %d jobs" % (groovy_script, len(job_configs))) data = { 'expected_num_jobs': len(job_configs), 'job_prefixes_and_names': { 'doc': (job_prefix, job_names), } } content = expand_template('snippet/reconfigure_jobs.groovy.em', data) write_groovy_script_and_configs( groovy_script, content, job_configs)
def configure_devel_jobs( config_url, rosdistro_name, source_build_name): """ Configure all Jenkins devel jobs. L{configure_release_job} will be invoked for source repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_source_build_files(config, rosdistro_name) build_file = build_files[source_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name) jenkins = connect(config.jenkins_url) view_name = get_devel_view_name(rosdistro_name, source_build_name) view = configure_devel_view(jenkins, view_name) repo_names = dist_file.repositories.keys() repo_names = build_file.filter_repositories(repo_names) job_names = [] for repo_name in sorted(repo_names): repo = dist_file.repositories[repo_name] if not repo.source_repository: print("Skipping repository '%s': no source section" % repo_name) continue if not repo.source_repository.version: print("Skipping repository '%s': no source version" % repo_name) continue for os_name, os_code_name, arch in targets: try: job_name = configure_devel_job( config_url, rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, view=view) job_names.append(job_name) except JobValidationError as e: print(e.message, file=sys.stderr) # delete obsolete jobs in this view remove_jobs(jenkins, '%s__' % view_name, job_names)
def main(argv=sys.argv[1:]): global templates parser = argparse.ArgumentParser( description="Generate a 'prerelease' script") add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_build_name(parser, 'source') add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_argument_output_dir(parser, required=True) group = parser.add_argument_group( 'Repositories in underlay workspace', description='The repositories in the underlay workspace will be ' + 'built and installed as well as built and tested. ' + 'Dependencies will be provided by binary packages.') group.add_argument( 'source_repos', nargs='*', default=[], metavar='REPO_NAME', help="A name of a 'repository' from the distribution file") group.add_argument( '--custom-branch', nargs='*', type=_repository_name_and_branch, default=[], metavar='REPO_NAME:BRANCH_OR_TAG_NAME', help="A name of a 'repository' from the distribution file followed " + 'by a colon and a branch / tag name') group.add_argument( '--custom-repo', nargs='*', type=_repository_name_and_type_and_url_and_branch, default=[], metavar='REPO_NAME:REPO_TYPE:REPO_URL:BRANCH_OR_TAG_NAME', help='The name, type, url and branch / tag name of a repository') add_overlay_arguments(parser) args = parser.parse_args(argv) print('Fetching buildfarm configuration...') config = get_config_index(args.config_url) build_files = get_source_build_files(config, args.rosdistro_name) build_file = build_files[args.source_build_name] print('Fetching rosdistro cache...') index = get_index(config.rosdistro_index_url) dist_cache = get_distribution_cache(index, args.rosdistro_name) dist_file = dist_cache.distribution_file # determine source repositories for underlay workspace repositories = {} for repo_name in args.source_repos: if repo_name in repositories: print("The repository '%s' appears multiple times" % repo_name, file=sys.stderr) return 1 try: repositories[repo_name] = \ dist_file.repositories[repo_name].source_repository except KeyError: print(("The repository '%s' was not found in the distribution " + "file") % repo_name, file=sys.stderr) return 1 for repo_name, custom_version in args.custom_branch: if repo_name in repositories: print("The repository '%s' appears multiple times" % repo_name, file=sys.stderr) return 1 try: source_repo = dist_file.repositories[repo_name].source_repository except KeyError: print(("The repository '%s' was not found in the distribution " + "file") % repo_name, file=sys.stderr) return 1 source_repo = deepcopy(source_repo) source_repo.version = custom_version repositories[repo_name] = source_repo for repo_name, repo_type, repo_url, version in args.custom_repo: if repo_name in repositories: print("The repository '%s' appears multiple times" % repo_name, file=sys.stderr) return 1 source_repo = RepositorySpecification( repo_name, { 'type': repo_type, 'url': repo_url, 'version': version, }) repositories[repo_name] = source_repo scms = [(repositories[k], 'catkin_workspace/src/%s' % k) for k in sorted(repositories.keys())] # collect all template snippets of specific types class IncludeHook(Hook): def __init__(self): Hook.__init__(self) self.scripts = [] def beforeInclude(self, *args, **kwargs): template_path = kwargs['file'].name print(template_path, file=sys.stderr) if template_path.endswith('/snippet/builder_shell.xml.em'): self.scripts.append(kwargs['locals']['script']) hook = IncludeHook() templates.template_hooks = [hook] # use random source repo to pass to devel job template source_repository = deepcopy(list(repositories.values())[0]) source_repository.name = 'prerelease' print('Evaluating job templates...') configure_devel_job( args.config_url, args.rosdistro_name, args.source_build_name, None, args.os_name, args.os_code_name, args.arch, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=False, views=False, source_repository=source_repository) templates.template_hooks = None # derive scripts for overlay workspace from underlay overlay_scripts = [] for script in hook.scripts: # skip cloning of ros_buildfarm repository if 'git clone' in script and '.git ros_buildfarm' in script: continue # skip build-and-install step if 'build and install' in script: continue # add prerelease overlay flag run_devel_job = '/run_devel_job.py' if run_devel_job in script: script = script.replace( run_devel_job, run_devel_job + ' --prerelease-overlay') # replace mounted workspace volume with overlay and underlay # used by: # - create_devel_task_generator.py needs to find packages in both # the underlay as well as the overlay workspace # - catkin_make_isolated_and_test.py needs to source the environment of # the underlay before building the overlay mount_volume = '-v $WORKSPACE/catkin_workspace:/tmp/catkin_workspace' if mount_volume in script: script = script.replace( mount_volume, mount_volume + ':ro ' + '-v $WORKSPACE/' + 'catkin_workspace_overlay:/tmp/catkin_workspace_overlay') # relocate all docker files docker_path = '$WORKSPACE/docker_' if docker_path in script: script = script.replace( docker_path, docker_path + 'overlay_') # rename all docker images name_suffix = '_prerelease' if name_suffix in script: script = script.replace( name_suffix, name_suffix + '_overlay') overlay_scripts.append(script) from ros_buildfarm import __file__ as ros_buildfarm_file data = deepcopy(args.__dict__) data.update({ 'scms': scms, 'scripts': hook.scripts, 'overlay_scripts': overlay_scripts, 'ros_buildfarm_python_path': os.path.dirname( os.path.dirname(os.path.abspath(ros_buildfarm_file))), 'python_executable': sys.executable, 'prerelease_script_path': os.path.dirname(os.path.abspath(__file__))}) if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) # generate multiple scripts for script_name in [ 'prerelease', 'prerelease_build_overlay', 'prerelease_build_underlay', 'prerelease_clone_overlay', 'prerelease_clone_underlay']: content = expand_template( 'prerelease/%s_script.sh.em' % script_name, data, options={BANGPATH_OPT: False}) script_file = os.path.join(args.output_dir, script_name + '.sh') with open(script_file, 'w') as h: h.write(content) os.chmod(script_file, os.stat(script_file).st_mode | stat.S_IEXEC) print('') print('Generated prerelease script - to execute it run:') if os.path.abspath(args.output_dir) != os.path.abspath(os.curdir): print(' cd %s' % args.output_dir) print(' ./prerelease.sh')
def configure_devel_job( config_url, rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, view=None): if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_source_build_files(config, rosdistro_name) build_file = build_files[source_build_name] if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name) repo_names = dist_file.repositories.keys() repo_names = build_file.filter_repositories(repo_names) if repo_name not in repo_names: return "Invalid repository name '%s' " % repo_name + \ 'choose one of the following: ' + \ ', '.join(sorted(repo_names)) repo = dist_file.repositories[repo_name] if not repo.source_repository: return "Repository '%s' has no source section" % repo_name if not repo.source_repository.version: return "Repository '%s' has no source version" % repo_name if os_name not in build_file.targets.keys(): return "Invalid OS name '%s' " % os_name + \ 'choose one of the following: ' + \ ', '.join(sorted(build_file.targets.keys())) if os_code_name not in build_file.targets[os_name].keys(): return "Invalid OS code name '%s' " % os_code_name + \ 'choose one of the following: ' + \ ', '.join(sorted(build_file.targets[os_name].keys())) if arch not in build_file.targets[os_name][os_code_name]: return "Invalid architecture '%s' " % arch + \ 'choose one of the following: ' + \ ', '.join(sorted( build_file.targets[os_name][os_code_name])) if dist_cache is None and build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: jenkins = connect(config.jenkins_url) if view is None: view_name = get_devel_view_name(rosdistro_name, source_build_name) configure_devel_view(jenkins, view_name) job_name = get_devel_job_name( rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch) job_config = _get_devel_job_config( config, rosdistro_name, source_build_name, build_file, os_name, os_code_name, arch, repo.source_repository, repo_name, dist_cache=dist_cache) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, job_name, job_config)
def configure_devel_jobs(config_url, rosdistro_name, source_build_name, groovy_script=None, dry_run=False, whitelist_repository_names=None): """ Configure all Jenkins devel jobs. L{configure_release_job} will be invoked for source repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_source_build_files(config, rosdistro_name) build_file = build_files[source_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return devel_view_name = get_devel_view_name(rosdistro_name, source_build_name, pull_request=False) pull_request_view_name = get_devel_view_name(rosdistro_name, source_build_name, pull_request=True) # all further configuration will be handled by either the Jenkins API # or by a generated groovy script from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if groovy_script is None else False view_configs = {} views = {} if build_file.test_commits_force is not False: views[devel_view_name] = configure_devel_view(jenkins, devel_view_name, dry_run=dry_run) if build_file.test_pull_requests_force is not False: views[pull_request_view_name] = configure_devel_view( jenkins, pull_request_view_name, dry_run=dry_run) if not jenkins: view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(view_configs), } repo_names = dist_file.repositories.keys() filtered_repo_names = build_file.filter_repositories(repo_names) devel_job_names = [] pull_request_job_names = [] job_configs = OrderedDict() for repo_name in sorted(repo_names): if whitelist_repository_names: if repo_name not in whitelist_repository_names: print( "Skipping repository '%s' not in explicitly passed list" % repo_name, file=sys.stderr) continue is_disabled = repo_name not in filtered_repo_names if is_disabled and build_file.skip_ignored_repositories: print("Skipping ignored repository '%s'" % repo_name, file=sys.stderr) continue repo = dist_file.repositories[repo_name] if not repo.source_repository: print("Skipping repository '%s': no source section" % repo_name) continue if not repo.source_repository.version: print("Skipping repository '%s': no source version" % repo_name) continue job_types = [] # check for testing commits if build_file.test_commits_force is False: print(("Skipping repository '%s': 'test_commits' is forced to " + "false in the build file") % repo_name) elif repo.source_repository.test_commits is False: print(("Skipping repository '%s': 'test_commits' of the " + "repository set to false") % repo_name) elif repo.source_repository.test_commits is None and \ not build_file.test_commits_default: print(("Skipping repository '%s': 'test_commits' defaults to " + "false in the build file") % repo_name) else: job_types.append('commit') if not is_disabled: # check for testing pull requests if build_file.test_pull_requests_force is False: # print(("Skipping repository '%s': 'test_pull_requests' " + # "is forced to false in the build file") % repo_name) pass elif repo.source_repository.test_pull_requests is False: # print(("Skipping repository '%s': 'test_pull_requests' of " + # "the repository set to false") % repo_name) pass elif repo.source_repository.test_pull_requests is None and \ not build_file.test_pull_requests_default: # print(("Skipping repository '%s': 'test_pull_requests' " + # "defaults to false in the build file") % repo_name) pass else: print("Pull request job for repository '%s'" % repo_name) job_types.append('pull_request') for job_type in job_types: pull_request = job_type == 'pull_request' for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_devel_job( config_url, rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, pull_request, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script, dry_run=dry_run) if not pull_request: devel_job_names.append(job_name) else: pull_request_job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(job_configs) groovy_data['job_prefixes_and_names'] = {} devel_job_prefix = '%s__' % devel_view_name pull_request_job_prefix = '%s__' % pull_request_view_name if not whitelist_repository_names: groovy_data['job_prefixes_and_names']['devel'] = \ (devel_job_prefix, devel_job_names) groovy_data['job_prefixes_and_names']['pull_request'] = \ (pull_request_job_prefix, pull_request_job_names) if groovy_script is None: # delete obsolete jobs in these views from ros_buildfarm.jenkins import remove_jobs print('Removing obsolete devel jobs') remove_jobs(jenkins, devel_job_prefix, devel_job_names, dry_run=dry_run) print('Removing obsolete pull request jobs') remove_jobs(jenkins, pull_request_job_prefix, pull_request_job_names, dry_run=dry_run) if groovy_script is not None: print( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % (groovy_script, len(view_configs), len(job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, job_configs, view_configs=view_configs)
def _get_blocked_releases_info(config_url, rosdistro_name, repo_names=None): import rosdistro from rosdistro.dependency_walker import DependencyWalker from catkin_pkg.package import InvalidPackage, parse_package_string prev_rosdistro_name = None config = get_config_index(config_url) index = rosdistro.get_index(config.rosdistro_index_url) valid_rosdistro_names = list(index.distributions.keys()) valid_rosdistro_names.sort() if rosdistro_name is None: rosdistro_name = valid_rosdistro_names[-1] print('Checking packages for "%s" distribution' % rosdistro_name) # skip distributions with a different type if the information is available distro_type = index.distributions[rosdistro_name].get('distribution_type') if distro_type is not None: valid_rosdistro_names = [ n for n in valid_rosdistro_names if distro_type == index.distributions[n].get('distribution_type')] # Find the previous distribution to the current one try: i = valid_rosdistro_names.index(rosdistro_name) except ValueError: print('Distribution key not found in list of valid distributions.', file=sys.stderr) exit(-1) if i == 0: print('No previous distribution found.', file=sys.stderr) exit(-1) prev_rosdistro_name = valid_rosdistro_names[i - 1] cache = rosdistro.get_distribution_cache(index, rosdistro_name) distro_file = cache.distribution_file prev_cache = rosdistro.get_distribution_cache(index, prev_rosdistro_name) prev_distribution = rosdistro.get_cached_distribution( index, prev_rosdistro_name, cache=prev_cache) prev_distro_file = prev_cache.distribution_file dependency_walker = DependencyWalker(prev_distribution) if repo_names is None: # Check missing dependencies for packages that were in the previous # distribution that have not yet been released in the current distribution # Filter repos without a version or a release repository keys = prev_distro_file.repositories.keys() prev_repo_names = set( repo for repo in keys if _is_released(repo, prev_distro_file)) repo_names = prev_repo_names ignored_inputs = [] else: prev_repo_names = set( repo for repo in repo_names if _is_released(repo, prev_distro_file)) ignored_inputs = list(set(repo_names).difference(prev_repo_names)) if len(ignored_inputs) > 0: print( 'Ignoring inputs for which repository info not found in previous distribution ' '(did you list a package instead of a repository?):') print('\n'.join( sorted('\t{0}'.format(repo) for repo in ignored_inputs))) keys = distro_file.repositories.keys() current_repo_names = set( repo for repo in keys if _is_released(repo, distro_file)) # Get a list of currently released packages current_package_names = set( pkg for repo in current_repo_names for pkg in distro_file.repositories[repo].release_repository.package_names) released_repos = prev_repo_names.intersection( current_repo_names) unreleased_repos = list(prev_repo_names.difference( current_repo_names)) if len(unreleased_repos) == 0: print('All inputs already released in {0}.'.format( rosdistro_name)) repos_info = defaultdict(dict) unprocessed_repos = prev_repo_names while unprocessed_repos: print('Processing repos:\n%s' % '\n'.join(['- %s' % r for r in sorted(unprocessed_repos)])) new_repos_to_process = set() # set containing repos that come up while processing others for repo_name in unprocessed_repos: repos_info[repo_name]['released'] = repo_name in released_repos if repo_name in released_repos: repo = distro_file.repositories[repo_name] version = repo.release_repository.version repos_info[repo_name]['version'] = version else: # Gather info on which required repos have not been released yet # Assume dependencies will be the same as in the previous distribution and find # which ones have been released repo = prev_distro_file.repositories[repo_name] release_repo = repo.release_repository package_dependencies = set() packages = release_repo.package_names # Accumulate all dependencies for those packages for package in packages: try: package_dependencies |= dependency_walker.get_recursive_depends( package, ['build', 'buildtool', 'run', 'test'], ros_packages_only=True, limit_depth=1) except AssertionError as e: print(e, file=sys.stderr) # For all package dependencies, check if they are released yet unreleased_pkgs = package_dependencies.difference( current_package_names) # Remove the packages which this repo provides unreleased_pkgs = unreleased_pkgs.difference(packages) # Get maintainer info and repo of unreleased packages maintainers = {} repos_blocked_by = set() for pkg_name in unreleased_pkgs: unreleased_repo_name = \ prev_distro_file.release_packages[pkg_name].repository_name repos_blocked_by.add(unreleased_repo_name) pkg_xml = prev_distribution.get_release_package_xml(pkg_name) if pkg_xml is not None: try: pkg = parse_package_string(pkg_xml) except InvalidPackage: pass else: pkg_maintainers = {m.name: m.email for m in pkg.maintainers} if unreleased_repo_name not in maintainers: maintainers[unreleased_repo_name] = {} maintainers[unreleased_repo_name].update(pkg_maintainers) if maintainers: repos_info[repo_name]['maintainers'] = maintainers repos_info[repo_name]['repos_blocked_by'] = {} for blocking_repo_name in repos_blocked_by: # Get url of blocking repos repo_url = None blocking_repo = prev_distro_file.repositories[blocking_repo_name] if blocking_repo.source_repository: repo_url = blocking_repo.source_repository.url elif blocking_repo.doc_repository: repo_url = blocking_repo.doc_repository.url repos_info[repo_name]['repos_blocked_by'].update( {blocking_repo_name: repo_url}) # Mark blocking relationship in other direction if blocking_repo_name not in repos_info: new_repos_to_process.add(blocking_repo_name) repos_info[blocking_repo_name] = {} if 'repos_blocking' not in repos_info[blocking_repo_name]: repos_info[blocking_repo_name]['repos_blocking'] = set([]) repos_info[blocking_repo_name]['repos_blocking'].add(repo_name) # Get url of repo repo_url = None if repo.source_repository: repo_url = repo.source_repository.url elif repo.doc_repository: repo_url = repo.doc_repository.url if repo_url: repos_info[repo_name]['url'] = repo_url new_repos_to_process.discard(repo_name) # this repo has been fully processed now for repo_name in repos_info.keys(): # Recursively get all repos being blocked by this repo recursive_blocks = set([]) repos_to_check = set([repo_name]) while repos_to_check: next_repo_to_check = repos_to_check.pop() blocks = repos_info[next_repo_to_check].get('repos_blocking', set([])) new_blocks = blocks - recursive_blocks repos_to_check |= new_blocks recursive_blocks |= new_blocks if recursive_blocks: repos_info[repo_name]['recursive_repos_blocking'] = recursive_blocks unprocessed_repos = new_repos_to_process return repos_info