def configure_devel_jobs(config_url, rosdistro_name, source_build_name, groovy_script=None, dry_run=False, whitelist_repository_names=None): """ Configure all Jenkins devel jobs. L{configure_release_job} will be invoked for source repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_source_build_files(config, rosdistro_name) build_file = build_files[source_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return devel_view_name = get_devel_view_name(rosdistro_name, source_build_name, pull_request=False) pull_request_view_name = get_devel_view_name(rosdistro_name, source_build_name, pull_request=True) # all further configuration will be handled by either the Jenkins API # or by a generated groovy script from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if groovy_script is None else False view_configs = {} views = {} if build_file.test_commits_force is not False: views[devel_view_name] = configure_devel_view(jenkins, devel_view_name, dry_run=dry_run) if build_file.test_pull_requests_force is not False: views[pull_request_view_name] = configure_devel_view( jenkins, pull_request_view_name, dry_run=dry_run) if not jenkins: view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(view_configs), } repo_names = dist_file.repositories.keys() filtered_repo_names = build_file.filter_repositories(repo_names) devel_job_names = [] pull_request_job_names = [] job_configs = OrderedDict() for repo_name in sorted(repo_names): if whitelist_repository_names: if repo_name not in whitelist_repository_names: print( "Skipping repository '%s' not in explicitly passed list" % repo_name, file=sys.stderr) continue is_disabled = repo_name not in filtered_repo_names if is_disabled and build_file.skip_ignored_repositories: print("Skipping ignored repository '%s'" % repo_name, file=sys.stderr) continue repo = dist_file.repositories[repo_name] if not repo.source_repository: print("Skipping repository '%s': no source section" % repo_name) continue if not repo.source_repository.version: print("Skipping repository '%s': no source version" % repo_name) continue job_types = [] # check for testing commits if build_file.test_commits_force is False: print(("Skipping repository '%s': 'test_commits' is forced to " + "false in the build file") % repo_name) elif repo.source_repository.test_commits is False: print(("Skipping repository '%s': 'test_commits' of the " + "repository set to false") % repo_name) elif repo.source_repository.test_commits is None and \ not build_file.test_commits_default: print(("Skipping repository '%s': 'test_commits' defaults to " + "false in the build file") % repo_name) else: job_types.append('commit') if not is_disabled: # check for testing pull requests if build_file.test_pull_requests_force is False: # print(("Skipping repository '%s': 'test_pull_requests' " + # "is forced to false in the build file") % repo_name) pass elif repo.source_repository.test_pull_requests is False: # print(("Skipping repository '%s': 'test_pull_requests' of " + # "the repository set to false") % repo_name) pass elif repo.source_repository.test_pull_requests is None and \ not build_file.test_pull_requests_default: # print(("Skipping repository '%s': 'test_pull_requests' " + # "defaults to false in the build file") % repo_name) pass else: print("Pull request job for repository '%s'" % repo_name) job_types.append('pull_request') for job_type in job_types: pull_request = job_type == 'pull_request' for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_devel_job( config_url, rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, pull_request, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script, dry_run=dry_run) if not pull_request: devel_job_names.append(job_name) else: pull_request_job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(job_configs) groovy_data['job_prefixes_and_names'] = {} devel_job_prefix = '%s__' % devel_view_name pull_request_job_prefix = '%s__' % pull_request_view_name if not whitelist_repository_names: groovy_data['job_prefixes_and_names']['devel'] = \ (devel_job_prefix, devel_job_names) groovy_data['job_prefixes_and_names']['pull_request'] = \ (pull_request_job_prefix, pull_request_job_names) if groovy_script is None: # delete obsolete jobs in these views from ros_buildfarm.jenkins import remove_jobs print('Removing obsolete devel jobs') remove_jobs(jenkins, devel_job_prefix, devel_job_names, dry_run=dry_run) print('Removing obsolete pull request jobs') remove_jobs(jenkins, pull_request_job_prefix, pull_request_job_names, dry_run=dry_run) if groovy_script is not None: print( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % (groovy_script, len(view_configs), len(job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, job_configs, view_configs=view_configs)
def configure_release_jobs(config_url, rosdistro_name, release_build_name, groovy_script=None, dry_run=False, whitelist_package_names=None): """ Configure all Jenkins release jobs. L{configure_release_job} will be invoked for every released package and target which matches the build file criteria. Additionally a job to import Debian packages into the Debian repository is created. """ config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) # get targets platforms = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): platforms.append((os_name, os_code_name)) print('The build file contains the following targets:') for os_name, os_code_name in platforms: print(' - %s %s: %s' % (os_name, os_code_name, ', '.join( build_file.targets[os_name][os_code_name]))) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return pkg_names = dist_file.release_packages.keys() filtered_pkg_names = build_file.filter_packages(pkg_names) explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names) if explicitly_ignored_pkg_names: print(('The following packages are being %s because of ' + 'white-/blacklisting:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(explicitly_ignored_pkg_names): print(' -', pkg_name) dist_cache = get_distribution_cache(index, rosdistro_name) if explicitly_ignored_pkg_names: # get direct dependencies from distro cache for each package direct_dependencies = {} for pkg_name in pkg_names: direct_dependencies[pkg_name] = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) or set([]) # find recursive downstream deps for all explicitly ignored packages ignored_pkg_names = set(explicitly_ignored_pkg_names) while True: implicitly_ignored_pkg_names = _get_downstream_package_names( ignored_pkg_names, direct_dependencies) if implicitly_ignored_pkg_names - ignored_pkg_names: ignored_pkg_names |= implicitly_ignored_pkg_names continue break implicitly_ignored_pkg_names = \ ignored_pkg_names - explicitly_ignored_pkg_names if implicitly_ignored_pkg_names: print(('The following packages are being %s because their ' + 'dependencies are being ignored:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(implicitly_ignored_pkg_names): print(' -', pkg_name) filtered_pkg_names = \ set(filtered_pkg_names) - implicitly_ignored_pkg_names # all further configuration will be handled by either the Jenkins API # or by a generated groovy script jenkins = False if groovy_script is None: from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) all_view_configs = {} all_job_configs = OrderedDict() job_name, job_config = configure_import_package_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config job_name, job_config = configure_sync_packages_to_main_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config for os_name, os_code_name in platforms: for arch in sorted(build_file.targets[os_name][os_code_name]): job_name, job_config = configure_sync_packages_to_testing_job( config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config targets = [] for os_name, os_code_name in platforms: targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) views = configure_release_views(jenkins, rosdistro_name, release_build_name, targets, dry_run=dry_run) if not jenkins: all_view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(views), } # binary jobs must be generated in topological order from catkin_pkg.package import parse_package_string from ros_buildfarm.common import topological_order_packages pkgs = {} for pkg_name in pkg_names: if pkg_name not in dist_cache.release_package_xmls: print("Skipping package '%s': no released package.xml in cache" % (pkg_name), file=sys.stderr) continue pkg_xml = dist_cache.release_package_xmls[pkg_name] pkg = parse_package_string(pkg_xml) pkgs[pkg_name] = pkg ordered_pkg_tuples = topological_order_packages(pkgs) other_build_files = [ v for k, v in build_files.items() if k != release_build_name ] all_source_job_names = [] all_binary_job_names = [] for pkg_name in [p.name for _, p in ordered_pkg_tuples]: if whitelist_package_names: if pkg_name not in whitelist_package_names: print( "Skipping package '%s' not in the explicitly passed list" % pkg_name, file=sys.stderr) continue pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] is_disabled = pkg_name not in filtered_pkg_names if is_disabled and build_file.skip_ignored_packages: print("Skipping ignored package '%s' in repository '%s'" % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository: print(("Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository.version: print(("Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name), file=sys.stderr) continue for os_name, os_code_name in platforms: other_build_files_same_platform = [] for other_build_file in other_build_files: if os_name not in other_build_file.targets: continue if os_code_name not in other_build_file.targets[os_name]: continue other_build_files_same_platform.append(other_build_file) try: source_job_names, binary_job_names, job_configs = \ configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, generate_import_package_job=False, generate_sync_packages_jobs=False, is_disabled=is_disabled, other_build_files_same_platform=other_build_files_same_platform, groovy_script=groovy_script, dry_run=dry_run) all_source_job_names += source_job_names all_binary_job_names += binary_job_names if groovy_script is not None: print('Configuration for jobs: ' + ', '.join(source_job_names + binary_job_names)) for source_job_name in source_job_names: all_job_configs[source_job_name] = job_configs[ source_job_name] for binary_job_name in binary_job_names: all_job_configs[binary_job_name] = job_configs[ binary_job_name] except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(all_job_configs) groovy_data['job_prefixes_and_names'] = {} # with an explicit list of packages we don't delete obsolete jobs if not whitelist_package_names: # delete obsolete binary jobs for os_name, os_code_name in platforms: for arch in build_file.targets[os_name][os_code_name]: binary_view = get_release_binary_view_name( rosdistro_name, release_build_name, os_name, os_code_name, arch) binary_job_prefix = '%s__' % binary_view excluded_job_names = set([ j for j in all_binary_job_names if j.startswith(binary_job_prefix) ]) if groovy_script is None: print("Removing obsolete binary jobs with prefix '%s'" % binary_job_prefix) from ros_buildfarm.jenkins import remove_jobs remove_jobs(jenkins, binary_job_prefix, excluded_job_names, dry_run=dry_run) else: binary_key = 'binary_%s_%s_%s' % \ (os_name, os_code_name, arch) groovy_data['job_prefixes_and_names'][binary_key] = \ (binary_job_prefix, excluded_job_names) # delete obsolete source jobs # requires knowledge about all other release build files for os_name, os_code_name in platforms: other_source_job_names = [] # get source job names for all other release build files for other_release_build_name in [ k for k in build_files.keys() if k != release_build_name ]: other_build_file = build_files[other_release_build_name] other_dist_file = get_distribution_file( index, rosdistro_name, other_build_file) if not other_dist_file: continue if os_name not in other_build_file.targets or \ os_code_name not in other_build_file.targets[os_name]: continue if other_build_file.skip_ignored_packages: filtered_pkg_names = other_build_file.filter_packages( pkg_names) else: filtered_pkg_names = pkg_names for pkg_name in sorted(filtered_pkg_names): pkg = other_dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = other_dist_file.repositories[repo_name] if not repo.release_repository: continue if not repo.release_repository.version: continue other_job_name = get_sourcedeb_job_name( rosdistro_name, other_release_build_name, pkg_name, os_name, os_code_name) other_source_job_names.append(other_job_name) source_view_prefix = get_release_source_view_name( rosdistro_name, os_name, os_code_name) source_job_prefix = '%s__' % source_view_prefix excluded_job_names = set([ j for j in (all_source_job_names + other_source_job_names) if j.startswith(source_job_prefix) ]) if groovy_script is None: print("Removing obsolete source jobs with prefix '%s'" % source_job_prefix) from ros_buildfarm.jenkins import remove_jobs remove_jobs(jenkins, source_job_prefix, excluded_job_names, dry_run=dry_run) else: source_key = 'source_%s_%s' % (os_name, os_code_name) groovy_data['job_prefixes_and_names'][source_key] = ( source_job_prefix, excluded_job_names) if groovy_script is not None: print( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % (groovy_script, len(all_view_configs), len(all_job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, all_job_configs, view_configs=all_view_configs)
def configure_doc_jobs( config_url, rosdistro_name, doc_build_name, groovy_script=None): """ Configure all Jenkins doc jobs. L{configure_doc_job} will be invoked for doc repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_doc_build_files(config, rosdistro_name) build_file = build_files[doc_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name) from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) views = [] views.append(configure_doc_view(jenkins, doc_view_name)) if groovy_script is not None: # all further configuration will be handled by the groovy script jenkins = False repo_names = dist_file.repositories.keys() filtered_repo_names = build_file.filter_repositories(repo_names) job_names = [] job_configs = {} for repo_name in sorted(repo_names): is_disabled = repo_name not in filtered_repo_names if is_disabled and build_file.skip_ignored_repositories: print("Skipping ignored repository '%s'" % repo_name, file=sys.stderr) continue repo = dist_file.repositories[repo_name] if not repo.doc_repository: print("Skipping repository '%s': no doc section" % repo_name) continue if not repo.doc_repository.version: print("Skipping repository '%s': no doc version" % repo_name) continue for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_doc_job( config_url, rosdistro_name, doc_build_name, repo_name, os_name, os_code_name, arch, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script) job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) job_prefix = '%s__' % doc_view_name if groovy_script is None: # delete obsolete jobs in this view from ros_buildfarm.jenkins import remove_jobs print('Removing obsolete doc jobs') remove_jobs(jenkins, job_prefix, job_names) else: print("Writing groovy script '%s' to reconfigure %d jobs" % (groovy_script, len(job_configs))) data = { 'expected_num_jobs': len(job_configs), 'job_prefixes_and_names': { 'doc': (job_prefix, job_names), } } content = expand_template('snippet/reconfigure_jobs.groovy.em', data) write_groovy_script_and_configs( groovy_script, content, job_configs)
def configure_devel_jobs( config_url, rosdistro_name, source_build_name, groovy_script=None): """ Configure all Jenkins devel jobs. L{configure_release_job} will be invoked for source repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_source_build_files(config, rosdistro_name) build_file = build_files[source_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return devel_view_name = get_devel_view_name( rosdistro_name, source_build_name, pull_request=False) pull_request_view_name = get_devel_view_name( rosdistro_name, source_build_name, pull_request=True) from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) views = [] if build_file.test_commits_force is not False: views.append(configure_devel_view(jenkins, devel_view_name)) if build_file.test_pull_requests_force is not False: views.append(configure_devel_view(jenkins, pull_request_view_name)) if groovy_script is not None: # all further configuration will be handled by the groovy script jenkins = False repo_names = dist_file.repositories.keys() filtered_repo_names = build_file.filter_repositories(repo_names) devel_job_names = [] pull_request_job_names = [] job_configs = {} for repo_name in sorted(repo_names): is_disabled = repo_name not in filtered_repo_names if is_disabled and build_file.skip_ignored_repositories: print("Skipping ignored repository '%s'" % repo_name, file=sys.stderr) continue repo = dist_file.repositories[repo_name] if not repo.source_repository: print("Skipping repository '%s': no source section" % repo_name) continue if not repo.source_repository.version: print("Skipping repository '%s': no source version" % repo_name) continue job_types = [] # check for testing commits if build_file.test_commits_force is False: print(("Skipping repository '%s': 'test_commits' is forced to " + "false in the build file") % repo_name) elif repo.source_repository.test_commits is False: print(("Skipping repository '%s': 'test_commits' of the " + "repository set to false") % repo_name) elif repo.source_repository.test_commits is None and \ not build_file.test_commits_default: print(("Skipping repository '%s': 'test_commits' defaults to " + "false in the build file") % repo_name) else: job_types.append('commit') if not is_disabled: # check for testing pull requests if build_file.test_pull_requests_force is False: # print(("Skipping repository '%s': 'test_pull_requests' " + # "is forced to false in the build file") % repo_name) pass elif repo.source_repository.test_pull_requests is False: # print(("Skipping repository '%s': 'test_pull_requests' of " + # "the repository set to false") % repo_name) pass elif repo.source_repository.test_pull_requests is None and \ not build_file.test_pull_requests_default: # print(("Skipping repository '%s': 'test_pull_requests' " + # "defaults to false in the build file") % repo_name) pass else: print("Pull request job for repository '%s'" % repo_name) job_types.append('pull_request') for job_type in job_types: pull_request = job_type == 'pull_request' for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_devel_job( config_url, rosdistro_name, source_build_name, repo_name, os_name, os_code_name, arch, pull_request, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script) if not pull_request: devel_job_names.append(job_name) else: pull_request_job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) devel_job_prefix = '%s__' % devel_view_name pull_request_job_prefix = '%s__' % pull_request_view_name if groovy_script is None: # delete obsolete jobs in these views from ros_buildfarm.jenkins import remove_jobs print('Removing obsolete devel jobs') remove_jobs(jenkins, devel_job_prefix, devel_job_names) print('Removing obsolete pull request jobs') remove_jobs( jenkins, pull_request_job_prefix, pull_request_job_names) else: print("Writing groovy script '%s' to reconfigure %d jobs" % (groovy_script, len(job_configs))) data = { 'expected_num_jobs': len(job_configs), 'job_prefixes_and_names': { 'devel': (devel_job_prefix, devel_job_names), 'pull_request': ( pull_request_job_prefix, pull_request_job_names), } } content = expand_template('snippet/reconfigure_jobs.groovy.em', data) write_groovy_script_and_configs( groovy_script, content, job_configs)
def _configure_ci_jobs(config, build_files, config_url, rosdistro_name, ci_build_name, groovy_script=None, dry_run=False): """Configure all Jenkins CI jobs for a specific CI build name.""" build_file = build_files[ci_build_name] index = get_index(config.rosdistro_index_url) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print("The build file '%s' contains the following targets:" % ci_build_name) for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return ci_view_name = get_ci_view_name(rosdistro_name) # all further configuration will be handled by either the Jenkins API # or by a generated groovy script from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if groovy_script is None else False view_configs = {} views = { ci_view_name: configure_ci_view(jenkins, ci_view_name, dry_run=dry_run) } if not jenkins: view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(view_configs), } ci_job_names = [] job_configs = OrderedDict() is_disabled = False for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_ci_job( config_url, rosdistro_name, ci_build_name, os_name, os_code_name, arch, config=config, build_file=build_file, index=index, dist_file=dist_file, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script, dry_run=dry_run, trigger_timer=build_file.jenkins_job_schedule) ci_job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(job_configs) groovy_data['job_prefixes_and_names'] = {} if groovy_script is not None: print("Writing groovy script '%s' to reconfigure %d jobs" % (groovy_script, len(job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, job_configs, view_configs)
def configure_release_jobs( config_url, rosdistro_name, release_build_name, groovy_script=None, dry_run=False, whitelist_package_names=None): """ Configure all Jenkins release jobs. L{configure_release_job} will be invoked for every released package and target which matches the build file criteria. Additionally a job to import Debian packages into the Debian repository is created. """ config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) # get targets platforms = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): platforms.append((os_name, os_code_name)) print('The build file contains the following targets:') for os_name, os_code_name in platforms: print(' - %s %s: %s' % (os_name, os_code_name, ', '.join( build_file.targets[os_name][os_code_name]))) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return pkg_names = dist_file.release_packages.keys() filtered_pkg_names = build_file.filter_packages(pkg_names) explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names) if explicitly_ignored_pkg_names: print(('The following packages are being %s because of ' + 'white-/blacklisting:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(explicitly_ignored_pkg_names): print(' -', pkg_name) dist_cache = None if build_file.notify_maintainers or \ build_file.abi_incompatibility_assumed or \ explicitly_ignored_pkg_names: dist_cache = get_distribution_cache(index, rosdistro_name) if explicitly_ignored_pkg_names: # get direct dependencies from distro cache for each package direct_dependencies = {} for pkg_name in pkg_names: direct_dependencies[pkg_name] = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) or set([]) # find recursive downstream deps for all explicitly ignored packages ignored_pkg_names = set(explicitly_ignored_pkg_names) while True: implicitly_ignored_pkg_names = _get_downstream_package_names( ignored_pkg_names, direct_dependencies) if implicitly_ignored_pkg_names - ignored_pkg_names: ignored_pkg_names |= implicitly_ignored_pkg_names continue break implicitly_ignored_pkg_names = \ ignored_pkg_names - explicitly_ignored_pkg_names if implicitly_ignored_pkg_names: print(('The following packages are being %s because their ' + 'dependencies are being ignored:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(implicitly_ignored_pkg_names): print(' -', pkg_name) filtered_pkg_names = \ set(filtered_pkg_names) - implicitly_ignored_pkg_names # all further configuration will be handled by either the Jenkins API # or by a generated groovy script jenkins = connect(config.jenkins_url) if groovy_script is None else False all_view_configs = {} all_job_configs = {} job_name, job_config = configure_import_package_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config job_name, job_config = configure_sync_packages_to_main_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config for os_name, os_code_name in platforms: for arch in sorted(build_file.targets[os_name][os_code_name]): job_name, job_config = configure_sync_packages_to_testing_job( config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if not jenkins: all_job_configs[job_name] = job_config targets = [] for os_name, os_code_name in platforms: targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) views = configure_release_views( jenkins, rosdistro_name, release_build_name, targets, dry_run=dry_run) if not jenkins: all_view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(views), } other_build_files = [v for k, v in build_files.items() if k != release_build_name] all_source_job_names = [] all_binary_job_names = [] for pkg_name in sorted(pkg_names): if whitelist_package_names: if pkg_name not in whitelist_package_names: print("Skipping package '%s' not in the explicitly passed list" % pkg_name, file=sys.stderr) continue pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] is_disabled = pkg_name not in filtered_pkg_names if is_disabled and build_file.skip_ignored_packages: print("Skipping ignored package '%s' in repository '%s'" % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository: print(("Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository.version: print(("Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name), file=sys.stderr) continue for os_name, os_code_name in platforms: other_build_files_same_platform = [] for other_build_file in other_build_files: if os_name not in other_build_file.targets: continue if os_code_name not in other_build_file.targets[os_name]: continue other_build_files_same_platform.append(other_build_file) try: source_job_names, binary_job_names, job_configs = \ configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, generate_import_package_job=False, generate_sync_packages_jobs=False, is_disabled=is_disabled, other_build_files_same_platform=other_build_files_same_platform, groovy_script=groovy_script, dry_run=dry_run) all_source_job_names += source_job_names all_binary_job_names += binary_job_names if groovy_script is not None: print('Configuration for jobs: ' + ', '.join(source_job_names + binary_job_names)) all_job_configs.update(job_configs) except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(all_job_configs) groovy_data['job_prefixes_and_names'] = {} # with an explicit list of packages we don't delete obsolete jobs if not whitelist_package_names: # delete obsolete binary jobs for os_name, os_code_name in platforms: for arch in build_file.targets[os_name][os_code_name]: binary_view = get_release_binary_view_name( rosdistro_name, release_build_name, os_name, os_code_name, arch) binary_job_prefix = '%s__' % binary_view excluded_job_names = set([ j for j in all_binary_job_names if j.startswith(binary_job_prefix)]) if groovy_script is None: print("Removing obsolete binary jobs with prefix '%s'" % binary_job_prefix) remove_jobs( jenkins, binary_job_prefix, excluded_job_names, dry_run=dry_run) else: binary_key = 'binary_%s_%s_%s' % \ (os_name, os_code_name, arch) groovy_data['job_prefixes_and_names'][binary_key] = \ (binary_job_prefix, excluded_job_names) # delete obsolete source jobs # requires knowledge about all other release build files for os_name, os_code_name in platforms: other_source_job_names = [] # get source job names for all other release build files for other_release_build_name in [ k for k in build_files.keys() if k != release_build_name]: other_build_file = build_files[other_release_build_name] other_dist_file = get_distribution_file( index, rosdistro_name, other_build_file) if not other_dist_file: continue if os_name not in other_build_file.targets or \ os_code_name not in other_build_file.targets[os_name]: continue if other_build_file.skip_ignored_packages: filtered_pkg_names = other_build_file.filter_packages( pkg_names) else: filtered_pkg_names = pkg_names for pkg_name in sorted(filtered_pkg_names): pkg = other_dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = other_dist_file.repositories[repo_name] if not repo.release_repository: continue if not repo.release_repository.version: continue other_job_name = get_sourcedeb_job_name( rosdistro_name, other_release_build_name, pkg_name, os_name, os_code_name) other_source_job_names.append(other_job_name) source_view_prefix = get_release_source_view_name( rosdistro_name, os_name, os_code_name) source_job_prefix = '%s__' % source_view_prefix excluded_job_names = set([ j for j in (all_source_job_names + other_source_job_names) if j.startswith(source_job_prefix)]) if groovy_script is None: print("Removing obsolete source jobs with prefix '%s'" % source_job_prefix) remove_jobs( jenkins, source_job_prefix, excluded_job_names, dry_run=dry_run) else: source_key = 'source_%s_%s' % (os_name, os_code_name) groovy_data['job_prefixes_and_names'][source_key] = ( source_job_prefix, excluded_job_names) if groovy_script is not None: print( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % (groovy_script, len(all_view_configs), len(all_job_configs))) content = expand_template( 'snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs( groovy_script, content, all_job_configs, view_configs=all_view_configs)
def configure_doc_jobs(config_url, rosdistro_name, doc_build_name, groovy_script=None, dry_run=False, whitelist_repository_names=None): """ Configure all Jenkins doc jobs. L{configure_doc_job} will be invoked for doc repository and target which matches the build file criteria. """ config = get_config_index(config_url) build_files = get_doc_build_files(config, rosdistro_name) build_file = build_files[doc_build_name] index = get_index(config.rosdistro_index_url) dist_cache = None if build_file.notify_maintainers: dist_cache = get_distribution_cache(index, rosdistro_name) # get targets targets = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) print('The build file contains the following targets:') for os_name, os_code_name, arch in targets: print(' -', os_name, os_code_name, arch) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name) # all further configuration will be handled by either the Jenkins API # or by a generated groovy script from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if groovy_script is None else False view_configs = {} views = {} views[doc_view_name] = configure_doc_view(jenkins, doc_view_name, dry_run=dry_run) if not jenkins: view_configs.update(views) groovy_data = { 'dry_run': dry_run, 'expected_num_views': len(view_configs), } repo_names = dist_file.repositories.keys() filtered_repo_names = build_file.filter_repositories(repo_names) job_names = [] job_configs = OrderedDict() for repo_name in sorted(repo_names): if whitelist_repository_names: if repo_name not in whitelist_repository_names: print( "Skipping repository '%s' not in explicitly passed list" % repo_name, file=sys.stderr) continue is_disabled = repo_name not in filtered_repo_names if is_disabled and build_file.skip_ignored_repositories: print("Skipping ignored repository '%s'" % repo_name, file=sys.stderr) continue repo = dist_file.repositories[repo_name] if not repo.doc_repository: print("Skipping repository '%s': no doc section" % repo_name) continue if not repo.doc_repository.version: print("Skipping repository '%s': no doc version" % repo_name) continue for os_name, os_code_name, arch in targets: try: job_name, job_config = configure_doc_job( config_url, rosdistro_name, doc_build_name, repo_name, os_name, os_code_name, arch, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, is_disabled=is_disabled, groovy_script=groovy_script, dry_run=dry_run) job_names.append(job_name) if groovy_script is not None: print("Configuration for job '%s'" % job_name) job_configs[job_name] = job_config except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data['expected_num_jobs'] = len(job_configs) groovy_data['job_prefixes_and_names'] = {} job_prefix = '%s__' % doc_view_name if not whitelist_repository_names: groovy_data['job_prefixes_and_names']['doc'] = (job_prefix, job_names) if groovy_script is None: # delete obsolete jobs in this view from ros_buildfarm.jenkins import remove_jobs print('Removing obsolete doc jobs') remove_jobs(jenkins, job_prefix, job_names, dry_run=dry_run) if groovy_script is not None: print( "Writing groovy script '%s' to reconfigure %d views and %d jobs" % (groovy_script, len(view_configs), len(job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, job_configs, view_configs=view_configs)
def configure_release_jobs(config_url, rosdistro_name, release_build_name, groovy_script=None): """ Configure all Jenkins release jobs. L{configure_release_job} will be invoked for every released package and target which matches the build file criteria. Additionally a job to import Debian packages into the Debian repository is created. """ config = get_config_index(config_url) build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] index = get_index(config.rosdistro_index_url) # get targets platforms = [] for os_name in build_file.targets.keys(): for os_code_name in build_file.targets[os_name].keys(): platforms.append((os_name, os_code_name)) print('The build file contains the following targets:') for os_name, os_code_name in platforms: print(' - %s %s: %s' % (os_name, os_code_name, ', '.join( build_file.targets[os_name][os_code_name]))) dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: print('No distribution file matches the build file') return pkg_names = dist_file.release_packages.keys() filtered_pkg_names = build_file.filter_packages(pkg_names) explicitly_ignored_pkg_names = set(pkg_names) - set(filtered_pkg_names) if explicitly_ignored_pkg_names: print(('The following packages are being %s because of ' + 'white-/blacklisting:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(explicitly_ignored_pkg_names): print(' -', pkg_name) dist_cache = None if build_file.notify_maintainers or \ build_file.abi_incompatibility_assumed or \ explicitly_ignored_pkg_names: dist_cache = get_distribution_cache(index, rosdistro_name) if explicitly_ignored_pkg_names: # get direct dependencies from distro cache for each package direct_dependencies = {} for pkg_name in pkg_names: direct_dependencies[pkg_name] = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) or set([]) # find recursive downstream deps for all explicitly ignored packages ignored_pkg_names = set(explicitly_ignored_pkg_names) while True: implicitly_ignored_pkg_names = _get_downstream_package_names( ignored_pkg_names, direct_dependencies) if implicitly_ignored_pkg_names - ignored_pkg_names: ignored_pkg_names |= implicitly_ignored_pkg_names continue break implicitly_ignored_pkg_names = \ ignored_pkg_names - explicitly_ignored_pkg_names if implicitly_ignored_pkg_names: print(('The following packages are being %s because their ' + 'dependencies are being ignored:') % ('ignored' if build_file.skip_ignored_packages else 'disabled')) for pkg_name in sorted(implicitly_ignored_pkg_names): print(' -', pkg_name) filtered_pkg_names = \ set(filtered_pkg_names) - implicitly_ignored_pkg_names jenkins = connect(config.jenkins_url) configure_import_package_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins) configure_sync_packages_to_main_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins) for os_name, os_code_name in platforms: for arch in sorted(build_file.targets[os_name][os_code_name]): configure_sync_packages_to_testing_job(config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins) targets = [] for os_name, os_code_name in platforms: targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) views = configure_release_views(jenkins, rosdistro_name, release_build_name, targets) if groovy_script is not None: # all further configuration will be handled by the groovy script jenkins = False all_source_job_names = [] all_binary_job_names = [] all_job_configs = {} for pkg_name in sorted(pkg_names): pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] is_disabled = pkg_name not in filtered_pkg_names if is_disabled and build_file.skip_ignored_packages: print("Skipping ignored package '%s' in repository '%s'" % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository: print(("Skipping package '%s' in repository '%s': no release " + "section") % (pkg_name, repo_name), file=sys.stderr) continue if not repo.release_repository.version: print(("Skipping package '%s' in repository '%s': no release " + "version") % (pkg_name, repo_name), file=sys.stderr) continue for os_name, os_code_name in platforms: try: source_job_names, binary_job_names, job_configs = \ configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=config, build_file=build_file, index=index, dist_file=dist_file, dist_cache=dist_cache, jenkins=jenkins, views=views, generate_import_package_job=False, generate_sync_packages_jobs=False, is_disabled=is_disabled, groovy_script=groovy_script) all_source_job_names += source_job_names all_binary_job_names += binary_job_names if groovy_script is not None: print('Configuration for jobs: ' + ', '.join(source_job_names + binary_job_names)) all_job_configs.update(job_configs) except JobValidationError as e: print(e.message, file=sys.stderr) groovy_data = { 'expected_num_jobs': len(all_job_configs), 'job_prefixes_and_names': {}, } # delete obsolete binary jobs for os_name, os_code_name in platforms: for arch in build_file.targets[os_name][os_code_name]: binary_view = get_release_binary_view_name(rosdistro_name, release_build_name, os_name, os_code_name, arch) binary_job_prefix = '%s__' % binary_view excluded_job_names = set([ j for j in all_binary_job_names if j.startswith(binary_job_prefix) ]) if groovy_script is None: print("Removing obsolete binary jobs with prefix '%s'" % binary_job_prefix) remove_jobs(jenkins, binary_job_prefix, excluded_job_names) else: binary_key = 'binary_%s_%s_%s' % (os_name, os_code_name, arch) groovy_data['job_prefixes_and_names'][binary_key] = \ (binary_job_prefix, excluded_job_names) # delete obsolete source jobs # requires knowledge about all other release build files for os_name, os_code_name in platforms: other_source_job_names = [] # get source job names for all other release build files for other_release_build_name in [ k for k in build_files.keys() if k != release_build_name ]: other_build_file = build_files[other_release_build_name] other_dist_file = get_distribution_file(index, rosdistro_name, other_build_file) if not other_dist_file: continue if os_name not in other_build_file.targets or \ os_code_name not in other_build_file.targets[os_name]: continue if other_build_file.skip_ignored_packages: filtered_pkg_names = other_build_file.filter_packages( pkg_names) else: filtered_pkg_names = pkg_names for pkg_name in sorted(filtered_pkg_names): pkg = other_dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = other_dist_file.repositories[repo_name] if not repo.release_repository: continue if not repo.release_repository.version: continue other_job_name = get_sourcedeb_job_name( rosdistro_name, other_release_build_name, pkg_name, os_name, os_code_name) other_source_job_names.append(other_job_name) source_view_prefix = get_release_source_view_name( rosdistro_name, os_name, os_code_name) source_job_prefix = '%s__' % source_view_prefix excluded_job_names = set([ j for j in (all_source_job_names + other_source_job_names) if j.startswith(source_job_prefix) ]) if groovy_script is None: print("Removing obsolete source jobs with prefix '%s'" % source_job_prefix) remove_jobs(jenkins, source_job_prefix, excluded_job_names) else: source_key = 'source_%s_%s' % (os_name, os_code_name) groovy_data['job_prefixes_and_names'][source_key] = ( source_job_prefix, excluded_job_names) if groovy_script is not None: print("Writing groovy script '%s' to reconfigure %d jobs" % (groovy_script, len(all_job_configs))) content = expand_template('snippet/reconfigure_jobs.groovy.em', groovy_data) write_groovy_script_and_configs(groovy_script, content, all_job_configs)