def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser(description="Generate a 'release' script") add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_build_name(parser, 'release') add_argument_package_name(parser) add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) parser.add_argument('--skip-install', action='store_true', help='Skip trying to install binarydeb') args = parser.parse_args(argv) # collect all template snippets of specific types class IncludeHook(Hook): def __init__(self): Hook.__init__(self) self.scripts = [] def beforeFile(self, *args, **kwargs): template_path = kwargs['file'].name if template_path.endswith('/release/binarydeb_job.xml.em'): self.scripts.append('--') if template_path.endswith('/snippet/builder_shell.xml.em'): self.scripts.append(kwargs['locals']['script']) hook = IncludeHook() from ros_buildfarm import templates templates.template_hooks = [hook] configure_release_job(args.config_url, args.rosdistro_name, args.release_build_name, args.package_name, args.os_name, args.os_code_name, jenkins=False, views=[], generate_import_package_job=False, generate_sync_packages_jobs=False, filter_arches=args.arch) templates.template_hooks = None source_job_name = get_sourcedeb_job_name(args.rosdistro_name, args.release_build_name, args.package_name, args.os_name, args.os_code_name) binary_job_name = get_binarydeb_job_name(args.rosdistro_name, args.release_build_name, args.package_name, args.os_name, args.os_code_name, args.arch) separator_index = hook.scripts.index('--') source_scripts = hook.scripts[:separator_index] binary_scripts = hook.scripts[separator_index + 1:] # inject additional argument to skip fetching sourcedeb from repo script_name = '/run_binarydeb_job.py ' additional_argument = '--skip-download-sourcedeb ' for i, script in enumerate(binary_scripts): offset = script.find(script_name) if offset != -1: offset += len(script_name) script = script[:offset] + additional_argument + script[offset:] binary_scripts[i] = script break # remove rm command for sourcedeb location rm_command = 'rm -fr $WORKSPACE/binarydeb' for i, script in enumerate(binary_scripts): offset = script.find(rm_command) if offset != -1: script = script[:offset] + script[offset + len(rm_command):] binary_scripts[i] = script break if args.skip_install: # remove install step script_name = '/create_binarydeb_install_task_generator.py ' for i, script in enumerate(binary_scripts): offset = script.find(script_name) if offset != -1: del binary_scripts[i] break value = expand_template('release/release_script.sh.em', { 'source_job_name': source_job_name, 'binary_job_name': binary_job_name, 'source_scripts': source_scripts, 'binary_scripts': binary_scripts }, options={BANGPATH_OPT: False}) value = value.replace('python3', sys.executable) print(value)
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description="Generate a 'release' script") add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_build_name(parser, 'release') add_argument_package_name(parser) add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) parser.add_argument( '--skip-install', action='store_true', help='Skip trying to install binarydeb') args = parser.parse_args(argv) # collect all template snippets of specific types class IncludeHook(Hook): def __init__(self): Hook.__init__(self) self.scripts = [] def beforeFile(self, *args, **kwargs): template_path = kwargs['file'].name if template_path.endswith('/release/binarydeb_job.xml.em'): self.scripts.append('--') if template_path.endswith('/snippet/builder_shell.xml.em'): self.scripts.append(kwargs['locals']['script']) hook = IncludeHook() from ros_buildfarm import templates templates.template_hooks = [hook] configure_release_job( args.config_url, args.rosdistro_name, args.release_build_name, args.package_name, args.os_name, args.os_code_name, jenkins=False, views=[], generate_import_package_job=False, generate_sync_packages_jobs=False, filter_arches=args.arch) templates.template_hooks = None source_job_name = get_sourcedeb_job_name( args.rosdistro_name, args.release_build_name, args.package_name, args.os_name, args.os_code_name) binary_job_name = get_binarydeb_job_name( args.rosdistro_name, args.release_build_name, args.package_name, args.os_name, args.os_code_name, args.arch) separator_index = hook.scripts.index('--') source_scripts = hook.scripts[:separator_index] binary_scripts = hook.scripts[separator_index + 1:] # inject additional argument to skip fetching sourcedeb from repo script_name = '/run_binarydeb_job.py ' additional_argument = '--skip-download-sourcedeb ' for i, script in enumerate(binary_scripts): offset = script.find(script_name) if offset != -1: offset += len(script_name) script = script[:offset] + additional_argument + script[offset:] binary_scripts[i] = script break # remove rm command for sourcedeb location rm_command = 'rm -fr $WORKSPACE/binarydeb' for i, script in enumerate(binary_scripts): offset = script.find(rm_command) if offset != -1: script = script[:offset] + script[offset + len(rm_command):] binary_scripts[i] = script break if args.skip_install: # remove install step script_name = '/create_binarydeb_install_task_generator.py ' for i, script in enumerate(binary_scripts): offset = script.find(script_name) if offset != -1: del binary_scripts[i] break value = expand_template( 'release/release_script.sh.em', { 'source_job_name': source_job_name, 'binary_job_name': binary_job_name, 'source_scripts': source_scripts, 'binary_scripts': binary_scripts}, options={BANGPATH_OPT: False}) value = value.replace('python3', sys.executable) print(value)
def configure_release_job(config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, views=None, generate_import_package_job=True, generate_sync_packages_jobs=True, is_disabled=False, other_build_files_same_platform=None, groovy_script=None, filter_arches=None, dry_run=False): """ Configure a Jenkins release job. The following jobs are created for each package: - M source jobs, one for each OS node name - M * N binary jobs, one for each combination of OS code name and arch """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: raise JobValidationError( 'No distribution file matches the build file') pkg_names = dist_file.release_packages.keys() if pkg_name not in pkg_names: raise JobValidationError("Invalid package name '%s' " % pkg_name + 'choose one of the following: ' + ', '.join(sorted(pkg_names))) pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] if not repo.release_repository: raise JobValidationError("Repository '%s' has no release section" % repo_name) if not repo.release_repository.version: raise JobValidationError("Repository '%s' has no release version" % repo_name) if os_name not in build_file.targets.keys(): raise JobValidationError("Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if dist_cache is None and \ (build_file.notify_maintainers or build_file.abi_incompatibility_assumed): dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if views is None: targets = [] targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) configure_release_views(jenkins, rosdistro_name, release_build_name, targets, dry_run=dry_run) if generate_import_package_job: configure_import_package_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if generate_sync_packages_jobs: configure_sync_packages_to_main_job(config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) for arch in build_file.targets[os_name][os_code_name]: configure_sync_packages_to_testing_job(config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) source_job_names = [] binary_job_names = [] job_configs = {} # sourcedeb job # since sourcedeb jobs are potentially being shared across multiple build # files the configuration has to take all of them into account in order to # generate a job which all build files agree on source_job_name = get_sourcedeb_job_name(rosdistro_name, release_build_name, pkg_name, os_name, os_code_name) # while the package is disabled in the current build file # it might be used by sibling build files is_source_disabled = is_disabled if is_source_disabled and other_build_files_same_platform: # check if sourcedeb job is used by any other build file with the same platform for other_build_file in other_build_files_same_platform: if other_build_file.filter_packages([pkg_name]): is_source_disabled = False break job_config = _get_sourcedeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache, is_disabled=is_source_disabled, other_build_files_same_platform=other_build_files_same_platform) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: from ros_buildfarm.jenkins import configure_job configure_job(jenkins, source_job_name, job_config, dry_run=dry_run) source_job_names.append(source_job_name) job_configs[source_job_name] = job_config dependency_names = [] if build_file.abi_incompatibility_assumed: dependency_names = _get_direct_dependencies(pkg_name, dist_cache, pkg_names) # if dependencies are not yet available in rosdistro cache # skip binary jobs if dependency_names is None: print(("Skipping binary jobs for package '%s' because it is not " + "yet in the rosdistro cache") % pkg_name, file=sys.stderr) return source_job_names, binary_job_names, job_configs # binarydeb jobs for arch in build_file.targets[os_name][os_code_name]: if filter_arches and arch not in filter_arches: continue job_name = get_binarydeb_job_name(rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, arch) upstream_job_names = [source_job_name] + [ get_binarydeb_job_name( rosdistro_name, release_build_name, dependency_name, os_name, os_code_name, arch) for dependency_name in dependency_names ] job_config = _get_binarydeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, arch, pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache, upstream_job_names=upstream_job_names, is_disabled=is_disabled) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, job_name, job_config, dry_run=dry_run) binary_job_names.append(job_name) job_configs[job_name] = job_config return source_job_names, binary_job_names, job_configs
def configure_release_job( config_url, rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, config=None, build_file=None, index=None, dist_file=None, dist_cache=None, jenkins=None, views=None, generate_import_package_job=True, generate_sync_packages_jobs=True, is_disabled=False, other_build_files_same_platform=None, groovy_script=None, filter_arches=None, dry_run=False): """ Configure a Jenkins release job. The following jobs are created for each package: - M source jobs, one for each OS node name - M * N binary jobs, one for each combination of OS code name and arch """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_release_build_files(config, rosdistro_name) build_file = build_files[release_build_name] if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: raise JobValidationError( 'No distribution file matches the build file') pkg_names = dist_file.release_packages.keys() if pkg_name not in pkg_names: raise JobValidationError( "Invalid package name '%s' " % pkg_name + 'choose one of the following: ' + ', '.join(sorted(pkg_names))) pkg = dist_file.release_packages[pkg_name] repo_name = pkg.repository_name repo = dist_file.repositories[repo_name] if not repo.release_repository: raise JobValidationError( "Repository '%s' has no release section" % repo_name) if not repo.release_repository.version: raise JobValidationError( "Repository '%s' has no release version" % repo_name) if os_name not in build_file.targets.keys(): raise JobValidationError( "Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if dist_cache is None and \ (build_file.notify_maintainers or build_file.abi_incompatibility_assumed): dist_cache = get_distribution_cache(index, rosdistro_name) if jenkins is None: jenkins = connect(config.jenkins_url) if views is None: targets = [] targets.append((os_name, os_code_name, 'source')) for arch in build_file.targets[os_name][os_code_name]: targets.append((os_name, os_code_name, arch)) configure_release_views( jenkins, rosdistro_name, release_build_name, targets, dry_run=dry_run) if generate_import_package_job: configure_import_package_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) if generate_sync_packages_jobs: configure_sync_packages_to_main_job( config_url, rosdistro_name, release_build_name, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) for arch in build_file.targets[os_name][os_code_name]: configure_sync_packages_to_testing_job( config_url, rosdistro_name, release_build_name, os_code_name, arch, config=config, build_file=build_file, jenkins=jenkins, dry_run=dry_run) source_job_names = [] binary_job_names = [] job_configs = {} # sourcedeb job # since sourcedeb jobs are potentially being shared across multiple build # files the configuration has to take all of them into account in order to # generate a job which all build files agree on source_job_name = get_sourcedeb_job_name( rosdistro_name, release_build_name, pkg_name, os_name, os_code_name) # while the package is disabled in the current build file # it might be used by sibling build files is_source_disabled = is_disabled if is_source_disabled and other_build_files_same_platform: # check if sourcedeb job is used by any other build file with the same platform for other_build_file in other_build_files_same_platform: if other_build_file.filter_packages([pkg_name]): is_source_disabled = False break job_config = _get_sourcedeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache, is_disabled=is_source_disabled, other_build_files_same_platform=other_build_files_same_platform) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, source_job_name, job_config, dry_run=dry_run) source_job_names.append(source_job_name) job_configs[source_job_name] = job_config dependency_names = [] if build_file.abi_incompatibility_assumed: dependency_names = _get_direct_dependencies( pkg_name, dist_cache, pkg_names) # if dependencies are not yet available in rosdistro cache # skip binary jobs if dependency_names is None: print(("Skipping binary jobs for package '%s' because it is not " + "yet in the rosdistro cache") % pkg_name, file=sys.stderr) return source_job_names, binary_job_names, job_configs # binarydeb jobs for arch in build_file.targets[os_name][os_code_name]: if filter_arches and arch not in filter_arches: continue job_name = get_binarydeb_job_name( rosdistro_name, release_build_name, pkg_name, os_name, os_code_name, arch) upstream_job_names = [source_job_name] + [ get_binarydeb_job_name( rosdistro_name, release_build_name, dependency_name, os_name, os_code_name, arch) for dependency_name in dependency_names] job_config = _get_binarydeb_job_config( config_url, rosdistro_name, release_build_name, config, build_file, os_name, os_code_name, arch, pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache, upstream_job_names=upstream_job_names, is_disabled=is_disabled) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: configure_job(jenkins, job_name, job_config, dry_run=dry_run) binary_job_names.append(job_name) job_configs[job_name] = job_config return source_job_names, binary_job_names, job_configs