def configure_ci_jobs( config_url, rosdistro_name, ci_build_names=None, groovy_script=None, dry_run=False): """Configure all Jenkins CI jobs.""" config = get_config_index(config_url) build_files = get_ci_build_files(config, rosdistro_name) if not ci_build_names: ci_build_names = build_files.keys() for ci_build_name in ci_build_names: _configure_ci_jobs( config, build_files, config_url, rosdistro_name, ci_build_name, groovy_script=groovy_script, dry_run=dry_run)
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description="Generate the 'CI' management jobs on Jenkins") # Positional add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_dry_run(parser) args = parser.parse_args(argv) config = get_index(args.config_url) build_files = get_ci_build_files(config, args.rosdistro_name) jenkins = connect(config.jenkins_url) configure_management_view(jenkins, dry_run=args.dry_run) group_name = get_ci_view_name(args.rosdistro_name) configure_reconfigure_jobs_job( jenkins, group_name, args, config, build_files, dry_run=args.dry_run)
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description="Generate the 'CI' management jobs on Jenkins") # Positional add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_dry_run(parser) args = parser.parse_args(argv) config = get_index(args.config_url) build_files = get_ci_build_files(config, args.rosdistro_name) jenkins = connect(config.jenkins_url) configure_management_view(jenkins, dry_run=args.dry_run) group_name = get_ci_view_name(args.rosdistro_name) configure_reconfigure_jobs_job(jenkins, group_name, args, config, build_files, dry_run=args.dry_run)
def configure_ci_job(config_url, rosdistro_name, ci_build_name, os_name, os_code_name, arch, config=None, build_file=None, index=None, dist_file=None, jenkins=None, views=None, is_disabled=False, groovy_script=None, build_targets=None, dry_run=False, underlay_source_paths=None, trigger_timer=None): """ Configure a single Jenkins CI job. This includes the following steps: - clone the ros_buildfarm repository - write the distribution repository keys into files - invoke the ci/run_ci_job.py script """ if config is None: config = get_config_index(config_url) if build_file is None: build_files = get_ci_build_files(config, rosdistro_name) build_file = build_files[ci_build_name] # Overwrite build_file.targets if build_targets is specified if build_targets is not None: build_file.targets = build_targets if index is None: index = get_index(config.rosdistro_index_url) if dist_file is None: dist_file = get_distribution_file(index, rosdistro_name, build_file) if not dist_file: raise JobValidationError( 'No distribution file matches the build file') if os_name not in build_file.targets.keys(): raise JobValidationError("Invalid OS name '%s' " % os_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets.keys()))) if os_code_name not in build_file.targets[os_name].keys(): raise JobValidationError( "Invalid OS code name '%s' " % os_code_name + 'choose one of the following: ' + ', '.join(sorted(build_file.targets[os_name].keys()))) if arch not in build_file.targets[os_name][os_code_name]: raise JobValidationError( "Invalid architecture '%s' " % arch + 'choose one of the following: %s' % ', '.join(sorted(build_file.targets[os_name][os_code_name]))) underlay_source_jobs = [ get_ci_job_name(rosdistro_name, os_name, os_code_name, arch, underlay_job) for underlay_job in build_file.underlay_from_ci_jobs ] underlay_source_paths = (underlay_source_paths or []) + \ ['$UNDERLAY%d_JOB_SPACE' % (index + 1) for index in range(len(underlay_source_jobs))] trigger_jobs = [ get_ci_job_name(rosdistro_name, os_name, os_code_name, arch, trigger_job) for trigger_job in build_file.jenkins_job_upstream_triggers ] if jenkins is None: from ros_buildfarm.jenkins import connect jenkins = connect(config.jenkins_url) if views is None: view_name = get_ci_view_name(rosdistro_name) configure_ci_view(jenkins, view_name, dry_run=dry_run) job_name = get_ci_job_name(rosdistro_name, os_name, os_code_name, arch, ci_build_name) job_config = _get_ci_job_config(index, rosdistro_name, build_file, os_name, os_code_name, arch, build_file.repos_files, build_file.repository_names, underlay_source_jobs, underlay_source_paths, trigger_timer, trigger_jobs, is_disabled=is_disabled) # jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero if isinstance(jenkins, object) and jenkins is not False: from ros_buildfarm.jenkins import configure_job configure_job(jenkins, job_name, job_config, dry_run=dry_run) return job_name, job_config
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description='Generate all jobs on Jenkins') add_argument_config_url(parser) parser.add_argument( '--ros-distro-names', nargs='*', metavar='ROS_DISTRO_NAME', default=[], help='The list of ROS distribution names if not generating all') parser.add_argument('--skip-rosdistro-cache-job', action='store_true', help='Skip generating the rosdistro-cache jobs') parser.add_argument( '--commit', action='store_true', help='Apply the changes to Jenkins instead of only showing them') args = parser.parse_args(argv) if args.commit: print('The following changes will be applied to the Jenkins server.') else: print('This is a dry run. The Jenkins configuration is not changed.') print('') config = get_index(args.config_url) if args.config_url.startswith('file:'): print( 'WARNING: Local file system path used for configuration. ', 'Configuration will not be accessible to jobs during execution. ', 'Consider using a web(http) hosted configuration repository.', file=sys.stderr) ros_distro_names = sorted(config.distributions.keys()) invalid_ros_distro_name = [ n for n in args.ros_distro_names if n not in ros_distro_names ] if invalid_ros_distro_name: parser.error( 'The following ROS distribution names are not part of the ' + 'buildfarm index: ' + ', '.join(sorted(invalid_ros_distro_name))) # try to connect to Jenkins master jenkins = connect(config.jenkins_url) configure_view(jenkins, 'Queue', filter_queue=False, dry_run=not args.commit) generate_check_agents_job(args.config_url, dry_run=not args.commit) if not args.ros_distro_names: generate_dashboard_job(args.config_url, dry_run=not args.commit) for doc_build_name in sorted(config.doc_builds.keys()): generate_doc_independent_job(args.config_url, doc_build_name, dry_run=not args.commit) selected_ros_distro_names = [ n for n in ros_distro_names if not args.ros_distro_names or n in args.ros_distro_names ] for ros_distro_name in selected_ros_distro_names: print(ros_distro_name) if not args.skip_rosdistro_cache_job: generate_rosdistro_cache_job(args.config_url, ros_distro_name, dry_run=not args.commit) generate_failing_jobs_job(args.config_url, ros_distro_name, dry_run=not args.commit) release_build_files = get_release_build_files(config, ros_distro_name) for release_build_name in release_build_files.keys(): generate_release_status_page_job(args.config_url, ros_distro_name, release_build_name, dry_run=not args.commit) generate_release_maintenance_jobs(args.config_url, ros_distro_name, release_build_name, dry_run=not args.commit) source_build_files = get_source_build_files(config, ros_distro_name) for source_build_name in source_build_files.keys(): generate_devel_maintenance_jobs(args.config_url, ros_distro_name, source_build_name, dry_run=not args.commit) ci_build_files = get_ci_build_files(config, ros_distro_name) for ci_build_name in ci_build_files.keys(): generate_ci_maintenance_jobs(args.config_url, ros_distro_name, ci_build_name, dry_run=not args.commit) doc_build_files = get_doc_build_files(config, ros_distro_name) for doc_build_name, doc_build_file in doc_build_files.items(): if doc_build_file.documentation_type == DOC_TYPE_ROSDOC: generate_doc_maintenance_jobs(args.config_url, ros_distro_name, doc_build_name, dry_run=not args.commit) elif doc_build_file.documentation_type == DOC_TYPE_MANIFEST: generate_doc_metadata_job(args.config_url, ros_distro_name, doc_build_name, dry_run=not args.commit) else: assert False, ("Unknown documentation type '%s' in doc " + "build file '%s'") % \ (doc_build_file.documentation_type, doc_build_name) generate_repos_status_page_jobs(args.config_url, ros_distro_name, dry_run=not args.commit) index = ros_distro_names.index(ros_distro_name) if index > 0: # generate compare pages for this rosdistro against all older ones generate_release_compare_page_job(args.config_url, ros_distro_name, ros_distro_names[:index], dry_run=not args.commit) generate_blocked_releases_page_job(args.config_url, ros_distro_name, dry_run=not args.commit) generate_blocked_source_entries_page_job(args.config_url, ros_distro_name, dry_run=not args.commit)
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser(description="Generate a 'CI' script") # Positional add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_build_name(parser, 'ci') add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_argument_build_tool(parser) a1 = add_argument_package_selection_args(parser) a2 = add_argument_build_tool_args(parser) a3 = add_argument_build_tool_test_args(parser) add_argument_repos_file_urls(parser) add_argument_skip_cleanup(parser) add_argument_test_branch(parser) parser.add_argument( '--underlay-source-path', nargs='*', metavar='DIR_NAME', help='Path to one or more install spaces to use as an underlay') remainder_args = extract_multiple_remainders(argv, (a1, a2, a3)) args = parser.parse_args(argv) for k, v in remainder_args.items(): setattr(args, k, v) # collect all template snippets of specific types class IncludeHook(Hook): def __init__(self): Hook.__init__(self) self.scms = [] self.scripts = [] self.parameters = {} if args.skip_cleanup: self.parameters['skip_cleanup'] = 'true' if args.repos_file_urls is not None: self.parameters['repos_file_urls'] = ' '.join( args.repos_file_urls) if args.test_branch is not None: self.parameters['test_branch'] = args.test_branch if args.package_selection_args is not None: self.parameters['package_selection_args'] = ' '.join( args.package_selection_args) if args.build_tool_args is not None: self.parameters['build_tool_args'] = ' '.join( args.build_tool_args) if args.build_tool_test_args is not None: self.parameters['build_tool_test_args'] = ' '.join( args.build_tool_test_args) def beforeInclude(self, *_, **kwargs): template_path = kwargs['file'].name if template_path.endswith('/snippet/scm.xml.em'): self.scms.append( (kwargs['locals']['repo_spec'], kwargs['locals']['path'])) if template_path.endswith('/snippet/builder_shell.xml.em'): script = kwargs['locals']['script'] # reuse existing ros_buildfarm folder if it exists if 'Clone ros_buildfarm' in script: lines = script.splitlines() lines.insert(0, 'if [ ! -d "ros_buildfarm" ]; then') lines += [ 'else', 'echo "Using existing ros_buildfarm folder"', 'fi', ] script = '\n'.join(lines) if args.build_tool and ' --build-tool ' in script: script = script.replace( ' --build-tool catkin_make_isolated', ' --build-tool ' + args.build_tool) self.scripts.append(script) if template_path.endswith( '/snippet/property_parameters-definition.xml.em'): for parameter in reversed(kwargs['locals']['parameters']): name = parameter['name'] value_type = parameter['type'] if value_type in ['string', 'text']: default_value = parameter['default_value'] elif value_type == 'boolean': default_value = 'true' if parameter.get( 'default_value', False) else 'false' else: continue self.parameters.setdefault(name, default_value) hook = IncludeHook() from ros_buildfarm import templates templates.template_hooks = [hook] config = get_config_index(args.config_url) build_files = get_ci_build_files(config, args.rosdistro_name) build_file = build_files[args.ci_build_name] underlay_source_paths = [ os.path.abspath(p) for p in args.underlay_source_path or [] ] configure_ci_job(args.config_url, args.rosdistro_name, args.ci_build_name, args.os_name, args.os_code_name, args.arch, config=config, build_file=build_file, jenkins=False, views=False, underlay_source_paths=underlay_source_paths) templates.template_hooks = None ci_job_name = get_ci_job_name(args.rosdistro_name, args.os_name, args.os_code_name, args.arch, 'script') value = expand_template('ci/ci_script.sh.em', { 'ci_job_name': ci_job_name, 'scms': hook.scms, 'scripts': hook.scripts, 'build_tool': args.build_tool or build_file.build_tool, 'parameters': hook.parameters }, options={BANGPATH_OPT: False}) value = value.replace('python3 ', sys.executable + ' ') print(value)
def main(argv=sys.argv[1:]): parser = argparse.ArgumentParser( description="Generate a 'CI' script") # Positional add_argument_config_url(parser) add_argument_rosdistro_name(parser) add_argument_build_name(parser, 'ci') add_argument_os_name(parser) add_argument_os_code_name(parser) add_argument_arch(parser) add_argument_build_ignore(parser) add_argument_build_tool(parser) add_argument_package_selection_args(parser) add_argument_repos_file_urls(parser) add_argument_skip_cleanup(parser) add_argument_test_branch(parser) parser.add_argument( '--underlay-source-path', nargs='*', metavar='DIR_NAME', help='Path to one or more install spaces to use as an underlay') args = parser.parse_args(argv) # collect all template snippets of specific types class IncludeHook(Hook): def __init__(self): Hook.__init__(self) self.scms = [] self.scripts = [] self.parameters = {} if args.skip_cleanup: self.parameters['skip_cleanup'] = 'true' if args.repos_file_urls is not None: self.parameters['repos_file_urls'] = ' '.join(args.repos_file_urls) if args.test_branch is not None: self.parameters['test_branch'] = args.test_branch if args.build_ignore is not None: self.parameters['build_ignore'] = ' '.join(args.build_ignore) if args.package_selection_args is not None: self.parameters['package_selection_args'] = ' '.join(args.package_selection_args) def beforeInclude(self, *_, **kwargs): template_path = kwargs['file'].name if template_path.endswith('/snippet/scm.xml.em'): self.scms.append( (kwargs['locals']['repo_spec'], kwargs['locals']['path'])) if template_path.endswith('/snippet/builder_shell.xml.em'): script = kwargs['locals']['script'] # reuse existing ros_buildfarm folder if it exists if 'Clone ros_buildfarm' in script: lines = script.splitlines() lines.insert(0, 'if [ ! -d "ros_buildfarm" ]; then') lines += [ 'else', 'echo "Using existing ros_buildfarm folder"', 'fi', ] script = '\n'.join(lines) if args.build_tool and ' --build-tool ' in script: script = script.replace( ' --build-tool catkin_make_isolated', ' --build-tool ' + args.build_tool) self.scripts.append(script) if template_path.endswith('/snippet/property_parameters-definition.xml.em'): for parameter in reversed(kwargs['locals']['parameters']): name = parameter['name'] value_type = parameter['type'] if value_type in ['string', 'text']: default_value = parameter['default_value'] elif value_type == 'boolean': default_value = 'true' if parameter.get( 'default_value', False) else 'false' else: continue self.parameters.setdefault(name, default_value) hook = IncludeHook() from ros_buildfarm import templates templates.template_hooks = [hook] config = get_config_index(args.config_url) build_files = get_ci_build_files(config, args.rosdistro_name) build_file = build_files[args.ci_build_name] underlay_source_paths = [os.path.abspath(p) for p in args.underlay_source_path or []] configure_ci_job( args.config_url, args.rosdistro_name, args.ci_build_name, args.os_name, args.os_code_name, args.arch, config=config, build_file=build_file, jenkins=False, views=False, underlay_source_paths=underlay_source_paths) templates.template_hooks = None ci_job_name = get_ci_job_name( args.rosdistro_name, args.os_name, args.os_code_name, args.arch, 'script') value = expand_template( 'ci/ci_script.sh.em', { 'ci_job_name': ci_job_name, 'scms': hook.scms, 'scripts': hook.scripts, 'build_tool': args.build_tool or build_file.build_tool, 'parameters': hook.parameters}, options={BANGPATH_OPT: False}) value = value.replace('python3 ', sys.executable + ' ') print(value)