def configure_ci_jobs(
config_url, rosdistro_name, ci_build_names=None,
groovy_script=None, dry_run=False):
"""Configure all Jenkins CI jobs."""
config = get_config_index(config_url)
build_files = get_ci_build_files(config, rosdistro_name)
if not ci_build_names:
ci_build_names = build_files.keys()
for ci_build_name in ci_build_names:
_configure_ci_jobs(
config, build_files, config_url, rosdistro_name, ci_build_name,
groovy_script=groovy_script, dry_run=dry_run)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate the 'CI' management jobs on Jenkins")
# Positional
add_argument_config_url(parser)
add_argument_rosdistro_name(parser)
add_argument_dry_run(parser)
args = parser.parse_args(argv)
config = get_index(args.config_url)
build_files = get_ci_build_files(config, args.rosdistro_name)
jenkins = connect(config.jenkins_url)
configure_management_view(jenkins, dry_run=args.dry_run)
group_name = get_ci_view_name(args.rosdistro_name)
configure_reconfigure_jobs_job(
jenkins, group_name, args, config, build_files, dry_run=args.dry_run)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate the 'CI' management jobs on Jenkins")
# Positional
add_argument_config_url(parser)
add_argument_rosdistro_name(parser)
add_argument_dry_run(parser)
args = parser.parse_args(argv)
config = get_index(args.config_url)
build_files = get_ci_build_files(config, args.rosdistro_name)
jenkins = connect(config.jenkins_url)
configure_management_view(jenkins, dry_run=args.dry_run)
group_name = get_ci_view_name(args.rosdistro_name)
configure_reconfigure_jobs_job(jenkins,
group_name,
args,
config,
build_files,
dry_run=args.dry_run)
def configure_ci_job(config_url,
rosdistro_name,
ci_build_name,
os_name,
os_code_name,
arch,
config=None,
build_file=None,
index=None,
dist_file=None,
jenkins=None,
views=None,
is_disabled=False,
groovy_script=None,
build_targets=None,
dry_run=False,
underlay_source_paths=None,
trigger_timer=None):
"""
Configure a single Jenkins CI job.
This includes the following steps:
- clone the ros_buildfarm repository
- write the distribution repository keys into files
- invoke the ci/run_ci_job.py script
"""
if config is None:
config = get_config_index(config_url)
if build_file is None:
build_files = get_ci_build_files(config, rosdistro_name)
build_file = build_files[ci_build_name]
# Overwrite build_file.targets if build_targets is specified
if build_targets is not None:
build_file.targets = build_targets
if index is None:
index = get_index(config.rosdistro_index_url)
if dist_file is None:
dist_file = get_distribution_file(index, rosdistro_name, build_file)
if not dist_file:
raise JobValidationError(
'No distribution file matches the build file')
if os_name not in build_file.targets.keys():
raise JobValidationError("Invalid OS name '%s' " % os_name +
'choose one of the following: ' +
', '.join(sorted(build_file.targets.keys())))
if os_code_name not in build_file.targets[os_name].keys():
raise JobValidationError(
"Invalid OS code name '%s' " % os_code_name +
'choose one of the following: ' +
', '.join(sorted(build_file.targets[os_name].keys())))
if arch not in build_file.targets[os_name][os_code_name]:
raise JobValidationError(
"Invalid architecture '%s' " % arch +
'choose one of the following: %s' %
', '.join(sorted(build_file.targets[os_name][os_code_name])))
underlay_source_jobs = [
get_ci_job_name(rosdistro_name, os_name, os_code_name, arch,
underlay_job)
for underlay_job in build_file.underlay_from_ci_jobs
]
underlay_source_paths = (underlay_source_paths or []) + \
['$UNDERLAY%d_JOB_SPACE' % (index + 1) for index in range(len(underlay_source_jobs))]
trigger_jobs = [
get_ci_job_name(rosdistro_name, os_name, os_code_name, arch,
trigger_job)
for trigger_job in build_file.jenkins_job_upstream_triggers
]
if jenkins is None:
from ros_buildfarm.jenkins import connect
jenkins = connect(config.jenkins_url)
if views is None:
view_name = get_ci_view_name(rosdistro_name)
configure_ci_view(jenkins, view_name, dry_run=dry_run)
job_name = get_ci_job_name(rosdistro_name, os_name, os_code_name, arch,
ci_build_name)
job_config = _get_ci_job_config(index,
rosdistro_name,
build_file,
os_name,
os_code_name,
arch,
build_file.repos_files,
build_file.repository_names,
underlay_source_jobs,
underlay_source_paths,
trigger_timer,
trigger_jobs,
is_disabled=is_disabled)
# jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
if isinstance(jenkins, object) and jenkins is not False:
from ros_buildfarm.jenkins import configure_job
configure_job(jenkins, job_name, job_config, dry_run=dry_run)
return job_name, job_config
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description='Generate all jobs on Jenkins')
add_argument_config_url(parser)
parser.add_argument(
'--ros-distro-names',
nargs='*',
metavar='ROS_DISTRO_NAME',
default=[],
help='The list of ROS distribution names if not generating all')
parser.add_argument('--skip-rosdistro-cache-job',
action='store_true',
help='Skip generating the rosdistro-cache jobs')
parser.add_argument(
'--commit',
action='store_true',
help='Apply the changes to Jenkins instead of only showing them')
args = parser.parse_args(argv)
if args.commit:
print('The following changes will be applied to the Jenkins server.')
else:
print('This is a dry run. The Jenkins configuration is not changed.')
print('')
config = get_index(args.config_url)
if args.config_url.startswith('file:'):
print(
'WARNING: Local file system path used for configuration. ',
'Configuration will not be accessible to jobs during execution. ',
'Consider using a web(http) hosted configuration repository.',
file=sys.stderr)
ros_distro_names = sorted(config.distributions.keys())
invalid_ros_distro_name = [
n for n in args.ros_distro_names if n not in ros_distro_names
]
if invalid_ros_distro_name:
parser.error(
'The following ROS distribution names are not part of the ' +
'buildfarm index: ' + ', '.join(sorted(invalid_ros_distro_name)))
# try to connect to Jenkins master
jenkins = connect(config.jenkins_url)
configure_view(jenkins,
'Queue',
filter_queue=False,
dry_run=not args.commit)
generate_check_agents_job(args.config_url, dry_run=not args.commit)
if not args.ros_distro_names:
generate_dashboard_job(args.config_url, dry_run=not args.commit)
for doc_build_name in sorted(config.doc_builds.keys()):
generate_doc_independent_job(args.config_url,
doc_build_name,
dry_run=not args.commit)
selected_ros_distro_names = [
n for n in ros_distro_names
if not args.ros_distro_names or n in args.ros_distro_names
]
for ros_distro_name in selected_ros_distro_names:
print(ros_distro_name)
if not args.skip_rosdistro_cache_job:
generate_rosdistro_cache_job(args.config_url,
ros_distro_name,
dry_run=not args.commit)
generate_failing_jobs_job(args.config_url,
ros_distro_name,
dry_run=not args.commit)
release_build_files = get_release_build_files(config, ros_distro_name)
for release_build_name in release_build_files.keys():
generate_release_status_page_job(args.config_url,
ros_distro_name,
release_build_name,
dry_run=not args.commit)
generate_release_maintenance_jobs(args.config_url,
ros_distro_name,
release_build_name,
dry_run=not args.commit)
source_build_files = get_source_build_files(config, ros_distro_name)
for source_build_name in source_build_files.keys():
generate_devel_maintenance_jobs(args.config_url,
ros_distro_name,
source_build_name,
dry_run=not args.commit)
ci_build_files = get_ci_build_files(config, ros_distro_name)
for ci_build_name in ci_build_files.keys():
generate_ci_maintenance_jobs(args.config_url,
ros_distro_name,
ci_build_name,
dry_run=not args.commit)
doc_build_files = get_doc_build_files(config, ros_distro_name)
for doc_build_name, doc_build_file in doc_build_files.items():
if doc_build_file.documentation_type == DOC_TYPE_ROSDOC:
generate_doc_maintenance_jobs(args.config_url,
ros_distro_name,
doc_build_name,
dry_run=not args.commit)
elif doc_build_file.documentation_type == DOC_TYPE_MANIFEST:
generate_doc_metadata_job(args.config_url,
ros_distro_name,
doc_build_name,
dry_run=not args.commit)
else:
assert False, ("Unknown documentation type '%s' in doc " +
"build file '%s'") % \
(doc_build_file.documentation_type, doc_build_name)
generate_repos_status_page_jobs(args.config_url,
ros_distro_name,
dry_run=not args.commit)
index = ros_distro_names.index(ros_distro_name)
if index > 0:
# generate compare pages for this rosdistro against all older ones
generate_release_compare_page_job(args.config_url,
ros_distro_name,
ros_distro_names[:index],
dry_run=not args.commit)
generate_blocked_releases_page_job(args.config_url,
ros_distro_name,
dry_run=not args.commit)
generate_blocked_source_entries_page_job(args.config_url,
ros_distro_name,
dry_run=not args.commit)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(description="Generate a 'CI' script")
# Positional
add_argument_config_url(parser)
add_argument_rosdistro_name(parser)
add_argument_build_name(parser, 'ci')
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_build_tool(parser)
a1 = add_argument_package_selection_args(parser)
a2 = add_argument_build_tool_args(parser)
a3 = add_argument_build_tool_test_args(parser)
add_argument_repos_file_urls(parser)
add_argument_skip_cleanup(parser)
add_argument_test_branch(parser)
parser.add_argument(
'--underlay-source-path',
nargs='*',
metavar='DIR_NAME',
help='Path to one or more install spaces to use as an underlay')
remainder_args = extract_multiple_remainders(argv, (a1, a2, a3))
args = parser.parse_args(argv)
for k, v in remainder_args.items():
setattr(args, k, v)
# collect all template snippets of specific types
class IncludeHook(Hook):
def __init__(self):
Hook.__init__(self)
self.scms = []
self.scripts = []
self.parameters = {}
if args.skip_cleanup:
self.parameters['skip_cleanup'] = 'true'
if args.repos_file_urls is not None:
self.parameters['repos_file_urls'] = ' '.join(
args.repos_file_urls)
if args.test_branch is not None:
self.parameters['test_branch'] = args.test_branch
if args.package_selection_args is not None:
self.parameters['package_selection_args'] = ' '.join(
args.package_selection_args)
if args.build_tool_args is not None:
self.parameters['build_tool_args'] = ' '.join(
args.build_tool_args)
if args.build_tool_test_args is not None:
self.parameters['build_tool_test_args'] = ' '.join(
args.build_tool_test_args)
def beforeInclude(self, *_, **kwargs):
template_path = kwargs['file'].name
if template_path.endswith('/snippet/scm.xml.em'):
self.scms.append(
(kwargs['locals']['repo_spec'], kwargs['locals']['path']))
if template_path.endswith('/snippet/builder_shell.xml.em'):
script = kwargs['locals']['script']
# reuse existing ros_buildfarm folder if it exists
if 'Clone ros_buildfarm' in script:
lines = script.splitlines()
lines.insert(0, 'if [ ! -d "ros_buildfarm" ]; then')
lines += [
'else',
'echo "Using existing ros_buildfarm folder"',
'fi',
]
script = '\n'.join(lines)
if args.build_tool and ' --build-tool ' in script:
script = script.replace(
' --build-tool catkin_make_isolated',
' --build-tool ' + args.build_tool)
self.scripts.append(script)
if template_path.endswith(
'/snippet/property_parameters-definition.xml.em'):
for parameter in reversed(kwargs['locals']['parameters']):
name = parameter['name']
value_type = parameter['type']
if value_type in ['string', 'text']:
default_value = parameter['default_value']
elif value_type == 'boolean':
default_value = 'true' if parameter.get(
'default_value', False) else 'false'
else:
continue
self.parameters.setdefault(name, default_value)
hook = IncludeHook()
from ros_buildfarm import templates
templates.template_hooks = [hook]
config = get_config_index(args.config_url)
build_files = get_ci_build_files(config, args.rosdistro_name)
build_file = build_files[args.ci_build_name]
underlay_source_paths = [
os.path.abspath(p) for p in args.underlay_source_path or []
]
configure_ci_job(args.config_url,
args.rosdistro_name,
args.ci_build_name,
args.os_name,
args.os_code_name,
args.arch,
config=config,
build_file=build_file,
jenkins=False,
views=False,
underlay_source_paths=underlay_source_paths)
templates.template_hooks = None
ci_job_name = get_ci_job_name(args.rosdistro_name, args.os_name,
args.os_code_name, args.arch, 'script')
value = expand_template('ci/ci_script.sh.em', {
'ci_job_name': ci_job_name,
'scms': hook.scms,
'scripts': hook.scripts,
'build_tool': args.build_tool or build_file.build_tool,
'parameters': hook.parameters
},
options={BANGPATH_OPT: False})
value = value.replace('python3 ', sys.executable + ' ')
print(value)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'CI' script")
# Positional
add_argument_config_url(parser)
add_argument_rosdistro_name(parser)
add_argument_build_name(parser, 'ci')
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_build_ignore(parser)
add_argument_build_tool(parser)
add_argument_package_selection_args(parser)
add_argument_repos_file_urls(parser)
add_argument_skip_cleanup(parser)
add_argument_test_branch(parser)
parser.add_argument(
'--underlay-source-path', nargs='*', metavar='DIR_NAME',
help='Path to one or more install spaces to use as an underlay')
args = parser.parse_args(argv)
# collect all template snippets of specific types
class IncludeHook(Hook):
def __init__(self):
Hook.__init__(self)
self.scms = []
self.scripts = []
self.parameters = {}
if args.skip_cleanup:
self.parameters['skip_cleanup'] = 'true'
if args.repos_file_urls is not None:
self.parameters['repos_file_urls'] = ' '.join(args.repos_file_urls)
if args.test_branch is not None:
self.parameters['test_branch'] = args.test_branch
if args.build_ignore is not None:
self.parameters['build_ignore'] = ' '.join(args.build_ignore)
if args.package_selection_args is not None:
self.parameters['package_selection_args'] = ' '.join(args.package_selection_args)
def beforeInclude(self, *_, **kwargs):
template_path = kwargs['file'].name
if template_path.endswith('/snippet/scm.xml.em'):
self.scms.append(
(kwargs['locals']['repo_spec'], kwargs['locals']['path']))
if template_path.endswith('/snippet/builder_shell.xml.em'):
script = kwargs['locals']['script']
# reuse existing ros_buildfarm folder if it exists
if 'Clone ros_buildfarm' in script:
lines = script.splitlines()
lines.insert(0, 'if [ ! -d "ros_buildfarm" ]; then')
lines += [
'else',
'echo "Using existing ros_buildfarm folder"',
'fi',
]
script = '\n'.join(lines)
if args.build_tool and ' --build-tool ' in script:
script = script.replace(
' --build-tool catkin_make_isolated',
' --build-tool ' + args.build_tool)
self.scripts.append(script)
if template_path.endswith('/snippet/property_parameters-definition.xml.em'):
for parameter in reversed(kwargs['locals']['parameters']):
name = parameter['name']
value_type = parameter['type']
if value_type in ['string', 'text']:
default_value = parameter['default_value']
elif value_type == 'boolean':
default_value = 'true' if parameter.get(
'default_value', False) else 'false'
else:
continue
self.parameters.setdefault(name, default_value)
hook = IncludeHook()
from ros_buildfarm import templates
templates.template_hooks = [hook]
config = get_config_index(args.config_url)
build_files = get_ci_build_files(config, args.rosdistro_name)
build_file = build_files[args.ci_build_name]
underlay_source_paths = [os.path.abspath(p) for p in args.underlay_source_path or []]
configure_ci_job(
args.config_url, args.rosdistro_name, args.ci_build_name,
args.os_name, args.os_code_name, args.arch,
config=config, build_file=build_file, jenkins=False, views=False,
underlay_source_paths=underlay_source_paths)
templates.template_hooks = None
ci_job_name = get_ci_job_name(
args.rosdistro_name, args.os_name,
args.os_code_name, args.arch, 'script')
value = expand_template(
'ci/ci_script.sh.em', {
'ci_job_name': ci_job_name,
'scms': hook.scms,
'scripts': hook.scripts,
'build_tool': args.build_tool or build_file.build_tool,
'parameters': hook.parameters},
options={BANGPATH_OPT: False})
value = value.replace('python3 ', sys.executable + ' ')
print(value)
