def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(description="Generate a 'release' script")
add_argument_config_url(parser)
add_argument_rosdistro_name(parser)
add_argument_build_name(parser, 'release')
add_argument_package_name(parser)
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
parser.add_argument('--skip-install',
action='store_true',
help='Skip trying to install binarydeb')
args = parser.parse_args(argv)
# collect all template snippets of specific types
class IncludeHook(Hook):
def __init__(self):
Hook.__init__(self)
self.scripts = []
def beforeFile(self, *args, **kwargs):
template_path = kwargs['file'].name
if template_path.endswith('/release/binarydeb_job.xml.em'):
self.scripts.append('--')
if template_path.endswith('/snippet/builder_shell.xml.em'):
self.scripts.append(kwargs['locals']['script'])
hook = IncludeHook()
from ros_buildfarm import templates
templates.template_hooks = [hook]
configure_release_job(args.config_url,
args.rosdistro_name,
args.release_build_name,
args.package_name,
args.os_name,
args.os_code_name,
jenkins=False,
views=[],
generate_import_package_job=False,
generate_sync_packages_jobs=False,
filter_arches=args.arch)
templates.template_hooks = None
source_job_name = get_sourcedeb_job_name(args.rosdistro_name,
args.release_build_name,
args.package_name, args.os_name,
args.os_code_name)
binary_job_name = get_binarydeb_job_name(args.rosdistro_name,
args.release_build_name,
args.package_name, args.os_name,
args.os_code_name, args.arch)
separator_index = hook.scripts.index('--')
source_scripts = hook.scripts[:separator_index]
binary_scripts = hook.scripts[separator_index + 1:]
# inject additional argument to skip fetching sourcedeb from repo
script_name = '/run_binarydeb_job.py '
additional_argument = '--skip-download-sourcedeb '
for i, script in enumerate(binary_scripts):
offset = script.find(script_name)
if offset != -1:
offset += len(script_name)
script = script[:offset] + additional_argument + script[offset:]
binary_scripts[i] = script
break
# remove rm command for sourcedeb location
rm_command = 'rm -fr $WORKSPACE/binarydeb'
for i, script in enumerate(binary_scripts):
offset = script.find(rm_command)
if offset != -1:
script = script[:offset] + script[offset + len(rm_command):]
binary_scripts[i] = script
break
if args.skip_install:
# remove install step
script_name = '/create_binarydeb_install_task_generator.py '
for i, script in enumerate(binary_scripts):
offset = script.find(script_name)
if offset != -1:
del binary_scripts[i]
break
value = expand_template('release/release_script.sh.em', {
'source_job_name': source_job_name,
'binary_job_name': binary_job_name,
'source_scripts': source_scripts,
'binary_scripts': binary_scripts
},
options={BANGPATH_OPT: False})
value = value.replace('python3', sys.executable)
print(value)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'release' script")
add_argument_config_url(parser)
add_argument_rosdistro_name(parser)
add_argument_build_name(parser, 'release')
add_argument_package_name(parser)
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
parser.add_argument(
'--skip-install',
action='store_true',
help='Skip trying to install binarydeb')
args = parser.parse_args(argv)
# collect all template snippets of specific types
class IncludeHook(Hook):
def __init__(self):
Hook.__init__(self)
self.scripts = []
def beforeFile(self, *args, **kwargs):
template_path = kwargs['file'].name
if template_path.endswith('/release/binarydeb_job.xml.em'):
self.scripts.append('--')
if template_path.endswith('/snippet/builder_shell.xml.em'):
self.scripts.append(kwargs['locals']['script'])
hook = IncludeHook()
from ros_buildfarm import templates
templates.template_hooks = [hook]
configure_release_job(
args.config_url, args.rosdistro_name, args.release_build_name,
args.package_name, args.os_name, args.os_code_name,
jenkins=False, views=[], generate_import_package_job=False,
generate_sync_packages_jobs=False, filter_arches=args.arch)
templates.template_hooks = None
source_job_name = get_sourcedeb_job_name(
args.rosdistro_name, args.release_build_name,
args.package_name, args.os_name, args.os_code_name)
binary_job_name = get_binarydeb_job_name(
args.rosdistro_name, args.release_build_name,
args.package_name, args.os_name, args.os_code_name, args.arch)
separator_index = hook.scripts.index('--')
source_scripts = hook.scripts[:separator_index]
binary_scripts = hook.scripts[separator_index + 1:]
# inject additional argument to skip fetching sourcedeb from repo
script_name = '/run_binarydeb_job.py '
additional_argument = '--skip-download-sourcedeb '
for i, script in enumerate(binary_scripts):
offset = script.find(script_name)
if offset != -1:
offset += len(script_name)
script = script[:offset] + additional_argument + script[offset:]
binary_scripts[i] = script
break
# remove rm command for sourcedeb location
rm_command = 'rm -fr $WORKSPACE/binarydeb'
for i, script in enumerate(binary_scripts):
offset = script.find(rm_command)
if offset != -1:
script = script[:offset] + script[offset + len(rm_command):]
binary_scripts[i] = script
break
if args.skip_install:
# remove install step
script_name = '/create_binarydeb_install_task_generator.py '
for i, script in enumerate(binary_scripts):
offset = script.find(script_name)
if offset != -1:
del binary_scripts[i]
break
value = expand_template(
'release/release_script.sh.em', {
'source_job_name': source_job_name,
'binary_job_name': binary_job_name,
'source_scripts': source_scripts,
'binary_scripts': binary_scripts},
options={BANGPATH_OPT: False})
value = value.replace('python3', sys.executable)
print(value)
def configure_release_job(config_url,
rosdistro_name,
release_build_name,
pkg_name,
os_name,
os_code_name,
config=None,
build_file=None,
index=None,
dist_file=None,
dist_cache=None,
jenkins=None,
views=None,
generate_import_package_job=True,
generate_sync_packages_jobs=True,
is_disabled=False,
other_build_files_same_platform=None,
groovy_script=None,
filter_arches=None,
dry_run=False):
"""
Configure a Jenkins release job.
The following jobs are created for each package:
- M source jobs, one for each OS node name
- M * N binary jobs, one for each combination of OS code name and arch
"""
if config is None:
config = get_config_index(config_url)
if build_file is None:
build_files = get_release_build_files(config, rosdistro_name)
build_file = build_files[release_build_name]
if index is None:
index = get_index(config.rosdistro_index_url)
if dist_file is None:
dist_file = get_distribution_file(index, rosdistro_name, build_file)
if not dist_file:
raise JobValidationError(
'No distribution file matches the build file')
pkg_names = dist_file.release_packages.keys()
if pkg_name not in pkg_names:
raise JobValidationError("Invalid package name '%s' " % pkg_name +
'choose one of the following: ' +
', '.join(sorted(pkg_names)))
pkg = dist_file.release_packages[pkg_name]
repo_name = pkg.repository_name
repo = dist_file.repositories[repo_name]
if not repo.release_repository:
raise JobValidationError("Repository '%s' has no release section" %
repo_name)
if not repo.release_repository.version:
raise JobValidationError("Repository '%s' has no release version" %
repo_name)
if os_name not in build_file.targets.keys():
raise JobValidationError("Invalid OS name '%s' " % os_name +
'choose one of the following: ' +
', '.join(sorted(build_file.targets.keys())))
if os_code_name not in build_file.targets[os_name].keys():
raise JobValidationError(
"Invalid OS code name '%s' " % os_code_name +
'choose one of the following: ' +
', '.join(sorted(build_file.targets[os_name].keys())))
if dist_cache is None and \
(build_file.notify_maintainers or
build_file.abi_incompatibility_assumed):
dist_cache = get_distribution_cache(index, rosdistro_name)
if jenkins is None:
from ros_buildfarm.jenkins import connect
jenkins = connect(config.jenkins_url)
if views is None:
targets = []
targets.append((os_name, os_code_name, 'source'))
for arch in build_file.targets[os_name][os_code_name]:
targets.append((os_name, os_code_name, arch))
configure_release_views(jenkins,
rosdistro_name,
release_build_name,
targets,
dry_run=dry_run)
if generate_import_package_job:
configure_import_package_job(config_url,
rosdistro_name,
release_build_name,
config=config,
build_file=build_file,
jenkins=jenkins,
dry_run=dry_run)
if generate_sync_packages_jobs:
configure_sync_packages_to_main_job(config_url,
rosdistro_name,
release_build_name,
config=config,
build_file=build_file,
jenkins=jenkins,
dry_run=dry_run)
for arch in build_file.targets[os_name][os_code_name]:
configure_sync_packages_to_testing_job(config_url,
rosdistro_name,
release_build_name,
os_code_name,
arch,
config=config,
build_file=build_file,
jenkins=jenkins,
dry_run=dry_run)
source_job_names = []
binary_job_names = []
job_configs = {}
# sourcedeb job
# since sourcedeb jobs are potentially being shared across multiple build
# files the configuration has to take all of them into account in order to
# generate a job which all build files agree on
source_job_name = get_sourcedeb_job_name(rosdistro_name,
release_build_name, pkg_name,
os_name, os_code_name)
# while the package is disabled in the current build file
# it might be used by sibling build files
is_source_disabled = is_disabled
if is_source_disabled and other_build_files_same_platform:
# check if sourcedeb job is used by any other build file with the same platform
for other_build_file in other_build_files_same_platform:
if other_build_file.filter_packages([pkg_name]):
is_source_disabled = False
break
job_config = _get_sourcedeb_job_config(
config_url,
rosdistro_name,
release_build_name,
config,
build_file,
os_name,
os_code_name,
pkg_name,
repo_name,
repo.release_repository,
dist_cache=dist_cache,
is_disabled=is_source_disabled,
other_build_files_same_platform=other_build_files_same_platform)
# jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
if isinstance(jenkins, object) and jenkins is not False:
from ros_buildfarm.jenkins import configure_job
configure_job(jenkins, source_job_name, job_config, dry_run=dry_run)
source_job_names.append(source_job_name)
job_configs[source_job_name] = job_config
dependency_names = []
if build_file.abi_incompatibility_assumed:
dependency_names = _get_direct_dependencies(pkg_name, dist_cache,
pkg_names)
# if dependencies are not yet available in rosdistro cache
# skip binary jobs
if dependency_names is None:
print(("Skipping binary jobs for package '%s' because it is not " +
"yet in the rosdistro cache") % pkg_name,
file=sys.stderr)
return source_job_names, binary_job_names, job_configs
# binarydeb jobs
for arch in build_file.targets[os_name][os_code_name]:
if filter_arches and arch not in filter_arches:
continue
job_name = get_binarydeb_job_name(rosdistro_name, release_build_name,
pkg_name, os_name, os_code_name,
arch)
upstream_job_names = [source_job_name] + [
get_binarydeb_job_name(
rosdistro_name, release_build_name, dependency_name, os_name,
os_code_name, arch) for dependency_name in dependency_names
]
job_config = _get_binarydeb_job_config(
config_url,
rosdistro_name,
release_build_name,
config,
build_file,
os_name,
os_code_name,
arch,
pkg_name,
repo_name,
repo.release_repository,
dist_cache=dist_cache,
upstream_job_names=upstream_job_names,
is_disabled=is_disabled)
# jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
if isinstance(jenkins, object) and jenkins is not False:
configure_job(jenkins, job_name, job_config, dry_run=dry_run)
binary_job_names.append(job_name)
job_configs[job_name] = job_config
return source_job_names, binary_job_names, job_configs
def configure_release_job(
config_url, rosdistro_name, release_build_name,
pkg_name, os_name, os_code_name,
config=None, build_file=None,
index=None, dist_file=None, dist_cache=None,
jenkins=None, views=None,
generate_import_package_job=True,
generate_sync_packages_jobs=True,
is_disabled=False, other_build_files_same_platform=None,
groovy_script=None,
filter_arches=None,
dry_run=False):
"""
Configure a Jenkins release job.
The following jobs are created for each package:
- M source jobs, one for each OS node name
- M * N binary jobs, one for each combination of OS code name and arch
"""
if config is None:
config = get_config_index(config_url)
if build_file is None:
build_files = get_release_build_files(config, rosdistro_name)
build_file = build_files[release_build_name]
if index is None:
index = get_index(config.rosdistro_index_url)
if dist_file is None:
dist_file = get_distribution_file(index, rosdistro_name, build_file)
if not dist_file:
raise JobValidationError(
'No distribution file matches the build file')
pkg_names = dist_file.release_packages.keys()
if pkg_name not in pkg_names:
raise JobValidationError(
"Invalid package name '%s' " % pkg_name +
'choose one of the following: ' + ', '.join(sorted(pkg_names)))
pkg = dist_file.release_packages[pkg_name]
repo_name = pkg.repository_name
repo = dist_file.repositories[repo_name]
if not repo.release_repository:
raise JobValidationError(
"Repository '%s' has no release section" % repo_name)
if not repo.release_repository.version:
raise JobValidationError(
"Repository '%s' has no release version" % repo_name)
if os_name not in build_file.targets.keys():
raise JobValidationError(
"Invalid OS name '%s' " % os_name +
'choose one of the following: ' +
', '.join(sorted(build_file.targets.keys())))
if os_code_name not in build_file.targets[os_name].keys():
raise JobValidationError(
"Invalid OS code name '%s' " % os_code_name +
'choose one of the following: ' +
', '.join(sorted(build_file.targets[os_name].keys())))
if dist_cache is None and \
(build_file.notify_maintainers or
build_file.abi_incompatibility_assumed):
dist_cache = get_distribution_cache(index, rosdistro_name)
if jenkins is None:
jenkins = connect(config.jenkins_url)
if views is None:
targets = []
targets.append((os_name, os_code_name, 'source'))
for arch in build_file.targets[os_name][os_code_name]:
targets.append((os_name, os_code_name, arch))
configure_release_views(
jenkins, rosdistro_name, release_build_name, targets,
dry_run=dry_run)
if generate_import_package_job:
configure_import_package_job(
config_url, rosdistro_name, release_build_name,
config=config, build_file=build_file, jenkins=jenkins,
dry_run=dry_run)
if generate_sync_packages_jobs:
configure_sync_packages_to_main_job(
config_url, rosdistro_name, release_build_name,
config=config, build_file=build_file, jenkins=jenkins,
dry_run=dry_run)
for arch in build_file.targets[os_name][os_code_name]:
configure_sync_packages_to_testing_job(
config_url, rosdistro_name, release_build_name,
os_code_name, arch,
config=config, build_file=build_file, jenkins=jenkins,
dry_run=dry_run)
source_job_names = []
binary_job_names = []
job_configs = {}
# sourcedeb job
# since sourcedeb jobs are potentially being shared across multiple build
# files the configuration has to take all of them into account in order to
# generate a job which all build files agree on
source_job_name = get_sourcedeb_job_name(
rosdistro_name, release_build_name,
pkg_name, os_name, os_code_name)
# while the package is disabled in the current build file
# it might be used by sibling build files
is_source_disabled = is_disabled
if is_source_disabled and other_build_files_same_platform:
# check if sourcedeb job is used by any other build file with the same platform
for other_build_file in other_build_files_same_platform:
if other_build_file.filter_packages([pkg_name]):
is_source_disabled = False
break
job_config = _get_sourcedeb_job_config(
config_url, rosdistro_name, release_build_name,
config, build_file, os_name, os_code_name,
pkg_name, repo_name, repo.release_repository, dist_cache=dist_cache,
is_disabled=is_source_disabled,
other_build_files_same_platform=other_build_files_same_platform)
# jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
if isinstance(jenkins, object) and jenkins is not False:
configure_job(jenkins, source_job_name, job_config, dry_run=dry_run)
source_job_names.append(source_job_name)
job_configs[source_job_name] = job_config
dependency_names = []
if build_file.abi_incompatibility_assumed:
dependency_names = _get_direct_dependencies(
pkg_name, dist_cache, pkg_names)
# if dependencies are not yet available in rosdistro cache
# skip binary jobs
if dependency_names is None:
print(("Skipping binary jobs for package '%s' because it is not " +
"yet in the rosdistro cache") % pkg_name, file=sys.stderr)
return source_job_names, binary_job_names, job_configs
# binarydeb jobs
for arch in build_file.targets[os_name][os_code_name]:
if filter_arches and arch not in filter_arches:
continue
job_name = get_binarydeb_job_name(
rosdistro_name, release_build_name,
pkg_name, os_name, os_code_name, arch)
upstream_job_names = [source_job_name] + [
get_binarydeb_job_name(
rosdistro_name, release_build_name,
dependency_name, os_name, os_code_name, arch)
for dependency_name in dependency_names]
job_config = _get_binarydeb_job_config(
config_url, rosdistro_name, release_build_name,
config, build_file, os_name, os_code_name, arch,
pkg_name, repo_name, repo.release_repository,
dist_cache=dist_cache, upstream_job_names=upstream_job_names,
is_disabled=is_disabled)
# jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
if isinstance(jenkins, object) and jenkins is not False:
configure_job(jenkins, job_name, job_config, dry_run=dry_run)
binary_job_names.append(job_name)
job_configs[job_name] = job_config
return source_job_names, binary_job_names, job_configs
