def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate the 'doc' management jobs on Jenkins")
add_argument_config_url(parser)
add_argument_rosdistro_name(parser)
add_argument_build_name(parser, 'doc')
args = parser.parse_args(argv)
config = get_index(args.config_url)
build_files = get_doc_build_files(config, args.rosdistro_name)
build_file = build_files[args.doc_build_name]
if build_file.documentation_type != DOC_TYPE_ROSDOC:
print(("The doc build file '%s' has the wrong documentation type to " +
"be used with this script") % args.doc_build_name,
file=sys.stderr)
return 1
jenkins = connect(config.jenkins_url)
configure_management_view(jenkins)
group_name = get_doc_view_name(
args.rosdistro_name, args.doc_build_name)
configure_reconfigure_jobs_job(
jenkins, group_name, args, config, build_file)
configure_trigger_jobs_job(jenkins, group_name, build_file)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate the 'doc' management jobs on Jenkins")
add_argument_config_url(parser)
add_argument_rosdistro_name(parser)
add_argument_build_name(parser, 'doc')
add_argument_dry_run(parser)
args = parser.parse_args(argv)
config = get_index(args.config_url)
build_files = get_doc_build_files(config, args.rosdistro_name)
build_file = build_files[args.doc_build_name]
if build_file.documentation_type != DOC_TYPE_ROSDOC:
print(("The doc build file '%s' has the wrong documentation type to " +
"be used with this script") % args.doc_build_name,
file=sys.stderr)
return 1
jenkins = connect(config.jenkins_url)
configure_management_view(jenkins, dry_run=args.dry_run)
group_name = get_doc_view_name(args.rosdistro_name, args.doc_build_name)
configure_reconfigure_jobs_job(jenkins,
group_name,
args,
config,
build_file,
dry_run=args.dry_run)
configure_trigger_jobs_job(jenkins,
group_name,
build_file,
dry_run=args.dry_run)
def get_job_config(args, config):
template_name = 'misc/rosdistro_cache_job.xml.em'
repository_args, script_generating_key_files = \
get_repositories_and_script_generating_key_files(config=config)
reconfigure_job_names = []
build_files = get_release_build_files(config, args.rosdistro_name)
for release_build_name in sorted(build_files.keys()):
group_name = get_release_job_prefix(args.rosdistro_name,
release_build_name)
job_name = '%s_%s' % (group_name, 'reconfigure-jobs')
reconfigure_job_names.append(job_name)
reconfigure_doc_job_names = []
build_files = get_doc_build_files(config, args.rosdistro_name)
for doc_build_name in sorted(build_files.keys()):
group_name = get_doc_view_name(args.rosdistro_name, doc_build_name)
job_name = '%s_%s' % (group_name, 'reconfigure-jobs')
reconfigure_doc_job_names.append(job_name)
reconfigure_source_job_names = []
build_files = get_source_build_files(config, args.rosdistro_name)
for source_build_name in sorted(build_files.keys()):
group_name = get_devel_view_name(args.rosdistro_name,
source_build_name)
job_name = '%s_%s' % (group_name, 'reconfigure-jobs')
reconfigure_source_job_names.append(job_name)
job_data = copy.deepcopy(args.__dict__)
job_data.update({
'ros_buildfarm_repository':
get_repository(),
'script_generating_key_files':
script_generating_key_files,
'rosdistro_index_url':
config.rosdistro_index_url,
'repository_args':
repository_args,
'reconfigure_job_names':
reconfigure_job_names,
'reconfigure_doc_job_names':
reconfigure_doc_job_names,
'reconfigure_source_job_names':
reconfigure_source_job_names,
'notification_emails':
config.distributions[args.rosdistro_name]['notification_emails'],
'git_ssh_credential_id':
config.git_ssh_credential_id,
})
job_config = expand_template(template_name, job_data)
return job_config
def get_job_config(args, config):
template_name = 'misc/rosdistro_cache_job.xml.em'
repository_args, script_generating_key_files = \
get_repositories_and_script_generating_key_files(config=config)
reconfigure_job_names = []
build_files = get_release_build_files(config, args.rosdistro_name)
for release_build_name in sorted(build_files.keys()):
group_name = get_release_job_prefix(
args.rosdistro_name, release_build_name)
job_name = '%s_%s' % (group_name, 'reconfigure-jobs')
reconfigure_job_names.append(job_name)
reconfigure_doc_job_names = []
build_files = get_doc_build_files(config, args.rosdistro_name)
for doc_build_name in sorted(build_files.keys()):
group_name = get_doc_view_name(
args.rosdistro_name, doc_build_name)
job_name = '%s_%s' % (group_name, 'reconfigure-jobs')
reconfigure_doc_job_names.append(job_name)
reconfigure_source_job_names = []
build_files = get_source_build_files(config, args.rosdistro_name)
for source_build_name in sorted(build_files.keys()):
group_name = get_devel_view_name(
args.rosdistro_name, source_build_name)
job_name = '%s_%s' % (group_name, 'reconfigure-jobs')
reconfigure_source_job_names.append(job_name)
job_data = copy.deepcopy(args.__dict__)
job_data.update({
'ros_buildfarm_repository': get_repository(),
'script_generating_key_files': script_generating_key_files,
'rosdistro_index_url': config.rosdistro_index_url,
'repository_args': repository_args,
'reconfigure_job_names': reconfigure_job_names,
'reconfigure_doc_job_names': reconfigure_doc_job_names,
'reconfigure_source_job_names': reconfigure_source_job_names,
'notification_emails':
config.distributions[args.rosdistro_name]['notification_emails'],
'git_ssh_credential_id': config.git_ssh_credential_id,
})
job_config = expand_template(template_name, job_data)
return job_config
def configure_doc_metadata_job(
config_url, rosdistro_name, doc_build_name,
config=None, build_file=None):
if config is None:
config = get_config_index(config_url)
if build_file is None:
build_files = get_doc_build_files(config, rosdistro_name)
build_file = build_files[doc_build_name]
from ros_buildfarm.jenkins import connect
jenkins = connect(config.jenkins_url)
job_name = get_doc_view_name(rosdistro_name, doc_build_name)
job_config = _get_doc_metadata_job_config(
config, config_url, rosdistro_name, doc_build_name, build_file)
# jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
if isinstance(jenkins, object) and jenkins is not False:
from ros_buildfarm.jenkins import configure_job
configure_job(jenkins, job_name, job_config)
def configure_doc_metadata_job(
config_url, rosdistro_name, doc_build_name,
config=None, build_file=None, dry_run=False):
if config is None:
config = get_config_index(config_url)
if build_file is None:
build_files = get_doc_build_files(config, rosdistro_name)
build_file = build_files[doc_build_name]
from ros_buildfarm.jenkins import connect
jenkins = connect(config.jenkins_url)
job_name = get_doc_view_name(rosdistro_name, doc_build_name)
job_config = _get_doc_metadata_job_config(
config, config_url, rosdistro_name, doc_build_name, build_file)
# jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
if isinstance(jenkins, object) and jenkins is not False:
from ros_buildfarm.jenkins import configure_job
configure_job(jenkins, job_name, job_config, dry_run=dry_run)
def configure_doc_jobs(
config_url, rosdistro_name, doc_build_name, groovy_script=None):
"""
Configure all Jenkins doc jobs.
L{configure_doc_job} will be invoked for doc repository and target
which matches the build file criteria.
"""
config = get_config_index(config_url)
build_files = get_doc_build_files(config, rosdistro_name)
build_file = build_files[doc_build_name]
index = get_index(config.rosdistro_index_url)
dist_cache = None
if build_file.notify_maintainers:
dist_cache = get_distribution_cache(index, rosdistro_name)
# get targets
targets = []
for os_name in build_file.targets.keys():
for os_code_name in build_file.targets[os_name].keys():
for arch in build_file.targets[os_name][os_code_name]:
targets.append((os_name, os_code_name, arch))
print('The build file contains the following targets:')
for os_name, os_code_name, arch in targets:
print(' -', os_name, os_code_name, arch)
dist_file = get_distribution_file(index, rosdistro_name, build_file)
if not dist_file:
print('No distribution file matches the build file')
return
doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name)
from ros_buildfarm.jenkins import connect
jenkins = connect(config.jenkins_url)
views = []
views.append(configure_doc_view(jenkins, doc_view_name))
if groovy_script is not None:
# all further configuration will be handled by the groovy script
jenkins = False
repo_names = dist_file.repositories.keys()
filtered_repo_names = build_file.filter_repositories(repo_names)
job_names = []
job_configs = {}
for repo_name in sorted(repo_names):
is_disabled = repo_name not in filtered_repo_names
if is_disabled and build_file.skip_ignored_repositories:
print("Skipping ignored repository '%s'" % repo_name,
file=sys.stderr)
continue
repo = dist_file.repositories[repo_name]
if not repo.doc_repository:
print("Skipping repository '%s': no doc section" % repo_name)
continue
if not repo.doc_repository.version:
print("Skipping repository '%s': no doc version" % repo_name)
continue
for os_name, os_code_name, arch in targets:
try:
job_name, job_config = configure_doc_job(
config_url, rosdistro_name, doc_build_name,
repo_name, os_name, os_code_name, arch,
config=config, build_file=build_file,
index=index, dist_file=dist_file,
dist_cache=dist_cache, jenkins=jenkins, views=views,
is_disabled=is_disabled,
groovy_script=groovy_script)
job_names.append(job_name)
if groovy_script is not None:
print("Configuration for job '%s'" % job_name)
job_configs[job_name] = job_config
except JobValidationError as e:
print(e.message, file=sys.stderr)
job_prefix = '%s__' % doc_view_name
if groovy_script is None:
# delete obsolete jobs in this view
from ros_buildfarm.jenkins import remove_jobs
print('Removing obsolete doc jobs')
remove_jobs(jenkins, job_prefix, job_names)
else:
print("Writing groovy script '%s' to reconfigure %d jobs" %
(groovy_script, len(job_configs)))
data = {
'expected_num_jobs': len(job_configs),
'job_prefixes_and_names': {
'doc': (job_prefix, job_names),
}
}
content = expand_template('snippet/reconfigure_jobs.groovy.em', data)
write_groovy_script_and_configs(
groovy_script, content, job_configs)
def configure_doc_job(
config_url, rosdistro_name, doc_build_name,
repo_name, os_name, os_code_name, arch,
config=None, build_file=None,
index=None, dist_file=None, dist_cache=None,
jenkins=None, views=None,
is_disabled=False,
groovy_script=None,
doc_repository=None):
"""
Configure a single Jenkins doc job.
This includes the following steps:
- clone the doc repository to use
- clone the ros_buildfarm repository
- write the distribution repository keys into files
- invoke the run_doc_job.py script
"""
if config is None:
config = get_config_index(config_url)
if build_file is None:
build_files = get_doc_build_files(config, rosdistro_name)
build_file = build_files[doc_build_name]
if index is None:
index = get_index(config.rosdistro_index_url)
if dist_file is None:
dist_file = get_distribution_file(index, rosdistro_name, build_file)
if not dist_file:
raise JobValidationError(
'No distribution file matches the build file')
repo_names = dist_file.repositories.keys()
if repo_name is not None:
if repo_name not in repo_names:
raise JobValidationError(
"Invalid repository name '%s' " % repo_name +
'choose one of the following: %s' %
', '.join(sorted(repo_names)))
repo = dist_file.repositories[repo_name]
if not repo.doc_repository:
raise JobValidationError(
"Repository '%s' has no doc section" % repo_name)
if not repo.doc_repository.version:
raise JobValidationError(
"Repository '%s' has no doc version" % repo_name)
doc_repository = repo.doc_repository
if os_name not in build_file.targets.keys():
raise JobValidationError(
"Invalid OS name '%s' " % os_name +
'choose one of the following: ' +
', '.join(sorted(build_file.targets.keys())))
if os_code_name not in build_file.targets[os_name].keys():
raise JobValidationError(
"Invalid OS code name '%s' " % os_code_name +
'choose one of the following: ' +
', '.join(sorted(build_file.targets[os_name].keys())))
if arch not in build_file.targets[os_name][os_code_name]:
raise JobValidationError(
"Invalid architecture '%s' " % arch +
'choose one of the following: %s' % ', '.join(sorted(
build_file.targets[os_name][os_code_name])))
if dist_cache is None and build_file.notify_maintainers:
dist_cache = get_distribution_cache(index, rosdistro_name)
if jenkins is None:
from ros_buildfarm.jenkins import connect
jenkins = connect(config.jenkins_url)
if views is None:
view_name = get_doc_view_name(
rosdistro_name, doc_build_name)
configure_doc_view(jenkins, view_name)
job_name = get_doc_job_name(
rosdistro_name, doc_build_name,
repo_name, os_name, os_code_name, arch)
job_config = _get_doc_job_config(
config, config_url, rosdistro_name, doc_build_name,
build_file, os_name, os_code_name, arch, doc_repository,
repo_name, dist_cache=dist_cache, is_disabled=is_disabled)
# jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
if isinstance(jenkins, object) and jenkins is not False:
from ros_buildfarm.jenkins import configure_job
configure_job(jenkins, job_name, job_config)
return job_name, job_config
def configure_doc_jobs(config_url,
rosdistro_name,
doc_build_name,
groovy_script=None,
dry_run=False,
whitelist_repository_names=None):
"""
Configure all Jenkins doc jobs.
L{configure_doc_job} will be invoked for doc repository and target
which matches the build file criteria.
"""
config = get_config_index(config_url)
build_files = get_doc_build_files(config, rosdistro_name)
build_file = build_files[doc_build_name]
index = get_index(config.rosdistro_index_url)
dist_cache = None
if build_file.notify_maintainers:
dist_cache = get_distribution_cache(index, rosdistro_name)
# get targets
targets = []
for os_name in build_file.targets.keys():
for os_code_name in build_file.targets[os_name].keys():
for arch in build_file.targets[os_name][os_code_name]:
targets.append((os_name, os_code_name, arch))
print('The build file contains the following targets:')
for os_name, os_code_name, arch in targets:
print(' -', os_name, os_code_name, arch)
dist_file = get_distribution_file(index, rosdistro_name, build_file)
if not dist_file:
print('No distribution file matches the build file')
return
doc_view_name = get_doc_view_name(rosdistro_name, doc_build_name)
# all further configuration will be handled by either the Jenkins API
# or by a generated groovy script
from ros_buildfarm.jenkins import connect
jenkins = connect(config.jenkins_url) if groovy_script is None else False
view_configs = {}
views = {}
views[doc_view_name] = configure_doc_view(jenkins,
doc_view_name,
dry_run=dry_run)
if not jenkins:
view_configs.update(views)
groovy_data = {
'dry_run': dry_run,
'expected_num_views': len(view_configs),
}
repo_names = dist_file.repositories.keys()
filtered_repo_names = build_file.filter_repositories(repo_names)
job_names = []
job_configs = OrderedDict()
for repo_name in sorted(repo_names):
if whitelist_repository_names:
if repo_name not in whitelist_repository_names:
print(
"Skipping repository '%s' not in explicitly passed list" %
repo_name,
file=sys.stderr)
continue
is_disabled = repo_name not in filtered_repo_names
if is_disabled and build_file.skip_ignored_repositories:
print("Skipping ignored repository '%s'" % repo_name,
file=sys.stderr)
continue
repo = dist_file.repositories[repo_name]
if not repo.doc_repository:
print("Skipping repository '%s': no doc section" % repo_name)
continue
if not repo.doc_repository.version:
print("Skipping repository '%s': no doc version" % repo_name)
continue
for os_name, os_code_name, arch in targets:
try:
job_name, job_config = configure_doc_job(
config_url,
rosdistro_name,
doc_build_name,
repo_name,
os_name,
os_code_name,
arch,
config=config,
build_file=build_file,
index=index,
dist_file=dist_file,
dist_cache=dist_cache,
jenkins=jenkins,
views=views,
is_disabled=is_disabled,
groovy_script=groovy_script,
dry_run=dry_run)
job_names.append(job_name)
if groovy_script is not None:
print("Configuration for job '%s'" % job_name)
job_configs[job_name] = job_config
except JobValidationError as e:
print(e.message, file=sys.stderr)
groovy_data['expected_num_jobs'] = len(job_configs)
groovy_data['job_prefixes_and_names'] = {}
job_prefix = '%s__' % doc_view_name
if not whitelist_repository_names:
groovy_data['job_prefixes_and_names']['doc'] = (job_prefix, job_names)
if groovy_script is None:
# delete obsolete jobs in this view
from ros_buildfarm.jenkins import remove_jobs
print('Removing obsolete doc jobs')
remove_jobs(jenkins, job_prefix, job_names, dry_run=dry_run)
if groovy_script is not None:
print(
"Writing groovy script '%s' to reconfigure %d views and %d jobs" %
(groovy_script, len(view_configs), len(job_configs)))
content = expand_template('snippet/reconfigure_jobs.groovy.em',
groovy_data)
write_groovy_script_and_configs(groovy_script,
content,
job_configs,
view_configs=view_configs)
def configure_doc_job(config_url,
rosdistro_name,
doc_build_name,
repo_name,
os_name,
os_code_name,
arch,
config=None,
build_file=None,
index=None,
dist_file=None,
dist_cache=None,
jenkins=None,
views=None,
is_disabled=False,
groovy_script=None,
doc_repository=None,
dry_run=False):
"""
Configure a single Jenkins doc job.
This includes the following steps:
- clone the doc repository to use
- clone the ros_buildfarm repository
- write the distribution repository keys into files
- invoke the run_doc_job.py script
"""
if config is None:
config = get_config_index(config_url)
if build_file is None:
build_files = get_doc_build_files(config, rosdistro_name)
build_file = build_files[doc_build_name]
if index is None:
index = get_index(config.rosdistro_index_url)
if dist_file is None:
dist_file = get_distribution_file(index, rosdistro_name, build_file)
if not dist_file:
raise JobValidationError(
'No distribution file matches the build file')
repo_names = dist_file.repositories.keys()
if repo_name is not None:
if repo_name not in repo_names:
raise JobValidationError("Invalid repository name '%s' " %
repo_name +
'choose one of the following: %s' %
', '.join(sorted(repo_names)))
repo = dist_file.repositories[repo_name]
if not repo.doc_repository:
raise JobValidationError("Repository '%s' has no doc section" %
repo_name)
if not repo.doc_repository.version:
raise JobValidationError("Repository '%s' has no doc version" %
repo_name)
doc_repository = repo.doc_repository
if os_name not in build_file.targets.keys():
raise JobValidationError("Invalid OS name '%s' " % os_name +
'choose one of the following: ' +
', '.join(sorted(build_file.targets.keys())))
if os_code_name not in build_file.targets[os_name].keys():
raise JobValidationError(
"Invalid OS code name '%s' " % os_code_name +
'choose one of the following: ' +
', '.join(sorted(build_file.targets[os_name].keys())))
if arch not in build_file.targets[os_name][os_code_name]:
raise JobValidationError(
"Invalid architecture '%s' " % arch +
'choose one of the following: %s' %
', '.join(sorted(build_file.targets[os_name][os_code_name])))
if dist_cache is None and build_file.notify_maintainers:
dist_cache = get_distribution_cache(index, rosdistro_name)
if jenkins is None:
from ros_buildfarm.jenkins import connect
jenkins = connect(config.jenkins_url)
if views is None:
view_name = get_doc_view_name(rosdistro_name, doc_build_name)
configure_doc_view(jenkins, view_name, dry_run=dry_run)
job_name = get_doc_job_name(rosdistro_name, doc_build_name, repo_name,
os_name, os_code_name, arch)
job_config = _get_doc_job_config(config,
config_url,
rosdistro_name,
doc_build_name,
build_file,
os_name,
os_code_name,
arch,
doc_repository,
repo_name,
dist_cache=dist_cache,
is_disabled=is_disabled)
# jenkinsapi.jenkins.Jenkins evaluates to false if job count is zero
if isinstance(jenkins, object) and jenkins is not False:
from ros_buildfarm.jenkins import configure_job
configure_job(jenkins, job_name, job_config, dry_run=dry_run)
return job_name, job_config