def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description='Create a workspace from vcs repos files.')
add_argument_repos_file_urls(parser, required=True)
add_argument_test_branch(parser)
parser.add_argument('--workspace-root',
help='The path of the desired workspace',
required=True)
args = parser.parse_args(argv)
ensure_workspace_exists(args.workspace_root)
with Scope('SUBSECTION', 'fetch repos files(s)'):
repos_files = []
for repos_file_url in args.repos_file_urls:
repos_file = os.path.join(args.workspace_root,
os.path.basename(repos_file_url))
print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file))
urlretrieve(repos_file_url, repos_file)
repos_files += [repos_file]
with Scope('SUBSECTION', 'import repositories'):
source_space = os.path.join(args.workspace_root, 'src')
for repos_file in repos_files:
print('Importing repositories from \'%s\'' % (repos_file))
import_repositories(source_space, repos_file, args.test_branch)
with Scope('SUBSECTION', 'vcs export --exact'):
# if a repo has been rebased against the default branch vcs can't detect the remote
export_repositories(args.workspace_root, check=not args.test_branch)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description='Create a workspace from vcs repos files.')
add_argument_rosdistro_name(parser)
add_argument_repos_file_urls(parser)
add_argument_repository_names(parser, optional=True)
add_argument_test_branch(parser)
parser.add_argument('--workspace-root',
help='The path of the desired workspace',
required=True)
args = parser.parse_args(argv)
assert args.repos_file_urls or args.repository_names
ensure_workspace_exists(args.workspace_root)
repos_files = []
if args.repository_names:
with Scope('SUBSECTION', 'get repository information from rosdistro'):
index = get_index(get_index_url())
dist = get_distribution(index, args.rosdistro_name)
data = {}
for repo_name in args.repository_names:
repo = dist.repositories[repo_name]
src_repo = repo.source_repository
repo_data = {
'type': src_repo.type,
'url': src_repo.url,
}
if src_repo.version is not None:
repo_data['version'] = src_repo.version
data[repo_name] = repo_data
repos_file = os.path.join(args.workspace_root,
'repositories-from-rosdistro.repos')
with open(repos_file, 'w') as h:
h.write(
yaml.safe_dump({'repositories': data},
default_flow_style=False))
repos_files.append(repos_file)
with Scope('SUBSECTION', 'fetch repos files(s)'):
for repos_file_url in args.repos_file_urls:
repos_file = os.path.join(args.workspace_root,
os.path.basename(repos_file_url))
print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file))
urlretrieve(repos_file_url, repos_file)
repos_files += [repos_file]
with Scope('SUBSECTION', 'import repositories'):
source_space = os.path.join(args.workspace_root, 'src')
for repos_file in repos_files:
print('Importing repositories from \'%s\'' % (repos_file))
import_repositories(source_space, repos_file, args.test_branch)
with Scope('SUBSECTION', 'vcs export --exact'):
# if a repo has been rebased against the default branch vcs can't detect the remote
export_repositories(args.workspace_root, check=not args.test_branch)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description='Create a workspace from vcs repos files.')
add_argument_build_ignore(parser)
add_argument_package_selection_args(parser)
add_argument_repos_file_urls(parser, required=True)
add_argument_test_branch(parser)
parser.add_argument(
'--workspace-root',
help='The path of the desired workspace',
required=True)
args = parser.parse_args(argv)
ensure_workspace_exists(args.workspace_root)
os.chdir(args.workspace_root)
with Scope('SUBSECTION', 'fetch repos files(s)'):
repos_files = []
for repos_file_url in args.repos_file_urls:
repos_file = os.path.join(args.workspace_root, os.path.basename(repos_file_url))
print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file))
urlretrieve(repos_file_url, repos_file)
repos_files += [repos_file]
with Scope('SUBSECTION', 'import repositories'):
source_space = os.path.join(args.workspace_root, 'src')
for repos_file in repos_files:
print('Importing repositories from \'%s\'' % (repos_file))
import_repositories(source_space, repos_file, args.test_branch)
with Scope('SUBSECTION', 'vcs export --exact'):
export_repositories(args.workspace_root)
with Scope('SUBSECTION', 'mark package(s) to ignore'):
if args.build_ignore:
source_space = os.path.join(args.workspace_root, 'src')
packages = locate_packages(source_space, args.build_ignore)
for package_name, package_root in packages.items():
print("Ignoring package '%s'" % (package_name,))
Path(package_root, 'COLCON_IGNORE').touch()
with Scope('SUBSECTION', 'select target package(s) in workspace'):
packages = locate_packages(source_space)
if args.package_selection_args:
selected_packages = locate_packages(
source_space, extra_args=args.package_selection_args)
to_ignore = packages.keys() - selected_packages.keys()
print('Ignoring %d packages to scope workspace' % len(to_ignore))
for package in to_ignore:
package_root = packages.pop(package)
Path(package_root, 'COLCON_IGNORE').touch()
print('There are %d packages which meet selection criteria' %
len(packages))
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description='Create a workspace from vcs repos files.')
add_argument_package_selection_args(parser)
add_argument_repos_file_urls(parser, required=True)
add_argument_test_branch(parser)
parser.add_argument('--workspace-root',
help='The path of the desired workspace',
required=True)
args = parser.parse_args(argv)
ensure_workspace_exists(args.workspace_root)
os.chdir(args.workspace_root)
with Scope('SUBSECTION', 'fetch repos files(s)'):
repos_files = []
for repos_file_url in args.repos_file_urls:
repos_file = os.path.join(args.workspace_root,
os.path.basename(repos_file_url))
print('Fetching \'%s\' to \'%s\'' % (repos_file_url, repos_file))
urlretrieve(repos_file_url, repos_file)
repos_files += [repos_file]
with Scope('SUBSECTION', 'import repositories'):
source_space = os.path.join(args.workspace_root, 'src')
for repos_file in repos_files:
print('Importing repositories from \'%s\'' % (repos_file))
import_repositories(source_space, repos_file, args.test_branch)
with Scope('SUBSECTION', 'vcs export --exact'):
export_repositories(args.workspace_root)
with Scope('SUBSECTION', 'mark packages with IGNORE files'):
packages = locate_packages(source_space)
if args.package_selection_args:
print('Using package selection arguments:',
args.package_selection_args)
selected_packages = locate_packages(
source_space, extra_args=args.package_selection_args)
to_ignore = packages.keys() - selected_packages.keys()
print('Ignoring %d packages' % len(to_ignore))
for package in sorted(to_ignore):
print('-', package)
package_root = packages.pop(package)
Path(package_root, 'COLCON_IGNORE').touch()
print('There are %d packages which meet selection criteria' %
len(packages))
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(description="Run the 'CI' job")
# Positional
add_argument_rosdistro_name(parser)
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_build_tool(parser, required=True)
add_argument_distribution_repository_key_files(parser)
add_argument_distribution_repository_urls(parser)
add_argument_dockerfile_dir(parser)
add_argument_env_vars(parser)
add_argument_install_packages(parser)
a1 = add_argument_package_selection_args(parser)
a2 = add_argument_build_tool_args(parser)
add_argument_repos_file_urls(parser)
add_argument_repository_names(parser, optional=True)
add_argument_ros_version(parser)
add_argument_skip_rosdep_keys(parser)
add_argument_test_branch(parser)
parser.add_argument(
'--workspace-mount-point',
nargs='*',
help='Locations within the docker image where the workspace(s) '
'will be mounted when the docker image is run.')
remainder_args = extract_multiple_remainders(argv, (a1, a2))
args = parser.parse_args(argv)
for k, v in remainder_args.items():
setattr(args, k, v)
assert args.repos_file_urls or args.repository_names
data = copy.deepcopy(args.__dict__)
data.update({
'distribution_repository_urls':
args.distribution_repository_urls,
'distribution_repository_keys':
get_distribution_repository_keys(
args.distribution_repository_urls,
args.distribution_repository_key_files),
'uid':
get_user_id(),
})
create_dockerfile('ci/ci_create_tasks.Dockerfile.em', data,
args.dockerfile_dir)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Run the 'CI' job")
# Positional
add_argument_rosdistro_name(parser)
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_build_ignore(parser)
add_argument_build_tool(parser, required=True)
add_argument_distribution_repository_key_files(parser)
add_argument_distribution_repository_urls(parser)
add_argument_dockerfile_dir(parser)
add_argument_env_vars(parser)
add_argument_install_packages(parser)
add_argument_package_selection_args(parser)
add_argument_repos_file_urls(parser, required=True)
add_argument_ros_version(parser)
add_argument_skip_rosdep_keys(parser)
add_argument_test_branch(parser)
parser.add_argument(
'--workspace-mount-point', nargs='*',
help='Locations within the docker image where the workspace(s) '
'will be mounted when the docker image is run.')
args = parser.parse_args(argv)
data = copy.deepcopy(args.__dict__)
data.update({
'distribution_repository_urls': args.distribution_repository_urls,
'distribution_repository_keys': get_distribution_repository_keys(
args.distribution_repository_urls,
args.distribution_repository_key_files),
'uid': get_user_id(),
})
create_dockerfile(
'ci/ci_create_tasks.Dockerfile.em', data, args.dockerfile_dir)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'Dockerfile' for the CI job")
# Positional
add_argument_rosdistro_name(parser)
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_distribution_repository_key_files(parser)
add_argument_distribution_repository_urls(parser)
add_argument_dockerfile_dir(parser)
add_argument_env_vars(parser)
add_argument_package_selection_args(parser)
add_argument_repos_file_urls(parser)
add_argument_repository_names(parser, optional=True)
add_argument_skip_rosdep_keys(parser)
add_argument_test_branch(parser)
parser.add_argument('--workspace-root',
nargs='+',
help='The root path of the workspace to compile')
args = parser.parse_args(argv)
assert args.repos_file_urls or args.repository_names
debian_pkg_names = [
'git',
'python3-apt',
'python3-colcon-metadata',
'python3-colcon-package-information',
'python3-colcon-package-selection',
'python3-colcon-recursive-crawl',
'python3-colcon-ros',
'python3-rosdep',
'python3-vcstool',
]
# get versions for build dependencies
apt_cache = Cache()
debian_pkg_versions = get_binary_package_versions(apt_cache,
debian_pkg_names)
# generate Dockerfile
data = {
'os_name':
args.os_name,
'os_code_name':
args.os_code_name,
'arch':
args.arch,
'distribution_repository_urls':
args.distribution_repository_urls,
'distribution_repository_keys':
get_distribution_repository_keys(
args.distribution_repository_urls,
args.distribution_repository_key_files),
'rosdistro_name':
args.rosdistro_name,
'custom_rosdep_urls': [],
'uid':
get_user_id(),
'build_environment_variables':
['%s=%s' % key_value for key_value in args.env_vars.items()],
'dependencies':
debian_pkg_names,
'dependency_versions':
debian_pkg_versions,
'repos_file_urls':
args.repos_file_urls,
'repository_names':
args.repository_names,
'test_branch':
args.test_branch,
'skip_rosdep_keys':
args.skip_rosdep_keys,
'package_selection_args':
args.package_selection_args,
'workspace_root':
args.workspace_root,
}
create_dockerfile('ci/create_workspace.Dockerfile.em', data,
args.dockerfile_dir)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(description="Generate a 'CI' script")
# Positional
add_argument_config_url(parser)
add_argument_rosdistro_name(parser)
add_argument_build_name(parser, 'ci')
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_build_tool(parser)
a1 = add_argument_package_selection_args(parser)
a2 = add_argument_build_tool_args(parser)
a3 = add_argument_build_tool_test_args(parser)
add_argument_repos_file_urls(parser)
add_argument_skip_cleanup(parser)
add_argument_test_branch(parser)
parser.add_argument(
'--underlay-source-path',
nargs='*',
metavar='DIR_NAME',
help='Path to one or more install spaces to use as an underlay')
remainder_args = extract_multiple_remainders(argv, (a1, a2, a3))
args = parser.parse_args(argv)
for k, v in remainder_args.items():
setattr(args, k, v)
# collect all template snippets of specific types
class IncludeHook(Hook):
def __init__(self):
Hook.__init__(self)
self.scms = []
self.scripts = []
self.parameters = {}
if args.skip_cleanup:
self.parameters['skip_cleanup'] = 'true'
if args.repos_file_urls is not None:
self.parameters['repos_file_urls'] = ' '.join(
args.repos_file_urls)
if args.test_branch is not None:
self.parameters['test_branch'] = args.test_branch
if args.package_selection_args is not None:
self.parameters['package_selection_args'] = ' '.join(
args.package_selection_args)
if args.build_tool_args is not None:
self.parameters['build_tool_args'] = ' '.join(
args.build_tool_args)
if args.build_tool_test_args is not None:
self.parameters['build_tool_test_args'] = ' '.join(
args.build_tool_test_args)
def beforeInclude(self, *_, **kwargs):
template_path = kwargs['file'].name
if template_path.endswith('/snippet/scm.xml.em'):
self.scms.append(
(kwargs['locals']['repo_spec'], kwargs['locals']['path']))
if template_path.endswith('/snippet/builder_shell.xml.em'):
script = kwargs['locals']['script']
# reuse existing ros_buildfarm folder if it exists
if 'Clone ros_buildfarm' in script:
lines = script.splitlines()
lines.insert(0, 'if [ ! -d "ros_buildfarm" ]; then')
lines += [
'else',
'echo "Using existing ros_buildfarm folder"',
'fi',
]
script = '\n'.join(lines)
if args.build_tool and ' --build-tool ' in script:
script = script.replace(
' --build-tool catkin_make_isolated',
' --build-tool ' + args.build_tool)
self.scripts.append(script)
if template_path.endswith(
'/snippet/property_parameters-definition.xml.em'):
for parameter in reversed(kwargs['locals']['parameters']):
name = parameter['name']
value_type = parameter['type']
if value_type in ['string', 'text']:
default_value = parameter['default_value']
elif value_type == 'boolean':
default_value = 'true' if parameter.get(
'default_value', False) else 'false'
else:
continue
self.parameters.setdefault(name, default_value)
hook = IncludeHook()
from ros_buildfarm import templates
templates.template_hooks = [hook]
config = get_config_index(args.config_url)
build_files = get_ci_build_files(config, args.rosdistro_name)
build_file = build_files[args.ci_build_name]
underlay_source_paths = [
os.path.abspath(p) for p in args.underlay_source_path or []
]
configure_ci_job(args.config_url,
args.rosdistro_name,
args.ci_build_name,
args.os_name,
args.os_code_name,
args.arch,
config=config,
build_file=build_file,
jenkins=False,
views=False,
underlay_source_paths=underlay_source_paths)
templates.template_hooks = None
ci_job_name = get_ci_job_name(args.rosdistro_name, args.os_name,
args.os_code_name, args.arch, 'script')
value = expand_template('ci/ci_script.sh.em', {
'ci_job_name': ci_job_name,
'scms': hook.scms,
'scripts': hook.scripts,
'build_tool': args.build_tool or build_file.build_tool,
'parameters': hook.parameters
},
options={BANGPATH_OPT: False})
value = value.replace('python3 ', sys.executable + ' ')
print(value)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'CI' script")
# Positional
add_argument_config_url(parser)
add_argument_rosdistro_name(parser)
add_argument_build_name(parser, 'ci')
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_build_ignore(parser)
add_argument_build_tool(parser)
add_argument_package_selection_args(parser)
add_argument_repos_file_urls(parser)
add_argument_skip_cleanup(parser)
add_argument_test_branch(parser)
parser.add_argument(
'--underlay-source-path', nargs='*', metavar='DIR_NAME',
help='Path to one or more install spaces to use as an underlay')
args = parser.parse_args(argv)
# collect all template snippets of specific types
class IncludeHook(Hook):
def __init__(self):
Hook.__init__(self)
self.scms = []
self.scripts = []
self.parameters = {}
if args.skip_cleanup:
self.parameters['skip_cleanup'] = 'true'
if args.repos_file_urls is not None:
self.parameters['repos_file_urls'] = ' '.join(args.repos_file_urls)
if args.test_branch is not None:
self.parameters['test_branch'] = args.test_branch
if args.build_ignore is not None:
self.parameters['build_ignore'] = ' '.join(args.build_ignore)
if args.package_selection_args is not None:
self.parameters['package_selection_args'] = ' '.join(args.package_selection_args)
def beforeInclude(self, *_, **kwargs):
template_path = kwargs['file'].name
if template_path.endswith('/snippet/scm.xml.em'):
self.scms.append(
(kwargs['locals']['repo_spec'], kwargs['locals']['path']))
if template_path.endswith('/snippet/builder_shell.xml.em'):
script = kwargs['locals']['script']
# reuse existing ros_buildfarm folder if it exists
if 'Clone ros_buildfarm' in script:
lines = script.splitlines()
lines.insert(0, 'if [ ! -d "ros_buildfarm" ]; then')
lines += [
'else',
'echo "Using existing ros_buildfarm folder"',
'fi',
]
script = '\n'.join(lines)
if args.build_tool and ' --build-tool ' in script:
script = script.replace(
' --build-tool catkin_make_isolated',
' --build-tool ' + args.build_tool)
self.scripts.append(script)
if template_path.endswith('/snippet/property_parameters-definition.xml.em'):
for parameter in reversed(kwargs['locals']['parameters']):
name = parameter['name']
value_type = parameter['type']
if value_type in ['string', 'text']:
default_value = parameter['default_value']
elif value_type == 'boolean':
default_value = 'true' if parameter.get(
'default_value', False) else 'false'
else:
continue
self.parameters.setdefault(name, default_value)
hook = IncludeHook()
from ros_buildfarm import templates
templates.template_hooks = [hook]
config = get_config_index(args.config_url)
build_files = get_ci_build_files(config, args.rosdistro_name)
build_file = build_files[args.ci_build_name]
underlay_source_paths = [os.path.abspath(p) for p in args.underlay_source_path or []]
configure_ci_job(
args.config_url, args.rosdistro_name, args.ci_build_name,
args.os_name, args.os_code_name, args.arch,
config=config, build_file=build_file, jenkins=False, views=False,
underlay_source_paths=underlay_source_paths)
templates.template_hooks = None
ci_job_name = get_ci_job_name(
args.rosdistro_name, args.os_name,
args.os_code_name, args.arch, 'script')
value = expand_template(
'ci/ci_script.sh.em', {
'ci_job_name': ci_job_name,
'scms': hook.scms,
'scripts': hook.scripts,
'build_tool': args.build_tool or build_file.build_tool,
'parameters': hook.parameters},
options={BANGPATH_OPT: False})
value = value.replace('python3 ', sys.executable + ' ')
print(value)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'Dockerfile' for the CI job")
# Positional
add_argument_rosdistro_name(parser)
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_build_ignore(parser)
add_argument_distribution_repository_key_files(parser)
add_argument_distribution_repository_urls(parser)
add_argument_dockerfile_dir(parser)
add_argument_env_vars(parser)
add_argument_package_selection_args(parser)
add_argument_repos_file_urls(parser, required=True)
add_argument_skip_rosdep_keys(parser)
add_argument_test_branch(parser)
parser.add_argument(
'--workspace-root',
nargs='+',
help='The root path of the workspace to compile')
args = parser.parse_args(argv)
debian_pkg_names = [
'git',
'python3-apt',
'python3-colcon-common-extensions',
'python3-rosdep',
'python3-vcstool',
]
# get versions for build dependencies
apt_cache = Cache()
debian_pkg_versions = get_binary_package_versions(
apt_cache, debian_pkg_names)
# generate Dockerfile
data = {
'os_name': args.os_name,
'os_code_name': args.os_code_name,
'arch': args.arch,
'distribution_repository_urls': args.distribution_repository_urls,
'distribution_repository_keys': get_distribution_repository_keys(
args.distribution_repository_urls,
args.distribution_repository_key_files),
'rosdistro_name': args.rosdistro_name,
'custom_rosdep_urls': [],
'uid': get_user_id(),
'build_environment_variables': args.env_vars,
'dependencies': debian_pkg_names,
'dependency_versions': debian_pkg_versions,
'repos_file_urls': args.repos_file_urls,
'test_branch': args.test_branch,
'skip_rosdep_keys': args.skip_rosdep_keys,
'build_ignore': args.build_ignore,
'package_selection_args': args.package_selection_args,
'workspace_root': args.workspace_root,
}
create_dockerfile(
'ci/create_workspace.Dockerfile.em', data, args.dockerfile_dir)
