def gather_rosdeps(
docker_client: DockerClient,
platform: Platform,
workspace: Path,
skip_rosdep_keys: List[str] = [],
custom_script: Optional[Path] = None,
custom_data_dir: Optional[Path] = None,
) -> None:
"""
Run the rosdep Docker image, which outputs a script for dependency installation.
:param docker_client: Will be used to run the container
:param platform: The name of the image produced by `build_rosdep_image`
:param workspace: Absolute path to the colcon source workspace.
:param custom_script: Optional absolute path of a script that does custom setup for rosdep
:param custom_data_dir: Optional absolute path of a directory containing custom data for setup
:return None
"""
out_path = rosdep_install_script(platform)
logger.info('Building rosdep collector image: %s', _IMG_NAME)
docker_client.build_image(
dockerfile_name='rosdep.Dockerfile',
tag=_IMG_NAME,
)
logger.info(
'Running rosdep collector image on workspace {}'.format(workspace))
volumes = {
workspace: '/ws',
}
if custom_script:
volumes[custom_script] = CUSTOM_SETUP
if custom_data_dir:
volumes[custom_data_dir] = CUSTOM_DATA
docker_client.run_container(
image_name=_IMG_NAME,
environment={
'CUSTOM_SETUP': CUSTOM_SETUP,
'OUT_PATH': str(out_path),
'OWNER_USER': str(os.getuid()),
'ROSDISTRO': platform.ros_distro,
'SKIP_ROSDEP_KEYS': ' '.join(skip_rosdep_keys),
'COLCON_DEFAULTS_FILE': 'defaults.yaml',
'TARGET_OS': '{}:{}'.format(platform.os_name, platform.os_distro),
},
volumes=volumes,
)
def __call__(
self,
platform: Platform,
docker_client: DockerClient,
ros_workspace_dir: Path,
options: PipelineStageOptions,
data_collector: DataCollector
):
"""
Run the inspection and output the dependency installation script.
Also recovers the size of the docker image generated.
:raises RuntimeError if the step was skipped when no dependency script has been
previously generated
"""
gather_rosdeps(
docker_client=docker_client,
platform=platform,
workspace=ros_workspace_dir,
skip_rosdep_keys=options.skip_rosdep_keys,
custom_script=options.custom_script,
custom_data_dir=options.custom_data_dir)
assert_install_rosdep_script_exists(ros_workspace_dir, platform)
img_size = docker_client.get_image_size(_IMG_NAME)
data_collector.add_size(self.name, img_size)
def cross_compile_pipeline(
args: argparse.Namespace,
data_collector: DataCollector,
):
platform = Platform(args.arch, args.os, args.rosdistro,
args.sysroot_base_image)
ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace)
skip_rosdep_keys = args.skip_rosdep_keys
custom_data_dir = _path_if(args.custom_data_dir)
custom_rosdep_script = _path_if(args.custom_rosdep_script)
custom_setup_script = _path_if(args.custom_setup_script)
sysroot_build_context = prepare_docker_build_environment(
platform=platform,
ros_workspace=ros_workspace_dir,
custom_setup_script=custom_setup_script,
custom_data_dir=custom_data_dir)
docker_client = DockerClient(args.sysroot_nocache,
default_docker_dir=sysroot_build_context,
colcon_defaults_file=args.colcon_defaults)
stages = [DependenciesStage(), CreateSysrootStage(), DockerBuildStage()]
customizations = PipelineStageConfigOptions(args.skip_rosdep_collection,
skip_rosdep_keys,
custom_rosdep_script,
custom_data_dir,
custom_setup_script)
for stage in stages:
with data_collector.timer('cross_compile_{}'.format(stage.name)):
stage(platform, docker_client, ros_workspace_dir, customizations)
def __call__(self, platform: Platform, docker_client: DockerClient,
ros_workspace_dir: Path, options: PipelineStageOptions,
data_collector: DataCollector):
create_workspace_sysroot_image(docker_client, platform)
img_size = docker_client.get_image_size(platform.sysroot_image_tag)
data_collector.add_size(self.name, img_size)
def cross_compile_pipeline(
args: argparse.Namespace,
data_collector: DataCollector,
platform: Platform,
):
ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace)
skip_rosdep_keys = args.skip_rosdep_keys
custom_data_dir = _path_if(args.custom_data_dir)
custom_rosdep_script = _path_if(args.custom_rosdep_script)
custom_setup_script = _path_if(args.custom_setup_script)
sysroot_build_context = prepare_docker_build_environment(
platform=platform,
ros_workspace=ros_workspace_dir,
custom_setup_script=custom_setup_script,
custom_data_dir=custom_data_dir)
docker_client = DockerClient(args.sysroot_nocache,
default_docker_dir=sysroot_build_context,
colcon_defaults_file=args.colcon_defaults)
options = PipelineStageOptions(skip_rosdep_keys, custom_rosdep_script,
custom_data_dir, custom_setup_script)
for stage in _PIPELINE:
if stage.name not in args.stages_skip:
with data_collector.timer('{}'.format(stage.name)):
stage(platform, docker_client, ros_workspace_dir, options,
data_collector)
def test_custom_rosdep_no_data_dir(tmpdir):
script_contents = """
cat > /test_rules.yaml <<EOF
definitely_does_not_exist:
ubuntu:
bionic: [successful_test]
EOF
echo "yaml file:/test_rules.yaml" > /etc/ros/rosdep/sources.list.d/22-test-rules.list
"""
ws = Path(str(tmpdir))
pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml'
pkg_xml.parent.mkdir(parents=True)
pkg_xml.write_text(CUSTOM_KEY_PKG_XML)
client = DockerClient()
platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing')
rosdep_setup = ws / 'rosdep_setup.sh'
rosdep_setup.write_text(script_contents)
gather_rosdeps(client, platform, workspace=ws, custom_script=rosdep_setup)
out_script = ws / rosdep_install_script(platform)
result = out_script.read_text().splitlines()
expected = [
'#!/bin/bash',
'set -euxo pipefail',
'#[apt] Installation commands:',
' apt-get install -y successful_test',
]
assert result == expected, 'Rosdep output did not meet expectations.'
def test_colcon_defaults(tmpdir):
ws = Path(str(tmpdir))
this_dir = Path(__file__).parent
src = ws / 'src'
src.mkdir()
shutil.copytree(str(this_dir / 'dummy_pkg_ros2_cpp'),
str(src / 'dummy_pkg_ros2_cpp'))
shutil.copytree(str(this_dir / 'dummy_pkg_ros2_py'),
str(src / 'dummy_pkg_ros2_py'))
client = DockerClient()
platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing')
out_script = ws / rosdep_install_script(platform)
# no defaults file should get everything
gather_rosdeps(client, platform, workspace=ws)
result = out_script.read_text()
assert 'ros-dashing-ament-cmake' in result
assert 'ros-dashing-rclcpp' in result
assert 'ros-dashing-rclpy' in result
# write defaults file and expect selective dependency output
(ws / 'defaults.yaml').write_text("""
list:
packages-select: [dummy_pkg_ros2_py]
""")
gather_rosdeps(client, platform, workspace=ws)
result = out_script.read_text()
assert 'ros-dashing-ament-cmake' not in result
assert 'ros-dashing-rclcpp' not in result
assert 'ros-dashing-rclpy' in result
def cross_compile_pipeline(args: argparse.Namespace, ):
platform = Platform(args.arch, args.os, args.rosdistro,
args.sysroot_base_image)
ros_workspace_dir = Path(args.ros_workspace)
skip_rosdep_keys = args.skip_rosdep_keys
custom_data_dir = _path_if(args.custom_data_dir)
custom_rosdep_script = _path_if(args.custom_rosdep_script)
custom_setup_script = _path_if(args.custom_setup_script)
sysroot_build_context = prepare_docker_build_environment(
platform=platform,
ros_workspace=ros_workspace_dir,
custom_setup_script=custom_setup_script,
custom_data_dir=custom_data_dir)
docker_client = DockerClient(args.sysroot_nocache,
default_docker_dir=sysroot_build_context,
colcon_defaults_file=args.colcon_defaults)
if not args.skip_rosdep_collection:
gather_rosdeps(docker_client=docker_client,
platform=platform,
workspace=ros_workspace_dir,
skip_rosdep_keys=skip_rosdep_keys,
custom_script=custom_rosdep_script,
custom_data_dir=custom_data_dir)
assert_install_rosdep_script_exists(ros_workspace_dir, platform)
create_workspace_sysroot_image(docker_client, platform)
run_emulated_docker_build(docker_client, platform, ros_workspace_dir)
def create_workspace_sysroot_image(self, docker_client: DockerClient) -> str:
"""Build the target sysroot docker image and return its full name."""
image_tag = self._platform.sysroot_image_tag
logger.info('Building sysroot image: %s', image_tag)
docker_client.build_image(
dockerfile_dir=self._target_sysroot,
dockerfile_name=ROS_DOCKERFILE_NAME,
tag=image_tag,
buildargs={
'BASE_IMAGE': self._platform.target_base_image,
'ROS_WORKSPACE': self._ros_workspace_relative_to_sysroot,
'ROS_VERSION': self._platform.ros_version,
'ROS_DISTRO': self._platform.ros_distro,
'TARGET_ARCH': self._platform.arch,
}
)
logger.info('Successfully created sysroot docker image: %s', image_tag)
def test_rosdep_bad_key(tmpdir):
ws = Path(str(tmpdir))
pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml'
pkg_xml.parent.mkdir(parents=True)
pkg_xml.write_text(CUSTOM_KEY_PKG_XML)
client = DockerClient()
platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing')
with pytest.raises(docker.errors.ContainerError):
gather_rosdeps(client, platform, workspace=ws)
def run_emulated_docker_build(docker_client: DockerClient, platform: Platform,
workspace_path: Path) -> None:
"""
Spin up a sysroot docker container and run an emulated build inside.
:param docker_client: Preconfigured to run Docker images.
:param platform: Information about the target platform.
:param workspace: Absolute path to the user's source workspace.
"""
docker_client.run_container(
image_name=platform.sysroot_image_tag,
environment={
'OWNER_USER': str(os.getuid()),
'ROS_DISTRO': platform.ros_distro,
'TARGET_ARCH': platform.arch,
},
volumes={
workspace_path: '/ros_ws',
},
)
def create_workspace_sysroot_image(
docker_client: DockerClient,
platform: Platform,
) -> None:
"""
Create the target platform sysroot image.
:param docker_client Docker client to use for building
:param platform Information about the target platform
:param build_context Directory containing all assets needed by sysroot.Dockerfile
"""
image_tag = platform.sysroot_image_tag
logger.info('Building sysroot image: %s', image_tag)
docker_client.build_image(dockerfile_name='sysroot.Dockerfile',
tag=image_tag,
buildargs={
'BASE_IMAGE': platform.target_base_image,
'ROS_VERSION': platform.ros_version,
})
logger.info('Successfully created sysroot docker image: %s', image_tag)
def test_parse_docker_build_output():
"""Test the SysrootCreator constructor assuming valid path setup."""
# Create mock directories and files
client = DockerClient()
log_generator_without_errors = [
{'stream': ' ---\\u003e a9eb17255234\\n'},
{'stream': 'Step 1 : VOLUME /data\\n'},
{'stream': ' ---\\u003e Running in abdc1e6896c6\\n'},
{'stream': ' ---\\u003e 713bca62012e\\n'},
{'stream': 'Removing intermediate container abdc1e6896c6\\n'},
{'stream': 'Step 2 : CMD [\\"/bin/sh\\"]\\n'},
{'stream': ' ---\\u003e Running in dba30f2a1a7e\\n'},
{'stream': ' ---\\u003e 032b8b2855fc\\n'},
{'stream': 'Removing intermediate container dba30f2a1a7e\\n'},
{'stream': 'Successfully built 032b8b2855fc\\n'},
]
# Just expect it not to raise
client._process_build_log(log_generator_without_errors)
log_generator_with_errors = [
{'stream': ' ---\\u003e a9eb17255234\\n'},
{'stream': 'Step 1 : VOLUME /data\\n'},
{'stream': ' ---\\u003e Running in abdc1e6896c6\\n'},
{'stream': ' ---\\u003e 713bca62012e\\n'},
{'stream': 'Removing intermediate container abdc1e6896c6\\n'},
{'stream': 'Step 2 : CMD [\\"/bin/sh\\"]\\n'},
{'error': ' ---\\COMMAND NOT FOUND\\n'},
]
with pytest.raises(docker.errors.BuildError):
client._process_build_log(log_generator_with_errors)
def test_dummy_skip_rosdep_multiple_keys_pkg(tmpdir):
ws = Path(str(tmpdir))
pkg_xml = ws / 'src' / 'dummy' / 'package.xml'
pkg_xml.parent.mkdir(parents=True)
pkg_xml.write_text(RCLCPP_PKG_XML)
client = DockerClient()
platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing')
out_script = ws / rosdep_install_script(platform)
skip_keys = ['ament_cmake', 'rclcpp']
gather_rosdeps(client, platform, workspace=ws, skip_rosdep_keys=skip_keys)
result = out_script.read_text()
assert 'ros-dashing-ament-cmake' not in result
assert 'ros-dashing-rclcpp' not in result
def test_dummy_skip_rosdep_keys_doesnt_exist_pkg(tmpdir):
ws = Path(str(tmpdir))
pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml'
pkg_xml.parent.mkdir(parents=True)
pkg_xml.write_text(CUSTOM_KEY_PKG_XML)
client = DockerClient()
platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing')
skip_keys = ['definitely_does_not_exist']
try:
gather_rosdeps(client,
platform,
workspace=ws,
skip_rosdep_keys=skip_keys)
except docker.errors.ContainerError:
assert False
def buildable_env(request, tmpdir):
"""Set up a temporary directory with everything needed to run the EmulatedDockerBuildStage."""
platform = Platform('aarch64', 'ubuntu', 'foxy')
ros_workspace = Path(str(tmpdir)) / 'ros_ws'
_touch_anywhere(ros_workspace / rosdep_install_script(platform))
build_context = prepare_docker_build_environment(platform, ros_workspace)
docker = DockerClient(disable_cache=False,
default_docker_dir=build_context)
options = default_pipeline_options()
data_collector = DataCollector()
CreateSysrootStage()(platform, docker, ros_workspace, options,
data_collector)
return BuildableEnv(platform, docker, ros_workspace, options,
data_collector)
def test_dummy_ros2_pkg(tmpdir):
ws = Path(str(tmpdir))
pkg_xml = ws / 'src' / 'dummy' / 'package.xml'
pkg_xml.parent.mkdir(parents=True)
pkg_xml.write_text(RCLCPP_PKG_XML)
client = DockerClient()
platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing')
out_script = ws / rosdep_install_script(platform)
test_collector = DataCollector()
stage = CollectDependencyListStage()
stage(platform, client, ws, default_pipeline_options(), test_collector)
result = out_script.read_text()
assert 'ros-dashing-ament-cmake' in result
assert 'ros-dashing-rclcpp' in result
def main():
"""Start the cross-compilation workflow."""
# Configuration
args = parse_args(sys.argv[1:])
platform = Platform(args.arch, args.os, args.rosdistro,
args.sysroot_base_image)
docker_client = DockerClient(args.sysroot_nocache)
sysroot_creator = SysrootCreator(
cc_root_dir=args.sysroot_path,
ros_workspace_dir=args.ros_workspace,
platform=platform,
custom_setup_script_path=args.custom_setup_script,
custom_data_dir=args.custom_data_dir)
sysroot_creator.create_workspace_sysroot_image(docker_client)
ros_workspace_dir = os.path.join(args.sysroot_path, 'sysroot',
args.ros_workspace)
run_emulated_docker_build(docker_client, platform.sysroot_image_tag,
ros_workspace_dir)
def test_dummy_ros2_pkg(tmpdir):
ws = Path(str(tmpdir))
pkg_xml = ws / 'src' / 'dummy' / 'package.xml'
pkg_xml.parent.mkdir(parents=True)
pkg_xml.write_text(RCLCPP_PKG_XML)
client = DockerClient()
platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing')
out_script = ws / rosdep_install_script(platform)
# a default set of customizations for the dependencies stage
customizations = PipelineStageConfigOptions(False, [], None, None, None)
temp_stage = DependenciesStage()
temp_stage(platform, client, ws, customizations)
result = out_script.read_text()
assert 'ros-dashing-ament-cmake' in result
assert 'ros-dashing-rclcpp' in result
def test_dummy_ros2_pkg(tmpdir):
ws = Path(str(tmpdir))
pkg_xml = ws / 'src' / 'dummy' / 'package.xml'
pkg_xml.parent.mkdir(parents=True)
pkg_xml.write_text(RCLCPP_PKG_XML)
client = DockerClient()
platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing')
out_script = ws / rosdep_install_script(platform)
gather_rosdeps(client, platform, workspace=ws)
result = out_script.read_text().splitlines()
expected = [
'#!/bin/bash',
'set -euxo pipefail',
'#[apt] Installation commands:',
' apt-get install -y ros-dashing-ament-cmake',
' apt-get install -y ros-dashing-rclcpp',
]
assert result == expected, 'Rosdep output did not meet expectations.'
def cross_compile_pipeline(
args: argparse.Namespace,
data_collector: DataCollector,
platform: Platform,
):
ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace)
skip_rosdep_keys = args.skip_rosdep_keys
custom_data_dir = _path_if(args.custom_data_dir)
custom_rosdep_script = _path_if(args.custom_rosdep_script)
custom_setup_script = _path_if(args.custom_setup_script)
custom_post_build_script = _path_if(args.custom_post_build_script)
colcon_defaults_file = _path_if(args.colcon_defaults)
sysroot_build_context = prepare_docker_build_environment(
platform=platform,
ros_workspace=ros_workspace_dir,
custom_setup_script=custom_setup_script,
custom_post_build_script=custom_post_build_script,
colcon_defaults_file=colcon_defaults_file,
custom_data_dir=custom_data_dir)
docker_client = DockerClient(args.sysroot_nocache,
default_docker_dir=sysroot_build_context)
options = PipelineStageOptions(skip_rosdep_keys, custom_rosdep_script,
custom_data_dir, custom_setup_script,
args.runtime_tag)
skip = set(args.stages_skip)
# Only package the runtime image if the user has specified a tag for it
if not args.runtime_tag:
skip.add(PackageRuntimeImageStage.NAME)
for stage in _PIPELINE:
if stage.name not in skip:
with data_collector.timer('{}'.format(stage.name)):
stage(platform, docker_client, ros_workspace_dir, options,
data_collector)
def test_custom_post_build_script(tmpdir):
created_filename = 'file-created-by-post-build'
platform = Platform('aarch64', 'ubuntu', 'foxy')
ros_workspace = Path(str(tmpdir)) / 'ros_ws'
_touch_anywhere(ros_workspace / rosdep_install_script(platform))
post_build_script = ros_workspace / 'post_build'
post_build_script.write_text("""
#!/bin/bash
echo "success" > {}
""".format(created_filename))
build_context = prepare_docker_build_environment(
platform, ros_workspace, custom_post_build_script=post_build_script)
docker = DockerClient(disable_cache=False,
default_docker_dir=build_context)
options = default_pipeline_options()
data_collector = DataCollector()
CreateSysrootStage()(platform, docker, ros_workspace, options,
data_collector)
EmulatedDockerBuildStage()(platform, docker, ros_workspace, options,
data_collector)
assert (ros_workspace / created_filename).is_file()
def test_custom_rosdep_no_data_dir(tmpdir):
script_contents = """
cat > /test_rules.yaml <<EOF
definitely_does_not_exist:
ubuntu:
bionic: [successful_test]
EOF
echo "yaml file:/test_rules.yaml" > /etc/ros/rosdep/sources.list.d/22-test-rules.list
"""
ws = Path(str(tmpdir))
pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml'
pkg_xml.parent.mkdir(parents=True)
pkg_xml.write_text(CUSTOM_KEY_PKG_XML)
client = DockerClient()
platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing')
rosdep_setup = ws / 'rosdep_setup.sh'
rosdep_setup.write_text(script_contents)
gather_rosdeps(client, platform, workspace=ws, custom_script=rosdep_setup)
out_script = ws / rosdep_install_script(platform)
result = out_script.read_text()
assert 'successful_test' in result
def cross_compile_pipeline(
args: argparse.Namespace,
):
platform = Platform(args.arch, args.os, args.rosdistro, args.sysroot_base_image)
ros_workspace_dir = Path(args.ros_workspace).resolve()
if not (ros_workspace_dir / 'src').is_dir():
raise ValueError(
'Specified workspace "{}" does not look like a colcon workspace '
'(there is no "src/" directory). Cannot continue'.format(ros_workspace_dir))
skip_rosdep_keys = args.skip_rosdep_keys
custom_data_dir = _path_if(args.custom_data_dir)
custom_rosdep_script = _path_if(args.custom_rosdep_script)
custom_setup_script = _path_if(args.custom_setup_script)
sysroot_build_context = prepare_docker_build_environment(
platform=platform,
ros_workspace=ros_workspace_dir,
custom_setup_script=custom_setup_script,
custom_data_dir=custom_data_dir)
docker_client = DockerClient(
args.sysroot_nocache,
default_docker_dir=sysroot_build_context,
colcon_defaults_file=args.colcon_defaults)
if not args.skip_rosdep_collection:
gather_rosdeps(
docker_client=docker_client,
platform=platform,
workspace=ros_workspace_dir,
skip_rosdep_keys=skip_rosdep_keys,
custom_script=custom_rosdep_script,
custom_data_dir=custom_data_dir)
assert_install_rosdep_script_exists(ros_workspace_dir, platform)
create_workspace_sysroot_image(docker_client, platform)
run_emulated_docker_build(docker_client, platform, ros_workspace_dir)
