def gather_rosdeps( docker_client: DockerClient, platform: Platform, workspace: Path, skip_rosdep_keys: List[str] = [], custom_script: Optional[Path] = None, custom_data_dir: Optional[Path] = None, ) -> None: """ Run the rosdep Docker image, which outputs a script for dependency installation. :param docker_client: Will be used to run the container :param platform: The name of the image produced by `build_rosdep_image` :param workspace: Absolute path to the colcon source workspace. :param custom_script: Optional absolute path of a script that does custom setup for rosdep :param custom_data_dir: Optional absolute path of a directory containing custom data for setup :return None """ out_path = rosdep_install_script(platform) logger.info('Building rosdep collector image: %s', _IMG_NAME) docker_client.build_image( dockerfile_name='rosdep.Dockerfile', tag=_IMG_NAME, ) logger.info( 'Running rosdep collector image on workspace {}'.format(workspace)) volumes = { workspace: '/ws', } if custom_script: volumes[custom_script] = CUSTOM_SETUP if custom_data_dir: volumes[custom_data_dir] = CUSTOM_DATA docker_client.run_container( image_name=_IMG_NAME, environment={ 'CUSTOM_SETUP': CUSTOM_SETUP, 'OUT_PATH': str(out_path), 'OWNER_USER': str(os.getuid()), 'ROSDISTRO': platform.ros_distro, 'SKIP_ROSDEP_KEYS': ' '.join(skip_rosdep_keys), 'COLCON_DEFAULTS_FILE': 'defaults.yaml', 'TARGET_OS': '{}:{}'.format(platform.os_name, platform.os_distro), }, volumes=volumes, )
def __call__( self, platform: Platform, docker_client: DockerClient, ros_workspace_dir: Path, options: PipelineStageOptions, data_collector: DataCollector ): """ Run the inspection and output the dependency installation script. Also recovers the size of the docker image generated. :raises RuntimeError if the step was skipped when no dependency script has been previously generated """ gather_rosdeps( docker_client=docker_client, platform=platform, workspace=ros_workspace_dir, skip_rosdep_keys=options.skip_rosdep_keys, custom_script=options.custom_script, custom_data_dir=options.custom_data_dir) assert_install_rosdep_script_exists(ros_workspace_dir, platform) img_size = docker_client.get_image_size(_IMG_NAME) data_collector.add_size(self.name, img_size)
def cross_compile_pipeline( args: argparse.Namespace, data_collector: DataCollector, ): platform = Platform(args.arch, args.os, args.rosdistro, args.sysroot_base_image) ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace) skip_rosdep_keys = args.skip_rosdep_keys custom_data_dir = _path_if(args.custom_data_dir) custom_rosdep_script = _path_if(args.custom_rosdep_script) custom_setup_script = _path_if(args.custom_setup_script) sysroot_build_context = prepare_docker_build_environment( platform=platform, ros_workspace=ros_workspace_dir, custom_setup_script=custom_setup_script, custom_data_dir=custom_data_dir) docker_client = DockerClient(args.sysroot_nocache, default_docker_dir=sysroot_build_context, colcon_defaults_file=args.colcon_defaults) stages = [DependenciesStage(), CreateSysrootStage(), DockerBuildStage()] customizations = PipelineStageConfigOptions(args.skip_rosdep_collection, skip_rosdep_keys, custom_rosdep_script, custom_data_dir, custom_setup_script) for stage in stages: with data_collector.timer('cross_compile_{}'.format(stage.name)): stage(platform, docker_client, ros_workspace_dir, customizations)
def __call__(self, platform: Platform, docker_client: DockerClient, ros_workspace_dir: Path, options: PipelineStageOptions, data_collector: DataCollector): create_workspace_sysroot_image(docker_client, platform) img_size = docker_client.get_image_size(platform.sysroot_image_tag) data_collector.add_size(self.name, img_size)
def cross_compile_pipeline( args: argparse.Namespace, data_collector: DataCollector, platform: Platform, ): ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace) skip_rosdep_keys = args.skip_rosdep_keys custom_data_dir = _path_if(args.custom_data_dir) custom_rosdep_script = _path_if(args.custom_rosdep_script) custom_setup_script = _path_if(args.custom_setup_script) sysroot_build_context = prepare_docker_build_environment( platform=platform, ros_workspace=ros_workspace_dir, custom_setup_script=custom_setup_script, custom_data_dir=custom_data_dir) docker_client = DockerClient(args.sysroot_nocache, default_docker_dir=sysroot_build_context, colcon_defaults_file=args.colcon_defaults) options = PipelineStageOptions(skip_rosdep_keys, custom_rosdep_script, custom_data_dir, custom_setup_script) for stage in _PIPELINE: if stage.name not in args.stages_skip: with data_collector.timer('{}'.format(stage.name)): stage(platform, docker_client, ros_workspace_dir, options, data_collector)
def test_custom_rosdep_no_data_dir(tmpdir): script_contents = """ cat > /test_rules.yaml <<EOF definitely_does_not_exist: ubuntu: bionic: [successful_test] EOF echo "yaml file:/test_rules.yaml" > /etc/ros/rosdep/sources.list.d/22-test-rules.list """ ws = Path(str(tmpdir)) pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml' pkg_xml.parent.mkdir(parents=True) pkg_xml.write_text(CUSTOM_KEY_PKG_XML) client = DockerClient() platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing') rosdep_setup = ws / 'rosdep_setup.sh' rosdep_setup.write_text(script_contents) gather_rosdeps(client, platform, workspace=ws, custom_script=rosdep_setup) out_script = ws / rosdep_install_script(platform) result = out_script.read_text().splitlines() expected = [ '#!/bin/bash', 'set -euxo pipefail', '#[apt] Installation commands:', ' apt-get install -y successful_test', ] assert result == expected, 'Rosdep output did not meet expectations.'
def test_colcon_defaults(tmpdir): ws = Path(str(tmpdir)) this_dir = Path(__file__).parent src = ws / 'src' src.mkdir() shutil.copytree(str(this_dir / 'dummy_pkg_ros2_cpp'), str(src / 'dummy_pkg_ros2_cpp')) shutil.copytree(str(this_dir / 'dummy_pkg_ros2_py'), str(src / 'dummy_pkg_ros2_py')) client = DockerClient() platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing') out_script = ws / rosdep_install_script(platform) # no defaults file should get everything gather_rosdeps(client, platform, workspace=ws) result = out_script.read_text() assert 'ros-dashing-ament-cmake' in result assert 'ros-dashing-rclcpp' in result assert 'ros-dashing-rclpy' in result # write defaults file and expect selective dependency output (ws / 'defaults.yaml').write_text(""" list: packages-select: [dummy_pkg_ros2_py] """) gather_rosdeps(client, platform, workspace=ws) result = out_script.read_text() assert 'ros-dashing-ament-cmake' not in result assert 'ros-dashing-rclcpp' not in result assert 'ros-dashing-rclpy' in result
def cross_compile_pipeline(args: argparse.Namespace, ): platform = Platform(args.arch, args.os, args.rosdistro, args.sysroot_base_image) ros_workspace_dir = Path(args.ros_workspace) skip_rosdep_keys = args.skip_rosdep_keys custom_data_dir = _path_if(args.custom_data_dir) custom_rosdep_script = _path_if(args.custom_rosdep_script) custom_setup_script = _path_if(args.custom_setup_script) sysroot_build_context = prepare_docker_build_environment( platform=platform, ros_workspace=ros_workspace_dir, custom_setup_script=custom_setup_script, custom_data_dir=custom_data_dir) docker_client = DockerClient(args.sysroot_nocache, default_docker_dir=sysroot_build_context, colcon_defaults_file=args.colcon_defaults) if not args.skip_rosdep_collection: gather_rosdeps(docker_client=docker_client, platform=platform, workspace=ros_workspace_dir, skip_rosdep_keys=skip_rosdep_keys, custom_script=custom_rosdep_script, custom_data_dir=custom_data_dir) assert_install_rosdep_script_exists(ros_workspace_dir, platform) create_workspace_sysroot_image(docker_client, platform) run_emulated_docker_build(docker_client, platform, ros_workspace_dir)
def create_workspace_sysroot_image(self, docker_client: DockerClient) -> str: """Build the target sysroot docker image and return its full name.""" image_tag = self._platform.sysroot_image_tag logger.info('Building sysroot image: %s', image_tag) docker_client.build_image( dockerfile_dir=self._target_sysroot, dockerfile_name=ROS_DOCKERFILE_NAME, tag=image_tag, buildargs={ 'BASE_IMAGE': self._platform.target_base_image, 'ROS_WORKSPACE': self._ros_workspace_relative_to_sysroot, 'ROS_VERSION': self._platform.ros_version, 'ROS_DISTRO': self._platform.ros_distro, 'TARGET_ARCH': self._platform.arch, } ) logger.info('Successfully created sysroot docker image: %s', image_tag)
def test_rosdep_bad_key(tmpdir): ws = Path(str(tmpdir)) pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml' pkg_xml.parent.mkdir(parents=True) pkg_xml.write_text(CUSTOM_KEY_PKG_XML) client = DockerClient() platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing') with pytest.raises(docker.errors.ContainerError): gather_rosdeps(client, platform, workspace=ws)
def run_emulated_docker_build(docker_client: DockerClient, platform: Platform, workspace_path: Path) -> None: """ Spin up a sysroot docker container and run an emulated build inside. :param docker_client: Preconfigured to run Docker images. :param platform: Information about the target platform. :param workspace: Absolute path to the user's source workspace. """ docker_client.run_container( image_name=platform.sysroot_image_tag, environment={ 'OWNER_USER': str(os.getuid()), 'ROS_DISTRO': platform.ros_distro, 'TARGET_ARCH': platform.arch, }, volumes={ workspace_path: '/ros_ws', }, )
def create_workspace_sysroot_image( docker_client: DockerClient, platform: Platform, ) -> None: """ Create the target platform sysroot image. :param docker_client Docker client to use for building :param platform Information about the target platform :param build_context Directory containing all assets needed by sysroot.Dockerfile """ image_tag = platform.sysroot_image_tag logger.info('Building sysroot image: %s', image_tag) docker_client.build_image(dockerfile_name='sysroot.Dockerfile', tag=image_tag, buildargs={ 'BASE_IMAGE': platform.target_base_image, 'ROS_VERSION': platform.ros_version, }) logger.info('Successfully created sysroot docker image: %s', image_tag)
def test_parse_docker_build_output(): """Test the SysrootCreator constructor assuming valid path setup.""" # Create mock directories and files client = DockerClient() log_generator_without_errors = [ {'stream': ' ---\\u003e a9eb17255234\\n'}, {'stream': 'Step 1 : VOLUME /data\\n'}, {'stream': ' ---\\u003e Running in abdc1e6896c6\\n'}, {'stream': ' ---\\u003e 713bca62012e\\n'}, {'stream': 'Removing intermediate container abdc1e6896c6\\n'}, {'stream': 'Step 2 : CMD [\\"/bin/sh\\"]\\n'}, {'stream': ' ---\\u003e Running in dba30f2a1a7e\\n'}, {'stream': ' ---\\u003e 032b8b2855fc\\n'}, {'stream': 'Removing intermediate container dba30f2a1a7e\\n'}, {'stream': 'Successfully built 032b8b2855fc\\n'}, ] # Just expect it not to raise client._process_build_log(log_generator_without_errors) log_generator_with_errors = [ {'stream': ' ---\\u003e a9eb17255234\\n'}, {'stream': 'Step 1 : VOLUME /data\\n'}, {'stream': ' ---\\u003e Running in abdc1e6896c6\\n'}, {'stream': ' ---\\u003e 713bca62012e\\n'}, {'stream': 'Removing intermediate container abdc1e6896c6\\n'}, {'stream': 'Step 2 : CMD [\\"/bin/sh\\"]\\n'}, {'error': ' ---\\COMMAND NOT FOUND\\n'}, ] with pytest.raises(docker.errors.BuildError): client._process_build_log(log_generator_with_errors)
def test_dummy_skip_rosdep_multiple_keys_pkg(tmpdir): ws = Path(str(tmpdir)) pkg_xml = ws / 'src' / 'dummy' / 'package.xml' pkg_xml.parent.mkdir(parents=True) pkg_xml.write_text(RCLCPP_PKG_XML) client = DockerClient() platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing') out_script = ws / rosdep_install_script(platform) skip_keys = ['ament_cmake', 'rclcpp'] gather_rosdeps(client, platform, workspace=ws, skip_rosdep_keys=skip_keys) result = out_script.read_text() assert 'ros-dashing-ament-cmake' not in result assert 'ros-dashing-rclcpp' not in result
def test_dummy_skip_rosdep_keys_doesnt_exist_pkg(tmpdir): ws = Path(str(tmpdir)) pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml' pkg_xml.parent.mkdir(parents=True) pkg_xml.write_text(CUSTOM_KEY_PKG_XML) client = DockerClient() platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing') skip_keys = ['definitely_does_not_exist'] try: gather_rosdeps(client, platform, workspace=ws, skip_rosdep_keys=skip_keys) except docker.errors.ContainerError: assert False
def buildable_env(request, tmpdir): """Set up a temporary directory with everything needed to run the EmulatedDockerBuildStage.""" platform = Platform('aarch64', 'ubuntu', 'foxy') ros_workspace = Path(str(tmpdir)) / 'ros_ws' _touch_anywhere(ros_workspace / rosdep_install_script(platform)) build_context = prepare_docker_build_environment(platform, ros_workspace) docker = DockerClient(disable_cache=False, default_docker_dir=build_context) options = default_pipeline_options() data_collector = DataCollector() CreateSysrootStage()(platform, docker, ros_workspace, options, data_collector) return BuildableEnv(platform, docker, ros_workspace, options, data_collector)
def test_dummy_ros2_pkg(tmpdir): ws = Path(str(tmpdir)) pkg_xml = ws / 'src' / 'dummy' / 'package.xml' pkg_xml.parent.mkdir(parents=True) pkg_xml.write_text(RCLCPP_PKG_XML) client = DockerClient() platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing') out_script = ws / rosdep_install_script(platform) test_collector = DataCollector() stage = CollectDependencyListStage() stage(platform, client, ws, default_pipeline_options(), test_collector) result = out_script.read_text() assert 'ros-dashing-ament-cmake' in result assert 'ros-dashing-rclcpp' in result
def main(): """Start the cross-compilation workflow.""" # Configuration args = parse_args(sys.argv[1:]) platform = Platform(args.arch, args.os, args.rosdistro, args.sysroot_base_image) docker_client = DockerClient(args.sysroot_nocache) sysroot_creator = SysrootCreator( cc_root_dir=args.sysroot_path, ros_workspace_dir=args.ros_workspace, platform=platform, custom_setup_script_path=args.custom_setup_script, custom_data_dir=args.custom_data_dir) sysroot_creator.create_workspace_sysroot_image(docker_client) ros_workspace_dir = os.path.join(args.sysroot_path, 'sysroot', args.ros_workspace) run_emulated_docker_build(docker_client, platform.sysroot_image_tag, ros_workspace_dir)
def test_dummy_ros2_pkg(tmpdir): ws = Path(str(tmpdir)) pkg_xml = ws / 'src' / 'dummy' / 'package.xml' pkg_xml.parent.mkdir(parents=True) pkg_xml.write_text(RCLCPP_PKG_XML) client = DockerClient() platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing') out_script = ws / rosdep_install_script(platform) # a default set of customizations for the dependencies stage customizations = PipelineStageConfigOptions(False, [], None, None, None) temp_stage = DependenciesStage() temp_stage(platform, client, ws, customizations) result = out_script.read_text() assert 'ros-dashing-ament-cmake' in result assert 'ros-dashing-rclcpp' in result
def test_dummy_ros2_pkg(tmpdir): ws = Path(str(tmpdir)) pkg_xml = ws / 'src' / 'dummy' / 'package.xml' pkg_xml.parent.mkdir(parents=True) pkg_xml.write_text(RCLCPP_PKG_XML) client = DockerClient() platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing') out_script = ws / rosdep_install_script(platform) gather_rosdeps(client, platform, workspace=ws) result = out_script.read_text().splitlines() expected = [ '#!/bin/bash', 'set -euxo pipefail', '#[apt] Installation commands:', ' apt-get install -y ros-dashing-ament-cmake', ' apt-get install -y ros-dashing-rclcpp', ] assert result == expected, 'Rosdep output did not meet expectations.'
def cross_compile_pipeline( args: argparse.Namespace, data_collector: DataCollector, platform: Platform, ): ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace) skip_rosdep_keys = args.skip_rosdep_keys custom_data_dir = _path_if(args.custom_data_dir) custom_rosdep_script = _path_if(args.custom_rosdep_script) custom_setup_script = _path_if(args.custom_setup_script) custom_post_build_script = _path_if(args.custom_post_build_script) colcon_defaults_file = _path_if(args.colcon_defaults) sysroot_build_context = prepare_docker_build_environment( platform=platform, ros_workspace=ros_workspace_dir, custom_setup_script=custom_setup_script, custom_post_build_script=custom_post_build_script, colcon_defaults_file=colcon_defaults_file, custom_data_dir=custom_data_dir) docker_client = DockerClient(args.sysroot_nocache, default_docker_dir=sysroot_build_context) options = PipelineStageOptions(skip_rosdep_keys, custom_rosdep_script, custom_data_dir, custom_setup_script, args.runtime_tag) skip = set(args.stages_skip) # Only package the runtime image if the user has specified a tag for it if not args.runtime_tag: skip.add(PackageRuntimeImageStage.NAME) for stage in _PIPELINE: if stage.name not in skip: with data_collector.timer('{}'.format(stage.name)): stage(platform, docker_client, ros_workspace_dir, options, data_collector)
def test_custom_post_build_script(tmpdir): created_filename = 'file-created-by-post-build' platform = Platform('aarch64', 'ubuntu', 'foxy') ros_workspace = Path(str(tmpdir)) / 'ros_ws' _touch_anywhere(ros_workspace / rosdep_install_script(platform)) post_build_script = ros_workspace / 'post_build' post_build_script.write_text(""" #!/bin/bash echo "success" > {} """.format(created_filename)) build_context = prepare_docker_build_environment( platform, ros_workspace, custom_post_build_script=post_build_script) docker = DockerClient(disable_cache=False, default_docker_dir=build_context) options = default_pipeline_options() data_collector = DataCollector() CreateSysrootStage()(platform, docker, ros_workspace, options, data_collector) EmulatedDockerBuildStage()(platform, docker, ros_workspace, options, data_collector) assert (ros_workspace / created_filename).is_file()
def test_custom_rosdep_no_data_dir(tmpdir): script_contents = """ cat > /test_rules.yaml <<EOF definitely_does_not_exist: ubuntu: bionic: [successful_test] EOF echo "yaml file:/test_rules.yaml" > /etc/ros/rosdep/sources.list.d/22-test-rules.list """ ws = Path(str(tmpdir)) pkg_xml = Path(ws) / 'src' / 'dummy' / 'package.xml' pkg_xml.parent.mkdir(parents=True) pkg_xml.write_text(CUSTOM_KEY_PKG_XML) client = DockerClient() platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing') rosdep_setup = ws / 'rosdep_setup.sh' rosdep_setup.write_text(script_contents) gather_rosdeps(client, platform, workspace=ws, custom_script=rosdep_setup) out_script = ws / rosdep_install_script(platform) result = out_script.read_text() assert 'successful_test' in result
def cross_compile_pipeline( args: argparse.Namespace, ): platform = Platform(args.arch, args.os, args.rosdistro, args.sysroot_base_image) ros_workspace_dir = Path(args.ros_workspace).resolve() if not (ros_workspace_dir / 'src').is_dir(): raise ValueError( 'Specified workspace "{}" does not look like a colcon workspace ' '(there is no "src/" directory). Cannot continue'.format(ros_workspace_dir)) skip_rosdep_keys = args.skip_rosdep_keys custom_data_dir = _path_if(args.custom_data_dir) custom_rosdep_script = _path_if(args.custom_rosdep_script) custom_setup_script = _path_if(args.custom_setup_script) sysroot_build_context = prepare_docker_build_environment( platform=platform, ros_workspace=ros_workspace_dir, custom_setup_script=custom_setup_script, custom_data_dir=custom_data_dir) docker_client = DockerClient( args.sysroot_nocache, default_docker_dir=sysroot_build_context, colcon_defaults_file=args.colcon_defaults) if not args.skip_rosdep_collection: gather_rosdeps( docker_client=docker_client, platform=platform, workspace=ros_workspace_dir, skip_rosdep_keys=skip_rosdep_keys, custom_script=custom_rosdep_script, custom_data_dir=custom_data_dir) assert_install_rosdep_script_exists(ros_workspace_dir, platform) create_workspace_sysroot_image(docker_client, platform) run_emulated_docker_build(docker_client, platform, ros_workspace_dir)