def __call__(
self,
platform: Platform,
docker_client: DockerClient,
ros_workspace_dir: Path,
options: PipelineStageOptions,
data_collector: DataCollector
):
"""
Run the inspection and output the dependency installation script.
Also recovers the size of the docker image generated.
:raises RuntimeError if the step was skipped when no dependency script has been
previously generated
"""
gather_rosdeps(
docker_client=docker_client,
platform=platform,
workspace=ros_workspace_dir,
skip_rosdep_keys=options.skip_rosdep_keys,
custom_script=options.custom_script,
custom_data_dir=options.custom_data_dir)
assert_install_rosdep_script_exists(ros_workspace_dir, platform)
img_size = docker_client.get_image_size(_IMG_NAME)
data_collector.add_size(self.name, img_size)
def __call__(self, platform: Platform, docker_client: DockerClient,
ros_workspace_dir: Path, options: PipelineStageOptions,
data_collector: DataCollector):
create_workspace_sysroot_image(docker_client, platform)
img_size = docker_client.get_image_size(platform.sysroot_image_tag)
data_collector.add_size(self.name, img_size)
def test_timer_can_time():
test_collector = DataCollector()
with test_collector.timer('test_time'):
pass
assert test_collector._data[0].complete
assert test_collector._data[0].value > 0
def test_timer_error_handling():
test_collector = DataCollector()
# The timer should not hide the exception, we expect it to add the datum value
with pytest.raises(Exception):
with test_collector.timer('test_time_fail'):
raise Exception
assert len(test_collector._data) > 0
assert test_collector._data[0].complete is False
def main():
"""Start the cross-compilation workflow."""
args = parse_args(sys.argv[1:])
ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace)
data_collector = DataCollector()
data_writer = DataWriter(ros_workspace_dir)
try:
with data_collector.timer('cross_compile_end_to_end'):
cross_compile_pipeline(args, data_collector)
finally:
data_writer.write(data_collector)
def test_data_printing(tmp_path, capfd):
platform = Platform('aarch64', 'ubuntu', 'foxy')
test_collector = DataCollector()
test_datum_a = Datum('test_stat_1', 3, 'tests', 130.243, True)
test_collector.add_datum(test_datum_a)
test_writer = DataWriter(tmp_path, 'test.json')
test_writer.write(test_collector, platform, True)
out, err = capfd.readouterr()
test_name = '------------'
assert test_name in out
def main():
"""Start the cross-compilation workflow."""
args = parse_args(sys.argv[1:])
platform = Platform(args.arch, args.os, args.rosdistro,
args.sysroot_base_image)
ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace)
data_collector = DataCollector()
data_writer = DataWriter(ros_workspace_dir, args.custom_metric_file)
try:
with data_collector.timer('end_to_end'):
cross_compile_pipeline(args, data_collector, platform)
finally:
data_writer.write(data_collector, platform, args.print_metrics)
def test_bad_workspace(tmpdir):
args = parse_args(
[str(tmpdir), '-a', 'aarch64', '-o', 'ubuntu', '-d', 'foxy'])
test_collector = DataCollector()
platform = Platform(args.arch, args.os, args.rosdistro)
with pytest.raises(ValueError):
cross_compile_pipeline(args, test_collector, platform)
def cross_compile_pipeline(
args: argparse.Namespace,
data_collector: DataCollector,
):
platform = Platform(args.arch, args.os, args.rosdistro,
args.sysroot_base_image)
ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace)
skip_rosdep_keys = args.skip_rosdep_keys
custom_data_dir = _path_if(args.custom_data_dir)
custom_rosdep_script = _path_if(args.custom_rosdep_script)
custom_setup_script = _path_if(args.custom_setup_script)
sysroot_build_context = prepare_docker_build_environment(
platform=platform,
ros_workspace=ros_workspace_dir,
custom_setup_script=custom_setup_script,
custom_data_dir=custom_data_dir)
docker_client = DockerClient(args.sysroot_nocache,
default_docker_dir=sysroot_build_context,
colcon_defaults_file=args.colcon_defaults)
stages = [DependenciesStage(), CreateSysrootStage(), DockerBuildStage()]
customizations = PipelineStageConfigOptions(args.skip_rosdep_collection,
skip_rosdep_keys,
custom_rosdep_script,
custom_data_dir,
custom_setup_script)
for stage in stages:
with data_collector.timer('cross_compile_{}'.format(stage.name)):
stage(platform, docker_client, ros_workspace_dir, customizations)
def cross_compile_pipeline(
args: argparse.Namespace,
data_collector: DataCollector,
platform: Platform,
):
ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace)
skip_rosdep_keys = args.skip_rosdep_keys
custom_data_dir = _path_if(args.custom_data_dir)
custom_rosdep_script = _path_if(args.custom_rosdep_script)
custom_setup_script = _path_if(args.custom_setup_script)
sysroot_build_context = prepare_docker_build_environment(
platform=platform,
ros_workspace=ros_workspace_dir,
custom_setup_script=custom_setup_script,
custom_data_dir=custom_data_dir)
docker_client = DockerClient(args.sysroot_nocache,
default_docker_dir=sysroot_build_context,
colcon_defaults_file=args.colcon_defaults)
options = PipelineStageOptions(skip_rosdep_keys, custom_rosdep_script,
custom_data_dir, custom_setup_script)
for stage in _PIPELINE:
if stage.name not in args.stages_skip:
with data_collector.timer('{}'.format(stage.name)):
stage(platform, docker_client, ros_workspace_dir, options,
data_collector)
def test_data_collection():
test_collector = DataCollector()
test_datum_a = Datum('test_stat_1', 3, 'tests', 130.452, True)
test_datum_b = Datum('test_stat_2', 4, 'tests', 130.455, True)
test_collector.add_datum(test_datum_a)
test_collector.add_datum(test_datum_b)
to_test_data = test_collector._data
assert to_test_data[0].name == 'test_stat_1'
assert to_test_data[1].name == 'test_stat_2'
assert to_test_data[0].value == 3
assert to_test_data[0].unit == 'tests'
assert abs(to_test_data[0].timestamp - 130.452) < 0.1
assert to_test_data[0].complete
def test_data_writing(tmp_path):
def load_json_validation(filename: Path) -> bool:
try:
with filename.open() as f:
json.load(f)
return True
except JSONDecodeError:
return False
test_collector = DataCollector()
test_datum_a = Datum('test_stat_1', 3, 'tests', 130.243, True)
test_datum_b = Datum('test_stat_2', 4, 'tests', 130.244, True)
test_collector.add_datum(test_datum_a)
test_collector.add_datum(test_datum_b)
test_writer = DataWriter(tmp_path)
test_writer.write(test_collector)
assert test_writer.write_file.exists()
assert load_json_validation(test_writer.write_file)
def test_mocked_cc_pipeline(tmpdir):
tmp = Path(str(tmpdir))
test_collector = DataCollector()
(tmp / 'src').mkdir()
args = parse_args([str(tmpdir), '-a', 'aarch64', '-o', 'ubuntu'])
with patch('ros_cross_compile.ros_cross_compile.DockerClient', Mock(
)) as docker_mock, patch(
'ros_cross_compile.dependencies.assert_install_rosdep_script_exists'
) as script_mock:
cross_compile_pipeline(args, test_collector)
assert script_mock.called
assert docker_mock.called
assert docker_mock().build_image.call_count == 2
assert docker_mock().run_container.call_count == 2
def buildable_env(request, tmpdir):
"""Set up a temporary directory with everything needed to run the EmulatedDockerBuildStage."""
platform = Platform('aarch64', 'ubuntu', 'foxy')
ros_workspace = Path(str(tmpdir)) / 'ros_ws'
_touch_anywhere(ros_workspace / rosdep_install_script(platform))
build_context = prepare_docker_build_environment(platform, ros_workspace)
docker = DockerClient(disable_cache=False,
default_docker_dir=build_context)
options = default_pipeline_options()
data_collector = DataCollector()
CreateSysrootStage()(platform, docker, ros_workspace, options,
data_collector)
return BuildableEnv(platform, docker, ros_workspace, options,
data_collector)
def test_relative_workspace(tmpdir):
# Change directory to the tmp dir and invoke using '.' as the
# workspace to check if relative paths work
tmp = Path(str(tmpdir))
test_collector = DataCollector()
(tmp / 'src').mkdir()
relative_dir = '.'
args = parse_args(
[relative_dir, '-a', 'aarch64', '-o', 'ubuntu', '-d', 'foxy'])
with chdir(str(tmp)), patch(
'ros_cross_compile.ros_cross_compile.DockerClient', Mock()
), patch(
'ros_cross_compile.dependencies.assert_install_rosdep_script_exists'
):
# should not raise an exception
cross_compile_pipeline(args, test_collector)
def test_dummy_ros2_pkg(tmpdir):
ws = Path(str(tmpdir))
pkg_xml = ws / 'src' / 'dummy' / 'package.xml'
pkg_xml.parent.mkdir(parents=True)
pkg_xml.write_text(RCLCPP_PKG_XML)
client = DockerClient()
platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing')
out_script = ws / rosdep_install_script(platform)
test_collector = DataCollector()
stage = CollectDependencyListStage()
stage(platform, client, ws, default_pipeline_options(), test_collector)
result = out_script.read_text()
assert 'ros-dashing-ament-cmake' in result
assert 'ros-dashing-rclcpp' in result
def cross_compile_pipeline(
args: argparse.Namespace,
data_collector: DataCollector,
platform: Platform,
):
ros_workspace_dir = _resolve_ros_workspace(args.ros_workspace)
skip_rosdep_keys = args.skip_rosdep_keys
custom_data_dir = _path_if(args.custom_data_dir)
custom_rosdep_script = _path_if(args.custom_rosdep_script)
custom_setup_script = _path_if(args.custom_setup_script)
custom_post_build_script = _path_if(args.custom_post_build_script)
colcon_defaults_file = _path_if(args.colcon_defaults)
sysroot_build_context = prepare_docker_build_environment(
platform=platform,
ros_workspace=ros_workspace_dir,
custom_setup_script=custom_setup_script,
custom_post_build_script=custom_post_build_script,
colcon_defaults_file=colcon_defaults_file,
custom_data_dir=custom_data_dir)
docker_client = DockerClient(args.sysroot_nocache,
default_docker_dir=sysroot_build_context)
options = PipelineStageOptions(skip_rosdep_keys, custom_rosdep_script,
custom_data_dir, custom_setup_script,
args.runtime_tag)
skip = set(args.stages_skip)
# Only package the runtime image if the user has specified a tag for it
if not args.runtime_tag:
skip.add(PackageRuntimeImageStage.NAME)
for stage in _PIPELINE:
if stage.name not in skip:
with data_collector.timer('{}'.format(stage.name)):
stage(platform, docker_client, ros_workspace_dir, options,
data_collector)
def test_custom_post_build_script(tmpdir):
created_filename = 'file-created-by-post-build'
platform = Platform('aarch64', 'ubuntu', 'foxy')
ros_workspace = Path(str(tmpdir)) / 'ros_ws'
_touch_anywhere(ros_workspace / rosdep_install_script(platform))
post_build_script = ros_workspace / 'post_build'
post_build_script.write_text("""
#!/bin/bash
echo "success" > {}
""".format(created_filename))
build_context = prepare_docker_build_environment(
platform, ros_workspace, custom_post_build_script=post_build_script)
docker = DockerClient(disable_cache=False,
default_docker_dir=build_context)
options = default_pipeline_options()
data_collector = DataCollector()
CreateSysrootStage()(platform, docker, ros_workspace, options,
data_collector)
EmulatedDockerBuildStage()(platform, docker, ros_workspace, options,
data_collector)
assert (ros_workspace / created_filename).is_file()
def test_collector_construction():
test_collector = DataCollector()
assert test_collector