def replace_targets_by_paths(x, layer_opts: LayerOpts):
'''
Converts target_tagger.bzl sigils to buck-out paths or Subvol objects.
JSON-serialized image features store single-item dicts of the form
{'__BUCK{_LAYER,}_TARGET': '//target:path'} whenever the compiler
requires a path to another target. This is because actual paths would
break Buck caching, and would not survive repo moves. Then, at runtime,
the compiler receives a dictionary of target-to-path mappings as
`--child-dependencies`, and performs the substitution in any image
feature JSON it consumes.
'''
if type(x) is dict:
if '__BUCK_TARGET' in x or '__BUCK_LAYER_TARGET' in x:
assert len(x) == 1, x
(sigil, target), = x.items()
path = layer_opts.target_to_path.get(target)
if not path:
raise RuntimeError(
f'{target} not in {layer_opts.target_to_path}')
return path if sigil == '__BUCK_TARGET' else find_built_subvol(
path,
subvolumes_dir=layer_opts.subvolumes_dir,
)
return {
k: replace_targets_by_paths(v, layer_opts)
for k, v in x.items()
}
elif type(x) is list:
return [replace_targets_by_paths(v, layer_opts) for v in x]
elif type(x) in [int, float, str, bool, type(None)]:
return x
assert False, f'Unknown {type(x)} for {x}' # pragma: no cover
def to_path(
self, *, target_to_path: Mapping[str, str], subvolumes_dir: str,
) -> str:
if self.type == 'layer':
out_path = target_to_path.get(self.source)
if out_path is None:
raise AssertionError(
f'MountItem could not resolve {self.source}'
)
subvol = find_built_subvol(out_path, subvolumes_dir=subvolumes_dir)
# If we allowed mounting a layer that has other mounts inside,
# it would force us to support nested mounts. We don't want to
# do this (yet).
if os.path.exists(subvol.path(META_MOUNTS_DIR)):
raise AssertionError(
f'Refusing to mount {subvol.path()} since that would '
'require the tooling to support nested mounts.'
)
return subvol.path()
elif self.type == 'host':
return self.source
else: # pragma: no cover
raise AssertionError(
f'Bad mount source "{self.type}" for {self.source}'
)
def test_find_built_subvol(self):
with open(
find_built_subvol(
# This works in @mode/opt since this artifact is baked into the XAR
os.path.join(
os.path.dirname(__file__),
'hello_world_base'), ).path('hello_world')) as f:
self.assertEqual('', f.read())
def test_rpm_metadata_from_subvol(self):
layer_path = os.path.join(os.path.dirname(__file__), 'child-layer')
child_subvol = find_built_subvol(layer_path)
a = RpmMetadata.from_subvol(child_subvol, 'rpm-test-mice')
self.assertEqual(a.name, 'rpm-test-mice')
self.assertEqual(a.epoch, 0)
self.assertEqual(a.version, '0.1')
self.assertEqual(a.release, 'a')
# not installed
with self.assertRaises(RuntimeError):
a = RpmMetadata.from_subvol(child_subvol, 'rpm-test-carrot')
# subvol with no RPM DB
layer_path = os.path.join(os.path.dirname(__file__), 'hello-layer')
hello_subvol = find_built_subvol(layer_path)
with self.assertRaisesRegex(ValueError, ' does not exist$'):
a = RpmMetadata.from_subvol(hello_subvol, 'rpm-test-mice')
def test_rpm_action_item_auto_downgrade(self):
parent_subvol = find_built_subvol(
(Path(__file__).dirname() / 'test-with-one-local-rpm').decode())
src_rpm = Path(__file__).dirname() / "rpm-test-cheese-1-1.rpm"
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
# ensure cheese2 is installed in the parent from rpm-test-cheese-2-1
assert os.path.isfile(
parent_subvol.path('/usr/share/rpm_test/cheese2.txt'))
# make sure the RPM we are installing is older in order to
# trigger the downgrade
src_data = RpmMetadata.from_file(src_rpm)
subvol_data = RpmMetadata.from_subvol(parent_subvol, src_data.name)
assert compare_rpm_versions(src_data, subvol_data) < 0
subvol = temp_subvolumes.snapshot(parent_subvol, 'rpm_action')
RpmActionItem.get_phase_builder(
[
RpmActionItem(
from_target='t',
source=src_rpm,
action=RpmAction.install,
)
],
DUMMY_LAYER_OPTS._replace(
yum_from_snapshot=Path(__file__).dirname() /
'yum-from-test-snapshot', ),
)(subvol)
subvol.run_as_root([
'rm',
'-rf',
subvol.path('dev'),
subvol.path('meta'),
subvol.path('var'),
])
self.assertEqual([
'(Dir)', {
'usr': [
'(Dir)', {
'share': [
'(Dir)', {
'rpm_test': [
'(Dir)', {
'cheese1.txt': ['(File d36)'],
}
],
}
],
}
],
}
], render_subvol(subvol))
def test_check_layers(self):
meta = {
'meta': [
'(Dir)', {
'private': [
'(Dir)', {
'opts': [
'(Dir)', {
'artifacts_may_require_repo':
['(File d2)'],
}
]
}
]
}
]
}
# The parent has a couple of directories.
self.assertEqual(
['(Dir)', {
'a': ['(Dir)', {
'b': ['(Dir)', {}]
}],
**meta
}],
render_sendstream(
find_built_subvol(self._resource_path(
'parent')).mark_readonly_and_get_sendstream()),
)
# The child is near-empty because the `remove_paths` cleaned it up.
self.assertEqual(
['(Dir)', {
**meta
}],
render_sendstream(
find_built_subvol(self._resource_path(
'child')).mark_readonly_and_get_sendstream()),
)
def target_subvol(self, target, mount_config=None):
with self.subTest(target):
# The mount configuration is very uniform, so we can check it here.
expected_config = {
'is_directory': True,
'build_source': {
'type': 'layer',
'source': '//fs_image/compiler/tests:' + target,
},
}
if mount_config:
expected_config.update(mount_config)
with open(TARGET_TO_PATH[target] + '/mountconfig.json') as infile:
self.assertEqual(expected_config, json.load(infile))
yield find_built_subvol(TARGET_TO_PATH[target])
def test_check_layers(self):
# The parent has a couple of directories.
self.assertEqual(
[
'(Dir)', {
'a': ['(Dir)', {
'b': ['(Dir)', {}]
}],
'meta': ['(Dir)', {}],
}
],
render_sendstream(
find_built_subvol(self._resource_path(
'parent')).mark_readonly_and_get_sendstream()),
)
# The child is near-empty because the `remove_paths` cleaned it up.
self.assertEqual(
['(Dir)', {
'meta': ['(Dir)', {}]
}],
render_sendstream(
find_built_subvol(self._resource_path(
'child')).mark_readonly_and_get_sendstream()),
)
def package_image(argv):
args = parse_args(argv)
assert not os.path.exists(args.output_path)
Format.make(args.format).package_full(
find_built_subvol(args.layer_path, subvolumes_dir=args.subvolumes_dir),
output_path=args.output_path,
opts=_Opts(
subvol_opts=SubvolOpts(readonly=not args.writable_subvolume), ),
)
# Paranoia: images are read-only after being built
os.chmod(
args.output_path,
stat.S_IMODE(os.stat(args.output_path).st_mode)
& ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH),
)
def test_rpm_build_item(self):
parent_subvol = find_built_subvol(
(Path(__file__).dirname() / 'toy-rpmbuild-setup').decode())
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
assert os.path.isfile(
parent_subvol.path('/rpmbuild/SOURCES/toy_src_file'))
assert os.path.isfile(
parent_subvol.path('/rpmbuild/SPECS/specfile.spec'))
subvol = temp_subvolumes.snapshot(parent_subvol, 'rpm_build')
item = RpmBuildItem(from_target='t', rpmbuild_dir='/rpmbuild')
RpmBuildItem.get_phase_builder(
[item],
DUMMY_LAYER_OPTS,
)(subvol)
self.assertEqual(item.phase_order(), PhaseOrder.RPM_BUILD)
assert os.path.isfile(subvol.path('/rpmbuild/RPMS/toy.rpm'))
def test_install_file_from_layer(self):
layer = find_built_subvol(
Path(__file__).dirname() / 'test-with-one-local-rpm')
path_in_layer = b'usr/share/rpm_test/cheese2.txt'
item = _install_file_item(
from_target='t',
source={
'layer': layer,
'path': '/' + path_in_layer.decode()
},
dest='cheese2',
)
self.assertEqual(0o444, item.mode)
self.assertEqual(Path(layer.path(path_in_layer)), item.source)
self.assertEqual(layer.path(path_in_layer), item.source)
self._check_item(
item,
{ProvidesFile(path='cheese2')},
{require_directory('/')},
)
'--no-logs-tmpfs', action='store_false', dest='logs_tmpfs',
help='Our production runtime always provides a user-writable `/logs` '
'in the container, so this wrapper simulates it by mounting a '
'tmpfs at that location by default. You may need this flag to '
'use `--no-snapshot` with an layer that lacks a `/logs` '
'mountpoint. NB: we do not supply a persistent writable mount '
'since that is guaranteed to break hermeticity and e.g. make '
'somebody\'s image tests very hard to debug.',
)
parser.add_argument(
'--forward-fd', type=int, action='append', default=[],
help='These FDs will be copied into the container with sequential '
'FD numbers starting from 3, in the order they were listed '
'on the command-line. Repeat to pass multiple FDs.',
)
parser.add_argument(
'cmd', nargs='*', default=['/bin/bash'],
help='The command to run (as PID 2) in the container',
)
opts = parser.parse_args(argv)
assert not opts.snapshot_into or opts.snapshot, opts
return opts
# The manual test is in the first paragraph of the top docblock.
if __name__ == '__main__': # pragma: no cover
opts = parse_opts(sys.argv[1:])
sys.exit(nspawn_in_subvol(
find_built_subvol(opts.layer), opts, check=False,
).returncode)
# When used as part of the `image.python_unittest` implementation, there
# is no good way to pass arguments to this nspawn wrapper. So, we
# package the `image.layer` as a resource, and the remaining arguments
# as Python source module. These are optional only to allow the kind of
# manual test shown above.
argv = []
packaged_layer = os.path.join(
os.path.dirname(__file__),
'nspawn-in-test-subvol-layer',
)
if os.path.exists(packaged_layer):
argv.extend(['--layer', packaged_layer])
import __image_python_unittest_spec__
argv.extend(__image_python_unittest_spec__.nspawn_in_subvol_args())
opts = parse_opts(argv + sys.argv[1:])
with rewrite_test_cmd(
opts.cmd,
next_fd=3 + len(opts.forward_fd),
) as (new_cmd, fd_to_forward):
opts.cmd = new_cmd
if fd_to_forward is not None:
opts.forward_fd.append(fd_to_forward)
ret = nspawn_in_subvol(find_built_subvol(opts.layer),
opts,
check=False)
# Only trigger SystemExit after the context was cleaned up.
sys.exit(ret.returncode)