def test_does_not_exist(self):
with tempfile.TemporaryDirectory() as td:
with self.assertRaisesRegex(AssertionError, 'No btrfs subvol'):
Subvol(td, already_exists=True)
sv = Subvol(td)
with self.assertRaisesRegex(AssertionError, 'exists is False'):
sv.run_as_root(['true'])
def test_out_of_subvol_symlink(self):
with tempfile.TemporaryDirectory() as td:
os.symlink('/dev/null', os.path.join(td, 'my_null'))
self.assertEqual(
os.path.join(td, 'my_null').encode(),
Subvol(td).path('my_null', no_dereference_leaf=True),
)
with self.assertRaisesRegex(AssertionError, 'outside the subvol'):
Subvol(td).path('my_null')
def test_out_of_subvol_symlink(self):
with temp_dir() as td:
os.symlink('/dev/null', td / 'my_null')
self.assertEqual(
td / 'my_null',
Subvol(td).path('my_null', no_dereference_leaf=True),
)
with self.assertRaisesRegex(AssertionError, 'outside the subvol'):
Subvol(td).path('my_null')
def build_image(args):
subvol = Subvol(os.path.join(args.subvolumes_dir, args.subvolume_rel_path))
for item in dependency_order_items(
itertools.chain(
gen_parent_layer_items(
args.child_layer_target,
args.parent_layer_json,
args.subvolumes_dir,
),
gen_items_for_features(
[args.child_feature_json],
make_target_filename_map(args.child_dependencies),
),
)):
item.build(subvol)
try:
return SubvolumeOnDisk.from_subvolume_path(
subvol.path().decode(),
args.subvolumes_dir,
args.subvolume_rel_path,
)
except Exception as ex:
raise RuntimeError(f'Serializing subvolume {subvol.path()}') from ex
def to_path(
self,
*,
target_to_path: Mapping[str, str],
subvolumes_dir: str,
) -> str:
if self.type == 'layer':
out_path = target_to_path.get(self.source)
if out_path is None:
raise AssertionError(
f'MountItem could not resolve {self.source}')
with open(os.path.join(out_path, 'layer.json')) as infile:
subvol = Subvol(SubvolumeOnDisk.from_json_file(
infile,
subvolumes_dir,
).subvolume_path(),
already_exists=True)
# If we allowed mounting a layer that has other mounts
# inside, it would force us to support nested mounts. We
# don't want to do this (yet).
if os.path.exists(subvol.path(META_MOUNTS_DIR)):
raise AssertionError(
f'Refusing to mount {subvol.path()} since that would '
'require the tooling to support nested mounts.')
return subvol.path()
elif self.type == 'host':
return self.source
else: # pragma: no cover
raise AssertionError(
f'Bad mount source "{self.type}" for {self.source}')
def build_image(args):
subvol = Subvol(os.path.join(args.subvolumes_dir, args.subvolume_rel_path))
dep_graph = DependencyGraph(itertools.chain(
gen_parent_layer_items(
args.child_layer_target,
args.parent_layer_json,
args.subvolumes_dir,
),
gen_items_for_features(
feature_paths=[args.child_feature_json],
target_to_path=make_target_path_map(args.child_dependencies),
yum_from_repo_snapshot=args.yum_from_repo_snapshot,
),
))
for phase in dep_graph.ordered_phases():
phase.build(subvol)
# We cannot validate or sort `ImageItem`s until the phases are
# materialized since the items may depend on the output of the phases.
for item in dep_graph.gen_dependency_order_items(subvol.path().decode()):
item.build(subvol)
# Build artifacts should never change.
subvol.set_readonly(True)
try:
return SubvolumeOnDisk.from_subvolume_path(
subvol.path().decode(),
args.subvolumes_dir,
)
except Exception as ex:
raise RuntimeError(f'Serializing subvolume {subvol.path()}') from ex
def package_full(self, svod: SubvolumeOnDisk, output_path: str):
# Future: rpm.common.create_ro, but it's kind of a big dep.
# Luckily `image_package` will promptly mark this read-only.
assert not os.path.exists(output_path)
with open(output_path, 'wb') as outfile, Subvol(
svod.subvolume_path(), already_exists=True,
).mark_readonly_and_write_sendstream_to_file(outfile):
pass
def find_built_subvol(layer_output, path_in_repo=None):
with open(os.path.join(layer_output, 'layer.json')) as infile:
return Subvol(
SubvolumeOnDisk.from_json_file(
infile,
subvolumes_dir(path_in_repo),
).subvolume_path(),
already_exists=True,
)
def package_full(self, svod: SubvolumeOnDisk, output_path: str):
assert not os.path.exists(output_path)
with open(output_path, 'wb') as outfile, subprocess.Popen(
['zstd', '--stdout'], stdin=subprocess.PIPE, stdout=outfile
) as zstd, Subvol(
svod.subvolume_path(), already_exists=True,
).mark_readonly_and_write_sendstream_to_file(zstd.stdin):
pass
check_popen_returncode(zstd)
def build_image(args):
subvol = Subvol(os.path.join(args.subvolumes_dir, args.subvolume_rel_path))
target_to_path = make_target_path_map(args.child_dependencies)
# This stack allows build items to hold temporary state on disk.
with ExitStack() as exit_stack:
dep_graph = DependencyGraph(
itertools.chain(
gen_parent_layer_items(
args.child_layer_target,
args.parent_layer_json,
args.subvolumes_dir,
),
gen_items_for_features(
exit_stack=exit_stack,
feature_paths=[args.child_feature_json],
target_to_path=target_to_path,
),
))
layer_opts = LayerOpts(
layer_target=args.child_layer_target,
yum_from_snapshot=args.yum_from_repo_snapshot,
build_appliance=None if not args.build_appliance_json else
get_subvolume_path(args.build_appliance_json, args.subvolumes_dir),
)
# Creating all the builders up-front lets phases validate their input
for builder in [
builder_maker(items, layer_opts)
for builder_maker, items in dep_graph.ordered_phases()
]:
builder(subvol)
# We cannot validate or sort `ImageItem`s until the phases are
# materialized since the items may depend on the output of the phases.
for item in dep_graph.gen_dependency_order_items(
subvol.path().decode()):
build_item(
item,
subvol=subvol,
target_to_path=target_to_path,
subvolumes_dir=args.subvolumes_dir,
)
# Build artifacts should never change. Run this BEFORE the exit_stack
# cleanup to enforce that the cleanup does not touch the image.
subvol.set_readonly(True)
try:
return SubvolumeOnDisk.from_subvolume_path(
# Converting to a path here does not seem too risky since this
# class shouldn't have a reason to follow symlinks in the subvol.
subvol.path().decode(),
args.subvolumes_dir,
)
# The complexity of covering this is high, but the only thing that can
# go wrong is a typo in the f-string.
except Exception as ex: # pragma: no cover
raise RuntimeError(f'Serializing subvolume {subvol.path()}') from ex
def package_full(self, svod: SubvolumeOnDisk, output_path: str):
Subvol(
svod.subvolume_path(), already_exists=True,
).mark_readonly_and_send_to_new_loopback(output_path)
# Paranoia: images are read-only after being built
os.chmod(
output_path,
stat.S_IMODE(os.stat(output_path).st_mode)
& ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH),
)
def test_run_as_root_return(self):
args = ['bash', '-c', 'echo -n my out; echo -n my err >&2']
r = Subvol('/dev/null/no-such-dir').run_as_root(
args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
_subvol_exists=False,
)
self.assertEqual(['sudo', '--'] + args, r.args)
self.assertEqual(0, r.returncode)
self.assertEqual(b'my out', r.stdout)
self.assertEqual(b'my err', r.stderr)
def test_path(self):
# We are only going to do path manipulations in this test.
sv = Subvol('/subvol/need/not/exist')
for bad_path in ['..', 'a/../../b/c/d', '../c/d/e']:
with self.assertRaisesRegex(AssertionError, 'outside the subvol'):
sv.path(bad_path)
self.assertEqual(sv.path('a/b'), sv.path('/a/b/'))
self.assertEqual(b'a/b', os.path.relpath(sv.path('a/b'), sv.path()))
self.assertTrue(not sv.path('.').endswith(b'/.'))
def find_built_subvol(
layer_output, *, path_in_repo=None, subvolumes_dir=None,
):
# It's OK for both to be None (uses the current file to find repo), but
# it's not OK to set both.
assert (path_in_repo is None) or (subvolumes_dir is None)
with open(Path(layer_output) / 'layer.json') as infile:
return Subvol(
SubvolumeOnDisk.from_json_file(
infile,
subvolumes_dir if subvolumes_dir
else _get_subvolumes_dir(path_in_repo),
).subvolume_path(),
already_exists=True,
)
def test_layer_from_demo_sendstreams(self):
# `btrfs_diff.demo_sendstream` produces a subvolume send-stream with
# fairly thorough coverage of filesystem features. This test grabs
# that send-stream, receives it into an `image_layer`, and validates
# that the send-stream of the **received** volume has the same
# rendering as the original send-stream was supposed to have.
#
# In other words, besides testing `image_layer`'s `from_sendstream`,
# this is also a test of idempotence for btrfs send+receive.
#
# Notes:
# - `compiler/tests/TARGETS` explains why `mutate_ops` is not here.
# - Currently, `mutate_ops` also uses `--no-data`, which would
# break this test of idempotence.
for op in ['create_ops']:
with self.target_subvol(op) as sod:
self.assertEqual(
render_demo_subvols(**{op: True}),
render_sendstream(
Subvol(sod.subvolume_path(), already_exists=True).
mark_readonly_and_get_sendstream(), ),
)
def builder(subvol: Subvol):
parent_subvol = Subvol(parent.path, already_exists=True)
subvol.snapshot(parent_subvol)
# This assumes that the parent has everything mounted already.
mount_item.clone_mounts(parent_subvol, subvol)
_ensure_meta_dir_exists(subvol)
def builder(subvol: Subvol) -> None:
# Go through the list of RPMs to install and change the action to
# downgrade if it is a local RPM with a lower version than what is
# installed.
# This is done in the builder because we need access to the subvol.
for nor in action_to_names_or_rpms[RpmAction.install].copy():
if isinstance(nor, _LocalRpm):
try:
old = RpmMetadata.from_subvol(subvol,
nor.metadata.name)
except (RuntimeError, ValueError):
# This can happen if the RPM DB does not exist in the
# subvolume or the package is not installed.
continue
if compare_rpm_versions(nor.metadata, old) <= 0:
action_to_names_or_rpms[RpmAction.install].remove(nor)
action_to_names_or_rpms[RpmAction.downgrade].add(nor)
for action, nors in action_to_names_or_rpms.items():
if not nors:
continue
# Future: `yum-from-snapshot` is actually designed to run
# unprivileged (but we have no nice abstraction for this).
if layer_opts.build_appliance is None:
subvol.run_as_root([
# Since `yum-from-snapshot` variants are generally
# Python binaries built from this very repo, in
# @mode/dev, we would run a symlink-PAR from the
# buck-out tree as `root`. This would leave behind
# root-owned `__pycache__` directories, which would
# break Buck's fragile cleanup, and cause us to leak old
# build artifacts. This eventually runs the host out of
# disk space. Un-deletable *.pyc files can also
# interfere with e.g. `test-image-layer`, since that
# test relies on there being just one `create_ops`
# subvolume in `buck-image-out` with the "received UUID"
# that was committed to VCS as part of the test
# sendstream.
'env',
'PYTHONDONTWRITEBYTECODE=1',
layer_opts.yum_from_snapshot,
*sum((['--protected-path', d]
for d in protected_path_set(subvol)), []),
'--install-root',
subvol.path(),
'--',
RPM_ACTION_TYPE_TO_YUM_CMD[action],
# Sort ensures determinism even if `yum` is
# order-dependent
'--assumeyes',
'--',
*sorted((nor.path if isinstance(nor, _LocalRpm
) else nor.encode())
for nor in nors),
])
else:
rpms, bind_ro_args = _rpms_and_bind_ro_args(nors)
_yum_using_build_appliance(
build_appliance=Subvol(
layer_opts.build_appliance,
already_exists=True,
),
nspawn_args=bind_ro_args,
install_root=subvol.path(),
protected_paths=protected_path_set(subvol),
yum_args=[
RPM_ACTION_TYPE_TO_YUM_CMD[action],
'--assumeyes',
# Sort ensures determinism even if `yum` is
# order-dependent
*sorted(rpms),
],
preserve_yum_cache=layer_opts.preserve_yum_cache,
)
def create(self, rel_path: Bytey) -> Subvol:
subvol = Subvol(self._rel_path(rel_path))
subvol.create()
self.subvols.append(subvol)
return subvol
def test_run_as_root_no_cwd(self):
sv = Subvol('/dev/null/no-such-dir')
sv.run_as_root(['true'], _subvol_exists=False)
with self.assertRaisesRegex(AssertionError, 'cwd= is not permitte'):
sv.run_as_root(['true'], _subvol_exists=False, cwd='.')
def test_create_and_snapshot_and_already_exists(self, temp_subvols):
p = temp_subvols.create('parent')
p2 = Subvol(p.path(), already_exists=True)
self.assertEqual(p.path(), p2.path())
temp_subvols.snapshot(p2, 'child')
def snapshot(self, source: Subvol, dest_rel_path: Bytey) -> Subvol:
dest = Subvol(self._rel_path(dest_rel_path))
dest.snapshot(source)
self.subvols.append(dest)
return dest
def caller_will_create(self, rel_path: Bytey) -> Subvol:
subvol = Subvol(self._rel_path(rel_path))
# If the caller fails to create it, our __exit__ is robust enough
# to ignore this subvolume.
self.subvols.append(subvol)
return subvol
def build_image(args):
# We want check the umask since it can affect the result of the
# `os.access` check for `image.install*` items. That said, having a
# umask that denies execute permission to "user" is likely to break this
# code earlier, since new directories wouldn't be traversible. At least
# this check gives a nice error message.
cur_umask = os.umask(0)
os.umask(cur_umask)
assert cur_umask & stat.S_IXUSR == 0, \
f'Refusing to run with pathological umask 0o{cur_umask:o}'
subvol = Subvol(os.path.join(args.subvolumes_dir, args.subvolume_rel_path))
layer_opts = LayerOpts(
layer_target=args.child_layer_target,
yum_from_snapshot=args.yum_from_repo_snapshot,
build_appliance=get_subvolume_path(
args.build_appliance_json,
args.subvolumes_dir,
) if args.build_appliance_json else None,
preserve_yum_cache=args.preserve_yum_cache,
artifacts_may_require_repo=args.artifacts_may_require_repo,
target_to_path=make_target_path_map(args.child_dependencies),
subvolumes_dir=args.subvolumes_dir,
)
# This stack allows build items to hold temporary state on disk.
with ExitStack() as exit_stack:
dep_graph = DependencyGraph(gen_items_for_features(
exit_stack=exit_stack,
features_or_paths=args.child_feature_json,
layer_opts=layer_opts,
),
layer_target=args.child_layer_target)
# Creating all the builders up-front lets phases validate their input
for builder in [
builder_maker(items, layer_opts)
for builder_maker, items in dep_graph.ordered_phases()
]:
builder(subvol)
# We cannot validate or sort `ImageItem`s until the phases are
# materialized since the items may depend on the output of the phases.
for item in dep_graph.gen_dependency_order_items(
PhasesProvideItem(
from_target=args.child_layer_target,
subvol=subvol,
)):
item.build(subvol, layer_opts)
# Build artifacts should never change. Run this BEFORE the exit_stack
# cleanup to enforce that the cleanup does not touch the image.
subvol.set_readonly(True)
try:
return SubvolumeOnDisk.from_subvolume_path(
# Converting to a path here does not seem too risky since this
# class shouldn't have a reason to follow symlinks in the subvol.
subvol.path().decode(),
args.subvolumes_dir,
)
# The complexity of covering this is high, but the only thing that can
# go wrong is a typo in the f-string.
except Exception as ex: # pragma: no cover
raise RuntimeError(f'Serializing subvolume {subvol.path()}') from ex
def builder(subvol: Subvol):
for action, rpms in action_to_rpms.items():
if not rpms:
continue
# Future: `yum-from-snapshot` is actually designed to run
# unprivileged (but we have no nice abstraction for this).
if layer_opts.build_appliance is None:
subvol.run_as_root([
# Since `yum-from-snapshot` variants are generally
# Python binaries built from this very repo, in
# @mode/dev, we would run a symlink-PAR from the
# buck-out tree as `root`. This would leave behind
# root-owned `__pycache__` directories, which would
# break Buck's fragile cleanup, and cause us to leak old
# build artifacts. This eventually runs the host out of
# disk space. Un-deletable *.pyc files can also
# interfere with e.g. `test-image-layer`, since that
# test relies on there being just one `create_ops`
# subvolume in `buck-image-out` with the "received UUID"
# that was committed to VCS as part of the test
# sendstream.
'env',
'PYTHONDONTWRITEBYTECODE=1',
layer_opts.yum_from_snapshot,
*sum((['--protected-path', d]
for d in _protected_path_set(subvol)), []),
'--install-root',
subvol.path(),
'--',
RPM_ACTION_TYPE_TO_YUM_CMD[action],
# Sort ensures determinism even if `yum` is
# order-dependent
'--assumeyes',
'--',
*sorted(rpms),
])
else:
'''
## Future
- implement image feature "manifold_support" with all
those bind-mounts below in mounts = [...]
- add features = ["manifold_support"] to fb_build_appliance
- call nspawn_in_subvol() instead of run_as_root() below
'''
svol = Subvol(
layer_opts.build_appliance,
already_exists=True,
)
mountpoints = mount_item.mountpoints_from_subvol_meta(svol)
bind_mount_args = sum(([
b'--bind-ro=' + svol.path(mp).replace(b':', b'\\:') +
b':' + b'/' + mp.encode()
] for mp in mountpoints), [])
protected_path_args = ' '.join(
sum((['--protected-path', d]
for d in _protected_path_set(subvol)), []))
# Without this, nspawn would look for the host systemd's
# cgroup setup, which breaks us in continuous integration
# containers, which may not have a `systemd` in the host
# container.
subvol.run_as_root([
'env', 'UNIFIED_CGROUP_HIERARCHY=yes',
'systemd-nspawn', '--quiet',
f'--directory={layer_opts.build_appliance}',
'--register=no', '--keep-unit', '--ephemeral',
b'--bind=' + subvol.path().replace(b':', b'\\:') +
b':/mnt', '--bind-ro=/dev/fuse',
'--bind-ro=/etc/fbwhoami', '--bind-ro=/etc/smc.tiers',
'--bind-ro=/var/facebook/rootcanal', *bind_mount_args,
'--capability=CAP_NET_ADMIN', 'sh', '-c',
('mkdir -p /mnt/var/cache/yum; '
'mount --bind /var/cache/yum /mnt/var/cache/yum; '
'/usr/bin/yum-from-fb-snapshot '
f'{protected_path_args}'
' --install-root /mnt -- '
f'{RPM_ACTION_TYPE_TO_YUM_CMD[action]} '
'--assumeyes -- '
f'{" ".join(sorted(rpms))}')
])
def build(self, subvol: Subvol):
subvol.snapshot(Subvol(self.path, already_exists=True))
def provides(self):
parent_subvol = Subvol(self.path, already_exists=True)
protected_paths = _protected_path_set(parent_subvol)
for prot_path in protected_paths:
yield ProvidesDoNotAccess(path=prot_path)
provided_root = False
# We need to traverse the parent image as root, so that we have
# permission to access everything.
for type_and_path in parent_subvol.run_as_root(
[
# -P is the analog of --no-dereference in GNU tools
#
# Filter out the protected paths at traversal time. If one of
# the paths has a very large or very slow mount, traversing it
# would have a devastating effect on build times, so let's avoid
# looking inside protected paths entirely. An alternative would
# be to `send` and to parse the sendstream, but this is ok too.
'find',
'-P',
self.path,
'(',
*itertools.dropwhile(
lambda x: x == '-o', # Drop the initial `-o`
itertools.chain.from_iterable([
# `normpath` removes the trailing / for protected dirs
'-o',
'-path',
os.path.join(self.path, os.path.normpath(p))
] for p in protected_paths),
),
')',
'-prune',
'-o',
'-printf',
'%y %p\\0',
],
stdout=subprocess.PIPE).stdout.split(b'\0'):
if not type_and_path: # after the trailing \0
continue
filetype, abspath = type_and_path.decode().split(' ', 1)
relpath = os.path.relpath(abspath, self.path)
# We already "provided" this path above, and it should have been
# filtered out by `find`.
assert not _is_path_protected(relpath, protected_paths), relpath
# Future: This provides all symlinks as files, while we should
# probably provide symlinks to valid directories inside the
# image as directories to be consistent with SymlinkToDirItem.
if filetype in ['b', 'c', 'p', 'f', 'l', 's']:
yield ProvidesFile(path=relpath)
elif filetype == 'd':
yield ProvidesDirectory(path=relpath)
else: # pragma: no cover
raise AssertionError(f'Unknown {filetype} for {abspath}')
if relpath == '.':
assert filetype == 'd'
provided_root = True
assert provided_root, 'parent layer {} lacks /'.format(self.path)