def ensure_meta_dir_exists(subvol: Subvol, layer_opts: LayerOpts):
subvol.run_as_root([
'mkdir',
'--mode=0755',
'--parents',
subvol.path(META_DIR),
])
# One might ask: why are we serializing this into the image instead
# of just putting a condition on `built_artifacts_require_repo`
# into our Buck macros? Two reasons:
# - In the case of build appliance images, it is possible for a
# @mode/dev (in-place) build to use **either** a @mode/dev, or a
# @mode/opt (standalone) build appliance. The only way to know
# to know if the appliance needs a repo mount is to have a marker
# in the image.
# - By marking the images, we avoid having to conditionally add
# `--bind-repo-ro` flags in a bunch of places in our codebase. The
# in-image marker enables `nspawn_in_subvol` to decide.
if os.path.exists(subvol.path(META_ARTIFACTS_REQUIRE_REPO)):
_validate_artifacts_require_repo(subvol, layer_opts, 'parent layer')
# I looked into adding an `allow_overwrite` flag to `serialize`, but
# it was too much hassle to do it right.
subvol.run_as_root(['rm', subvol.path(META_ARTIFACTS_REQUIRE_REPO)])
procfs_serde.serialize(
layer_opts.artifacts_may_require_repo,
subvol,
META_ARTIFACTS_REQUIRE_REPO,
)
def builder(subvol: Subvol):
subvol.create()
# Guarantee standard / permissions. This could be a setting,
# but in practice, probably any other choice would be wrong.
subvol.run_as_root(['chmod', '0755', subvol.path()])
subvol.run_as_root(['chown', 'root:root', subvol.path()])
ensure_meta_dir_exists(subvol, layer_opts)
def build(self, subvol: Subvol, layer_opts: LayerOpts):
mount_dir = os.path.join(META_MOUNTS_DIR, self.mountpoint, MOUNT_MARKER)
for name, data in (
# NB: Not exporting self.mountpoint since it's implicit in the path.
('is_directory', self.is_directory),
('build_source', self.build_source._asdict()),
('runtime_source', json.loads(self.runtime_source)),
):
procfs_serde.serialize(data, subvol, os.path.join(mount_dir, name))
source_path = self.build_source.to_path(
target_to_path=layer_opts.target_to_path,
subvolumes_dir=layer_opts.subvolumes_dir,
)
# Support mounting directories and non-directories... This check
# follows symlinks for the mount source, which seems correct.
is_dir = os.path.isdir(source_path)
assert is_dir == self.is_directory, self
if is_dir:
subvol.run_as_root([
'mkdir', '--mode=0755', subvol.path(self.mountpoint),
])
else: # Regular files, device nodes, FIFOs, you name it.
# `touch` lacks a `--mode` argument, but the mode of this
# mountpoint will be shadowed anyway, so let it be whatever.
subvol.run_as_root(['touch', subvol.path(self.mountpoint)])
ro_rbind_mount(source_path, subvol, self.mountpoint)
def ro_rbind_mount(src: AnyStr, subvol: Subvol, dest_in_subvol: AnyStr):
# Even though `fs_image` currently does not support mount nesting, the
# mount must be recursive so that host mounts propagate as expected (we
# don't want to have to know if a source host directory contains
# sub-mounts).
subvol.run_as_root([
'mount',
'-o',
'ro,rbind',
src,
subvol.path(dest_in_subvol),
])
# Performing mount/unmount operations inside the subvol must not be able
# to affect the host system, so the tree must be marked at least
# `rslave`. It would be defensible to use `rprivate`, but IMO this is
# more surprising than `rslave` in the case of host mounts -- normal
# filesystem operations on the host are visible to the container, which
# suggests that mount changes must be, also.
#
# IMPORTANT: Even on fairly recent versions of `util-linux`, merging
# this into the first `mount` invocation above does NOT work. Just
# leave this ugly 2-call version as is.
#
# NB: We get slave (not private) propagation since `set_up_volume.sh`
# sets propagation to shared on the parent mount `buck-image-out/volume`.
subvol.run_as_root(['mount', '--make-rslave', subvol.path(dest_in_subvol)])
def build(self, subvol: Subvol, layer_opts: LayerOpts):
dest = subvol.path(self.dest)
# The compiler should have detected any collisons, so `--no-clobber`
# is just a failsafe. `--no-dereference` is also a failsafe since
# we ban symlinks above.
#
# Opportunistic reflinking & mandatory sparsification are easy
# efficiency wins.
#
# Don't bother preserving metadata since we explicitly set mode &
# ownership ... and our build setup lets timestamp float (for now).
subvol.run_as_root([
'cp',
'--recursive',
'--no-clobber',
'--no-dereference',
'--reflink=auto',
'--sparse=always',
'--no-preserve=all',
self.source,
dest,
])
build_stat_options(self, subvol, dest, do_not_set_mode=True)
# Group by mode to make as few shell calls as possible.
for mode_str, modes_and_paths in itertools.groupby(
sorted((mode_to_str(i.mode), i.provides.path)
for i in self.paths), lambda x: x[0]):
# `chmod` follows symlinks, and there's no option to stop it.
# However, `customize_fields` should have failed on symlinks.
subvol.run_as_root([
'chmod', mode_str,
*(subvol.path(p) for _, p in modes_and_paths)
])
def build(self, subvol: Subvol, layer_opts: LayerOpts):
outer_dir = self.path_to_make.split('/', 1)[0]
inner_dir = subvol.path(os.path.join(self.into_dir, self.path_to_make))
subvol.run_as_root(['mkdir', '-p', inner_dir])
build_stat_options(
self,
subvol,
subvol.path(os.path.join(self.into_dir, outer_dir)),
)
def build(self, subvol: Subvol, layer_opts: LayerOpts):
# The compiler should have caught this, this is just paranoia.
if self.pre_existing_dest:
subvol.run_as_root(["test", "-d", subvol.path(self.dest)])
if self.omit_outer_dir:
# Like `ls`, but NUL-separated. Needs `root` since the repo
# user may not be able to access the source subvol.
sources = [
self.source / p for p in subvol.run_as_root(
[
'find',
self.source,
'-mindepth',
'1',
'-maxdepth',
'1',
'-printf',
'%f\\0',
],
stdout=subprocess.PIPE).stdout.strip(b'\0').split(b'\0')
]
else:
sources = [self.source]
# Option rationales:
# - The compiler should have detected any collisons on the
# destination, so `--no-clobber` is just a failsafe.
# - `--no-dereference` is needed since our contract is to copy
# each symlink's destination text verbatim. Not doing this
# would also risk following absolute symlinks, reaching OUTSIDE
# of the source subvolume!
# - `--reflink=always` aids efficiency and, more importantly,
# preserves "cloned extent" relationships that existed within
# the source subtree.
# - `--sparse=auto` is implied by `--reflink=always`. The two
# together ought to preserve the original sparseness layout,
# - `--preserve=all` keeps as much original metadata as possible,
# including hardlinks.
subvol.run_as_root([
'cp',
'--recursive',
'--no-clobber',
'--no-dereference',
'--reflink=always',
'--sparse=auto',
'--preserve=all',
*sources,
subvol.path(self.dest),
])
def gen_subvolume_subtree_provides(subvol: Subvol, subtree: Path):
'Yields "Provides" instances for a path `subtree` in `subvol`.'
# "Provides" classes use image-absolute paths that are `str` (for now).
# Accept any string type to ease future migrations.
subtree = os.path.join('/', Path(subtree).decode())
protected_paths = protected_path_set(subvol)
for prot_path in protected_paths:
rel_to_subtree = os.path.relpath(os.path.join('/', prot_path), subtree)
if not has_leading_dot_dot(rel_to_subtree):
yield ProvidesDoNotAccess(path=rel_to_subtree)
subtree_full_path = subvol.path(subtree).decode()
subtree_exists = False
# Traverse the subvolume as root, so that we have permission to access
# everything.
for type_and_path in subvol.run_as_root([
# -P is the analog of --no-dereference in GNU tools
#
# Filter out the protected paths at traversal time. If one of the
# paths has a very large or very slow mount, traversing it would
# have a devastating effect on build times, so let's avoid looking
# inside protected paths entirely. An alternative would be to
# `send` and to parse the sendstream, but this is ok too.
'find', '-P', subtree_full_path, '(', *itertools.dropwhile(
lambda x: x == '-o', # Drop the initial `-o`
itertools.chain.from_iterable([
# `normpath` removes the trailing / for protected dirs
'-o', '-path', subvol.path(os.path.normpath(p))
] for p in protected_paths),
), ')', '-prune', '-o', '-printf', '%y %p\\0',
], stdout=subprocess.PIPE).stdout.split(b'\0'):
if not type_and_path: # after the trailing \0
continue
filetype, abspath = type_and_path.decode().split(' ', 1)
relpath = os.path.relpath(abspath, subtree_full_path)
assert not has_leading_dot_dot(relpath), (abspath, subtree_full_path)
# We already "provided" this path above, and it should have been
# filtered out by `find`.
assert not is_path_protected(relpath, protected_paths), relpath
# Future: This provides all symlinks as files, while we should
# probably provide symlinks to valid directories inside the image as
# directories to be consistent with SymlinkToDirItem.
if filetype in ['b', 'c', 'p', 'f', 'l', 's']:
yield ProvidesFile(path=relpath)
elif filetype == 'd':
yield ProvidesDirectory(path=relpath)
else: # pragma: no cover
raise AssertionError(f'Unknown {filetype} for {abspath}')
if relpath == '.':
subtree_exists = True
# We should've gotten a CalledProcessError from `find`.
assert subtree_exists, f'{subtree} does not exist in {subvol.path()}'
def build(self, subvol: Subvol, layer_opts: LayerOpts):
if layer_opts.build_appliance:
work_dir = generate_work_dir()
full_path = Path(work_dir) / self.into_dir / self.path_to_make
opts = new_nspawn_opts(
cmd=['mkdir', '-p', full_path],
layer=layer_opts.build_appliance,
bindmount_rw=[(subvol.path(), work_dir)],
user=pwd.getpwnam('root'),
)
run_non_booted_nspawn(opts, PopenArgs())
else:
inner_dir = subvol.path(
os.path.join(self.into_dir, self.path_to_make))
subvol.run_as_root(['mkdir', '-p', inner_dir])
outer_dir = self.path_to_make.split('/', 1)[0]
build_stat_options(
self, subvol, subvol.path(os.path.join(self.into_dir, outer_dir)),
)
def build(self, subvol: Subvol, layer_opts: LayerOpts):
dest = subvol.path(self.dest)
# Best-practice would tell us to do `subvol.path(self.source)`.
# However, this will trigger the paranoid check in the `path()`
# implementation if any component of `source` inside the image is an
# absolute symlink. We are not writing to `source`, so that
# safeguard isn't useful here.
#
# We DO check below that the relative symlink we made does not point
# outside the image. However, a non-chrooted process resolving our
# well-formed relative link might still traverse pre-existing
# absolute symlinks on the filesystem, and go outside of the image
# root.
abs_source = subvol.path() / self.source
# Make all symlinks relative because this makes it easy to inspect
# the subvolums from outside the container. We can add an
# `absolute` option if needed.
rel_source = os.path.relpath(abs_source, dest.dirname())
assert os.path.normpath(dest / rel_source).startswith(subvol.path()), \
'{self}: A symlink to {rel_source} would point outside the image'
if layer_opts.build_appliance:
build_appliance = layer_opts.build_appliance
work_dir = generate_work_dir()
rel_dest = work_dir + '/' + self.dest
opts = new_nspawn_opts(
cmd=[
'ln',
'--symbolic',
'--no-dereference',
rel_source,
rel_dest,
],
layer=build_appliance,
bindmount_rw=[(subvol.path(), work_dir)],
user=pwd.getpwnam('root'),
)
run_non_booted_nspawn(opts, PopenArgs())
else:
subvol.run_as_root(
['ln', '--symbolic', '--no-dereference', rel_source, dest]
)
def builder(subvol: Subvol):
protected_paths = protected_path_set(subvol)
# Reverse-lexicographic order deletes inner paths before
# deleting the outer paths, thus minimizing conflicts between
# `remove_paths` items.
for item in sorted(
items,
reverse=True,
key=lambda i: i.__sort_key(),
):
if is_path_protected(item.path, protected_paths):
# For META_DIR, this is never reached because of
# make_path_normal_relative's check, but for other
# protected paths, this is required.
raise AssertionError(
f'Cannot remove protected {item}: {protected_paths}')
# This ensures that there are no symlinks in item.path that
# might take us outside of the subvolume. Since recursive
# `rm` does not follow symlinks, it is OK if the inode at
# `item.path` is a symlink (or one of its sub-paths).
path = subvol.path(item.path, no_dereference_leaf=True)
if not os.path.lexists(path):
if item.action == RemovePathAction.assert_exists:
raise AssertionError(f'Path does not exist: {item}')
elif item.action == RemovePathAction.if_exists:
continue
else: # pragma: no cover
raise AssertionError(f'Unknown {item.action}')
subvol.run_as_root([
'rm',
'-r',
# This prevents us from making removes outside of the
# per-repo loopback, which is an important safeguard.
# It does not stop us from reaching into other subvols,
# but since those have random IDs in the path, this is
# nearly impossible to do by accident.
'--one-file-system',
path,
])
pass
