def ensure_meta_dir_exists(subvol: Subvol, layer_opts: LayerOpts): subvol.run_as_root([ 'mkdir', '--mode=0755', '--parents', subvol.path(META_DIR), ]) # One might ask: why are we serializing this into the image instead # of just putting a condition on `built_artifacts_require_repo` # into our Buck macros? Two reasons: # - In the case of build appliance images, it is possible for a # @mode/dev (in-place) build to use **either** a @mode/dev, or a # @mode/opt (standalone) build appliance. The only way to know # to know if the appliance needs a repo mount is to have a marker # in the image. # - By marking the images, we avoid having to conditionally add # `--bind-repo-ro` flags in a bunch of places in our codebase. The # in-image marker enables `nspawn_in_subvol` to decide. if os.path.exists(subvol.path(META_ARTIFACTS_REQUIRE_REPO)): _validate_artifacts_require_repo(subvol, layer_opts, 'parent layer') # I looked into adding an `allow_overwrite` flag to `serialize`, but # it was too much hassle to do it right. subvol.run_as_root(['rm', subvol.path(META_ARTIFACTS_REQUIRE_REPO)]) procfs_serde.serialize( layer_opts.artifacts_may_require_repo, subvol, META_ARTIFACTS_REQUIRE_REPO, )
def builder(subvol: Subvol): subvol.create() # Guarantee standard / permissions. This could be a setting, # but in practice, probably any other choice would be wrong. subvol.run_as_root(['chmod', '0755', subvol.path()]) subvol.run_as_root(['chown', 'root:root', subvol.path()]) ensure_meta_dir_exists(subvol, layer_opts)
def build(self, subvol: Subvol, layer_opts: LayerOpts): mount_dir = os.path.join(META_MOUNTS_DIR, self.mountpoint, MOUNT_MARKER) for name, data in ( # NB: Not exporting self.mountpoint since it's implicit in the path. ('is_directory', self.is_directory), ('build_source', self.build_source._asdict()), ('runtime_source', json.loads(self.runtime_source)), ): procfs_serde.serialize(data, subvol, os.path.join(mount_dir, name)) source_path = self.build_source.to_path( target_to_path=layer_opts.target_to_path, subvolumes_dir=layer_opts.subvolumes_dir, ) # Support mounting directories and non-directories... This check # follows symlinks for the mount source, which seems correct. is_dir = os.path.isdir(source_path) assert is_dir == self.is_directory, self if is_dir: subvol.run_as_root([ 'mkdir', '--mode=0755', subvol.path(self.mountpoint), ]) else: # Regular files, device nodes, FIFOs, you name it. # `touch` lacks a `--mode` argument, but the mode of this # mountpoint will be shadowed anyway, so let it be whatever. subvol.run_as_root(['touch', subvol.path(self.mountpoint)]) ro_rbind_mount(source_path, subvol, self.mountpoint)
def ro_rbind_mount(src: AnyStr, subvol: Subvol, dest_in_subvol: AnyStr): # Even though `fs_image` currently does not support mount nesting, the # mount must be recursive so that host mounts propagate as expected (we # don't want to have to know if a source host directory contains # sub-mounts). subvol.run_as_root([ 'mount', '-o', 'ro,rbind', src, subvol.path(dest_in_subvol), ]) # Performing mount/unmount operations inside the subvol must not be able # to affect the host system, so the tree must be marked at least # `rslave`. It would be defensible to use `rprivate`, but IMO this is # more surprising than `rslave` in the case of host mounts -- normal # filesystem operations on the host are visible to the container, which # suggests that mount changes must be, also. # # IMPORTANT: Even on fairly recent versions of `util-linux`, merging # this into the first `mount` invocation above does NOT work. Just # leave this ugly 2-call version as is. # # NB: We get slave (not private) propagation since `set_up_volume.sh` # sets propagation to shared on the parent mount `buck-image-out/volume`. subvol.run_as_root(['mount', '--make-rslave', subvol.path(dest_in_subvol)])
def build(self, subvol: Subvol, layer_opts: LayerOpts): dest = subvol.path(self.dest) # The compiler should have detected any collisons, so `--no-clobber` # is just a failsafe. `--no-dereference` is also a failsafe since # we ban symlinks above. # # Opportunistic reflinking & mandatory sparsification are easy # efficiency wins. # # Don't bother preserving metadata since we explicitly set mode & # ownership ... and our build setup lets timestamp float (for now). subvol.run_as_root([ 'cp', '--recursive', '--no-clobber', '--no-dereference', '--reflink=auto', '--sparse=always', '--no-preserve=all', self.source, dest, ]) build_stat_options(self, subvol, dest, do_not_set_mode=True) # Group by mode to make as few shell calls as possible. for mode_str, modes_and_paths in itertools.groupby( sorted((mode_to_str(i.mode), i.provides.path) for i in self.paths), lambda x: x[0]): # `chmod` follows symlinks, and there's no option to stop it. # However, `customize_fields` should have failed on symlinks. subvol.run_as_root([ 'chmod', mode_str, *(subvol.path(p) for _, p in modes_and_paths) ])
def build(self, subvol: Subvol, layer_opts: LayerOpts): outer_dir = self.path_to_make.split('/', 1)[0] inner_dir = subvol.path(os.path.join(self.into_dir, self.path_to_make)) subvol.run_as_root(['mkdir', '-p', inner_dir]) build_stat_options( self, subvol, subvol.path(os.path.join(self.into_dir, outer_dir)), )
def build(self, subvol: Subvol, layer_opts: LayerOpts): # The compiler should have caught this, this is just paranoia. if self.pre_existing_dest: subvol.run_as_root(["test", "-d", subvol.path(self.dest)]) if self.omit_outer_dir: # Like `ls`, but NUL-separated. Needs `root` since the repo # user may not be able to access the source subvol. sources = [ self.source / p for p in subvol.run_as_root( [ 'find', self.source, '-mindepth', '1', '-maxdepth', '1', '-printf', '%f\\0', ], stdout=subprocess.PIPE).stdout.strip(b'\0').split(b'\0') ] else: sources = [self.source] # Option rationales: # - The compiler should have detected any collisons on the # destination, so `--no-clobber` is just a failsafe. # - `--no-dereference` is needed since our contract is to copy # each symlink's destination text verbatim. Not doing this # would also risk following absolute symlinks, reaching OUTSIDE # of the source subvolume! # - `--reflink=always` aids efficiency and, more importantly, # preserves "cloned extent" relationships that existed within # the source subtree. # - `--sparse=auto` is implied by `--reflink=always`. The two # together ought to preserve the original sparseness layout, # - `--preserve=all` keeps as much original metadata as possible, # including hardlinks. subvol.run_as_root([ 'cp', '--recursive', '--no-clobber', '--no-dereference', '--reflink=always', '--sparse=auto', '--preserve=all', *sources, subvol.path(self.dest), ])
def gen_subvolume_subtree_provides(subvol: Subvol, subtree: Path): 'Yields "Provides" instances for a path `subtree` in `subvol`.' # "Provides" classes use image-absolute paths that are `str` (for now). # Accept any string type to ease future migrations. subtree = os.path.join('/', Path(subtree).decode()) protected_paths = protected_path_set(subvol) for prot_path in protected_paths: rel_to_subtree = os.path.relpath(os.path.join('/', prot_path), subtree) if not has_leading_dot_dot(rel_to_subtree): yield ProvidesDoNotAccess(path=rel_to_subtree) subtree_full_path = subvol.path(subtree).decode() subtree_exists = False # Traverse the subvolume as root, so that we have permission to access # everything. for type_and_path in subvol.run_as_root([ # -P is the analog of --no-dereference in GNU tools # # Filter out the protected paths at traversal time. If one of the # paths has a very large or very slow mount, traversing it would # have a devastating effect on build times, so let's avoid looking # inside protected paths entirely. An alternative would be to # `send` and to parse the sendstream, but this is ok too. 'find', '-P', subtree_full_path, '(', *itertools.dropwhile( lambda x: x == '-o', # Drop the initial `-o` itertools.chain.from_iterable([ # `normpath` removes the trailing / for protected dirs '-o', '-path', subvol.path(os.path.normpath(p)) ] for p in protected_paths), ), ')', '-prune', '-o', '-printf', '%y %p\\0', ], stdout=subprocess.PIPE).stdout.split(b'\0'): if not type_and_path: # after the trailing \0 continue filetype, abspath = type_and_path.decode().split(' ', 1) relpath = os.path.relpath(abspath, subtree_full_path) assert not has_leading_dot_dot(relpath), (abspath, subtree_full_path) # We already "provided" this path above, and it should have been # filtered out by `find`. assert not is_path_protected(relpath, protected_paths), relpath # Future: This provides all symlinks as files, while we should # probably provide symlinks to valid directories inside the image as # directories to be consistent with SymlinkToDirItem. if filetype in ['b', 'c', 'p', 'f', 'l', 's']: yield ProvidesFile(path=relpath) elif filetype == 'd': yield ProvidesDirectory(path=relpath) else: # pragma: no cover raise AssertionError(f'Unknown {filetype} for {abspath}') if relpath == '.': subtree_exists = True # We should've gotten a CalledProcessError from `find`. assert subtree_exists, f'{subtree} does not exist in {subvol.path()}'
def build(self, subvol: Subvol, layer_opts: LayerOpts): if layer_opts.build_appliance: work_dir = generate_work_dir() full_path = Path(work_dir) / self.into_dir / self.path_to_make opts = new_nspawn_opts( cmd=['mkdir', '-p', full_path], layer=layer_opts.build_appliance, bindmount_rw=[(subvol.path(), work_dir)], user=pwd.getpwnam('root'), ) run_non_booted_nspawn(opts, PopenArgs()) else: inner_dir = subvol.path( os.path.join(self.into_dir, self.path_to_make)) subvol.run_as_root(['mkdir', '-p', inner_dir]) outer_dir = self.path_to_make.split('/', 1)[0] build_stat_options( self, subvol, subvol.path(os.path.join(self.into_dir, outer_dir)), )
def build(self, subvol: Subvol, layer_opts: LayerOpts): dest = subvol.path(self.dest) # Best-practice would tell us to do `subvol.path(self.source)`. # However, this will trigger the paranoid check in the `path()` # implementation if any component of `source` inside the image is an # absolute symlink. We are not writing to `source`, so that # safeguard isn't useful here. # # We DO check below that the relative symlink we made does not point # outside the image. However, a non-chrooted process resolving our # well-formed relative link might still traverse pre-existing # absolute symlinks on the filesystem, and go outside of the image # root. abs_source = subvol.path() / self.source # Make all symlinks relative because this makes it easy to inspect # the subvolums from outside the container. We can add an # `absolute` option if needed. rel_source = os.path.relpath(abs_source, dest.dirname()) assert os.path.normpath(dest / rel_source).startswith(subvol.path()), \ '{self}: A symlink to {rel_source} would point outside the image' if layer_opts.build_appliance: build_appliance = layer_opts.build_appliance work_dir = generate_work_dir() rel_dest = work_dir + '/' + self.dest opts = new_nspawn_opts( cmd=[ 'ln', '--symbolic', '--no-dereference', rel_source, rel_dest, ], layer=build_appliance, bindmount_rw=[(subvol.path(), work_dir)], user=pwd.getpwnam('root'), ) run_non_booted_nspawn(opts, PopenArgs()) else: subvol.run_as_root( ['ln', '--symbolic', '--no-dereference', rel_source, dest] )
def builder(subvol: Subvol): protected_paths = protected_path_set(subvol) # Reverse-lexicographic order deletes inner paths before # deleting the outer paths, thus minimizing conflicts between # `remove_paths` items. for item in sorted( items, reverse=True, key=lambda i: i.__sort_key(), ): if is_path_protected(item.path, protected_paths): # For META_DIR, this is never reached because of # make_path_normal_relative's check, but for other # protected paths, this is required. raise AssertionError( f'Cannot remove protected {item}: {protected_paths}') # This ensures that there are no symlinks in item.path that # might take us outside of the subvolume. Since recursive # `rm` does not follow symlinks, it is OK if the inode at # `item.path` is a symlink (or one of its sub-paths). path = subvol.path(item.path, no_dereference_leaf=True) if not os.path.lexists(path): if item.action == RemovePathAction.assert_exists: raise AssertionError(f'Path does not exist: {item}') elif item.action == RemovePathAction.if_exists: continue else: # pragma: no cover raise AssertionError(f'Unknown {item.action}') subvol.run_as_root([ 'rm', '-r', # This prevents us from making removes outside of the # per-repo loopback, which is an important safeguard. # It does not stop us from reaching into other subvols, # but since those have random IDs in the path, this is # nearly impossible to do by accident. '--one-file-system', path, ]) pass