def _create_sockets_inside_netns(
target_pid: int, num_socks: int,
) -> List[socket.socket]:
'''
Creates TCP stream socket inside the container.
Returns the socket.socket() object.
'''
with listen_temporary_unix_socket() as (
unix_sock_path, list_sock
), subprocess.Popen([
# NB: /usr/local/fbcode/bin must come first because /bin/python3
# may be very outdated
'sudo', 'env', 'PATH=/usr/local/fbcode/bin:/bin',
'nsenter', '--net', '--target', str(target_pid),
# NB: We pass our listening socket as FD 1 to avoid dealing with
# the `sudo` option of `-C`. Nothing here writes to `stdout`:
*_make_sockets_and_send_via(unix_sock_fd=1, num_socks=num_socks),
], stdout=list_sock.fileno()) as sock_proc:
repo_server_socks = [
socket.socket(fileno=fd)
for fd in recv_fds_from_unix_sock(unix_sock_path, num_socks)
]
assert len(repo_server_socks) == num_socks, len(repo_server_socks)
check_popen_returncode(sock_proc)
return repo_server_socks
def popen_as_root(
self,
args,
*,
_subvol_exists=True,
stdout=None,
check=True,
**kwargs,
):
if 'cwd' in kwargs:
raise AssertionError(
'cwd= is not permitted as an argument to run_as_root, '
'because that makes it too easy to accidentally traverse '
'a symlink from inside the container and touch the host '
'filesystem. Best practice: wrap your path with '
'Subvol.path() as close as possible to its site of use.')
if 'pass_fds' in kwargs:
# Future: if you add support for this, see the note on how to
# improve `receive`, too.
#
# Why doesn't `pass_fds` just work? `sudo` closes all FDs in a
# (likely misguided) attempt to improve security. `sudo -C` can
# help here, but it's disabled by default.
raise NotImplementedError( # pragma: no cover
'But there is a straightforward fix -- you would need to '
'move the usage of our FD-passing wrapper from '
'nspawn_in_subvol.py to this function.')
if _subvol_exists != self._exists:
raise AssertionError(
f'{self.path()} exists is {self._exists}, not {_subvol_exists}'
)
# Ban our subcommands from writing to stdout, since many of our
# tools (e.g. make-demo-sendstream, compiler) write structured
# data to stdout to be usable in pipelines.
if stdout is None:
stdout = 2
# The '--' is to avoid `args` from accidentally being parsed as
# environment variables or `sudo` options.
with subprocess.Popen(
# Clobber any pre-existing `TMP` because in the context of Buck,
# this is often set to something inside the repo's `buck-out`
# (as documented in https://buck.build/rule/genrule.html).
# Using the in-repo temporary directory causes a variety of
# issues, including (i) `yum` leaking root-owned files into
# `buck-out`, breaking `buck clean`, and (ii) `systemd-nspawn`
# bugging out with "Failed to create inaccessible file node"
# when we use `--bind-repo-ro`.
['sudo', 'TMP=', '--', *args],
stdout=stdout,
**kwargs,
) as pr:
yield pr
if check:
check_popen_returncode(pr)
def reader(self, sid: str) -> ContextManager[StorageInput]:
# We currently waste significant time per read waiting for CLIs
# to start, which is terrible for small reads (most system
# RPMs are small).
path = self._path_for_storage_id(self.strip_key(sid))
log_prefix = f'{self.__class__.__name__}'
with subprocess.Popen(self._read_cmd(
path=path,
), env=self._configured_env(), stdout=subprocess.PIPE) as proc:
log.debug(f'{log_prefix} - Started {path} GET proc')
yield StorageInput(input=proc.stdout)
log.debug(f'{log_prefix} - Waiting for {path} GET')
log.debug(f'{log_prefix} - Exit code {proc.returncode} from {path} GET')
# No `finally`: this doesn't need to run if the context block raises.
check_popen_returncode(proc)
def remover(self) -> ContextManager[_StorageRemover]:
rm = _StorageRemover(storage=self, procs=[])
try:
yield rm
finally:
last_ex = None # We'll re-raise the last exception.
for proc in rm.procs:
# Ensure we wait for each process, no matter what.
try:
assert proc.returncode is None # Not yet waited for
proc.wait()
# Unit-testing this error-within-error case is hard, but all
# it would verify is that we properly re-raise `ex`. I
# tested this by hand in an interpreter, see P60127851.
except Exception as ex: # pragma: no cover
last_ex = ex
# Raise the **last** of the "wait()" exceptions.
if last_ex is not None: # pragma: no cover
raise last_ex
# Check return codes after all processes have been waited for to
# avoid creating zombies in the event that the caller catches.
for proc in rm.procs:
check_popen_returncode(proc)
def get_id_and_release_resources():
# Wait for `cli` to exit cleanly to make sure the
# `sid` is available to read after the `yield`.
try:
proc.stdin.close()
log.debug(f'{log_prefix} - Wait for {path} PUT')
proc.wait()
log.debug(
f'{log_prefix} - Exit code {proc.returncode}'
f' from {path} PUT'
)
check_popen_returncode(proc)
# Clean up even on KeyboardInterrupt -- we cannot assume
# that the blob was stored unless `cli` exited cleanly.
#
# The reason we need this clunky `except` is that we never
# pass the sid to `_CommitCallback`, so it can't clean up.
except BaseException:
try:
# Daemonize the cleanup: do NOT wait, do not check
# the return code. Future: Following the idea in
# remove(), we could plop this cleanup on the
# innermost remover.
subprocess.run(['setsid'] + self._remove_cmd(
path=self._path_for_storage_id(sid),
), env=self._configured_env(), stdout=2)
# To cover this, I'd need `setsid` or `cli` not to
# exist, neither is a useful test. The validity of the
# f-string is ensured by `flake8`.
except Exception: # pragma: no cover
# Log & ignore: we'll re-raise the original exception
log.exception(
f'{log_prefix} - While cleaning up partial {sid}'
)
raise
yield sid
def yum_from_snapshot(
*,
storage_cfg: str,
snapshot_dir: Path,
install_root: Path,
protected_paths: List[str],
yum_args: List[str],
):
# The paths that have trailing slashes are directories, others are
# files. There's a separate code path for protecting some files above.
# The rationale is that those files are not guaranteed to exist.
protected_paths.extend([
'/var/log/yum.log', # Created above if it doesn't exist
# See the `_isolate_yum_and_wait_until_ready` docblock for how (and
# why) this list was produced. All are assumed to exist on the host
# -- otherwise, we'd be in the awkard situation of either leaving
# them unprotected, or creating them on the host to protect them.
'/etc/yum.repos.d/',
'/etc/yum/',
'/var/cache/yum/',
'/var/lib/yum/',
'/etc/pki/rpm-gpg/',
'/etc/rpm/',
'/var/lib/rpm/',
# Harcode `IMAGE/meta` because it should ALWAYS be off-limits --
# even though the compiler will redundantly tell us to protect it.
'meta/',
])
# These user-specified arguments could really mess up hermeticity.
for bad_arg in ['--installroot', '--config', '--setopt', '--downloaddir']:
for arg in yum_args:
assert arg != '-c'
assert not arg.startswith(bad_arg), f'{arg} is prohibited'
with _temp_fifo() as netns_fifo, _temp_fifo(
# Lets the child wait for yum_conf to be ready. This could
# be done via an `flock` on `yum_conf.name`, but that's not
# robust on some network filesystems, so let's use a pipe.
) as ready_fifo, \
tempfile.NamedTemporaryFile('w', suffix='yum') as out_yum_conf, \
_dummy_dev() as dummy_dev, \
_dummies_for_protected_paths(
protected_paths,
) as protected_path_to_dummy, \
subprocess.Popen([
'sudo',
# Cannot do --pid or --cgroup without extra work (nspawn).
# Note that `--mount` implies `mount --make-rprivate /` for
# all recent `util-linux` releases (since 2.27 circa 2015).
'unshare', '--mount', '--uts', '--ipc', '--net',
*_isolate_yum_and_wait_until_ready(
install_root, dummy_dev, protected_path_to_dummy,
netns_fifo, ready_fifo,
),
'yum-from-snapshot', # argv[0]
'yum',
# Most `yum` options are isolated by our `YumConfIsolator`.
'--config', out_yum_conf.name,
# NB: We omit `--downloaddir` because the default behavior
# is to put any downloaded RPMs in `$installroot/$cachedir`,
# which is reasonable, and easy to clean up in a post-pass.
*yum_args,
]) as yum_proc, \
open(
# ORDER IS IMPORTANT: In case of error, this must be closed
# before `proc.__exit__` calls `wait`, or we'll deadlock.
ready_fifo, 'w'
) as ready_out:
# To start the repo server we must obtain a socket that belongs to
# the network namespace of the `yum` container, and we must bring up
# the loopback device to later bind to it. Since this parent
# process has low privileges, we do this via a `sudo` helper.
with open(netns_fifo, 'r') as netns_in:
netns_path = netns_in.read()
with listen_temporary_unix_socket(
) as (unix_sock_path, list_sock), subprocess.Popen(
[
# NB: /usr/local/fbcode/bin must come first because /bin/python3
# may be very outdated
'sudo',
'env',
'PATH=/usr/local/fbcode/bin:/bin',
'nsenter',
'--net=' + netns_path,
# NB: We pass our listening socket as FD 1 to avoid dealing with
# the `sudo` option of `-C`. Nothing here writes to `stdout`:
*_make_socket_and_send_via(unix_sock_fd=1),
],
stdout=list_sock.fileno()) as sock_proc:
repo_server_sock_fd, = recv_fds_from_unix_sock(unix_sock_path, 1)
repo_server_sock = socket.socket(fileno=repo_server_sock_fd)
check_popen_returncode(sock_proc)
# Binds the socket to the loopback inside yum's netns
repo_server_sock.bind(('127.0.0.1', 0))
host, port = repo_server_sock.getsockname()
log.info(f'Bound {netns_path} socket to {host}:{port}')
# The server takes ownership of the socket, so we don't enter it here.
with _repo_server(
repo_server_sock, storage_cfg, snapshot_dir
) as server_proc, \
open(snapshot_dir / 'yum.conf') as in_yum_conf, \
_prepare_isolated_yum_conf(
in_yum_conf, out_yum_conf, install_root, host, port
):
log.info('Waiting for repo server to listen')
while server_proc.poll() is None:
if repo_server_sock.getsockopt(
socket.SOL_SOCKET,
socket.SO_ACCEPTCONN,
):
break
time.sleep(0.1)
log.info('Ready to run yum')
ready_out.write('ready') # `yum` can run now.
ready_out.close() # Proceed past the inner `read`.
# Wait **before** we tear down all the `yum.conf` isolation.
yum_proc.wait()
check_popen_returncode(yum_proc)
def yum_dnf_from_snapshot(
*,
yum_dnf: YumDnf,
snapshot_dir: Path,
protected_paths: List[str],
yum_dnf_args: List[str],
debug: bool = False,
):
_ensure_fs_image_container()
_ensure_private_network()
prog_name = yum_dnf.value
# This path convention must match how `write_yum_dnf_conf.py` and
# `rpm_repo_snapshot.bzl` set up their output.
conf_path = snapshot_dir / f'{prog_name}/etc/{prog_name}/{prog_name}.conf'
install_root = _install_root(conf_path, yum_dnf_args)
# The paths that have trailing slashes are directories, others are
# files. There's a separate code path for protecting some files above.
# The rationale is that those files are not guaranteed to exist.
protected_paths.extend([
# See the `_isolate_yum_dnf` docblock for how (and why) this list
# was produced. All are assumed to exist on the host -- otherwise,
# we'd be in the awkard situation of leaving them unprotected, or
# creating them on the host to protect them.
'/etc/yum.repos.d/', # dnf ALSO needs this isolated
f'/etc/{prog_name}/', # A duplicate for the `yum` case
'/etc/pki/rpm-gpg/',
'/etc/rpm/',
# Harcode `IMAGE/meta` because it should ALWAYS be off-limits --
# even though the compiler will redundantly tell us to protect it.
'meta/',
] + (
# On Fedora, `yum` is just a symlink to `dnf`, so `/etc/yum` is missing
['/etc/yum/'] if (has_yum() and not yum_is_dnf()) else []))
# Only isolate the host DBs and log if we are NOT installing to /.
install_to_fs_root = os.path.realpath(install_root) == b'/'
if not install_to_fs_root:
# Ensure the log exists, so we can guarantee we don't write to the host.
log_path = f'/var/log/{prog_name}.log'
subprocess.check_call(['sudo', 'touch', log_path])
protected_paths.extend([
log_path,
f'/var/lib/{prog_name}/',
'/var/lib/rpm/',
# Future: We should isolate the cache even when installing to /
# because the snapshot should have its own cache, and should not
# pollute the OS cache. However, right now our cache handling
# is pretty broken, so this change is deferred.
f'/var/cache/{prog_name}/',
])
for arg in yum_dnf_args:
assert arg != '-c' and not arg.startswith('--config'), \
f'If you change --config, you will no longer use the repo snapshot'
with _dummy_dev() as dummy_dev, \
_dummies_for_protected_paths(
protected_paths,
) as protected_path_to_dummy, \
subprocess.Popen([
'sudo',
# We need `--mount` so as not to leak our `--protect-path`
# bind mounts outside of the package manager invocation.
#
# Note that `--mount` implies `mount --make-rprivate /` for
# all recent `util-linux` releases (since 2.27 circa 2015).
#
# We omit `--net` because `yum-dnf-from-snapshot` should
# only be running in a private-network `nspawn_in_subvol` at
# this point, and `inject_repo_server.py` servers listen on
# sockets that are outside of this `unshare` (the latter
# could be changed but requires laboriously punching through
# some abstraction boundaries).
#
# `--uts` and `--ipc` are set just because they're free to
# namespace. We couldn't do `--pid` or `--cgroup` without
# significant extra work, which has no clear value (i.e.
# we'd effectively need to use `systemd-nspawn` here).
'unshare', '--mount', '--uts', '--ipc',
*_isolate_yum_dnf(
yum_dnf, install_root, dummy_dev, protected_path_to_dummy,
),
'yum-dnf-from-snapshot', # argv[0]
prog_name,
# Help us debug CI issues that don't reproduce locally.
'--verbose',
# Config options get isolated by our `YumDnfConfIsolator`
# when `write-yum-dnf-conf` builds this file. Note that
# `yum` doesn't work if the config path is relative.
f'--config={Path(os.path.abspath(conf_path))}',
f'--installroot={install_root}',
# NB: We omit `--downloaddir` because the default behavior
# is to put any downloaded RPMs in `$installroot/$cachedir`,
# which is reasonable, and easy to clean up in a post-pass.
*yum_dnf_args,
]) as yum_dnf_proc:
# Wait **before** we tear down all the `yum` / `dnf` isolation.
yum_dnf_proc.wait()
check_popen_returncode(yum_dnf_proc)
def package_full(self, subvol: Subvol, output_path: str, opts: _Opts):
with create_ro(output_path, 'wb') as outfile, subprocess.Popen(
['zstd', '--stdout'], stdin=subprocess.PIPE, stdout=outfile
) as zst, subvol.mark_readonly_and_write_sendstream_to_file(zst.stdin):
pass
check_popen_returncode(zst)