def test_cli(self):
with temp_dir() as td:
p = b'Hello, world!' # Write ~1.67 chunks of this phrase
f_in = BytesIO(p * int(cli._CHUNK_SIZE * 5 / (3 * len(p))))
f_sid = BytesIO()
cli.main([
'--storage',
Path.json_dumps({
'kind': 'filesystem',
'key': 'test',
'base_dir': td / 'storage',
}), 'put'
],
from_file=f_in,
to_file=f_sid)
self.assertTrue(f_sid.getvalue().endswith(b'\n'))
sid = f_sid.getvalue()[:-1]
f_out = BytesIO()
cli.main([
'--storage',
Path.json_dumps({
'kind': 'filesystem',
'key': 'test',
'base_dir': td / 'storage',
}), 'get', sid
],
from_file=None,
to_file=f_out)
self.assertEqual(f_in.getvalue(), f_out.getvalue())
def sign_rpm(rpm_path: Path, gpg_signing_key: str) -> None:
'Signs an RPM with the provided key data'
with tempfile.TemporaryDirectory() as td:
gpg = gnupg.GPG(gnupghome=td)
res = gpg.import_keys(gpg_signing_key)
assert res.count == 1, 'Only 1 private key can be imported for signing'
# Paths inside the container for passing artifacts to and fro
work_dir = Path(generate_work_dir())
package_dir = work_dir / 'package'
gpg_dir = work_dir / 'gpg'
opts = new_nspawn_opts(
cmd=[
'/usr/bin/rpmsign',
f'--define=_gpg_name {res.fingerprints[0]}',
'--addsign',
Path(package_dir / os.path.basename(rpm_path)).shell_quote(),
],
layer=_build_appliance(),
bindmount_ro=[
(td, gpg_dir),
],
bindmount_rw=[
(os.path.dirname(rpm_path), package_dir),
],
user=pwd.getpwnam('root'),
setenv=[f'GNUPGHOME={gpg_dir.shell_quote()}'],
)
run_non_booted_nspawn(opts, PopenArgs())
def test_yum_is_dnf(self):
# Setup for yum not being the same as dnf, modeled after fb
with temp_dir() as td:
yum_path = Path(td / 'yum').touch()
with mock.patch('shutil.which') as mock_which:
mock_which.return_value = None
self.assertFalse(yum_is_dnf())
mock_which.return_value = yum_path.decode()
self.assertFalse(yum_is_dnf())
# Setup for yum being the same as dnf, modeled after fedora
# where `/bin/yum -> dnf-3`
with temp_dir() as td:
dnf_name = 'dnf-3'
dnf_path = Path(td / dnf_name).touch()
yum_path = td / 'yum'
# Symlink to the name for a relative symlink that ends up
# as yum -> dnf-3
os.symlink(dnf_name, yum_path)
with mock.patch('shutil.which') as mock_which:
mock_paths = {dnf_name: dnf_path, 'yum': yum_path}
mock_which.side_effect = lambda p: mock_paths[p].decode()
self.assertTrue(yum_is_dnf())
def test_rpm_action_item_auto_downgrade(self):
parent_subvol = find_built_subvol(
(Path(__file__).dirname() / 'test-with-one-local-rpm').decode())
src_rpm = Path(__file__).dirname() / "rpm-test-cheese-1-1.rpm"
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
# ensure cheese2 is installed in the parent from rpm-test-cheese-2-1
assert os.path.isfile(
parent_subvol.path('/usr/share/rpm_test/cheese2.txt'))
# make sure the RPM we are installing is older in order to
# trigger the downgrade
src_data = RpmMetadata.from_file(src_rpm)
subvol_data = RpmMetadata.from_subvol(parent_subvol, src_data.name)
assert compare_rpm_versions(src_data, subvol_data) < 0
subvol = temp_subvolumes.snapshot(parent_subvol, 'rpm_action')
RpmActionItem.get_phase_builder(
[
RpmActionItem(
from_target='t',
source=src_rpm,
action=RpmAction.install,
)
],
DUMMY_LAYER_OPTS._replace(
yum_from_snapshot=Path(__file__).dirname() /
'yum-from-test-snapshot', ),
)(subvol)
subvol.run_as_root([
'rm',
'-rf',
subvol.path('dev'),
subvol.path('meta'),
subvol.path('var'),
])
self.assertEqual([
'(Dir)', {
'usr': [
'(Dir)', {
'share': [
'(Dir)', {
'rpm_test': [
'(Dir)', {
'cheese1.txt': ['(File d36)'],
}
],
}
],
}
],
}
], render_subvol(subvol))
def gold_demo_sendstreams():
with Path.resource(
__package__,
'gold_demo_sendstreams.pickle',
exe=False,
) as pickle_path, open(pickle_path, 'rb') as f:
return pickle.load(f)
def main(argv, from_file: BytesIO, to_file: BytesIO):
parser = argparse.ArgumentParser(
description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
Storage.add_argparse_arg(
parser, '--storage', required=True,
help='JSON blob for creating a Storage instance.',
)
parser.add_argument('--debug', action='store_true', help='Log more?')
subparsers = parser.add_subparsers(help='Sub-commands have help.')
parser_get = subparsers.add_parser('get', help='Download blob to stdout')
parser_get.add_argument('storage_id', help='String of the form KEY:ID')
parser_get.set_defaults(to_file=to_file)
parser_get.set_defaults(func=get)
parser_put = subparsers.add_parser(
'put', help='Write a blob from stdin, print its ID to stdout',
)
parser_put.set_defaults(from_file=from_file)
parser_put.set_defaults(to_file=to_file) # For the storage ID
parser_put.set_defaults(func=put)
args = Path.parse_args(parser, argv)
init_logging(debug=args.debug)
args.func(args)
def test_units_enabled(self):
# Get a list of the available .wants dirs for all targets to validate
available_targets = [
Path(avail) for avail in glob.glob(PROV_ROOT / "*.wants")
]
# spec[1] is the target name, skip if None
for unit, target, *_ in unit_test_specs:
# Make sure it's enabled where it should be
if target:
enabled_in_target = PROV_ROOT / _twant(target) / unit
self.assertTrue(os.path.islink(enabled_in_target),
enabled_in_target)
self.assertTrue(os.path.isfile(enabled_in_target),
enabled_in_target)
# make sure it's *not* enabled where it shouldn't be
for avail_target in [
avail for avail in available_targets
if target and avail.basename() != _twant(target)
]:
unit_in_target_wants = avail_target / unit
self.assertFalse(os.path.exists(avail_target / unit),
unit_in_target_wants)
def populate_versionlock_conf(
yum_dnf: YumDnf,
out_dir: Path,
install_dir: Path,
):
with create_ro(out_dir / 'versionlock.conf', 'w') as outf:
outf.write(
textwrap.dedent(f'''\
[main]
enabled = 1
locklist = {install_dir.decode()}/versionlock.list
'''))
# Write an empty lock-list. This will be bind-mounted in at runtime.
with create_ro(out_dir / 'versionlock.list', 'w'):
pass
# Side-load the appropriate versionlock plugin, we currently don't have
# a good way to install this via an RPM.
with Path.resource(
__package__, f'{yum_dnf.value}_versionlock.gz', exe=False,
) as p, \
gzip.open(p) as rf, \
create_ro(out_dir / 'versionlock.py', 'wb') as wf:
wf.write(rf.read())
def _install(
self, *, protected_paths, install_args=None
):
if install_args is None:
install_args = _INSTALL_ARGS
install_root = Path(tempfile.mkdtemp())
try:
# IMAGE_ROOT/meta/ is always required since it's always protected
for p in set(protected_paths) | {'meta/'}:
if p.endswith('/'):
os.makedirs(install_root / p)
else:
os.makedirs(os.path.dirname(install_root / p))
with open(install_root / p, 'wb'):
pass
# Note: this can't use `_yum_using_build_appliance` because that
# would lose coverage info on `yum_dnf_from_snapshot.py`. On
# the other hand, running this test against the host is fragile
# since it depends on the system packages available on CI
# containers. For this reason, this entire test is an
# `image.python_unittest` that runs in a build appliance.
yum_dnf_from_snapshot.yum_dnf_from_snapshot(
yum_dnf=self._YUM_DNF,
snapshot_dir=_SNAPSHOT_DIR,
protected_paths=protected_paths,
yum_dnf_args=[
f'--installroot={install_root}',
*install_args,
]
)
yield install_root
finally:
assert os.path.realpath(install_root) != b'/'
# Courtesy of `yum`, the `install_root` is now owned by root.
subprocess.run(['sudo', 'rm', '-rf', install_root], check=True)
def setUpClass(cls):
td_ctx = temp_dir()
cls._shadow = td_ctx.__enter__()
# NB: This may leak on SystemExit et al
cls.addClassCleanup(td_ctx.__exit__, None, None, None)
os.environ['FS_IMAGE_SHADOWED_PATHS_ROOT'] = f'{cls._shadow}'
lib_ctx = Path.resource(__package__,
'librename_shadowed.so',
exe=False)
lib_path = lib_ctx.__enter__()
# NB: This may leak a tempfile on SystemExit et al
cls.addClassCleanup(lib_ctx.__exit__, None, None, None)
lib = ctypes.cdll.LoadLibrary(lib_path)
cls._get_shadowed_original = lib.get_shadowed_original
cls._get_shadowed_original.restype = ctypes.c_char_p
cls._get_shadowed_original.argtypes = [ctypes.c_char_p]
cls._get_shadowed_rename_dest = lib.get_shadowed_rename_dest
cls._get_shadowed_rename_dest.restype = ctypes.c_char_p
cls._get_shadowed_rename_dest.argtypes = [
ctypes.c_char_p, ctypes.c_char_p
]
cls._rename = lib.rename
cls._rename.restype = ctypes.c_int
cls._rename.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
def test_rpm_action_item_remove_local(self):
# We expect the removal to be based just on the name of the RPM
# in the metadata, so removing cheese-2 should be fine via either:
for ver in [1, 2]:
self._check_cheese_removal(
Path(__file__).dirname() / f'rpm-test-cheese-{ver}-1.rpm',
)
def _check_protected_dir(self, subvol, protected_dir):
protected_dir = Path(protected_dir)
write_to_protected = _builder(_touch_cmd(protected_dir / 'ALIEN'))
with self.assertRaises(subprocess.CalledProcessError):
write_to_protected(subvol)
self.assertTrue(os.path.isdir(subvol.path(protected_dir)))
self.assertFalse(os.path.exists(subvol.path(protected_dir / 'ALIEN')))
def customize_fields(kwargs): # noqa: B902
cmd = kwargs.pop('cmd')
assert all(isinstance(c, (str, bytes)) for c in cmd), cmd
kwargs['cmd'] = tuple(cmd)
assert isinstance(kwargs['user'], str), kwargs['user']
kwargs['serve_rpm_snapshots'] = tuple(
Path(s) for s in kwargs.pop('serve_rpm_snapshots'))
def _image_source_path(
layer_opts: LayerOpts,
*,
source: AnyStr = None,
layer: Subvol = None,
path: AnyStr = None,
) -> Path:
assert (source is None) ^ (layer is None), (source, layer, path)
source = Path.or_none(source)
# Absolute `path` is still relative to `source` or `layer`
path = Path((path and path.lstrip('/')) or '.')
if source:
return (source / path).normpath()
if os.path.exists(layer.path(META_ARTIFACTS_REQUIRE_REPO)):
_validate_artifacts_require_repo(layer, layer_opts, 'image.source')
return Path(layer.path(path))
def customize_fields(cls, kwargs):
super().customize_fields(kwargs)
coerce_path_field_normal_relative(kwargs, 'dest')
customize_stat_options(kwargs, default_mode=None) # Defaulted later
source = kwargs['source']
dest = kwargs['dest']
# The 3 separate `*_mode` arguments must be set instead of `mode` for
# directory sources.
popped_args = ['mode', 'exe_mode', 'data_mode', 'dir_mode']
mode, dir_mode, exe_mode, data_mode = (kwargs.pop(a, None)
for a in popped_args)
st_source = os.stat(source, follow_symlinks=False)
if stat.S_ISDIR(st_source.st_mode):
assert mode is None, f'Cannot use `mode` for directory sources.'
kwargs['paths'] = tuple(
_recurse_into_source(
Path(source),
Path(dest),
dir_mode=dir_mode or _DIR_MODE,
exe_mode=exe_mode or _EXE_MODE,
data_mode=data_mode or _DATA_MODE,
))
elif stat.S_ISREG(st_source.st_mode):
assert {dir_mode, exe_mode, data_mode} == {None}, \
'Cannot use `{dir,exe,data}_mode` for file sources.'
if mode is None:
# This tests whether the build repo user can execute the
# file. This is a very natural test for build artifacts,
# and files in the repo. Note that this can be affected if
# the ambient umask is pathological, which is why
# `compiler.py` checks the umask.
mode = _EXE_MODE if os.access(source, os.X_OK) else _DATA_MODE
kwargs['paths'] = (_InstallablePath(
source=source,
provides=ProvidesFile(path=dest),
mode=mode,
), )
else:
raise RuntimeError(
f'{source} must be a regular file or directory, got {st_source}'
)
def gen_subvolume_subtree_provides(subvol: Subvol, subtree: Path):
'Yields "Provides" instances for a path `subtree` in `subvol`.'
# "Provides" classes use image-absolute paths that are `str` (for now).
# Accept any string type to ease future migrations.
subtree = os.path.join('/', Path(subtree).decode())
protected_paths = protected_path_set(subvol)
for prot_path in protected_paths:
rel_to_subtree = os.path.relpath(os.path.join('/', prot_path), subtree)
if not has_leading_dot_dot(rel_to_subtree):
yield ProvidesDoNotAccess(path=rel_to_subtree)
subtree_full_path = subvol.path(subtree).decode()
subtree_exists = False
# Traverse the subvolume as root, so that we have permission to access
# everything.
for type_and_path in subvol.run_as_root([
# -P is the analog of --no-dereference in GNU tools
#
# Filter out the protected paths at traversal time. If one of the
# paths has a very large or very slow mount, traversing it would
# have a devastating effect on build times, so let's avoid looking
# inside protected paths entirely. An alternative would be to
# `send` and to parse the sendstream, but this is ok too.
'find', '-P', subtree_full_path, '(', *itertools.dropwhile(
lambda x: x == '-o', # Drop the initial `-o`
itertools.chain.from_iterable([
# `normpath` removes the trailing / for protected dirs
'-o', '-path', subvol.path(os.path.normpath(p))
] for p in protected_paths),
), ')', '-prune', '-o', '-printf', '%y %p\\0',
], stdout=subprocess.PIPE).stdout.split(b'\0'):
if not type_and_path: # after the trailing \0
continue
filetype, abspath = type_and_path.decode().split(' ', 1)
relpath = os.path.relpath(abspath, subtree_full_path)
assert not has_leading_dot_dot(relpath), (abspath, subtree_full_path)
# We already "provided" this path above, and it should have been
# filtered out by `find`.
assert not is_path_protected(relpath, protected_paths), relpath
# Future: This provides all symlinks as files, while we should
# probably provide symlinks to valid directories inside the image as
# directories to be consistent with SymlinkToDirItem.
if filetype in ['b', 'c', 'p', 'f', 'l', 's']:
yield ProvidesFile(path=relpath)
elif filetype == 'd':
yield ProvidesDirectory(path=relpath)
else: # pragma: no cover
raise AssertionError(f'Unknown {filetype} for {abspath}')
if relpath == '.':
subtree_exists = True
# We should've gotten a CalledProcessError from `find`.
assert subtree_exists, f'{subtree} does not exist in {subvol.path()}'
def test_receive(self, temp_subvols):
new_subvol_name = 'differs_from_create_ops'
sv = temp_subvols.caller_will_create(new_subvol_name)
with open(Path(__file__).dirname() / 'create_ops.sendstream') as f, \
sv.receive(f):
pass
self.assertEqual(
render_demo_subvols(create_ops=new_subvol_name),
render_sendstream(sv.mark_readonly_and_get_sendstream()),
)
def test_rpm_action_conflict(self):
layer_opts = DUMMY_LAYER_OPTS._replace(
yum_from_snapshot='required but ignored')
# Test both install-install, install-remove, and install-downgrade
# conflicts.
for rpm_actions in (
(('cat', RpmAction.install), ('cat', RpmAction.install)),
(
('dog', RpmAction.remove_if_exists),
('dog', RpmAction.install),
),
):
with self.assertRaisesRegex(RuntimeError, 'RPM action conflict '):
# Note that we don't need to run the builder to hit the error
RpmActionItem.get_phase_builder(
[
RpmActionItem(from_target='t', name=r, action=a)
for r, a in rpm_actions
],
layer_opts,
)
with self.assertRaisesRegex(RuntimeError, 'RPM action conflict '):
# An extra test case for local RPM name conflicts (filenames are
# different but RPM names are the same)
RpmActionItem.get_phase_builder(
[
RpmActionItem(
from_target='t',
source=Path(__file__).dirname() /
"rpm-test-cheese-2-1.rpm",
action=RpmAction.install,
),
RpmActionItem(
from_target='t',
source=Path(__file__).dirname() /
"rpm-test-cheese-1-1.rpm",
action=RpmAction.remove_if_exists,
),
],
layer_opts,
)
def test_install_file_from_layer(self):
layer = find_built_subvol(
Path(__file__).dirname() / 'test-with-one-local-rpm')
path_in_layer = b'usr/share/rpm_test/cheese2.txt'
item = _install_file_item(
from_target='t',
source={
'layer': layer,
'path': '/' + path_in_layer.decode()
},
dest='cheese2',
)
self.assertEqual(0o444, item.mode)
self.assertEqual(Path(layer.path(path_in_layer)), item.source)
self.assertEqual(layer.path(path_in_layer), item.source)
self._check_item(
item,
{ProvidesFile(path='cheese2')},
{require_directory('/')},
)
def _dummies_for_protected_paths(
protected_paths: Iterable[str], ) -> Mapping[Path, Path]:
'''
Some locations (some host yum/dnf directories, and install root /meta/
and mountpoints) should be off-limits to writes by RPMs. We enforce
that by bind-mounting an empty file or directory on top of each one.
'''
with temp_dir() as td, tempfile.NamedTemporaryFile() as tf:
# NB: There may be duplicates in protected_paths, so we normalize.
# If the duplicates include both a file and a directory, this picks
# one arbitrarily, and if the type on disk is different, we will
# fail at mount time. This doesn't seem worth an explicit check.
yield {
Path(p).normpath(): (td if p.endswith('/') else Path(tf.name))
for p in protected_paths
}
# NB: The bind mount is read-only, so this is just paranoia. If it
# were left RW, we'd need to check its owner / permissions too.
for expected, actual in (([], td.listdir()), (b'', tf.read())):
assert expected == actual, \
f'Some RPM wrote {actual} to {protected_paths}'
def check_call(infile, subvolumes_dir):
if Path(infile.name).dirname().basename() != sigil_dirname:
return orig_from_json_file(infile, subvolumes_dir)
test_case.assertEqual(parent_layer_file, infile.name)
test_case.assertEqual(_SUBVOLS_DIR, subvolumes_dir)
class FakeSubvolumeOnDisk:
def subvolume_path(self):
return subvolume_path.decode()
return FakeSubvolumeOnDisk()
def _ensure_private_network():
'''
Normally, we run under `systemd-nspawn --private-network`. We don't
want to run in environments with network access because in these cases
it's very possible that `yum` / `dnf` will end up doing something
non-deterministic by reaching out to the network.
'''
# From `/usr/include/uapi/linux/if_arp.h`
allowed_types = {
768, # ARPHRD_TUNNEL
769, # ARPHRD_TUNNEL6
772, # ARPHRD_LOOPBACK
}
net = Path('/sys/class/net')
for iface in net.listdir():
with open(net / iface / 'type') as infile:
iface_type = int(infile.read())
# Not covered because we don't want to rely on the CI container
# having a network interface.
if iface_type not in allowed_types: # pragma: no cover
raise RuntimeError(
'Refusing to run without --private-network, found '
f'unknown interface {iface} of type {iface_type}.')
def test_receive_sendstream(self):
item = ReceiveSendstreamItem(
from_target='t',
source=Path(__file__).dirname() / 'create_ops.sendstream',
)
self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order())
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
new_subvol_name = 'differs_from_create_ops'
subvol = temp_subvolumes.caller_will_create(new_subvol_name)
item.get_phase_builder([item], DUMMY_LAYER_OPTS)(subvol)
self.assertEqual(
render_demo_subvols(create_ops=new_subvol_name),
render_sendstream(subvol.mark_readonly_and_get_sendstream()),
)
def test_snapshot(self):
with temp_repos.temp_repos_steps(
gpg_signing_key=temp_repos.get_test_signing_key(),
repo_change_steps=[{'dog': temp_repos.SAMPLE_STEPS[0]['dog']}]
) as repos_root, temp_dir() as td:
with open(td / 'fake_gpg_key', 'w'):
pass
whitelist_dir = td / 'gpg_whitelist'
os.mkdir(whitelist_dir)
shutil.copy(td / 'fake_gpg_key', whitelist_dir)
storage_dict = {
'key': 'test',
'kind': 'filesystem',
'base_dir': td / 'storage',
}
snapshot_repo([
'--repo-universe=fakeverse',
'--repo-name=dog',
'--repo-url=' + (repos_root / "0/dog").file_url(),
f'--gpg-key-whitelist-dir={whitelist_dir}',
'--gpg-url=' + (td / 'fake_gpg_key').file_url(),
f'--snapshot-dir={td / "snap"}',
f'--storage={Path.json_dumps(storage_dict)}',
'--db=' + Path.json_dumps({
'kind': 'sqlite',
'db_path': td / 'db.sqlite3',
}),
'--threads=4',
])
# This test simply checks the overall integration, so we don't
# bother looking inside the DB or Storage, or inspecting the
# details of the snapshot -- those should all be covered by
# lower-level tests.
with sqlite3.connect(RepoSnapshot.fetch_sqlite_from_storage(
Storage.make(**storage_dict),
td / 'snap',
td / 'snapshot.sql3',
)) as db:
self.assertEqual({
'dog-pkgs/rpm-test-carrot-2-rc0.x86_64.rpm',
'dog-pkgs/rpm-test-mice-0.1-a.x86_64.rpm',
'dog-pkgs/rpm-test-milk-1.41-42.x86_64.rpm',
'dog-pkgs/rpm-test-mutable-a-f.x86_64.rpm',
}, {
path for path, in db.execute(
'SELECT "path" FROM "rpm";'
).fetchall()
})
def _install_root(conf_path: Path, yum_dnf_args: Iterable[str]) -> Path:
# Peek at the `yum` / `dnf` args, which take precedence over the config.
p = argparse.ArgumentParser(allow_abbrev=False, add_help=False)
p.add_argument('--installroot', type=Path.from_argparse)
args, _ = p.parse_known_args(yum_dnf_args)
if args.installroot:
return args.installroot
# For our wrapper to be transparent, the `installroot` semantics have to
# match that of `yum` / `dnf`, so the argument is optional, with a
# fallback to the config file, and then to `/`.
cp = ConfigParser()
with open(conf_path) as conf_in:
cp.read_file(conf_in)
return Path(cp['main'].get('installroot', '/'))
def get_volume_for_current_repo(min_free_bytes, artifacts_dir):
'''
Multiple repos need to be able to concurrently build images on the same
host. The cleanest way to achieve such isolation is to supply each repo
with its own volume, which will store the repo's image build outputs.
It is easiest to back this volume with a loop device. The appropriate
size of the loop device depends on the expected size of the target being
built. To address this this by ensuring that prior to every build, the
volume has at least a specified amount of space. The default in
`image_layer` is large enough for most builds, but really huge
`image_layer` targets can further increase their requested
`min_free_bytes`.
Image-build tooling **must never** access paths in this volume without
going through this function. Otherwise, the volume will not get
remounted correctly if the host containing the repo got rebooted.
PRE-CONDITION: `artifacts_dir` exists and is writable by `root`.
'''
if not os.path.exists(artifacts_dir): # pragma: no cover
raise RuntimeError(f'{artifacts_dir} must exist')
volume_dir = os.path.join(artifacts_dir, VOLUME_DIR)
with Path.resource(__package__, 'set_up_volume.sh', exe=True) as binary:
subprocess.check_call([
# While Buck probably does not call this concurrently under normal
# circumstances, the worst-case outcome is that we lose or corrupt
# the whole buld cache, so add some locking to be on the safe side.
'flock',
os.path.join(
artifacts_dir, '.lock.set_up_volume.sh.never.rm.or.mv',
),
'sudo',
binary,
str(int(min_free_bytes)), # Accepts floats & ints
os.path.join(artifacts_dir, IMAGE_FILE),
volume_dir,
])
# We prefer to have the volume owned by the repo user, instead of root:
# - The trusted repo user has to be able to access the built
# subvolumes, but nobody else should be able to (they might contain
# setuid binaries & similar). Thus, subvols ought to have wrapper
# directories owned by the user, with mode 0700.
# - This reduces the number of places we have to `sudo` to create
# directories inside the subvolume.
subprocess.check_call([
'sudo', 'chown', f'{os.getuid()}:{os.getgid()}', volume_dir,
])
return volume_dir
def spec(self, busybox_path: Path) -> str:
format_kwargs = {
**self._asdict(),
'quoted_contents':
shlex.quote(f'{self.name} {self.version} {self.release}' if self.
override_contents is None else self.override_contents),
'quoted_busybox_path':
busybox_path.shell_quote(),
}
common_spec = textwrap.dedent('''\
Summary: The "{name}" package.
Name: rpm-test-{name}
Version: {version}
Release: {release}
Provides: virtual-{name}
License: MIT
Group: Facebook/Script
Vendor: Facebook, Inc.
Packager: [email protected]
%description
%install
mkdir -p "$RPM_BUILD_ROOT"/rpm_test
echo {quoted_contents} > "$RPM_BUILD_ROOT"/rpm_test/{name}.txt
mkdir -p "$RPM_BUILD_ROOT"/bin
''').format(**format_kwargs)
return common_spec + textwrap.dedent(('''\
%files
/rpm_test/{name}.txt
''') if not self.test_post_install else (
'''\
cp {quoted_busybox_path} "$RPM_BUILD_ROOT"/bin/sh
%post
'''
# yum-dnf-from-snapshot prepares /dev in a subtle way to protect
# host system from side-effects of rpm post-install scripts.
# The command below lets us test that /dev/null is prepared
# properly: if "echo > /dev/null" fails, tests will catch the
# absence of post.txt
'''\
echo > /dev/null && echo 'stuff' > \
"$RPM_BUILD_ROOT"/rpm_test/post.txt
%files
/bin/sh
/rpm_test/{name}.txt
''')).format(**format_kwargs)
def find_built_subvol(
layer_output, *, path_in_repo=None, subvolumes_dir=None,
):
# It's OK for both to be None (uses the current file to find repo), but
# it's not OK to set both.
assert (path_in_repo is None) or (subvolumes_dir is None)
with open(Path(layer_output) / 'layer.json') as infile:
return Subvol(
SubvolumeOnDisk.from_json_file(
infile,
subvolumes_dir if subvolumes_dir
else _get_subvolumes_dir(path_in_repo),
).subvolume_path(),
already_exists=True,
)
def _populate_temp_file_and_rename(dest_path: Path, *, mode='w'):
with tempfile.NamedTemporaryFile(
mode=mode,
dir=dest_path.dirname(),
delete=False,
) as tf:
try:
yield tf
except BaseException:
os.unlink(tf.name)
raise
# NB: This will fail to replace a directory, preventing us from
# transparently converting JSON to BZL databases without using
# `--out-db`. This is fine since the DB paths should look different
# anyhow (`dirname` vs `filename.bzl`).
os.rename(tf.name, dest_path)
def _yum_using_build_appliance(
*,
build_appliance: Subvol,
nspawn_args: List[str],
install_root: Path,
protected_paths: Iterable[str],
yum_args: List[str],
preserve_yum_cache: bool,
) -> None:
work_dir = '/work' + base64.urlsafe_b64encode(
uuid.uuid4().bytes # base64 instead of hex saves 10 bytes
).decode().strip('=')
mount_var_cache_yum = '' if preserve_yum_cache else f'''
mkdir -p {work_dir}/var/cache/yum ; \
mount --bind /var/cache/yum {work_dir}/var/cache/yum ;
'''
opts = nspawn_in_subvol_parse_opts([
'--layer',
'UNUSED',
'--user',
'root',
# You can see below --no-private-network in conjunction with
# --cap-net-admin. CAP_NET_ADMIN is not intended to administer the
# host's network stack. See how yum_from_snapshot() brings loopback
# interface up under protection of "unshare --net".
'--cap-net-admin',
'--no-private-network',
'--bindmount-rw',
install_root.decode(),
work_dir,
*nspawn_args,
'--',
'sh',
'-uec',
f'''
{mount_var_cache_yum}
/yum-from-snapshot \
{' '.join(
'--protected-path=' + shlex.quote(p) for p in protected_paths
)} \
--install-root {work_dir} \
-- {' '.join(shlex.quote(arg) for arg in yum_args)}
''',
])
# Don't pollute stdout with logging
nspawn_in_subvol(build_appliance, opts, stdout=sys.stderr)
