def test_make_dirs(self):
self._check_item(
MakeDirsItem(from_target='t', into_dir='x', path_to_make='y/z'),
{ProvidesDirectory(path='x/y'),
ProvidesDirectory(path='x/y/z')},
{require_directory('x')},
)
def temp_filesystem_provides(p=''):
'Captures what is provided by _temp_filesystem, if installed at `p` '
'inside the image.'
return {
ProvidesDirectory(path=f'{p}/a'),
ProvidesDirectory(path=f'{p}/a/b'),
ProvidesDirectory(path=f'{p}/a/b/c'),
ProvidesDirectory(path=f'{p}/a/d'),
ProvidesFile(path=f'{p}/a/E'),
ProvidesFile(path=f'{p}/a/d/F'),
ProvidesFile(path=f'{p}/a/b/c/G'),
}
def test_stat_options(self):
self._check_item(
MakeDirsItem(
from_target='t',
into_dir='x',
path_to_make='y/z',
mode=0o733,
user_group='cat:dog',
),
{ProvidesDirectory(path='x/y'),
ProvidesDirectory(path='x/y/z')},
{require_directory('x')},
)
def test_phases_provide(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
parent = temp_subvolumes.create('parent')
# Permit _populate_temp_filesystem to make writes.
parent.run_as_root([
'chown', '--no-dereference', f'{os.geteuid()}:{os.getegid()}',
parent.path(),
])
populate_temp_filesystem(parent.path().decode())
with self.assertRaises(subprocess.CalledProcessError):
list(gen_subvolume_subtree_provides(parent, 'no_such/path'))
for create_meta in [False, True]:
# Check that we properly handle ignoring a /meta if it's present
if create_meta:
parent.run_as_root(['mkdir', parent.path('meta')])
self._check_item(
PhasesProvideItem(from_target='t', subvol=parent),
temp_filesystem_provides() | {
ProvidesDirectory(path='/'),
ProvidesDoNotAccess(path='/meta'),
},
set(),
)
def test_clone_pre_existing_dest(self):
ci = self._clone_item('/foo/bar', '/', pre_existing_dest=True)
self.assertEqual('', ci.dest)
self._check_item(
ci,
{
ProvidesDirectory(path='bar'),
ProvidesDirectory(path='bar/baz'),
ProvidesFile(path='bar/baz/bar'),
ProvidesFile(path='bar/even_more_hello_world.tar'),
},
{require_directory('/')},
)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_pre_existing_dest')
self._check_clone_bar(ci, subvol)
def _recurse_into_source(
source_dir: Path,
dest_dir: str,
*,
dir_mode: Mode,
exe_mode: Mode,
data_mode: Mode,
) -> Iterable[_InstallablePath]:
'Yields paths in top-down order, making recursive copying easy.'
yield _InstallablePath(
source=source_dir,
provides=ProvidesDirectory(path=dest_dir.decode()),
mode=dir_mode,
)
with os.scandir(source_dir) as it:
for e in it:
source = source_dir / e.name
dest = dest_dir / e.name
if e.is_dir(follow_symlinks=False):
yield from _recurse_into_source(
source,
dest,
dir_mode=dir_mode,
exe_mode=exe_mode,
data_mode=data_mode,
)
elif e.is_file(follow_symlinks=False):
yield _InstallablePath(
source=source,
provides=ProvidesFile(path=dest.decode()),
# Same `os.access` rationale as in `customize_fields`.
mode=exe_mode if os.access(source, os.X_OK) else data_mode,
)
else:
raise RuntimeError(f'{source}: neither a file nor a directory')
def gen_subvolume_subtree_provides(subvol: Subvol, subtree: Path):
'Yields "Provides" instances for a path `subtree` in `subvol`.'
# "Provides" classes use image-absolute paths that are `str` (for now).
# Accept any string type to ease future migrations.
subtree = os.path.join('/', Path(subtree).decode())
protected_paths = protected_path_set(subvol)
for prot_path in protected_paths:
rel_to_subtree = os.path.relpath(os.path.join('/', prot_path), subtree)
if not has_leading_dot_dot(rel_to_subtree):
yield ProvidesDoNotAccess(path=rel_to_subtree)
subtree_full_path = subvol.path(subtree).decode()
subtree_exists = False
# Traverse the subvolume as root, so that we have permission to access
# everything.
for type_and_path in subvol.run_as_root([
# -P is the analog of --no-dereference in GNU tools
#
# Filter out the protected paths at traversal time. If one of the
# paths has a very large or very slow mount, traversing it would
# have a devastating effect on build times, so let's avoid looking
# inside protected paths entirely. An alternative would be to
# `send` and to parse the sendstream, but this is ok too.
'find', '-P', subtree_full_path, '(', *itertools.dropwhile(
lambda x: x == '-o', # Drop the initial `-o`
itertools.chain.from_iterable([
# `normpath` removes the trailing / for protected dirs
'-o', '-path', subvol.path(os.path.normpath(p))
] for p in protected_paths),
), ')', '-prune', '-o', '-printf', '%y %p\\0',
], stdout=subprocess.PIPE).stdout.split(b'\0'):
if not type_and_path: # after the trailing \0
continue
filetype, abspath = type_and_path.decode().split(' ', 1)
relpath = os.path.relpath(abspath, subtree_full_path)
assert not has_leading_dot_dot(relpath), (abspath, subtree_full_path)
# We already "provided" this path above, and it should have been
# filtered out by `find`.
assert not is_path_protected(relpath, protected_paths), relpath
# Future: This provides all symlinks as files, while we should
# probably provide symlinks to valid directories inside the image as
# directories to be consistent with SymlinkToDirItem.
if filetype in ['b', 'c', 'p', 'f', 'l', 's']:
yield ProvidesFile(path=relpath)
elif filetype == 'd':
yield ProvidesDirectory(path=relpath)
else: # pragma: no cover
raise AssertionError(f'Unknown {filetype} for {abspath}')
if relpath == '.':
subtree_exists = True
# We should've gotten a CalledProcessError from `find`.
assert subtree_exists, f'{subtree} does not exist in {subvol.path()}'
def test_symlink(self):
self._check_item(
SymlinkToDirItem(from_target='t', source='x', dest='y'),
{ProvidesDirectory(path='y')},
{require_directory('/'),
require_directory('/x')},
)
self._check_item(
SymlinkToFileItem(from_target='t',
source='source_file',
dest='dest_symlink'),
{ProvidesFile(path='dest_symlink')},
{require_directory('/'),
require_file('/source_file')},
)
def provides(self):
# We own ZST decompression, tarfile handles other gz, bz2, etc.
import tarfile # Lazy since only this method needs it.
with open_for_read_decompress(self.source) as tf, \
tarfile.open(fileobj=tf, mode='r|') as f:
for item in f:
path = os.path.join(
self.into_dir,
make_path_normal_relative(item.name),
)
if item.isdir():
# We do NOT provide the installation directory, and the
# image build script tarball extractor takes pains (e.g.
# `tar --no-overwrite-dir`) not to touch the extraction
# directory.
if os.path.normpath(os.path.relpath(path,
self.into_dir)) != '.':
yield ProvidesDirectory(path=path)
else:
yield ProvidesFile(path=path)
def test_clone_omit_outer_dir(self):
ci = self._clone_item(
'/foo/bar',
'/bar',
omit_outer_dir=True,
pre_existing_dest=True,
)
self.assertEqual('bar', ci.dest)
self._check_item(
ci,
{
ProvidesDirectory(path='bar/baz'),
ProvidesFile(path='bar/baz/bar'),
ProvidesFile(path='bar/even_more_hello_world.tar'),
},
{require_directory('/bar')},
)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_omit_outer_dir')
subvol.run_as_root(['mkdir', subvol.path('bar')])
self._check_clone_bar(ci, subvol)
def provides(self):
yield ProvidesDirectory(path=self.dest)
def test_mount_item(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes, \
tempfile.TemporaryDirectory() as source_dir:
runtime_source = {'so': 'me', 'arbitrary': {'j': 'son'}}
mount_config = {
'is_directory': True,
'build_source': {
'type': 'layer',
'source': '//fake:path'
},
'runtime_source': runtime_source,
}
with open(os.path.join(source_dir, 'mountconfig.json'), 'w') as f:
json.dump(mount_config, f)
self._check_item(
self._make_mount_item(
mountpoint='can/haz',
target=source_dir,
mount_config=mount_config,
),
{ProvidesDoNotAccess(path='can/haz')},
{require_directory('can')},
)
# Make a subvolume that we will mount inside `mounter`
mountee = temp_subvolumes.create('moun:tee/volume')
mountee.run_as_root(['tee', mountee.path('kitteh')],
input=b'cheez')
# These sub-mounts inside `mountee` act as canaries to make sure
# that (a) `mounter` receives the sub-mounts as a consequence of
# mounting `mountee` recursively, (b) that unmounting one in
# `mounter` does not affect the original in `mountee` -- i.e.
# that rslave propagation is set up correctly, (c) that
# unmounting in `mountee` immediately affects `mounter`.
#
# In practice, our build artifacts should NEVER be mutated after
# construction (and the only un-mount is implicitly, and
# seemingly safely, performed by `btrfs subvolume delete`).
# However, ensuring that we have correct `rslave` propagation is
# a worthwhile safeguard for host mounts, where an errant
# `umount` by a user inside their repo could otherwise break
# their host.
for submount in ('submount1', 'submount2'):
mountee.run_as_root(['mkdir', mountee.path(submount)])
mountee.run_as_root([
'mount', '-o', 'bind,ro', source_dir,
mountee.path(submount)
])
self.assertTrue(
os.path.exists(mountee.path(submount +
'/mountconfig.json')))
# Make the JSON file normally in "buck-out" that refers to `mountee`
mountee_subvolumes_dir = self._write_layer_json_into(
mountee, source_dir)
# Mount <mountee> at <mounter>/meow
mounter = temp_subvolumes.caller_will_create('mount:er/volume')
root_item = FilesystemRootItem(from_target='t')
root_item.get_phase_builder([root_item], DUMMY_LAYER_OPTS)(mounter)
mount_meow = self._make_mount_item(
mountpoint='meow',
target=source_dir,
mount_config=mount_config,
)
self.assertEqual(
runtime_source,
json.loads(mount_meow.runtime_source),
)
with self.assertRaisesRegex(AssertionError, ' could not resolve '):
mount_meow.build_source.to_path(
target_to_path={},
subvolumes_dir=mountee_subvolumes_dir,
)
mount_meow.build(
mounter,
DUMMY_LAYER_OPTS._replace(
target_to_path={'//fake:path': source_dir},
subvolumes_dir=mountee_subvolumes_dir,
))
# This checks the subvolume **contents**, but not the mounts.
# Ensure the build created a mountpoint, and populated metadata.
self._check_subvol_mounts_meow(mounter)
# `mountee` was also mounted at `/meow`
with open(mounter.path('meow/kitteh')) as f:
self.assertEqual('cheez', f.read())
def check_mountee_mounter_submounts(submount_presence):
for submount, (in_mountee, in_mounter) in submount_presence:
self.assertEqual(
in_mountee,
os.path.exists(
mountee.path(submount + '/mountconfig.json')),
f'{submount}, {in_mountee}')
self.assertEqual(
in_mounter,
os.path.exists(
mounter.path('meow/' + submount +
'/mountconfig.json')),
f'{submount}, {in_mounter}')
# Both sub-mounts are accessible in both places now.
check_mountee_mounter_submounts([
('submount1', (True, True)),
('submount2', (True, True)),
])
# Unmounting `submount1` from `mountee` also affects `mounter`.
mountee.run_as_root(['umount', mountee.path('submount1')])
check_mountee_mounter_submounts([
('submount1', (False, False)),
('submount2', (True, True)),
])
# Unmounting `submount2` from `mounter` doesn't affect `mountee`.
mounter.run_as_root(['umount', mounter.path('meow/submount2')])
check_mountee_mounter_submounts([
('submount1', (False, False)),
('submount2', (True, False)),
])
# Check that we read back the `mounter` metadata, mark `/meow`
# inaccessible, and do not emit a `ProvidesFile` for `kitteh`.
self._check_item(
PhasesProvideItem(from_target='t', subvol=mounter),
{
ProvidesDirectory(path='/'),
ProvidesDoNotAccess(path='/meta'),
ProvidesDoNotAccess(path='/meow'),
},
set(),
)
# Check that we successfully clone mounts from the parent layer.
mounter_child = temp_subvolumes.caller_will_create('child/volume')
ParentLayerItem.get_phase_builder(
[ParentLayerItem(from_target='t', subvol=mounter)],
DUMMY_LAYER_OPTS,
)(mounter_child)
# The child has the same mount, and the same metadata
self._check_subvol_mounts_meow(mounter_child)
# Check that we refuse to create nested mounts.
nested_mounter = temp_subvolumes.create('nested_mounter')
nested_item = MountItem(
layer_opts=DUMMY_LAYER_OPTS,
from_target='t',
mountpoint='/whatever',
target=None,
mount_config={
'is_directory': True,
'build_source': {
'type': 'layer',
'source': '//:fake'
},
},
)
with tempfile.TemporaryDirectory() as d:
mounter_subvolumes_dir = self._write_layer_json_into(
mounter, d)
with self.assertRaisesRegex(
AssertionError,
'Refusing .* nested mount',
):
nested_item.build(
nested_mounter,
DUMMY_LAYER_OPTS._replace(
target_to_path={'//:fake': d},
subvolumes_dir=mounter_subvolumes_dir,
))
def test_install_file_command_recursive(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('d')])
with temp_dir() as td:
with open(td / 'data.txt', 'w') as df:
print('Hello', file=df)
os.mkdir(td / 'subdir')
with open(td / 'subdir/exe.sh', 'w') as ef:
print('#!/bin/sh\necho "Hello"', file=ef)
os.chmod(td / 'subdir/exe.sh', 0o100)
dir_item = _install_file_item(
from_target='t',
source={'source': td},
dest='/d/a',
)
ps = [
_InstallablePath(
td,
ProvidesDirectory(path='d/a'),
'u+rwx,og+rx',
),
_InstallablePath(
td / 'data.txt',
ProvidesFile(path='d/a/data.txt'),
'a+r',
),
_InstallablePath(
td / 'subdir',
ProvidesDirectory(path='d/a/subdir'),
'u+rwx,og+rx',
),
_InstallablePath(
td / 'subdir/exe.sh',
ProvidesFile(path='d/a/subdir/exe.sh'),
'a+rx',
),
]
self.assertEqual(sorted(ps), sorted(dir_item.paths))
self.assertEqual(td, dir_item.source)
self._check_item(dir_item, {p.provides
for p in ps},
{require_directory('d')})
# This implicitly checks that `a` precedes its contents.
dir_item.build(subvol, DUMMY_LAYER_OPTS)
self.assertEqual(
[
'(Dir)', {
'd': [
'(Dir)', {
'a': [
'(Dir)', {
'data.txt': ['(File m444 d6)'],
'subdir': [
'(Dir)', {
'exe.sh': ['(File m555 d23)'],
}
],
}
]
}
]
}
],
render_subvol(subvol),
)
def provides(self):
inner_dir = os.path.join(self.into_dir, self.path_to_make)
while inner_dir != self.into_dir:
yield ProvidesDirectory(path=inner_dir)
inner_dir = os.path.dirname(inner_dir)