def temp_filesystem_provides(p=''):
'Captures what is provided by _temp_filesystem, if installed at `p` '
'inside the image.'
return {
ProvidesDirectory(path=f'{p}/a'),
ProvidesDirectory(path=f'{p}/a/b'),
ProvidesDirectory(path=f'{p}/a/b/c'),
ProvidesDirectory(path=f'{p}/a/d'),
ProvidesFile(path=f'{p}/a/E'),
ProvidesFile(path=f'{p}/a/d/F'),
ProvidesFile(path=f'{p}/a/b/c/G'),
}
def test_clone_special_files(self):
with TempSubvolumes(sys.argv[0]) as temp_subvols:
src_subvol = temp_subvols.create('test_clone_special_files_src')
dest_subvol = temp_subvols.create('test_clone_special_files_dest')
src_subvol.run_as_root(['mkfifo', src_subvol.path('fifo')])
src_subvol.run_as_root([
'mknod',
src_subvol.path('null'),
'c',
'1',
'3',
])
for name in ['fifo', 'null']:
ci = self._clone_item(name, name, subvol=src_subvol)
self.assertEqual(name, ci.dest)
self._check_item(
ci,
{ProvidesFile(path=name)},
{require_directory('/')},
)
ci.build(dest_subvol, DUMMY_LAYER_OPTS)
src_r = render_subvol(src_subvol)
dest_r = render_subvol(dest_subvol)
self.assertEqual(src_r, dest_r)
self.assertEqual(
['(Dir)', {
'fifo': ['(FIFO)'],
'null': ['(Char 103)'],
}], dest_r)
def test_clone_pre_existing_dest(self):
ci = self._clone_item('/foo/bar', '/', pre_existing_dest=True)
self.assertEqual('', ci.dest)
self._check_item(
ci,
{
ProvidesDirectory(path='bar'),
ProvidesDirectory(path='bar/baz'),
ProvidesFile(path='bar/baz/bar'),
ProvidesFile(path='bar/even_more_hello_world.tar'),
},
{require_directory('/')},
)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_pre_existing_dest')
self._check_clone_bar(ci, subvol)
def _recurse_into_source(
source_dir: Path,
dest_dir: str,
*,
dir_mode: Mode,
exe_mode: Mode,
data_mode: Mode,
) -> Iterable[_InstallablePath]:
'Yields paths in top-down order, making recursive copying easy.'
yield _InstallablePath(
source=source_dir,
provides=ProvidesDirectory(path=dest_dir.decode()),
mode=dir_mode,
)
with os.scandir(source_dir) as it:
for e in it:
source = source_dir / e.name
dest = dest_dir / e.name
if e.is_dir(follow_symlinks=False):
yield from _recurse_into_source(
source,
dest,
dir_mode=dir_mode,
exe_mode=exe_mode,
data_mode=data_mode,
)
elif e.is_file(follow_symlinks=False):
yield _InstallablePath(
source=source,
provides=ProvidesFile(path=dest.decode()),
# Same `os.access` rationale as in `customize_fields`.
mode=exe_mode if os.access(source, os.X_OK) else data_mode,
)
else:
raise RuntimeError(f'{source}: neither a file nor a directory')
def test_clone_hardlinks(self):
with TempSubvolumes(sys.argv[0]) as temp_subvols:
src_subvol = temp_subvols.create('test_clone_hardlinks_src')
dest_subvol = temp_subvols.create('test_clone_hardlinks_dest')
src_subvol.run_as_root(['touch', src_subvol.path('a')])
src_subvol.run_as_root([
'ln',
src_subvol.path('a'),
src_subvol.path('b'),
])
ci = self._clone_item(
'/',
'/',
omit_outer_dir=True,
pre_existing_dest=True,
subvol=src_subvol,
)
self.assertEqual('', ci.dest)
self._check_item(
ci,
{
ProvidesFile(path='a'),
ProvidesFile(path='b'),
# This looks like a bug (there's no /meta on disk here) but
# it's really just an artifact of how this path is
# protected. Read: This Is Fine (TM).
ProvidesDoNotAccess(path='/meta'),
},
{require_directory('/')})
ci.build(dest_subvol, DUMMY_LAYER_OPTS)
src_r = render_subvol(src_subvol)
dest_r = render_subvol(dest_subvol)
self.assertEqual(src_r, dest_r)
self.assertEqual(
[
'(Dir)',
{
# Witness that they have the same (rendered) inode # of "0"
'a': [['(File)', 0]],
'b': [['(File)', 0]],
}
],
dest_r)
def gen_subvolume_subtree_provides(subvol: Subvol, subtree: Path):
'Yields "Provides" instances for a path `subtree` in `subvol`.'
# "Provides" classes use image-absolute paths that are `str` (for now).
# Accept any string type to ease future migrations.
subtree = os.path.join('/', Path(subtree).decode())
protected_paths = protected_path_set(subvol)
for prot_path in protected_paths:
rel_to_subtree = os.path.relpath(os.path.join('/', prot_path), subtree)
if not has_leading_dot_dot(rel_to_subtree):
yield ProvidesDoNotAccess(path=rel_to_subtree)
subtree_full_path = subvol.path(subtree).decode()
subtree_exists = False
# Traverse the subvolume as root, so that we have permission to access
# everything.
for type_and_path in subvol.run_as_root([
# -P is the analog of --no-dereference in GNU tools
#
# Filter out the protected paths at traversal time. If one of the
# paths has a very large or very slow mount, traversing it would
# have a devastating effect on build times, so let's avoid looking
# inside protected paths entirely. An alternative would be to
# `send` and to parse the sendstream, but this is ok too.
'find', '-P', subtree_full_path, '(', *itertools.dropwhile(
lambda x: x == '-o', # Drop the initial `-o`
itertools.chain.from_iterable([
# `normpath` removes the trailing / for protected dirs
'-o', '-path', subvol.path(os.path.normpath(p))
] for p in protected_paths),
), ')', '-prune', '-o', '-printf', '%y %p\\0',
], stdout=subprocess.PIPE).stdout.split(b'\0'):
if not type_and_path: # after the trailing \0
continue
filetype, abspath = type_and_path.decode().split(' ', 1)
relpath = os.path.relpath(abspath, subtree_full_path)
assert not has_leading_dot_dot(relpath), (abspath, subtree_full_path)
# We already "provided" this path above, and it should have been
# filtered out by `find`.
assert not is_path_protected(relpath, protected_paths), relpath
# Future: This provides all symlinks as files, while we should
# probably provide symlinks to valid directories inside the image as
# directories to be consistent with SymlinkToDirItem.
if filetype in ['b', 'c', 'p', 'f', 'l', 's']:
yield ProvidesFile(path=relpath)
elif filetype == 'd':
yield ProvidesDirectory(path=relpath)
else: # pragma: no cover
raise AssertionError(f'Unknown {filetype} for {abspath}')
if relpath == '.':
subtree_exists = True
# We should've gotten a CalledProcessError from `find`.
assert subtree_exists, f'{subtree} does not exist in {subvol.path()}'
def test_clone_omit_outer_dir(self):
ci = self._clone_item(
'/foo/bar',
'/bar',
omit_outer_dir=True,
pre_existing_dest=True,
)
self.assertEqual('bar', ci.dest)
self._check_item(
ci,
{
ProvidesDirectory(path='bar/baz'),
ProvidesFile(path='bar/baz/bar'),
ProvidesFile(path='bar/even_more_hello_world.tar'),
},
{require_directory('/bar')},
)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_omit_outer_dir')
subvol.run_as_root(['mkdir', subvol.path('bar')])
self._check_clone_bar(ci, subvol)
def test_install_file(self):
with tempfile.NamedTemporaryFile() as tf:
os.chmod(tf.name, stat.S_IXUSR)
exe_item = _install_file_item(
from_target='t',
source={'source': tf.name},
dest='d/c',
)
ep = _InstallablePath(Path(tf.name), ProvidesFile(path='d/c'), 'a+rx')
self.assertEqual((ep, ), exe_item.paths)
self.assertEqual(tf.name.encode(), exe_item.source)
self._check_item(exe_item, {ep.provides}, {require_directory('d')})
# Checks `image.source(path=...)`
with temp_dir() as td:
os.mkdir(td / 'b')
open(td / 'b/q', 'w').close()
data_item = _install_file_item(
from_target='t',
source={
'source': td,
'path': '/b/q'
},
dest='d',
)
dp = _InstallablePath(td / 'b/q', ProvidesFile(path='d'), 'a+r')
self.assertEqual((dp, ), data_item.paths)
self.assertEqual(td / 'b/q', data_item.source)
self._check_item(data_item, {dp.provides}, {require_directory('/')})
# NB: We don't need to get coverage for this check on ALL the items
# because the presence of the ProvidesDoNotAccess items it the real
# safeguard -- e.g. that's what prevents TarballItem from writing
# to /meta/ or other protected paths.
with self.assertRaisesRegex(AssertionError, 'cannot start with meta/'):
_install_file_item(
from_target='t',
source={'source': 'a/b/c'},
dest='/meta/foo',
)
def test_clone_file(self):
ci = self._clone_item('/rpm_test/hello_world.tar', '/cloned_hello.tar')
self.assertEqual('cloned_hello.tar', ci.dest)
self._check_item(
ci,
{ProvidesFile(path='cloned_hello.tar')},
{require_directory('/')},
)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_file')
ci.build(subvol, DUMMY_LAYER_OPTS)
r = render_subvol(subvol)
ino, = pop_path(r, 'cloned_hello.tar')
self.assertRegex(ino, '(File m444 d[0-9]+)')
self.assertEqual(['(Dir)', {}], r)
def test_symlink(self):
self._check_item(
SymlinkToDirItem(from_target='t', source='x', dest='y'),
{ProvidesDirectory(path='y')},
{require_directory('/'),
require_directory('/x')},
)
self._check_item(
SymlinkToFileItem(from_target='t',
source='source_file',
dest='dest_symlink'),
{ProvidesFile(path='dest_symlink')},
{require_directory('/'),
require_file('/source_file')},
)
def test_install_file_from_layer(self):
layer = find_built_subvol(
Path(__file__).dirname() / 'test-with-one-local-rpm')
path_in_layer = b'rpm_test/cheese2.txt'
item = _install_file_item(
from_target='t',
source={
'layer': layer,
'path': '/' + path_in_layer.decode()
},
dest='cheese2',
)
source_path = layer.path(path_in_layer)
p = _InstallablePath(source_path, ProvidesFile(path='cheese2'), 'a+r')
self.assertEqual((p, ), item.paths)
self.assertEqual(source_path, item.source)
self._check_item(item, {p.provides}, {require_directory('/')})
def customize_fields(cls, kwargs):
super().customize_fields(kwargs)
coerce_path_field_normal_relative(kwargs, 'dest')
customize_stat_options(kwargs, default_mode=None) # Defaulted later
source = kwargs['source']
dest = kwargs['dest']
# The 3 separate `*_mode` arguments must be set instead of `mode` for
# directory sources.
popped_args = ['mode', 'exe_mode', 'data_mode', 'dir_mode']
mode, dir_mode, exe_mode, data_mode = (kwargs.pop(a, None)
for a in popped_args)
st_source = os.stat(source, follow_symlinks=False)
if stat.S_ISDIR(st_source.st_mode):
assert mode is None, f'Cannot use `mode` for directory sources.'
kwargs['paths'] = tuple(
_recurse_into_source(
Path(source),
Path(dest),
dir_mode=dir_mode or _DIR_MODE,
exe_mode=exe_mode or _EXE_MODE,
data_mode=data_mode or _DATA_MODE,
))
elif stat.S_ISREG(st_source.st_mode):
assert {dir_mode, exe_mode, data_mode} == {None}, \
'Cannot use `{dir,exe,data}_mode` for file sources.'
if mode is None:
# This tests whether the build repo user can execute the
# file. This is a very natural test for build artifacts,
# and files in the repo. Note that this can be affected if
# the ambient umask is pathological, which is why
# `compiler.py` checks the umask.
mode = _EXE_MODE if os.access(source, os.X_OK) else _DATA_MODE
kwargs['paths'] = (_InstallablePath(
source=source,
provides=ProvidesFile(path=dest),
mode=mode,
), )
else:
raise RuntimeError(
f'{source} must be a regular file or directory, got {st_source}'
)
def provides(self):
# We own ZST decompression, tarfile handles other gz, bz2, etc.
import tarfile # Lazy since only this method needs it.
with open_for_read_decompress(self.source) as tf, \
tarfile.open(fileobj=tf, mode='r|') as f:
for item in f:
path = os.path.join(
self.into_dir,
make_path_normal_relative(item.name),
)
if item.isdir():
# We do NOT provide the installation directory, and the
# image build script tarball extractor takes pains (e.g.
# `tar --no-overwrite-dir`) not to touch the extraction
# directory.
if os.path.normpath(os.path.relpath(path,
self.into_dir)) != '.':
yield ProvidesDirectory(path=path)
else:
yield ProvidesFile(path=path)
def provides(self):
yield ProvidesFile(path=self.dest)
def test_install_file_command_recursive(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('d')])
with temp_dir() as td:
with open(td / 'data.txt', 'w') as df:
print('Hello', file=df)
os.mkdir(td / 'subdir')
with open(td / 'subdir/exe.sh', 'w') as ef:
print('#!/bin/sh\necho "Hello"', file=ef)
os.chmod(td / 'subdir/exe.sh', 0o100)
dir_item = _install_file_item(
from_target='t',
source={'source': td},
dest='/d/a',
)
ps = [
_InstallablePath(
td,
ProvidesDirectory(path='d/a'),
'u+rwx,og+rx',
),
_InstallablePath(
td / 'data.txt',
ProvidesFile(path='d/a/data.txt'),
'a+r',
),
_InstallablePath(
td / 'subdir',
ProvidesDirectory(path='d/a/subdir'),
'u+rwx,og+rx',
),
_InstallablePath(
td / 'subdir/exe.sh',
ProvidesFile(path='d/a/subdir/exe.sh'),
'a+rx',
),
]
self.assertEqual(sorted(ps), sorted(dir_item.paths))
self.assertEqual(td, dir_item.source)
self._check_item(dir_item, {p.provides
for p in ps},
{require_directory('d')})
# This implicitly checks that `a` precedes its contents.
dir_item.build(subvol, DUMMY_LAYER_OPTS)
self.assertEqual(
[
'(Dir)', {
'd': [
'(Dir)', {
'a': [
'(Dir)', {
'data.txt': ['(File m444 d6)'],
'subdir': [
'(Dir)', {
'exe.sh': ['(File m555 d23)'],
}
],
}
]
}
]
}
],
render_subvol(subvol),
)