def test_item_predecessors(self):
dg = DependencyGraph(PATH_TO_ITEM.values(), layer_target='t-34')
self.assertEqual(
_fs_root_phases(FilesystemRootItem(from_target='t-34')),
list(dg.ordered_phases()),
)
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
phases_provide = PhasesProvideItem(from_target='t', subvol=subvol)
ns = dg._prep_item_predecessors(phases_provide)
path_to_item = {'/': phases_provide, **PATH_TO_ITEM}
self.assertEqual(
ns.item_to_predecessors, {
path_to_item[k]: {path_to_item[v]
for v in vs}
for k, vs in {
'/a/b/c': {'/'},
'/a/d/e': {'/a/b/c'},
'/a/b/c/F': {'/a/b/c'},
'/a/d/e/G': {'/a/d/e'},
}.items()
})
self.assertEqual(
ns.predecessor_to_items, {
path_to_item[k]: {path_to_item[v]
for v in vs}
for k, vs in {
'/': {'/a/b/c'},
'/a/b/c': {'/a/d/e', '/a/b/c/F'},
'/a/b/c/F': set(),
'/a/d/e': {'/a/d/e/G'},
'/a/d/e/G': set(),
}.items()
})
self.assertEqual(ns.items_without_predecessors, {path_to_item['/']})
def test_clone_special_files(self):
with TempSubvolumes(sys.argv[0]) as temp_subvols:
src_subvol = temp_subvols.create('test_clone_special_files_src')
dest_subvol = temp_subvols.create('test_clone_special_files_dest')
src_subvol.run_as_root(['mkfifo', src_subvol.path('fifo')])
src_subvol.run_as_root([
'mknod',
src_subvol.path('null'),
'c',
'1',
'3',
])
for name in ['fifo', 'null']:
ci = self._clone_item(name, name, subvol=src_subvol)
self.assertEqual(name, ci.dest)
self._check_item(
ci,
{ProvidesFile(path=name)},
{require_directory('/')},
)
ci.build(dest_subvol, DUMMY_LAYER_OPTS)
src_r = render_subvol(src_subvol)
dest_r = render_subvol(dest_subvol)
self.assertEqual(src_r, dest_r)
self.assertEqual(
['(Dir)', {
'fifo': ['(FIFO)'],
'null': ['(Char 103)'],
}], dest_r)
def test_phases_provide(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
parent = temp_subvolumes.create('parent')
# Permit _populate_temp_filesystem to make writes.
parent.run_as_root([
'chown', '--no-dereference', f'{os.geteuid()}:{os.getegid()}',
parent.path(),
])
populate_temp_filesystem(parent.path().decode())
with self.assertRaises(subprocess.CalledProcessError):
list(gen_subvolume_subtree_provides(parent, 'no_such/path'))
for create_meta in [False, True]:
# Check that we properly handle ignoring a /meta if it's present
if create_meta:
parent.run_as_root(['mkdir', parent.path('meta')])
self._check_item(
PhasesProvideItem(from_target='t', subvol=parent),
temp_filesystem_provides() | {
ProvidesDirectory(path='/'),
ProvidesDoNotAccess(path='/meta'),
},
set(),
)
def _check_cheese_removal(self, local_rpm_path: Path):
parent_subvol = layer_resource_subvol(
__package__,
'test-with-one-local-rpm',
)
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
# ensure cheese2 is installed in the parent from rpm-test-cheese-2-1
assert os.path.isfile(parent_subvol.path('/rpm_test/cheese2.txt'))
subvol = temp_subvolumes.snapshot(parent_subvol, 'remove_cheese')
RpmActionItem.get_phase_builder(
[
RpmActionItem(
from_target='t',
source=local_rpm_path,
action=RpmAction.remove_if_exists,
)
],
self._opts(),
)(subvol)
subvol.run_as_root([
'rm',
'-rf',
subvol.path('dev'),
subvol.path('etc'),
subvol.path('meta'),
subvol.path('var'),
])
self.assertEqual(
[
'(Dir)',
{
# No more `rpm_test/cheese2.txt` here.
}
],
render_subvol(subvol))
def test_filesystem_root(self):
item = FilesystemRootItem(from_target='t')
self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order())
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.caller_will_create('fs-root')
item.get_phase_builder([item], DUMMY_LAYER_OPTS)(subvol)
self.assertEqual(
[
'(Dir)', {
'meta': [
'(Dir)', {
'private': [
'(Dir)', {
'opts': [
'(Dir)', {
'artifacts_may_require_repo':
['(File d2)'],
}
],
}
]
}
]
}
],
render_subvol(subvol),
)
def test_install_order(self):
dg = DependencyGraph(si.ID_TO_ITEM.values(), layer_target='ttt')
builders_and_phases = list(dg.ordered_phases())
self.assertEqual([
(
FilesystemRootItem.get_phase_builder,
(si.ID_TO_ITEM['/'], ),
),
(
RpmActionItem.get_phase_builder,
(
si.ID_TO_ITEM['.rpms/remove_if_exists/rpm-test-carrot'],
si.ID_TO_ITEM['.rpms/remove_if_exists/rpm-test-milk'],
),
),
(
RpmActionItem.get_phase_builder,
(
si.ID_TO_ITEM['.rpms/install/rpm-test-mice'],
si.ID_TO_ITEM['.rpms/install/rpm-test-cheese-2-1.rpm'],
),
),
(
RemovePathItem.get_phase_builder,
(
si.ID_TO_ITEM['.remove_if_exists/path/to/remove'],
si.ID_TO_ITEM['.remove_assert_exists/path/to/remove'],
si.
ID_TO_ITEM['.remove_assert_exists/another/path/to/remove'],
),
),
], builders_and_phases)
phase_items = [i for _, items in builders_and_phases for i in items]
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
doi = list(
dg.gen_dependency_order_items(
PhasesProvideItem(from_target='t', subvol=subvol), ))
self.assertEqual(set(si.ID_TO_ITEM.values()), set(doi + phase_items))
self.assertEqual(
len(si.ID_TO_ITEM),
len(doi) + len(phase_items),
msg='Duplicate items?',
)
id_to_idx = {
k: doi.index(v)
for k, v in si.ID_TO_ITEM.items() if v not in phase_items
}
# The 2 mounts are not ordered in any way with respect to the
# `foo/bar` tree, so any of these 3 can be the first.
mount_idxs = sorted([id_to_idx['host_etc'], id_to_idx['meownt']])
if mount_idxs == [0, 1]:
self.assertEqual(2, id_to_idx['foo/bar'])
elif 0 in mount_idxs:
self.assertEqual(1, id_to_idx['foo/bar'])
else:
self.assertEqual(0, id_to_idx['foo/bar'])
self.assertLess(id_to_idx['foo/borf/beep'],
id_to_idx['foo/borf/hello_world'])
def _temp_resource_subvol(self, name: str):
parent_sv = find_built_subvol(load_location(__package__, name))
with TempSubvolumes(sys.argv[0]) as temp_subvols:
# Cannot use `.snapshot()` since that doesn't handle mounts.
child_sv = temp_subvols.caller_will_create(name)
ParentLayerItem.get_phase_builder([
ParentLayerItem(from_target='t', subvol=parent_sv),
], DUMMY_LAYER_OPTS)(child_sv)
yield child_sv
def test_clone_nonexistent_source(self):
ci = self._clone_item('/no_such_path', '/none_such')
self.assertEqual('none_such', ci.dest)
with self.assertRaises(subprocess.CalledProcessError):
self._check_item(ci, set(), {require_directory('/')})
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_nonexistent_source')
with self.assertRaises(subprocess.CalledProcessError):
ci.build(subvol, DUMMY_LAYER_OPTS)
def test_install_file_command(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes, \
tempfile.NamedTemporaryFile() as empty_tf:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('d')])
_install_file_item(
from_target='t',
source={
'source': empty_tf.name
},
dest='/d/empty',
).build(subvol, DUMMY_LAYER_OPTS)
self.assertEqual(
['(Dir)', {
'd': ['(Dir)', {
'empty': ['(File m444)']
}]
}],
render_subvol(subvol),
)
# Fail to write to a nonexistent dir
with self.assertRaises(subprocess.CalledProcessError):
_install_file_item(
from_target='t',
source={
'source': empty_tf.name
},
dest='/no_dir/empty',
).build(subvol, DUMMY_LAYER_OPTS)
# Running a second copy to the same destination. This just
# overwrites the previous file, because we have a build-time
# check for this, and a run-time check would add overhead.
_install_file_item(
from_target='t',
source={
'source': empty_tf.name
},
dest='/d/empty',
# A non-default mode & owner shows that the file was
# overwritten, and also exercises HasStatOptions.
mode='u+rw',
user_group='12:34',
).build(subvol, DUMMY_LAYER_OPTS)
self.assertEqual(
['(Dir)', {
'd': ['(Dir)', {
'empty': ['(File m600 o12:34)']
}]
}],
render_subvol(subvol),
)
def test_receive_sendstream(self):
item = ReceiveSendstreamItem(
from_target='t',
source=Path(__file__).dirname() / 'create_ops.sendstream',
)
self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order())
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
new_subvol_name = 'differs_from_create_ops'
subvol = temp_subvolumes.caller_will_create(new_subvol_name)
item.get_phase_builder([item], DUMMY_LAYER_OPTS)(subvol)
self.assertEqual(
render_demo_subvols(create_ops=new_subvol_name),
render_sendstream(subvol.mark_readonly_and_get_sendstream()),
)
def _test_make_dirs_command(self, layer_opts):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('d')])
MakeDirsItem(
from_target='t',
path_to_make='/a/b/',
into_dir='/d',
user_group='77:88',
mode='u+rx',
).build(subvol, layer_opts)
self.assertEqual([
'(Dir)', {
'd': [
'(Dir)', {
'a': [
'(Dir m500 o77:88)', {
'b': ['(Dir m500 o77:88)', {}],
}
],
}
],
}
], render_subvol(subvol))
# The "should never happen" cases -- since we have build-time
# checks, for simplicity/speed, our runtime clobbers permissions
# of preexisting directories, and quietly creates non-existent
# ones with default permissions.
MakeDirsItem(from_target='t',
path_to_make='a',
into_dir='/no_dir',
user_group='4:0').build(subvol, layer_opts)
MakeDirsItem(from_target='t',
path_to_make='a/new',
into_dir='/d',
user_group='5:0').build(subvol, layer_opts)
self.assertEqual(['(Dir)', {
'd': ['(Dir)', {
# permissions overwritten for this whole tree
'a': ['(Dir o5:0)', {
'b': ['(Dir o5:0)', {}], 'new': ['(Dir o5:0)', {}],
}],
}],
'no_dir': ['(Dir)', { # default permissions!
'a': ['(Dir o4:0)', {}],
}],
}], render_subvol(subvol))
def test_clone_file(self):
ci = self._clone_item('/rpm_test/hello_world.tar', '/cloned_hello.tar')
self.assertEqual('cloned_hello.tar', ci.dest)
self._check_item(
ci,
{ProvidesFile(path='cloned_hello.tar')},
{require_directory('/')},
)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_file')
ci.build(subvol, DUMMY_LAYER_OPTS)
r = render_subvol(subvol)
ino, = pop_path(r, 'cloned_hello.tar')
self.assertRegex(ino, '(File m444 d[0-9]+)')
self.assertEqual(['(Dir)', {}], r)
def test_clone_pre_existing_dest(self):
ci = self._clone_item('/foo/bar', '/', pre_existing_dest=True)
self.assertEqual('', ci.dest)
self._check_item(
ci,
{
ProvidesDirectory(path='bar'),
ProvidesDirectory(path='bar/baz'),
ProvidesFile(path='bar/baz/bar'),
ProvidesFile(path='bar/even_more_hello_world.tar'),
},
{require_directory('/')},
)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_pre_existing_dest')
self._check_clone_bar(ci, subvol)
def _snapshot_subvol(src_subvol: Subvol,
snapshot_into: Optional[AnyStr]) -> Iterable[Subvol]:
if snapshot_into:
nspawn_subvol = Subvol(snapshot_into)
nspawn_subvol.snapshot(src_subvol)
clone_mounts(src_subvol, nspawn_subvol)
yield nspawn_subvol
else:
with TempSubvolumes() as tmp_subvols:
# To make it easier to debug where a temporary subvolume came
# from, make make its name resemble that of its source.
tmp_name = os.path.normpath(src_subvol.path())
tmp_name = os.path.basename(os.path.dirname(tmp_name)) or \
os.path.basename(tmp_name)
nspawn_subvol = tmp_subvols.snapshot(src_subvol, tmp_name)
clone_mounts(src_subvol, nspawn_subvol)
yield nspawn_subvol
def test_clone_hardlinks(self):
with TempSubvolumes(sys.argv[0]) as temp_subvols:
src_subvol = temp_subvols.create('test_clone_hardlinks_src')
dest_subvol = temp_subvols.create('test_clone_hardlinks_dest')
src_subvol.run_as_root(['touch', src_subvol.path('a')])
src_subvol.run_as_root([
'ln',
src_subvol.path('a'),
src_subvol.path('b'),
])
ci = self._clone_item(
'/',
'/',
omit_outer_dir=True,
pre_existing_dest=True,
subvol=src_subvol,
)
self.assertEqual('', ci.dest)
self._check_item(
ci,
{
ProvidesFile(path='a'),
ProvidesFile(path='b'),
# This looks like a bug (there's no /meta on disk here) but
# it's really just an artifact of how this path is
# protected. Read: This Is Fine (TM).
ProvidesDoNotAccess(path='/meta'),
},
{require_directory('/')})
ci.build(dest_subvol, DUMMY_LAYER_OPTS)
src_r = render_subvol(src_subvol)
dest_r = render_subvol(dest_subvol)
self.assertEqual(src_r, dest_r)
self.assertEqual(
[
'(Dir)',
{
# Witness that they have the same (rendered) inode # of "0"
'a': [['(File)', 0]],
'b': [['(File)', 0]],
}
],
dest_r)
def test_clone_demo_sendstream(self):
src_subvol = layer_resource_subvol(__package__, 'create_ops')
ci = self._clone_item(
'/',
'/',
omit_outer_dir=True,
pre_existing_dest=True,
subvol=src_subvol,
)
self.assertEqual({require_directory('/')}, set(ci.requires()))
self.assertGreater(len(set(ci.provides())), 1)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
dest_subvol = temp_subvols.create('create_ops')
ci.build(dest_subvol, DUMMY_LAYER_OPTS)
self.assertEqual(
render_subvol(src_subvol),
render_subvol(dest_subvol),
)
def test_parent_layer(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
parent = temp_subvolumes.create('parent')
item = ParentLayerItem(from_target='t', subvol=parent)
self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order())
MakeDirsItem(
from_target='t',
into_dir='/',
path_to_make='a/b',
).build(parent, DUMMY_LAYER_OPTS)
parent_content = ['(Dir)', {'a': ['(Dir)', {'b': ['(Dir)', {}]}]}]
self.assertEqual(parent_content, render_subvol(parent))
# Take a snapshot and add one more directory.
child = temp_subvolumes.caller_will_create('child')
item.get_phase_builder([item], DUMMY_LAYER_OPTS)(child)
MakeDirsItem(
from_target='t',
into_dir='a',
path_to_make='c',
).build(child, DUMMY_LAYER_OPTS)
# The parent is unchanged.
self.assertEqual(parent_content, render_subvol(parent))
child_content = copy.deepcopy(parent_content)
child_content[1]['a'][1]['c'] = ['(Dir)', {}]
# Since the parent lacked a /meta, the child added it.
child_content[1]['meta'] = [
'(Dir)', {
'private': [
'(Dir)', {
'opts': [
'(Dir)', {
'artifacts_may_require_repo':
['(File d2)']
}
]
}
]
}
]
self.assertEqual(child_content, render_subvol(child))
def test_rpm_action_item_auto_downgrade(self):
parent_subvol = layer_resource_subvol(
__package__,
'test-with-one-local-rpm',
)
src_rpm = Path(__file__).dirname() / "rpm-test-cheese-1-1.rpm"
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
# ensure cheese2 is installed in the parent from rpm-test-cheese-2-1
assert os.path.isfile(parent_subvol.path('/rpm_test/cheese2.txt'))
# make sure the RPM we are installing is older in order to
# trigger the downgrade
src_data = RpmMetadata.from_file(src_rpm)
subvol_data = RpmMetadata.from_subvol(parent_subvol, src_data.name)
assert compare_rpm_versions(src_data, subvol_data) < 0
subvol = temp_subvolumes.snapshot(parent_subvol, 'rpm_action')
RpmActionItem.get_phase_builder(
[
RpmActionItem(
from_target='t',
source=src_rpm,
action=RpmAction.install,
)
],
self._opts(),
)(subvol)
subvol.run_as_root([
'rm',
'-rf',
subvol.path('dev'),
subvol.path('etc'),
subvol.path('meta'),
subvol.path('var'),
])
self.assertEqual([
'(Dir)', {
'rpm_test': ['(Dir)', {
'cheese1.txt': ['(File d42)'],
}],
}
], render_subvol(subvol))
def test_clone_omit_outer_dir(self):
ci = self._clone_item(
'/foo/bar',
'/bar',
omit_outer_dir=True,
pre_existing_dest=True,
)
self.assertEqual('bar', ci.dest)
self._check_item(
ci,
{
ProvidesDirectory(path='bar/baz'),
ProvidesFile(path='bar/baz/bar'),
ProvidesFile(path='bar/even_more_hello_world.tar'),
},
{require_directory('/bar')},
)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_omit_outer_dir')
subvol.run_as_root(['mkdir', subvol.path('bar')])
self._check_clone_bar(ci, subvol)
def test_gen_dependency_graph(self):
dg = DependencyGraph(PATH_TO_ITEM.values(), layer_target='t-72')
self.assertEqual(
_fs_root_phases(FilesystemRootItem(from_target='t-72')),
list(dg.ordered_phases()),
)
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
self.assertIn(
tuple(
dg.gen_dependency_order_items(
PhasesProvideItem(from_target='t', subvol=subvol), )),
{
tuple(PATH_TO_ITEM[p] for p in paths)
for paths in [
# A few orders are valid, don't make the test fragile.
['/a/b/c', '/a/b/c/F', '/a/d/e', '/a/d/e/G'],
['/a/b/c', '/a/d/e', '/a/b/c/F', '/a/d/e/G'],
['/a/b/c', '/a/d/e', '/a/d/e/G', '/a/b/c/F'],
]
},
)
def make_demo_sendstreams(path_in_repo: bytes):
with TempSubvolumes(path_in_repo) as subvols:
res = {}
with _populate_sendstream_dict(res.setdefault('create_ops', {})) as d:
create_ops = _make_create_ops_subvolume(subvols, b'create_ops')
d['sendstream'] = create_ops.mark_readonly_and_get_sendstream()
with _populate_sendstream_dict(res.setdefault('mutate_ops', {})) as d:
d['sendstream'] = _make_mutate_ops_subvolume(
subvols,
create_ops,
b'mutate_ops',
).mark_readonly_and_get_sendstream(
parent=create_ops,
# The resulting send-stream will have `update_extent`
# instead of `write`, which is one way of making sure that
# `update_extent` in `parse_sendstream.py` is covered.
no_data=True,
)
return res
def test_cycle_detection(self):
def requires_provides_directory_class(requires_dir, provides_dir):
@dataclass(init=False, frozen=True)
class RequiresProvidesDirectory(ImageItem):
def requires(self):
yield require_directory(requires_dir)
def provides(self):
yield ProvidesDirectory(path=provides_dir)
return RequiresProvidesDirectory
# `dg_ok`: dependency-sorting will work without a cycle
first = FilesystemRootItem(from_target='')
second = requires_provides_directory_class('/', 'a')(from_target='')
third = MakeDirsItem(from_target='', into_dir='a', path_to_make='b/c')
dg_ok = DependencyGraph([second, first, third], layer_target='t')
self.assertEqual(_fs_root_phases(first), list(dg_ok.ordered_phases()))
# `dg_bad`: changes `second` to get a cycle
dg_bad = DependencyGraph([
requires_provides_directory_class('a/b', 'a')(from_target=''),
first,
third,
],
layer_target='t')
self.assertEqual(_fs_root_phases(first), list(dg_bad.ordered_phases()))
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
provides_root = PhasesProvideItem(from_target='t', subvol=subvol)
self.assertEqual(
[second, third],
list(dg_ok.gen_dependency_order_items(provides_root)),
)
with self.assertRaisesRegex(AssertionError, '^Cycle in '):
list(dg_bad.gen_dependency_order_items(provides_root))
def test_phase_order(self):
class FakeRemovePaths:
get_phase_builder = 'kittycat'
def phase_order(self):
return PhaseOrder.REMOVE_PATHS
first = FilesystemRootItem(from_target='')
second = FakeRemovePaths()
third = MakeDirsItem(from_target='', into_dir='/', path_to_make='a/b')
dg = DependencyGraph([second, first, third], layer_target='t')
self.assertEqual(
_fs_root_phases(first) + [
(FakeRemovePaths.get_phase_builder, (second, )),
],
list(dg.ordered_phases()),
)
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
self.assertEqual([third],
list(
dg.gen_dependency_order_items(
PhasesProvideItem(from_target='t',
subvol=subvol), )))
def _test_rpm_action_item_install_local_setup(self):
parent_subvol = layer_resource_subvol(__package__, 'test-with-no-rpm')
local_rpm_path = Path(__file__).dirname() / 'rpm-test-cheese-2-1.rpm'
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.snapshot(parent_subvol, 'add_cheese')
RpmActionItem.get_phase_builder(
[
RpmActionItem(
from_target='t',
source=local_rpm_path,
action=RpmAction.install,
)
],
self._opts(),
)(subvol)
r = render_subvol(subvol)
self.assertEqual(['(Dir)', {
'cheese2.txt': ['(File d45)'],
}], pop_path(r, 'rpm_test'))
yield r
def test_mount_item(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes, \
tempfile.TemporaryDirectory() as source_dir:
runtime_source = {'so': 'me', 'arbitrary': {'j': 'son'}}
mount_config = {
'is_directory': True,
'build_source': {
'type': 'layer',
'source': '//fake:path'
},
'runtime_source': runtime_source,
}
with open(os.path.join(source_dir, 'mountconfig.json'), 'w') as f:
json.dump(mount_config, f)
self._check_item(
self._make_mount_item(
mountpoint='can/haz',
target=source_dir,
mount_config=mount_config,
),
{ProvidesDoNotAccess(path='can/haz')},
{require_directory('can')},
)
# Make a subvolume that we will mount inside `mounter`
mountee = temp_subvolumes.create('moun:tee/volume')
mountee.run_as_root(['tee', mountee.path('kitteh')],
input=b'cheez')
# These sub-mounts inside `mountee` act as canaries to make sure
# that (a) `mounter` receives the sub-mounts as a consequence of
# mounting `mountee` recursively, (b) that unmounting one in
# `mounter` does not affect the original in `mountee` -- i.e.
# that rslave propagation is set up correctly, (c) that
# unmounting in `mountee` immediately affects `mounter`.
#
# In practice, our build artifacts should NEVER be mutated after
# construction (and the only un-mount is implicitly, and
# seemingly safely, performed by `btrfs subvolume delete`).
# However, ensuring that we have correct `rslave` propagation is
# a worthwhile safeguard for host mounts, where an errant
# `umount` by a user inside their repo could otherwise break
# their host.
for submount in ('submount1', 'submount2'):
mountee.run_as_root(['mkdir', mountee.path(submount)])
mountee.run_as_root([
'mount', '-o', 'bind,ro', source_dir,
mountee.path(submount)
])
self.assertTrue(
os.path.exists(mountee.path(submount +
'/mountconfig.json')))
# Make the JSON file normally in "buck-out" that refers to `mountee`
mountee_subvolumes_dir = self._write_layer_json_into(
mountee, source_dir)
# Mount <mountee> at <mounter>/meow
mounter = temp_subvolumes.caller_will_create('mount:er/volume')
root_item = FilesystemRootItem(from_target='t')
root_item.get_phase_builder([root_item], DUMMY_LAYER_OPTS)(mounter)
mount_meow = self._make_mount_item(
mountpoint='meow',
target=source_dir,
mount_config=mount_config,
)
self.assertEqual(
runtime_source,
json.loads(mount_meow.runtime_source),
)
with self.assertRaisesRegex(AssertionError, ' could not resolve '):
mount_meow.build_source.to_path(
target_to_path={},
subvolumes_dir=mountee_subvolumes_dir,
)
mount_meow.build(
mounter,
DUMMY_LAYER_OPTS._replace(
target_to_path={'//fake:path': source_dir},
subvolumes_dir=mountee_subvolumes_dir,
))
# This checks the subvolume **contents**, but not the mounts.
# Ensure the build created a mountpoint, and populated metadata.
self._check_subvol_mounts_meow(mounter)
# `mountee` was also mounted at `/meow`
with open(mounter.path('meow/kitteh')) as f:
self.assertEqual('cheez', f.read())
def check_mountee_mounter_submounts(submount_presence):
for submount, (in_mountee, in_mounter) in submount_presence:
self.assertEqual(
in_mountee,
os.path.exists(
mountee.path(submount + '/mountconfig.json')),
f'{submount}, {in_mountee}')
self.assertEqual(
in_mounter,
os.path.exists(
mounter.path('meow/' + submount +
'/mountconfig.json')),
f'{submount}, {in_mounter}')
# Both sub-mounts are accessible in both places now.
check_mountee_mounter_submounts([
('submount1', (True, True)),
('submount2', (True, True)),
])
# Unmounting `submount1` from `mountee` also affects `mounter`.
mountee.run_as_root(['umount', mountee.path('submount1')])
check_mountee_mounter_submounts([
('submount1', (False, False)),
('submount2', (True, True)),
])
# Unmounting `submount2` from `mounter` doesn't affect `mountee`.
mounter.run_as_root(['umount', mounter.path('meow/submount2')])
check_mountee_mounter_submounts([
('submount1', (False, False)),
('submount2', (True, False)),
])
# Check that we read back the `mounter` metadata, mark `/meow`
# inaccessible, and do not emit a `ProvidesFile` for `kitteh`.
self._check_item(
PhasesProvideItem(from_target='t', subvol=mounter),
{
ProvidesDirectory(path='/'),
ProvidesDoNotAccess(path='/meta'),
ProvidesDoNotAccess(path='/meow'),
},
set(),
)
# Check that we successfully clone mounts from the parent layer.
mounter_child = temp_subvolumes.caller_will_create('child/volume')
ParentLayerItem.get_phase_builder(
[ParentLayerItem(from_target='t', subvol=mounter)],
DUMMY_LAYER_OPTS,
)(mounter_child)
# The child has the same mount, and the same metadata
self._check_subvol_mounts_meow(mounter_child)
# Check that we refuse to create nested mounts.
nested_mounter = temp_subvolumes.create('nested_mounter')
nested_item = MountItem(
layer_opts=DUMMY_LAYER_OPTS,
from_target='t',
mountpoint='/whatever',
target=None,
mount_config={
'is_directory': True,
'build_source': {
'type': 'layer',
'source': '//:fake'
},
},
)
with tempfile.TemporaryDirectory() as d:
mounter_subvolumes_dir = self._write_layer_json_into(
mounter, d)
with self.assertRaisesRegex(
AssertionError,
'Refusing .* nested mount',
):
nested_item.build(
nested_mounter,
DUMMY_LAYER_OPTS._replace(
target_to_path={'//:fake': d},
subvolumes_dir=mounter_subvolumes_dir,
))
def test_remove_item(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes, \
tempfile.NamedTemporaryFile() as empty_tf:
subvol = temp_subvolumes.create('remove_action')
self.assertEqual(['(Dir)', {}], render_subvol(subvol))
MakeDirsItem(
from_target='t', path_to_make='/a/b/c', into_dir='/',
).build(subvol, DUMMY_LAYER_OPTS)
for d in ['d', 'e']:
InstallFileItem(
from_target='t', source=empty_tf.name, dest=f'/a/b/c/{d}',
).build(subvol, DUMMY_LAYER_OPTS)
MakeDirsItem(
from_target='t', path_to_make='/f/g', into_dir='/',
).build(subvol, DUMMY_LAYER_OPTS)
# Checks that `rm` won't follow symlinks
SymlinkToDirItem(
from_target='t', source='/f', dest='/a/b/f_sym',
).build(subvol, DUMMY_LAYER_OPTS_BA)
for d in ['h', 'i']:
InstallFileItem(
from_target='t', source=empty_tf.name, dest=f'/f/{d}',
).build(subvol, DUMMY_LAYER_OPTS)
SymlinkToDirItem(
from_target='t', source='/f/i', dest='/f/i_sym',
).build(subvol, DUMMY_LAYER_OPTS_BA)
intact_subvol = ['(Dir)', {
'a': ['(Dir)', {
'b': ['(Dir)', {
'c': ['(Dir)', {
'd': ['(File m444)'],
'e': ['(File m444)'],
}],
'f_sym': ['(Symlink ../../f)'],
}],
}],
'f': ['(Dir)', {
'g': ['(Dir)', {}],
'h': ['(File m444)'],
'i': ['(File m444)'],
'i_sym': ['(Symlink i)'],
}],
}]
self.assertEqual(intact_subvol, render_subvol(subvol))
# We refuse to touch protected paths, even with "if_exists". If
# the paths started with 'meta', they would trip the check in
# `_make_path_normal_relative`, so we mock-protect 'xyz'.
for prot_path in ['xyz', 'xyz/potato/carrot']:
with unittest.mock.patch(
'fs_image.compiler.items.remove_path.protected_path_set',
side_effect=lambda sv: protected_path_set(sv) | {'xyz'},
), self.assertRaisesRegex(
AssertionError, f'Cannot remove protected .*{prot_path}',
):
RemovePathItem.get_phase_builder([RemovePathItem(
from_target='t',
action=RemovePathAction.if_exists,
path=prot_path,
)], DUMMY_LAYER_OPTS)(subvol)
# Check handling of non-existent paths without removing anything
remove = RemovePathItem(
from_target='t',
action=RemovePathAction.if_exists,
path='/does/not/exist',
)
self.assertEqual(PhaseOrder.REMOVE_PATHS, remove.phase_order())
RemovePathItem.get_phase_builder([remove], DUMMY_LAYER_OPTS)(subvol)
with self.assertRaisesRegex(AssertionError, 'does not exist'):
RemovePathItem.get_phase_builder([
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/does/not/exist',
),
], DUMMY_LAYER_OPTS)(subvol)
self.assertEqual(intact_subvol, render_subvol(subvol))
# Now remove most of the subvolume.
RemovePathItem.get_phase_builder([
# These 3 removes are not covered by a recursive remove.
# And we leave behind /f/i, which lets us know that neither
# `f_sym` nor `i_sym` were followed during their deletion.
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/f/i_sym',
),
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/f/h',
),
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/f/g',
),
# The next 3 items are intentionally sequenced so that if
# they were applied in the given order, they would fail.
RemovePathItem(
from_target='t',
action=RemovePathAction.if_exists,
path='/a/b/c/e',
),
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
# The surrounding items don't delete /a/b/c/d, e.g. so
# this recursive remove is still tested.
path='/a/b/',
),
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/a/b/c/e',
),
], DUMMY_LAYER_OPTS)(subvol)
self.assertEqual(['(Dir)', {
'a': ['(Dir)', {}],
'f': ['(Dir)', {'i': ['(File m444)']}],
}], render_subvol(subvol))
def test_mount_item_file_from_host(self):
mount_config = {
'is_directory': False,
'build_source': {
'type': 'host',
'source': '/dev/null'
},
}
with self.assertRaisesRegex(AssertionError, 'must be located under'):
_mount_item_new('t', mount_config)
bad_mount_config = mount_config.copy()
bad_mount_config['runtime_source'] = bad_mount_config['build_source']
with self.assertRaisesRegex(AssertionError,
'Only `build_source` may '):
_mount_item_new('//dummy/host_mounts:t', bad_mount_config)
mount_item = _mount_item_new('//dummy/host_mounts:t', mount_config)
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('mounter')
mount_item.build(
subvol,
DUMMY_LAYER_OPTS._replace(
target_to_path={},
subvolumes_dir='unused',
))
self.assertEqual(
[
'(Dir)',
{
'lala': ['(File)'
], # An empty mountpoint for /dev/null
'meta': [
'(Dir)',
{
'private': [
'(Dir)',
{
# No `opts/artifacts_may_require_repo` here because we
# directly created the subvol instead of using an Item.
'mount': [
'(Dir)', {
'lala': [
'(Dir)', {
'MOUNT': [
'(Dir)', {
'is_directory':
['(File d2)'],
'build_source':
[
'(Dir)', {
'type':
[
'(File d5)'
],
'source':
[
f'(File d{len("/dev/null") + 1})'
],
}
],
}
]
}
]
}
],
}
]
}
],
}
],
render_subvol(subvol))
for filename, contents in (
('is_directory', '0\n'),
('build_source/type', 'host\n'),
('build_source/source', '/dev/null\n'),
):
with open(
subvol.path(
os.path.join(
'meta/private/mount/lala/MOUNT',
filename,
))) as f:
self.assertEqual(contents, f.read())
def _test_symlink_command(self, layer_opts):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('dir')])
# We need a source file to validate a SymlinkToFileItem
with tempfile.NamedTemporaryFile() as tf:
InstallFileItem(
from_target='t',
source=tf.name,
dest='/file',
).build(subvol, layer_opts)
SymlinkToDirItem(from_target='t',
source='/dir',
dest='/dir_symlink').build(subvol, layer_opts)
SymlinkToFileItem(from_target='t',
source='file',
dest='/file_symlink').build(subvol, layer_opts)
# Make a couple of absolute symlinks to test our behavior on
# linking to paths that contain those.
subvol.run_as_root([
'bash', '-c', f'''\
ln -s /file {subvol.path('abs_link_to_file').shell_quote()}
mkdir {subvol.path('my_dir').shell_quote()}
touch {subvol.path('my_dir/inner').shell_quote()}
ln -s /my_dir {subvol.path('my_dir_link').shell_quote()}
'''
])
# A simple case: we link to an absolute link.
SymlinkToFileItem(
from_target='t',
source='/abs_link_to_file',
dest='/link_to_abs_link',
).build(subvol, layer_opts)
# This link traverses a directory that is an absolute link. The
# resulting relative symlink is not traversible from outside the
# container.
SymlinkToFileItem(
from_target='t',
source='my_dir_link/inner',
dest='/dir/inner_link',
).build(subvol, layer_opts)
self.assertEqual([
'(Dir)', {
'dir': [
'(Dir)', {
'inner_link': ['(Symlink ../my_dir_link/inner)'],
}
],
'dir_symlink': ['(Symlink dir)'],
'file': ['(File m444)'],
'file_symlink': ['(Symlink file)'],
'abs_link_to_file': ['(Symlink /file)'],
'my_dir': ['(Dir)', {
'inner': ['(File)']
}],
'my_dir_link': ['(Symlink /my_dir)'],
'link_to_abs_link': ['(Symlink abs_link_to_file)'],
}
], render_subvol(subvol))
def _check_rpm_action_item(self, layer_opts):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('rpm_action')
self.assertEqual(['(Dir)', {}], render_subvol(subvol))
# The empty action is a no-op
RpmActionItem.get_phase_builder([], layer_opts)(subvol)
self.assertEqual(['(Dir)', {}], render_subvol(subvol))
# `yum-dnf-from-snapshot` needs a `/meta` directory to work
subvol.run_as_root(['mkdir', subvol.path('meta')])
self.assertEqual(
# No `opts/artifacts_may_require_repo` here because we directly
# created the subvol instead of using an Item.
['(Dir)', {'meta': ['(Dir)', {}]}], render_subvol(subvol),
)
# Specifying RPM versions is prohibited
with self.assertRaises(subprocess.CalledProcessError):
RpmActionItem.get_phase_builder(
[RpmActionItem(
from_target='m',
name='rpm-test-milk-2.71',
source=None,
action=RpmAction.install,
)],
layer_opts,
)(subvol)
# Cannot pass both `name` and `source`
with self.assertRaisesRegex(
AssertionError,
'Exactly one of `name` or `source` must be set .*',
):
RpmActionItem.get_phase_builder(
[RpmActionItem(
from_target='m',
name='rpm-test-milk',
source='rpm-test-milk',
action=RpmAction.install,
)],
layer_opts,
)(subvol)
RpmActionItem.get_phase_builder(
[
RpmActionItem(
from_target='t', name=n, action=RpmAction.install,
) for n in [
# This specific RPM contains `/bin/sh` and a
# post-install script to test `/dev/null` isolation.
'rpm-test-milk',
'rpm-test-carrot',
]
],
layer_opts,
)(subvol)
# Clean up the `dnf`, `yum` & `rpm` litter before checking the
# packages. Maybe fixme: We end up not asserting ownership /
# permissions / etc on directories like /var and /dev.
subvol.run_as_root([
'rm', '-rf',
# Annotate all paths since `sudo rm -rf` is scary.
subvol.path('var/lib/rpm'),
subvol.path('var/lib/yum'),
subvol.path('var/lib/dnf'),
subvol.path('var/log/yum.log'),
*(subvol.path('var/log/' + log) for log in [
'yum.log', 'dnf.log', 'dnf.librepo.log', 'dnf.rpm.log',
'hawkey.log',
]),
subvol.path('usr/lib/.build-id'),
subvol.path('bin/sh'),
])
if self._YUM_DNF == YumDnf.dnf:
subvol.run_as_root([
'rmdir', subvol.path('etc/dnf/modules.d'),
subvol.path('etc/dnf'), subvol.path('etc'),
])
# The way that `RpmActionItem` nspawns into build_appliance must
# gurantee that `/var/cache/{dnf,yum}` is empty. The next two
# lines test that the cache directory is empty because `rmdir`
# fails if it is not. It is important that the cache of built
# images be empty, to avoid unnecessarily increasing the
# distributed image size.
rm_cmd = ['rmdir'] if (
layer_opts.build_appliance
and not layer_opts.preserve_yum_dnf_cache
) else ['rm', '-rf']
subvol.run_as_root(rm_cmd + [
subvol.path(f'var/cache/{self._YUM_DNF.value}')
])
subvol.run_as_root([
'rmdir',
subvol.path('dev'), # made by yum_dnf_from_snapshot.py
subvol.path('meta'),
subvol.path('var/cache'),
subvol.path('var/lib'),
subvol.path('var/log'),
subvol.path('var/tmp'),
subvol.path('var'),
subvol.path('usr/lib'),
subvol.path('bin'),
])
self.assertEqual(['(Dir)', {
'rpm_test': ['(Dir)', {
'carrot.txt': ['(File d13)'],
'milk.txt': ['(File d12)'],
'post.txt': ['(File d6)'],
}],
'usr': ['(Dir)', {}],
}], render_subvol(subvol))
def test_paths_to_reqs_provs(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
provides_root = PhasesProvideItem(from_target='t', subvol=subvol)
expected = {
'/meta':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDoNotAccess(path='/meta'),
provides_root)
},
item_reqs=set(),
),
'/':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='/'), provides_root)
},
item_reqs={
ItemReq(require_directory('/'), PATH_TO_ITEM['/a/b/c'])
},
),
'/a':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a'),
PATH_TO_ITEM['/a/b/c'])
},
item_reqs={
ItemReq(require_directory('a'), PATH_TO_ITEM['/a/d/e'])
},
),
'/a/b':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a/b'),
PATH_TO_ITEM['/a/b/c'])
},
item_reqs=set(),
),
'/a/b/c':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a/b/c'),
PATH_TO_ITEM['/a/b/c'])
},
item_reqs={
ItemReq(require_directory('a/b/c'),
PATH_TO_ITEM['/a/b/c/F'])
},
),
'/a/b/c/F':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesFile(path='a/b/c/F'),
PATH_TO_ITEM['/a/b/c/F'])
},
item_reqs=set(),
),
'/a/d':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a/d'),
PATH_TO_ITEM['/a/d/e'])
},
item_reqs=set(),
),
'/a/d/e':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a/d/e'),
PATH_TO_ITEM['/a/d/e'])
},
item_reqs={
ItemReq(require_directory('a/d/e'),
PATH_TO_ITEM['/a/d/e/G'])
},
),
'/a/d/e/G':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesFile(path='a/d/e/G'),
PATH_TO_ITEM['/a/d/e/G'])
},
item_reqs=set(),
),
}
self.assertEqual(
ValidatedReqsProvs([provides_root, *PATH_TO_ITEM.values()
]).path_to_reqs_provs, expected)