def test_item_predecessors(self):
dg = DependencyGraph(PATH_TO_ITEM.values(), layer_target='t-34')
self.assertEqual(
_fs_root_phases(FilesystemRootItem(from_target='t-34')),
list(dg.ordered_phases()),
)
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
phases_provide = PhasesProvideItem(from_target='t', subvol=subvol)
ns = dg._prep_item_predecessors(phases_provide)
path_to_item = {'/': phases_provide, **PATH_TO_ITEM}
self.assertEqual(
ns.item_to_predecessors, {
path_to_item[k]: {path_to_item[v]
for v in vs}
for k, vs in {
'/a/b/c': {'/'},
'/a/d/e': {'/a/b/c'},
'/a/b/c/F': {'/a/b/c'},
'/a/d/e/G': {'/a/d/e'},
}.items()
})
self.assertEqual(
ns.predecessor_to_items, {
path_to_item[k]: {path_to_item[v]
for v in vs}
for k, vs in {
'/': {'/a/b/c'},
'/a/b/c': {'/a/d/e', '/a/b/c/F'},
'/a/b/c/F': set(),
'/a/d/e': {'/a/d/e/G'},
'/a/d/e/G': set(),
}.items()
})
self.assertEqual(ns.items_without_predecessors, {path_to_item['/']})
def test_filesystem_root(self):
item = FilesystemRootItem(from_target='t')
self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order())
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.caller_will_create('fs-root')
item.get_phase_builder([item], DUMMY_LAYER_OPTS)(subvol)
self.assertEqual(
[
'(Dir)', {
'meta': [
'(Dir)', {
'private': [
'(Dir)', {
'opts': [
'(Dir)', {
'artifacts_may_require_repo':
['(File d2)'],
}
],
}
]
}
]
}
],
render_subvol(subvol),
)
def _make_mutate_ops_subvolume(
subvols: TempSubvolumes,
create_ops: Subvol,
path: bytes,
) -> Subvol:
'Exercise the send-stream ops that are unique to snapshots.'
subvol = subvols.snapshot(create_ops, path) # snapshot
run = partial(subvol.run_as_root, cwd=subvol.path())
run(['rm', 'hello/world']) # unlink
run(['rmdir', 'dir_to_remove/']) # rmdir
run([ # remove_xattr
'setfattr',
'--remove=user.test_attr',
'hello/',
])
# You would think this would emit a `rename`, but for files, the
# sendstream instead `link`s to the new location, and unlinks the old.
run(['mv', 'goodbye', 'farewell']) # NOT a rename, {,un}link
run(['mv', 'hello/', 'hello_renamed/']) # yes, a rename!
run( # write
['dd', 'of=hello_renamed/een'],
input=b'push\n',
)
# This is a no-op because `btfs send` does not support `chattr` at
# present. However, it's good to have a canary so that our tests start
# failing the moment it is supported -- that will remind us to update
# the mock VFS. NB: The absolute path to `chattr` is a clowny hack to
# work around a clowny hack, to work around clowny hacks. Don't ask.
run(['/usr/bin/chattr', '+a', 'hello_renamed/een'])
return subvol
def test_phase_order(self):
class FakeRemovePaths:
get_phase_builder = 'kittycat'
def phase_order(self):
return PhaseOrder.REMOVE_PATHS
first = FilesystemRootItem(from_target='')
second = FakeRemovePaths()
third = MakeDirsItem(from_target='', into_dir='/', path_to_make='a/b')
dg = DependencyGraph([second, first, third])
self.assertEqual(
_fs_root_phases(first) + [
(FakeRemovePaths.get_phase_builder, (second,)),
],
list(dg.ordered_phases()),
)
# We had a phase other than PARENT_LAYER, so the dependency sorting
# will need to inspect the resulting subvolume -- let it be empty.
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
self.assertEqual([third], list(dg.gen_dependency_order_items(
subvol.path().decode(),
)))
def test_install_order(self):
dg = DependencyGraph(si.ID_TO_ITEM.values(), layer_target='ttt')
builders_and_phases = list(dg.ordered_phases())
self.assertEqual([
(
FilesystemRootItem.get_phase_builder,
(si.ID_TO_ITEM['/'], ),
),
(
RpmActionItem.get_phase_builder,
(
si.ID_TO_ITEM['.rpms/remove_if_exists/rpm-test-carrot'],
si.ID_TO_ITEM['.rpms/remove_if_exists/rpm-test-milk'],
),
),
(
RpmActionItem.get_phase_builder,
(
si.ID_TO_ITEM['.rpms/install/rpm-test-mice'],
si.ID_TO_ITEM['.rpms/install/rpm-test-cheese-2-1.rpm'],
),
),
(
RemovePathItem.get_phase_builder,
(
si.ID_TO_ITEM['.remove_if_exists/path/to/remove'],
si.ID_TO_ITEM['.remove_assert_exists/path/to/remove'],
si.
ID_TO_ITEM['.remove_assert_exists/another/path/to/remove'],
),
),
], builders_and_phases)
phase_items = [i for _, items in builders_and_phases for i in items]
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
doi = list(
dg.gen_dependency_order_items(
PhasesProvideItem(from_target='t', subvol=subvol), ))
self.assertEqual(set(si.ID_TO_ITEM.values()), set(doi + phase_items))
self.assertEqual(
len(si.ID_TO_ITEM),
len(doi) + len(phase_items),
msg='Duplicate items?',
)
id_to_idx = {
k: doi.index(v)
for k, v in si.ID_TO_ITEM.items() if v not in phase_items
}
# The 2 mounts are not ordered in any way with respect to the
# `foo/bar` tree, so any of these 3 can be the first.
mount_idxs = sorted([id_to_idx['host_etc'], id_to_idx['meownt']])
if mount_idxs == [0, 1]:
self.assertEqual(2, id_to_idx['foo/bar'])
elif 0 in mount_idxs:
self.assertEqual(1, id_to_idx['foo/bar'])
else:
self.assertEqual(0, id_to_idx['foo/bar'])
self.assertLess(id_to_idx['foo/borf/beep'],
id_to_idx['foo/borf/hello_world'])
def test_filesystem_root(self):
self._check_item(
FilesystemRootItem(from_target='t'),
{ProvidesDirectory(path='/')},
set(),
)
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.caller_will_create('fs-root')
FilesystemRootItem(from_target='t').build(subvol)
self.assertEqual(['(Dir)', {}], _render_subvol(subvol))
def test_install_file_command(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('d')])
_install_file_item(
from_target='t',
source={
'source': '/dev/null'
},
dest='/d/null',
).build(subvol, DUMMY_LAYER_OPTS)
self.assertEqual(
['(Dir)', {
'd': ['(Dir)', {
'null': ['(File m444)']
}]
}],
render_subvol(subvol),
)
# Fail to write to a nonexistent dir
with self.assertRaises(subprocess.CalledProcessError):
_install_file_item(
from_target='t',
source={
'source': '/dev/null'
},
dest='/no_dir/null',
).build(subvol, DUMMY_LAYER_OPTS)
# Running a second copy to the same destination. This just
# overwrites the previous file, because we have a build-time
# check for this, and a run-time check would add overhead.
_install_file_item(
from_target='t',
source={
'source': '/dev/null'
},
dest='/d/null',
# A non-default mode & owner shows that the file was
# overwritten, and also exercises HasStatOptions.
mode='u+rw',
user_group='12:34',
).build(subvol, DUMMY_LAYER_OPTS)
self.assertEqual(
['(Dir)', {
'd': ['(Dir)', {
'null': ['(File m600 o12:34)']
}]
}],
render_subvol(subvol),
)
def test_rpm_action_item_auto_downgrade(self):
parent_subvol = find_built_subvol(
(Path(__file__).dirname() / 'test-with-one-local-rpm').decode())
src_rpm = Path(__file__).dirname() / "rpm-test-cheese-1-1.rpm"
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
# ensure cheese2 is installed in the parent from rpm-test-cheese-2-1
assert os.path.isfile(
parent_subvol.path('/usr/share/rpm_test/cheese2.txt'))
# make sure the RPM we are installing is older in order to
# trigger the downgrade
src_data = RpmMetadata.from_file(src_rpm)
subvol_data = RpmMetadata.from_subvol(parent_subvol, src_data.name)
assert compare_rpm_versions(src_data, subvol_data) < 0
subvol = temp_subvolumes.snapshot(parent_subvol, 'rpm_action')
RpmActionItem.get_phase_builder(
[
RpmActionItem(
from_target='t',
source=src_rpm,
action=RpmAction.install,
)
],
DUMMY_LAYER_OPTS._replace(
yum_from_snapshot=Path(__file__).dirname() /
'yum-from-test-snapshot', ),
)(subvol)
subvol.run_as_root([
'rm',
'-rf',
subvol.path('dev'),
subvol.path('meta'),
subvol.path('var'),
])
self.assertEqual([
'(Dir)', {
'usr': [
'(Dir)', {
'share': [
'(Dir)', {
'rpm_test': [
'(Dir)', {
'cheese1.txt': ['(File d36)'],
}
],
}
],
}
],
}
], render_subvol(subvol))
def test_copy_file_command(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('d')])
CopyFileItem(
# `dest` has a rsync-convention trailing /
from_target='t',
source='/dev/null',
dest='/d/',
).build(subvol)
self.assertEqual(
['(Dir)', {
'd': ['(Dir)', {
'null': ['(File m755)']
}]
}],
_render_subvol(subvol),
)
# Fail to write to a nonexistent dir
with self.assertRaises(subprocess.CalledProcessError):
CopyFileItem(
from_target='t',
source='/dev/null',
dest='/no_dir/',
).build(subvol)
# Running a second copy to the same destination. This just
# overwrites the previous file, because we have a build-time
# check for this, and a run-time check would add overhead.
CopyFileItem(
# Test this works without the rsync-covnvention /, too
from_target='t',
source='/dev/null',
dest='/d/null',
# A non-default mode & owner shows that the file was
# overwritten, and also exercises HasStatOptions.
mode='u+rw',
user='******',
group='34',
).build(subvol)
self.assertEqual(
['(Dir)', {
'd': ['(Dir)', {
'null': ['(File m600 o12:34)']
}]
}],
_render_subvol(subvol),
)
def test_make_dirs_command(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('d')])
MakeDirsItem(
from_target='t',
path_to_make='/a/b/',
into_dir='/d',
user='******',
group='88',
mode='u+rx',
).build(subvol)
self.assertEqual([
'(Dir)', {
'd': [
'(Dir)', {
'a': [
'(Dir m500 o77:88)', {
'b': ['(Dir m500 o77:88)', {}],
}
],
}
],
}
], _render_subvol(subvol))
# The "should never happen" cases -- since we have build-time
# checks, for simplicity/speed, our runtime clobbers permissions
# of preexisting directories, and quietly creates non-existent
# ones with default permissions.
MakeDirsItem(from_target='t',
path_to_make='a',
into_dir='/no_dir',
user='******').build(subvol)
MakeDirsItem(from_target='t',
path_to_make='a/new',
into_dir='/d',
user='******').build(subvol)
self.assertEqual(['(Dir)', {
'd': ['(Dir)', {
# permissions overwritten for this whole tree
'a': ['(Dir o5:0)', {
'b': ['(Dir o5:0)', {}], 'new': ['(Dir o5:0)', {}],
}],
}],
'no_dir': ['(Dir)', { # default permissions!
'a': ['(Dir o4:0)', {}],
}],
}], _render_subvol(subvol))
def test_receive_sendstream(self):
item = ReceiveSendstreamItem(
from_target='t',
source=Path(__file__).dirname() / 'create_ops.sendstream',
)
self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order())
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
new_subvol_name = 'differs_from_create_ops'
subvol = temp_subvolumes.caller_will_create(new_subvol_name)
item.get_phase_builder([item], DUMMY_LAYER_OPTS)(subvol)
self.assertEqual(
render_demo_subvols(create_ops=new_subvol_name),
render_sendstream(subvol.mark_readonly_and_get_sendstream()),
)
def _snapshot_subvol(src_subvol, snapshot_into):
if snapshot_into:
nspawn_subvol = Subvol(snapshot_into)
nspawn_subvol.snapshot(src_subvol)
clone_mounts(src_subvol, nspawn_subvol)
yield nspawn_subvol
else:
with TempSubvolumes() as tmp_subvols:
# To make it easier to debug where a temporary subvolume came
# from, make make its name resemble that of its source.
tmp_name = os.path.normpath(src_subvol.path())
tmp_name = os.path.basename(os.path.dirname(tmp_name)) or \
os.path.basename(tmp_name)
nspawn_subvol = tmp_subvols.snapshot(src_subvol, tmp_name)
clone_mounts(src_subvol, nspawn_subvol)
yield nspawn_subvol
def make_demo_sendstreams(path_in_repo: bytes):
with TempSubvolumes(path_in_repo) as subvols:
res = {}
with _populate_sendstream_dict(res.setdefault('create_ops', {})) as d:
create_ops = _make_create_ops_subvolume(subvols, b'create_ops')
d['sendstream'] = create_ops.mark_readonly_and_get_sendstream(
no_data=True, )
with _populate_sendstream_dict(res.setdefault('mutate_ops', {})) as d:
d['sendstream'] = _make_mutate_ops_subvolume(
subvols,
create_ops,
b'mutate_ops',
).mark_readonly_and_get_sendstream(parent=create_ops)
return res
def test_package_image_as_squashfs(self):
with self._package_image(
self._sibling_path('create_ops.layer'), 'squashfs',
) as out_path, TempSubvolumes(sys.argv[0]) as temp_subvolumes, \
tempfile.NamedTemporaryFile() as temp_sendstream:
subvol = temp_subvolumes.create('subvol')
with Unshare([Namespace.MOUNT, Namespace.PID]) as unshare, \
tempfile.TemporaryDirectory() as mount_dir:
subprocess.check_call(nsenter_as_root(
unshare, 'mount', '-t', 'squashfs', '-o', 'loop',
out_path, mount_dir,
))
# `unsquashfs` would have been cleaner than `mount` +
# `rsync`, and faster too, but unfortunately it corrupts
# device nodes as of v4.3.
subprocess.check_call(nsenter_as_root(
unshare, 'rsync', '--archive', '--hard-links',
'--sparse', '--xattrs', mount_dir + '/', subvol.path(),
))
with subvol.mark_readonly_and_write_sendstream_to_file(
temp_sendstream
):
pass
original_render = _render_sendstream_path(
self._sibling_path('create_ops-original.sendstream'),
)
# SquashFS does not preserve the original's cloned extents of
# zeros, nor the zero-hole-zero patter. In all cases, it
# (efficiently) transmutes the whole file into 1 sparse hole.
self.assertEqual(original_render[1].pop('56KB_nuls'), [
'(File d57344(create_ops@56KB_nuls_clone:0+49152@0/' +
'create_ops@56KB_nuls_clone:49152+8192@49152))'
])
original_render[1]['56KB_nuls'] = ['(File h57344)']
self.assertEqual(original_render[1].pop('56KB_nuls_clone'), [
'(File d57344(create_ops@56KB_nuls:0+49152@0/' +
'create_ops@56KB_nuls:49152+8192@49152))'
])
original_render[1]['56KB_nuls_clone'] = ['(File h57344)']
self.assertEqual(original_render[1].pop('zeros_hole_zeros'), [
'(File d16384h16384d16384)'
])
original_render[1]['zeros_hole_zeros'] = ['(File h49152)']
self.assertEqual(
original_render, _render_sendstream_path(temp_sendstream.name),
)
def test_parent_layer(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
parent = temp_subvolumes.create('parent')
item = ParentLayerItem(from_target='t', subvol=parent)
self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order())
MakeDirsItem(
from_target='t',
into_dir='/',
path_to_make='a/b',
).build(parent, DUMMY_LAYER_OPTS)
parent_content = ['(Dir)', {'a': ['(Dir)', {'b': ['(Dir)', {}]}]}]
self.assertEqual(parent_content, render_subvol(parent))
# Take a snapshot and add one more directory.
child = temp_subvolumes.caller_will_create('child')
item.get_phase_builder([item], DUMMY_LAYER_OPTS)(child)
MakeDirsItem(
from_target='t',
into_dir='a',
path_to_make='c',
).build(child, DUMMY_LAYER_OPTS)
# The parent is unchanged.
self.assertEqual(parent_content, render_subvol(parent))
child_content = copy.deepcopy(parent_content)
child_content[1]['a'][1]['c'] = ['(Dir)', {}]
# Since the parent lacked a /meta, the child added it.
child_content[1]['meta'] = [
'(Dir)', {
'private': [
'(Dir)', {
'opts': [
'(Dir)', {
'artifacts_may_require_repo':
['(File d2)']
}
]
}
]
}
]
self.assertEqual(child_content, render_subvol(child))
def test_rpm_build_item(self):
parent_subvol = find_built_subvol(
(Path(__file__).dirname() / 'toy-rpmbuild-setup').decode())
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
assert os.path.isfile(
parent_subvol.path('/rpmbuild/SOURCES/toy_src_file'))
assert os.path.isfile(
parent_subvol.path('/rpmbuild/SPECS/specfile.spec'))
subvol = temp_subvolumes.snapshot(parent_subvol, 'rpm_build')
item = RpmBuildItem(from_target='t', rpmbuild_dir='/rpmbuild')
RpmBuildItem.get_phase_builder(
[item],
DUMMY_LAYER_OPTS,
)(subvol)
self.assertEqual(item.phase_order(), PhaseOrder.RPM_BUILD)
assert os.path.isfile(subvol.path('/rpmbuild/RPMS/toy.rpm'))
def _make_mutate_ops_subvolume(
subvols: TempSubvolumes,
create_ops: Subvol,
path: bytes,
) -> Subvol:
'Exercise the send-stream ops that are unique to snapshots.'
subvol = subvols.snapshot(create_ops, path) # snapshot
run = subvol.run_as_root
# `cwd` is intentionally prohibited with `run_as_root`
def p(sv_path):
return subvol.path(sv_path).decode()
run(['rm', p('hello/world')]) # unlink
run(['rmdir', p('dir_to_remove/')]) # rmdir
run([ # remove_xattr
'setfattr',
'--remove=user.test_attr',
p('hello/'),
])
# You would think this would emit a `rename`, but for files, the
# sendstream instead `link`s to the new location, and unlinks the old.
run(['mv', p('goodbye'), p('farewell')]) # NOT a rename, {,un}link
run(['mv', p('hello/'), p('hello_renamed/')]) # yes, a rename!
run( # write
['dd', 'of=' + p('hello_renamed/een')],
input=b'push\n',
)
# This is a no-op because `btfs send` does not support `chattr` at
# present. However, it's good to have a canary so that our tests start
# failing the moment it is supported -- that will remind us to update
# the mock VFS. NB: The absolute path to `chattr` is a clowny hack to
# work around a clowny hack, to work around clowny hacks. Don't ask.
run(['/usr/bin/chattr', '+a', p('hello_renamed/een')])
# Besides files with trailing holes, one can also get `truncate`
# sendstream commands in incremental sendstreams by having a snapshot
# truncate relative a file relative to the parent.
run(['truncate', '-s', '2', p('hello_big_hole')])
return subvol
def test_symlink_command(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('dir')])
# We need a source file to validate a SymlinkToFileItem
InstallFileItem(
from_target='t', source='/dev/null', dest='/file',
).build(subvol, DUMMY_LAYER_OPTS)
SymlinkToDirItem(
from_target='t', source='/dir', dest='/dir_symlink'
).build(subvol, DUMMY_LAYER_OPTS)
SymlinkToFileItem(
from_target='t', source='file', dest='/file_symlink'
).build(subvol, DUMMY_LAYER_OPTS)
self.assertEqual(['(Dir)', {
'dir': ['(Dir)', {}],
'dir_symlink': ['(Symlink /dir)'],
'file': ['(File m444)'],
'file_symlink': ['(Symlink /file)'],
}], render_subvol(subvol))
def test_gen_dependency_graph(self):
dg = DependencyGraph(PATH_TO_ITEM.values(), layer_target='t-72')
self.assertEqual(
_fs_root_phases(FilesystemRootItem(from_target='t-72')),
list(dg.ordered_phases()),
)
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
self.assertIn(
tuple(
dg.gen_dependency_order_items(
PhasesProvideItem(from_target='t', subvol=subvol), )),
{
tuple(PATH_TO_ITEM[p] for p in paths)
for paths in [
# A few orders are valid, don't make the test fragile.
['/a/b/c', '/a/b/c/F', '/a/d/e', '/a/d/e/G'],
['/a/b/c', '/a/d/e', '/a/b/c/F', '/a/d/e/G'],
['/a/b/c', '/a/d/e', '/a/d/e/G', '/a/b/c/F'],
]
},
)
def make_demo_sendstreams(path_in_repo: bytes):
with TempSubvolumes(path_in_repo) as subvols:
res = {}
with _populate_sendstream_dict(res.setdefault('create_ops', {})) as d:
create_ops = _make_create_ops_subvolume(subvols, b'create_ops')
d['sendstream'] = create_ops.mark_readonly_and_get_sendstream()
with _populate_sendstream_dict(res.setdefault('mutate_ops', {})) as d:
d['sendstream'] = _make_mutate_ops_subvolume(
subvols,
create_ops,
b'mutate_ops',
).mark_readonly_and_get_sendstream(
parent=create_ops,
# The resulting send-stream will have `update_extent`
# instead of `write`, which is one way of making sure that
# `update_extent` in `parse_sendstream.py` is covered.
no_data=True,
)
return res
def test_phases_provide(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
parent = temp_subvolumes.create('parent')
# Permit _populate_temp_filesystem to make writes.
parent.run_as_root([
'chown',
'--no-dereference',
f'{os.geteuid()}:{os.getegid()}',
parent.path(),
])
populate_temp_filesystem(parent.path().decode())
for create_meta in [False, True]:
# Check that we properly handle ignoring a /meta if it's present
if create_meta:
parent.run_as_root(['mkdir', parent.path('meta')])
self._check_item(
PhasesProvideItem(from_target='t', subvol=parent),
temp_filesystem_provides() | {
ProvidesDirectory(path='/'),
ProvidesDoNotAccess(path='/meta'),
},
set(),
)
def test_cycle_detection(self):
def requires_provides_directory_class(requires_dir, provides_dir):
class RequiresProvidesDirectory(metaclass=ImageItem):
def requires(self):
yield require_directory(requires_dir)
def provides(self):
yield ProvidesDirectory(path=provides_dir)
return RequiresProvidesDirectory
# `dg_ok`: dependency-sorting will work without a cycle
first = FilesystemRootItem(from_target='')
second = requires_provides_directory_class('/', 'a')(from_target='')
third = MakeDirsItem(from_target='', into_dir='a', path_to_make='b/c')
dg_ok = DependencyGraph([second, first, third], layer_target='t')
self.assertEqual(_fs_root_phases(first), list(dg_ok.ordered_phases()))
# `dg_bad`: changes `second` to get a cycle
dg_bad = DependencyGraph([
requires_provides_directory_class('a/b', 'a')(from_target=''),
first,
third,
],
layer_target='t')
self.assertEqual(_fs_root_phases(first), list(dg_bad.ordered_phases()))
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
provides_root = PhasesProvideItem(from_target='t', subvol=subvol)
self.assertEqual(
[second, third],
list(dg_ok.gen_dependency_order_items(provides_root)),
)
with self.assertRaisesRegex(AssertionError, '^Cycle in '):
list(dg_bad.gen_dependency_order_items(provides_root))
def test_parent_layer(self):
# First, get reasonable coverage of enumerating files and
# directories without a real btrfs subvol.
with self._temp_filesystem() as parent_path:
self._check_item(
ParentLayerItem(from_target='t', path=parent_path),
self._temp_filesystem_provides() | {
ProvidesDirectory(path='/'),
},
set(),
)
# Now exercise actually making a btrfs snapshot.
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
parent = temp_subvolumes.create('parent')
MakeDirsItem(
from_target='t',
into_dir='/',
path_to_make='a/b',
).build(parent)
parent_content = ['(Dir)', {'a': ['(Dir)', {'b': ['(Dir)', {}]}]}]
self.assertEqual(parent_content, _render_subvol(parent))
# Take a snapshot and add one more directory.
child = temp_subvolumes.caller_will_create('child')
ParentLayerItem(from_target='t', path=parent.path()).build(child)
MakeDirsItem(
from_target='t',
into_dir='a',
path_to_make='c',
).build(child)
# The parent is unchanged.
self.assertEqual(parent_content, _render_subvol(parent))
child_content = copy.deepcopy(parent_content)
child_content[1]['a'][1]['c'] = ['(Dir)', {}]
self.assertEqual(child_content, _render_subvol(child))
def test_phase_order(self):
class FakeRemovePaths:
get_phase_builder = 'kittycat'
def phase_order(self):
return PhaseOrder.REMOVE_PATHS
first = FilesystemRootItem(from_target='')
second = FakeRemovePaths()
third = MakeDirsItem(from_target='', into_dir='/', path_to_make='a/b')
dg = DependencyGraph([second, first, third], layer_target='t')
self.assertEqual(
_fs_root_phases(first) + [
(FakeRemovePaths.get_phase_builder, (second, )),
],
list(dg.ordered_phases()),
)
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
self.assertEqual([third],
list(
dg.gen_dependency_order_items(
PhasesProvideItem(from_target='t',
subvol=subvol), )))
def test_remove_item(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('remove_action')
self.assertEqual(['(Dir)', {}], render_subvol(subvol))
MakeDirsItem(
from_target='t',
path_to_make='/a/b/c',
into_dir='/',
).build(subvol, DUMMY_LAYER_OPTS)
for d in ['d', 'e']:
InstallFileItem(
from_target='t',
source='/dev/null',
dest=f'/a/b/c/{d}',
).build(subvol, DUMMY_LAYER_OPTS)
MakeDirsItem(
from_target='t',
path_to_make='/f/g',
into_dir='/',
).build(subvol, DUMMY_LAYER_OPTS)
# Checks that `rm` won't follow symlinks
SymlinkToDirItem(
from_target='t',
source='/f',
dest='/a/b/f_sym',
).build(subvol, DUMMY_LAYER_OPTS)
for d in ['h', 'i']:
InstallFileItem(
from_target='t',
source='/dev/null',
dest=f'/f/{d}',
).build(subvol, DUMMY_LAYER_OPTS)
SymlinkToDirItem(
from_target='t',
source='/f/i',
dest='/f/i_sym',
).build(subvol, DUMMY_LAYER_OPTS)
intact_subvol = [
'(Dir)', {
'a': [
'(Dir)', {
'b': [
'(Dir)', {
'c': [
'(Dir)', {
'd': ['(File m444)'],
'e': ['(File m444)'],
}
],
'f_sym': ['(Symlink /f)'],
}
],
}
],
'f': [
'(Dir)', {
'g': ['(Dir)', {}],
'h': ['(File m444)'],
'i': ['(File m444)'],
'i_sym': ['(Symlink /f/i)'],
}
],
}
]
self.assertEqual(intact_subvol, render_subvol(subvol))
# We refuse to touch protected paths, even with "if_exists". If
# the paths started with 'meta', they would trip the check in
# `_make_path_normal_relative`, so we mock-protect 'xyz'.
for prot_path in ['xyz', 'xyz/potato/carrot']:
with unittest.mock.patch(
'fs_image.compiler.items.remove_path.protected_path_set',
side_effect=lambda sv: protected_path_set(sv) |
{'xyz'},
), self.assertRaisesRegex(
AssertionError,
f'Cannot remove protected .*{prot_path}',
):
RemovePathItem.get_phase_builder([
RemovePathItem(
from_target='t',
action=RemovePathAction.if_exists,
path=prot_path,
)
], DUMMY_LAYER_OPTS)(subvol)
# Check handling of non-existent paths without removing anything
remove = RemovePathItem(
from_target='t',
action=RemovePathAction.if_exists,
path='/does/not/exist',
)
self.assertEqual(PhaseOrder.REMOVE_PATHS, remove.phase_order())
RemovePathItem.get_phase_builder([remove],
DUMMY_LAYER_OPTS)(subvol)
with self.assertRaisesRegex(AssertionError, 'does not exist'):
RemovePathItem.get_phase_builder([
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/does/not/exist',
),
], DUMMY_LAYER_OPTS)(subvol)
self.assertEqual(intact_subvol, render_subvol(subvol))
# Now remove most of the subvolume.
RemovePathItem.get_phase_builder(
[
# These 3 removes are not covered by a recursive remove.
# And we leave behind /f/i, which lets us know that neither
# `f_sym` nor `i_sym` were followed during their deletion.
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/f/i_sym',
),
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/f/h',
),
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/f/g',
),
# The next 3 items are intentionally sequenced so that if
# they were applied in the given order, they would fail.
RemovePathItem(
from_target='t',
action=RemovePathAction.if_exists,
path='/a/b/c/e',
),
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
# The surrounding items don't delete /a/b/c/d, e.g. so
# this recursive remove is still tested.
path='/a/b/',
),
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/a/b/c/e',
),
],
DUMMY_LAYER_OPTS)(subvol)
self.assertEqual([
'(Dir)', {
'a': ['(Dir)', {}],
'f': ['(Dir)', {
'i': ['(File m444)']
}],
}
], render_subvol(subvol))
def _make_create_ops_subvolume(subvols: TempSubvolumes, path: bytes) -> Subvol:
'Exercise all the send-stream ops that can occur on a new subvolume.'
subvol = subvols.create(path)
run = subvol.run_as_root
# `cwd` is intentionally prohibited with `run_as_root`
def p(sv_path):
return subvol.path(sv_path).decode()
# Due to an odd `btrfs send` implementation detail, creating a file or
# directory emits a rename from a temporary name to the final one.
run(['mkdir', p('hello')]) # mkdir,rename
run(['mkdir', p('dir_to_remove')])
run(['touch', p('hello/world')]) # mkfile,utimes,chmod,chown
run([ # set_xattr
'setfattr',
'-n',
'user.test_attr',
'-v',
'chickens',
p('hello/'),
])
run(['mknod', p('buffered'), 'b', '1337', '31415']) # mknod
run(['chmod', 'og-r', p('buffered')]) # chmod a device
run(['mknod', p('unbuffered'), 'c', '1337', '31415'])
run(['mkfifo', p('fifo')]) # mkfifo
run([
'python3',
'-c',
(
'import os, sys, socket as s\n'
'dir, base = os.path.split(sys.argv[1])\n'
# Otherwise, we can easily get "AF_UNIX path too long"
'os.chdir(os.path.join(".", dir))\n'
'with s.socket(s.AF_UNIX, s.SOCK_STREAM) as sock:\n'
' sock.bind(base)\n' # mksock
),
p('unix_sock')
])
run(['ln', p('hello/world'), p('goodbye')]) # link
run(['ln', '-s', 'hello/world', p('bye_symlink')]) # symlink
run([ # update_extent
# 56KB was chosen so that `btrfs send` emits more than 1 write,
# specifically 48KB + 8KB.
'dd',
'if=/dev/zero',
'of=' + p('56KB_nuls'),
'bs=1024',
'count=56',
])
run([ # clone
'cp',
'--reflink=always',
p('56KB_nuls'),
p('56KB_nuls_clone'),
])
# Make a file with a 16KB hole in the middle.
run([
'dd',
'if=/dev/zero',
'of=' + p('zeros_hole_zeros'),
'bs=1024',
'count=16',
])
run(['truncate', '-s', str(32 * 1024), p('zeros_hole_zeros')])
run([
'dd',
'if=/dev/zero',
'of=' + p('zeros_hole_zeros'),
'oflag=append',
'conv=notrunc',
'bs=1024',
'count=16',
])
# A trailing hole exercises the `truncate` sendstream command.
run(['bash', '-c', 'echo hello > ' + shlex.quote(p('hello_big_hole'))])
run(['truncate', '-s', '1G', p('hello_big_hole')])
# This just serves to show that `btrfs send` ignores nested subvolumes.
# There is no mention of `nested_subvol` in the send-stream.
nested_subvol = subvols.create(p('nested_subvol'))
nested_subvol.run_as_root(['touch', nested_subvol.path('borf')])
nested_subvol.run_as_root(['mkdir', nested_subvol.path('beep')])
return subvol
def _make_create_ops_subvolume(subvols: TempSubvolumes, path: bytes) -> Subvol:
'Exercise all the send-stream ops that can occur on a new subvolume.'
subvol = subvols.create(path)
run = partial(subvol.run_as_root, cwd=subvol.path())
# Due to an odd `btrfs send` implementation detail, creating a file or
# directory emits a rename from a temporary name to the final one.
run(['mkdir', 'hello']) # mkdir,rename
run(['mkdir', 'dir_to_remove'])
run(['touch', 'hello/world']) # mkfile,utimes,chmod,chown
run([ # set_xattr
'setfattr',
'-n',
'user.test_attr',
'-v',
'chickens',
'hello/',
])
run(['mknod', 'buffered', 'b', '1337', '31415']) # mknod
run(['chmod', 'og-r', 'buffered']) # chmod a device
run(['mknod', 'unbuffered', 'c', '1337', '31415'])
run(['mkfifo', 'fifo']) # mkfifo
run([
'python3',
'-c',
(
'import socket as s\n'
'with s.socket(s.AF_UNIX, s.SOCK_STREAM) as sock:\n'
' sock.bind("unix_sock")\n' # mksock
)
])
run(['ln', 'hello/world', 'goodbye']) # link
run(['ln', '-s', 'hello/world', 'bye_symlink']) # symlink
run([ # update_extent
# 56KB was chosen so that `btrfs send` emits more than 1 write,
# specifically 48KB + 8KB.
'dd',
'if=/dev/zero',
'of=56KB_nuls',
'bs=1024',
'count=56',
])
run([ # clone
'cp',
'--reflink=always',
'56KB_nuls',
'56KB_nuls_clone',
])
# Make a file with a 16KB hole in the middle.
run(['dd', 'if=/dev/zero', 'of=zeros_hole_zeros', 'bs=1024', 'count=16'])
run(['truncate', '-s', str(32 * 1024), 'zeros_hole_zeros'])
run([
'dd',
'if=/dev/zero',
'of=zeros_hole_zeros',
'oflag=append',
'conv=notrunc',
'bs=1024',
'count=16',
])
# This just serves to show that `btrfs send` ignores nested subvolumes.
# There is no mention of `nested_subvol` in the send-stream.
nested_subvol = subvols.create(subvol.path(b'nested_subvol'))
run2 = partial(nested_subvol.run_as_root, cwd=nested_subvol.path())
run2(['touch', 'borf'])
run2(['mkdir', 'beep'])
return subvol
def test_multi_rpm_action(self):
# This works in @mode/opt since this binary is baked into the XAR
yum = os.path.join(os.path.dirname(__file__), 'yum-from-test-snapshot')
mice = MultiRpmAction.new(
frozenset(['rpm-test-mice']),
RpmActionType.install,
yum,
)
carrot = MultiRpmAction.new(
frozenset(['rpm-test-carrot']),
RpmActionType.install,
yum,
)
buggy_mice = mice._replace(yum_from_snapshot='bug')
with self.assertRaises(AssertionError):
buggy_mice.union(carrot)
action = mice.union(carrot)
self.assertEqual({'rpm-test-mice', 'rpm-test-carrot'}, action.rpms)
self.assertEqual(carrot, action._replace(rpms={'rpm-test-carrot'}))
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('rpm_action')
self.assertEqual(['(Dir)', {}], _render_subvol(subvol))
# The empty action is a no-op
MultiRpmAction.new(
frozenset(),
RpmActionType.install,
None,
).build(subvol)
self.assertEqual(['(Dir)', {}], _render_subvol(subvol))
# Specifying RPM versions is prohibited
with self.assertRaises(subprocess.CalledProcessError):
MultiRpmAction.new(
frozenset(['rpm-test-mice-2']),
RpmActionType.install,
yum,
).build(subvol)
action.build(subvol)
# Clean up the `yum` & `rpm` litter before checking the packages.
subvol.run_as_root([
'rm',
'-rf',
# Annotate all paths since `sudo rm -rf` is scary.
subvol.path('var/cache/yum'),
subvol.path('var/lib/yum'),
subvol.path('var/lib/rpm'),
subvol.path('var/log/yum.log'),
])
subvol.run_as_root([
'rmdir',
'var/cache',
'var/lib',
'var/log',
'var',
],
cwd=subvol.path())
self.assertEqual([
'(Dir)', {
'usr': [
'(Dir)', {
'share': [
'(Dir)', {
'rpm_test': [
'(Dir)', {
'carrot.txt': ['(File d13)'],
'mice.txt': ['(File d11)'],
}
],
}
],
}
],
}
], _render_subvol(subvol))
def test_tarball_command(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('d')])
# Fail on pre-existing files
subvol.run_as_root(['touch', subvol.path('d/exists')])
with tempfile.NamedTemporaryFile() as t:
with tarfile.TarFile(t.name, 'w') as tar_obj:
tar_obj.addfile(tarfile.TarInfo('exists'))
with self.assertRaises(subprocess.CalledProcessError):
TarballItem(
from_target='t',
into_dir='/d',
tarball=t.name,
).build(subvol)
# Adding new files & directories works. Overwriting a
# pre-existing directory leaves the owner+mode of the original
# directory intact.
subvol.run_as_root(['mkdir', subvol.path('d/old_dir')])
subvol.run_as_root(['chown', '123:456', subvol.path('d/old_dir')])
subvol.run_as_root(['chmod', '0301', subvol.path('d/old_dir')])
with tempfile.NamedTemporaryFile() as t:
with tarfile.TarFile(t.name, 'w') as tar_obj:
tar_obj.addfile(tarfile.TarInfo('new_file'))
new_dir = tarfile.TarInfo('new_dir')
new_dir.type = tarfile.DIRTYPE
tar_obj.addfile(new_dir)
old_dir = tarfile.TarInfo('old_dir')
old_dir.type = tarfile.DIRTYPE
# These will not be applied because old_dir exists
old_dir.uid = 0
old_dir.gid = 0
old_dir.mode = 0o755
tar_obj.addfile(old_dir)
# Fail when the destination does not exist
with self.assertRaises(subprocess.CalledProcessError):
TarballItem(
from_target='t',
into_dir='/no_dir',
tarball=t.name,
).build(subvol)
# Check the subvolume content before and after unpacking
content = [
'(Dir)', {
'd': [
'(Dir)', {
'exists': ['(File)'],
'old_dir': ['(Dir m301 o123:456)', {}],
}
]
}
]
self.assertEqual(content, _render_subvol(subvol))
TarballItem(
from_target='t',
into_dir='/d',
tarball=t.name,
).build(subvol)
content[1]['d'][1].update({
'new_dir': ['(Dir m644)', {}],
'new_file': ['(File)'],
})
self.assertEqual(content, _render_subvol(subvol))
def test_paths_to_reqs_provs(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
provides_root = PhasesProvideItem(from_target='t', subvol=subvol)
expected = {
'/meta':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDoNotAccess(path='/meta'),
provides_root)
},
item_reqs=set(),
),
'/':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='/'), provides_root)
},
item_reqs={
ItemReq(require_directory('/'), PATH_TO_ITEM['/a/b/c'])
},
),
'/a':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a'),
PATH_TO_ITEM['/a/b/c'])
},
item_reqs={
ItemReq(require_directory('a'), PATH_TO_ITEM['/a/d/e'])
},
),
'/a/b':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a/b'),
PATH_TO_ITEM['/a/b/c'])
},
item_reqs=set(),
),
'/a/b/c':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a/b/c'),
PATH_TO_ITEM['/a/b/c'])
},
item_reqs={
ItemReq(require_directory('a/b/c'),
PATH_TO_ITEM['/a/b/c/F'])
},
),
'/a/b/c/F':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesFile(path='a/b/c/F'),
PATH_TO_ITEM['/a/b/c/F'])
},
item_reqs=set(),
),
'/a/d':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a/d'),
PATH_TO_ITEM['/a/d/e'])
},
item_reqs=set(),
),
'/a/d/e':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a/d/e'),
PATH_TO_ITEM['/a/d/e'])
},
item_reqs={
ItemReq(require_directory('a/d/e'),
PATH_TO_ITEM['/a/d/e/G'])
},
),
'/a/d/e/G':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesFile(path='a/d/e/G'),
PATH_TO_ITEM['/a/d/e/G'])
},
item_reqs=set(),
),
}
self.assertEqual(
ValidatedReqsProvs([provides_root, *PATH_TO_ITEM.values()
]).path_to_reqs_provs, expected)
