def _temp_resource_subvol(self, name: str):
parent_sv = find_built_subvol(load_location(__package__, name))
with TempSubvolumes(sys.argv[0]) as temp_subvols:
# Cannot use `.snapshot()` since that doesn't handle mounts.
child_sv = temp_subvols.caller_will_create(name)
ParentLayerItem.get_phase_builder([
ParentLayerItem(from_target='t', subvol=parent_sv),
], DUMMY_LAYER_OPTS)(child_sv)
yield child_sv
def test_clone_nonexistent_source(self):
ci = self._clone_item('/no_such_path', '/none_such')
self.assertEqual('none_such', ci.dest)
with self.assertRaises(subprocess.CalledProcessError):
self._check_item(ci, set(), {require_directory('/')})
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_nonexistent_source')
with self.assertRaises(subprocess.CalledProcessError):
ci.build(subvol, DUMMY_LAYER_OPTS)
def test_install_file_command(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes, \
tempfile.NamedTemporaryFile() as empty_tf:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('d')])
_install_file_item(
from_target='t',
source={
'source': empty_tf.name
},
dest='/d/empty',
).build(subvol, DUMMY_LAYER_OPTS)
self.assertEqual(
['(Dir)', {
'd': ['(Dir)', {
'empty': ['(File m444)']
}]
}],
render_subvol(subvol),
)
# Fail to write to a nonexistent dir
with self.assertRaises(subprocess.CalledProcessError):
_install_file_item(
from_target='t',
source={
'source': empty_tf.name
},
dest='/no_dir/empty',
).build(subvol, DUMMY_LAYER_OPTS)
# Running a second copy to the same destination. This just
# overwrites the previous file, because we have a build-time
# check for this, and a run-time check would add overhead.
_install_file_item(
from_target='t',
source={
'source': empty_tf.name
},
dest='/d/empty',
# A non-default mode & owner shows that the file was
# overwritten, and also exercises HasStatOptions.
mode='u+rw',
user_group='12:34',
).build(subvol, DUMMY_LAYER_OPTS)
self.assertEqual(
['(Dir)', {
'd': ['(Dir)', {
'empty': ['(File m600 o12:34)']
}]
}],
render_subvol(subvol),
)
def test_receive_sendstream(self):
item = ReceiveSendstreamItem(
from_target='t',
source=Path(__file__).dirname() / 'create_ops.sendstream',
)
self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order())
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
new_subvol_name = 'differs_from_create_ops'
subvol = temp_subvolumes.caller_will_create(new_subvol_name)
item.get_phase_builder([item], DUMMY_LAYER_OPTS)(subvol)
self.assertEqual(
render_demo_subvols(create_ops=new_subvol_name),
render_sendstream(subvol.mark_readonly_and_get_sendstream()),
)
def _test_make_dirs_command(self, layer_opts):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('d')])
MakeDirsItem(
from_target='t',
path_to_make='/a/b/',
into_dir='/d',
user_group='77:88',
mode='u+rx',
).build(subvol, layer_opts)
self.assertEqual([
'(Dir)', {
'd': [
'(Dir)', {
'a': [
'(Dir m500 o77:88)', {
'b': ['(Dir m500 o77:88)', {}],
}
],
}
],
}
], render_subvol(subvol))
# The "should never happen" cases -- since we have build-time
# checks, for simplicity/speed, our runtime clobbers permissions
# of preexisting directories, and quietly creates non-existent
# ones with default permissions.
MakeDirsItem(from_target='t',
path_to_make='a',
into_dir='/no_dir',
user_group='4:0').build(subvol, layer_opts)
MakeDirsItem(from_target='t',
path_to_make='a/new',
into_dir='/d',
user_group='5:0').build(subvol, layer_opts)
self.assertEqual(['(Dir)', {
'd': ['(Dir)', {
# permissions overwritten for this whole tree
'a': ['(Dir o5:0)', {
'b': ['(Dir o5:0)', {}], 'new': ['(Dir o5:0)', {}],
}],
}],
'no_dir': ['(Dir)', { # default permissions!
'a': ['(Dir o4:0)', {}],
}],
}], render_subvol(subvol))
def test_clone_file(self):
ci = self._clone_item('/rpm_test/hello_world.tar', '/cloned_hello.tar')
self.assertEqual('cloned_hello.tar', ci.dest)
self._check_item(
ci,
{ProvidesFile(path='cloned_hello.tar')},
{require_directory('/')},
)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_file')
ci.build(subvol, DUMMY_LAYER_OPTS)
r = render_subvol(subvol)
ino, = pop_path(r, 'cloned_hello.tar')
self.assertRegex(ino, '(File m444 d[0-9]+)')
self.assertEqual(['(Dir)', {}], r)
def test_clone_pre_existing_dest(self):
ci = self._clone_item('/foo/bar', '/', pre_existing_dest=True)
self.assertEqual('', ci.dest)
self._check_item(
ci,
{
ProvidesDirectory(path='bar'),
ProvidesDirectory(path='bar/baz'),
ProvidesFile(path='bar/baz/bar'),
ProvidesFile(path='bar/even_more_hello_world.tar'),
},
{require_directory('/')},
)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_pre_existing_dest')
self._check_clone_bar(ci, subvol)
def _snapshot_subvol(src_subvol: Subvol,
snapshot_into: Optional[AnyStr]) -> Iterable[Subvol]:
if snapshot_into:
nspawn_subvol = Subvol(snapshot_into)
nspawn_subvol.snapshot(src_subvol)
clone_mounts(src_subvol, nspawn_subvol)
yield nspawn_subvol
else:
with TempSubvolumes() as tmp_subvols:
# To make it easier to debug where a temporary subvolume came
# from, make make its name resemble that of its source.
tmp_name = os.path.normpath(src_subvol.path())
tmp_name = os.path.basename(os.path.dirname(tmp_name)) or \
os.path.basename(tmp_name)
nspawn_subvol = tmp_subvols.snapshot(src_subvol, tmp_name)
clone_mounts(src_subvol, nspawn_subvol)
yield nspawn_subvol
def test_clone_hardlinks(self):
with TempSubvolumes(sys.argv[0]) as temp_subvols:
src_subvol = temp_subvols.create('test_clone_hardlinks_src')
dest_subvol = temp_subvols.create('test_clone_hardlinks_dest')
src_subvol.run_as_root(['touch', src_subvol.path('a')])
src_subvol.run_as_root([
'ln',
src_subvol.path('a'),
src_subvol.path('b'),
])
ci = self._clone_item(
'/',
'/',
omit_outer_dir=True,
pre_existing_dest=True,
subvol=src_subvol,
)
self.assertEqual('', ci.dest)
self._check_item(
ci,
{
ProvidesFile(path='a'),
ProvidesFile(path='b'),
# This looks like a bug (there's no /meta on disk here) but
# it's really just an artifact of how this path is
# protected. Read: This Is Fine (TM).
ProvidesDoNotAccess(path='/meta'),
},
{require_directory('/')})
ci.build(dest_subvol, DUMMY_LAYER_OPTS)
src_r = render_subvol(src_subvol)
dest_r = render_subvol(dest_subvol)
self.assertEqual(src_r, dest_r)
self.assertEqual(
[
'(Dir)',
{
# Witness that they have the same (rendered) inode # of "0"
'a': [['(File)', 0]],
'b': [['(File)', 0]],
}
],
dest_r)
def test_clone_demo_sendstream(self):
src_subvol = layer_resource_subvol(__package__, 'create_ops')
ci = self._clone_item(
'/',
'/',
omit_outer_dir=True,
pre_existing_dest=True,
subvol=src_subvol,
)
self.assertEqual({require_directory('/')}, set(ci.requires()))
self.assertGreater(len(set(ci.provides())), 1)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
dest_subvol = temp_subvols.create('create_ops')
ci.build(dest_subvol, DUMMY_LAYER_OPTS)
self.assertEqual(
render_subvol(src_subvol),
render_subvol(dest_subvol),
)
def test_parent_layer(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
parent = temp_subvolumes.create('parent')
item = ParentLayerItem(from_target='t', subvol=parent)
self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order())
MakeDirsItem(
from_target='t',
into_dir='/',
path_to_make='a/b',
).build(parent, DUMMY_LAYER_OPTS)
parent_content = ['(Dir)', {'a': ['(Dir)', {'b': ['(Dir)', {}]}]}]
self.assertEqual(parent_content, render_subvol(parent))
# Take a snapshot and add one more directory.
child = temp_subvolumes.caller_will_create('child')
item.get_phase_builder([item], DUMMY_LAYER_OPTS)(child)
MakeDirsItem(
from_target='t',
into_dir='a',
path_to_make='c',
).build(child, DUMMY_LAYER_OPTS)
# The parent is unchanged.
self.assertEqual(parent_content, render_subvol(parent))
child_content = copy.deepcopy(parent_content)
child_content[1]['a'][1]['c'] = ['(Dir)', {}]
# Since the parent lacked a /meta, the child added it.
child_content[1]['meta'] = [
'(Dir)', {
'private': [
'(Dir)', {
'opts': [
'(Dir)', {
'artifacts_may_require_repo':
['(File d2)']
}
]
}
]
}
]
self.assertEqual(child_content, render_subvol(child))
def test_rpm_action_item_auto_downgrade(self):
parent_subvol = layer_resource_subvol(
__package__,
'test-with-one-local-rpm',
)
src_rpm = Path(__file__).dirname() / "rpm-test-cheese-1-1.rpm"
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
# ensure cheese2 is installed in the parent from rpm-test-cheese-2-1
assert os.path.isfile(parent_subvol.path('/rpm_test/cheese2.txt'))
# make sure the RPM we are installing is older in order to
# trigger the downgrade
src_data = RpmMetadata.from_file(src_rpm)
subvol_data = RpmMetadata.from_subvol(parent_subvol, src_data.name)
assert compare_rpm_versions(src_data, subvol_data) < 0
subvol = temp_subvolumes.snapshot(parent_subvol, 'rpm_action')
RpmActionItem.get_phase_builder(
[
RpmActionItem(
from_target='t',
source=src_rpm,
action=RpmAction.install,
)
],
self._opts(),
)(subvol)
subvol.run_as_root([
'rm',
'-rf',
subvol.path('dev'),
subvol.path('etc'),
subvol.path('meta'),
subvol.path('var'),
])
self.assertEqual([
'(Dir)', {
'rpm_test': ['(Dir)', {
'cheese1.txt': ['(File d42)'],
}],
}
], render_subvol(subvol))
def test_clone_omit_outer_dir(self):
ci = self._clone_item(
'/foo/bar',
'/bar',
omit_outer_dir=True,
pre_existing_dest=True,
)
self.assertEqual('bar', ci.dest)
self._check_item(
ci,
{
ProvidesDirectory(path='bar/baz'),
ProvidesFile(path='bar/baz/bar'),
ProvidesFile(path='bar/even_more_hello_world.tar'),
},
{require_directory('/bar')},
)
with TempSubvolumes(sys.argv[0]) as temp_subvols:
subvol = temp_subvols.create('test_clone_omit_outer_dir')
subvol.run_as_root(['mkdir', subvol.path('bar')])
self._check_clone_bar(ci, subvol)
def _make_mutate_ops_subvolume(
subvols: TempSubvolumes,
create_ops: Subvol,
path: bytes,
) -> Subvol:
'Exercise the send-stream ops that are unique to snapshots.'
subvol = subvols.snapshot(create_ops, path) # snapshot
run = subvol.run_as_root
# `cwd` is intentionally prohibited with `run_as_root`
def p(sv_path):
return subvol.path(sv_path).decode()
run(['rm', p('hello/world')]) # unlink
run(['rmdir', p('dir_to_remove/')]) # rmdir
run([ # remove_xattr
'setfattr',
'--remove=user.test_attr',
p('hello/'),
])
# You would think this would emit a `rename`, but for files, the
# sendstream instead `link`s to the new location, and unlinks the old.
run(['mv', p('goodbye'), p('farewell')]) # NOT a rename, {,un}link
run(['mv', p('hello/'), p('hello_renamed/')]) # yes, a rename!
run( # write
['dd', 'of=' + p('hello_renamed/een')],
input=b'push\n',
)
# This is a no-op because `btfs send` does not support `chattr` at
# present. However, it's good to have a canary so that our tests start
# failing the moment it is supported -- that will remind us to update
# the mock VFS. NB: The absolute path to `chattr` is a clowny hack to
# work around a clowny hack, to work around clowny hacks. Don't ask.
run(['/usr/bin/chattr', '+a', p('hello_renamed/een')])
# Besides files with trailing holes, one can also get `truncate`
# sendstream commands in incremental sendstreams by having a snapshot
# truncate relative a file relative to the parent.
run(['truncate', '-s', '2', p('hello_big_hole')])
return subvol
def test_gen_dependency_graph(self):
dg = DependencyGraph(PATH_TO_ITEM.values(), layer_target='t-72')
self.assertEqual(
_fs_root_phases(FilesystemRootItem(from_target='t-72')),
list(dg.ordered_phases()),
)
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
self.assertIn(
tuple(
dg.gen_dependency_order_items(
PhasesProvideItem(from_target='t', subvol=subvol), )),
{
tuple(PATH_TO_ITEM[p] for p in paths)
for paths in [
# A few orders are valid, don't make the test fragile.
['/a/b/c', '/a/b/c/F', '/a/d/e', '/a/d/e/G'],
['/a/b/c', '/a/d/e', '/a/b/c/F', '/a/d/e/G'],
['/a/b/c', '/a/d/e', '/a/d/e/G', '/a/b/c/F'],
]
},
)
def make_demo_sendstreams(path_in_repo: bytes):
with TempSubvolumes(path_in_repo) as subvols:
res = {}
with _populate_sendstream_dict(res.setdefault('create_ops', {})) as d:
create_ops = _make_create_ops_subvolume(subvols, b'create_ops')
d['sendstream'] = create_ops.mark_readonly_and_get_sendstream()
with _populate_sendstream_dict(res.setdefault('mutate_ops', {})) as d:
d['sendstream'] = _make_mutate_ops_subvolume(
subvols,
create_ops,
b'mutate_ops',
).mark_readonly_and_get_sendstream(
parent=create_ops,
# The resulting send-stream will have `update_extent`
# instead of `write`, which is one way of making sure that
# `update_extent` in `parse_sendstream.py` is covered.
no_data=True,
)
return res
def test_cycle_detection(self):
def requires_provides_directory_class(requires_dir, provides_dir):
@dataclass(init=False, frozen=True)
class RequiresProvidesDirectory(ImageItem):
def requires(self):
yield require_directory(requires_dir)
def provides(self):
yield ProvidesDirectory(path=provides_dir)
return RequiresProvidesDirectory
# `dg_ok`: dependency-sorting will work without a cycle
first = FilesystemRootItem(from_target='')
second = requires_provides_directory_class('/', 'a')(from_target='')
third = MakeDirsItem(from_target='', into_dir='a', path_to_make='b/c')
dg_ok = DependencyGraph([second, first, third], layer_target='t')
self.assertEqual(_fs_root_phases(first), list(dg_ok.ordered_phases()))
# `dg_bad`: changes `second` to get a cycle
dg_bad = DependencyGraph([
requires_provides_directory_class('a/b', 'a')(from_target=''),
first,
third,
],
layer_target='t')
self.assertEqual(_fs_root_phases(first), list(dg_bad.ordered_phases()))
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
provides_root = PhasesProvideItem(from_target='t', subvol=subvol)
self.assertEqual(
[second, third],
list(dg_ok.gen_dependency_order_items(provides_root)),
)
with self.assertRaisesRegex(AssertionError, '^Cycle in '):
list(dg_bad.gen_dependency_order_items(provides_root))
def _test_rpm_action_item_install_local_setup(self):
parent_subvol = layer_resource_subvol(__package__, 'test-with-no-rpm')
local_rpm_path = Path(__file__).dirname() / 'rpm-test-cheese-2-1.rpm'
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.snapshot(parent_subvol, 'add_cheese')
RpmActionItem.get_phase_builder(
[
RpmActionItem(
from_target='t',
source=local_rpm_path,
action=RpmAction.install,
)
],
self._opts(),
)(subvol)
r = render_subvol(subvol)
self.assertEqual(['(Dir)', {
'cheese2.txt': ['(File d45)'],
}], pop_path(r, 'rpm_test'))
yield r
def test_phase_order(self):
class FakeRemovePaths:
get_phase_builder = 'kittycat'
def phase_order(self):
return PhaseOrder.REMOVE_PATHS
first = FilesystemRootItem(from_target='')
second = FakeRemovePaths()
third = MakeDirsItem(from_target='', into_dir='/', path_to_make='a/b')
dg = DependencyGraph([second, first, third], layer_target='t')
self.assertEqual(
_fs_root_phases(first) + [
(FakeRemovePaths.get_phase_builder, (second, )),
],
list(dg.ordered_phases()),
)
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
self.assertEqual([third],
list(
dg.gen_dependency_order_items(
PhasesProvideItem(from_target='t',
subvol=subvol), )))
def test_paths_to_reqs_provs(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('subvol')
provides_root = PhasesProvideItem(from_target='t', subvol=subvol)
expected = {
'/meta':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDoNotAccess(path='/meta'),
provides_root)
},
item_reqs=set(),
),
'/':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='/'), provides_root)
},
item_reqs={
ItemReq(require_directory('/'), PATH_TO_ITEM['/a/b/c'])
},
),
'/a':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a'),
PATH_TO_ITEM['/a/b/c'])
},
item_reqs={
ItemReq(require_directory('a'), PATH_TO_ITEM['/a/d/e'])
},
),
'/a/b':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a/b'),
PATH_TO_ITEM['/a/b/c'])
},
item_reqs=set(),
),
'/a/b/c':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a/b/c'),
PATH_TO_ITEM['/a/b/c'])
},
item_reqs={
ItemReq(require_directory('a/b/c'),
PATH_TO_ITEM['/a/b/c/F'])
},
),
'/a/b/c/F':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesFile(path='a/b/c/F'),
PATH_TO_ITEM['/a/b/c/F'])
},
item_reqs=set(),
),
'/a/d':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a/d'),
PATH_TO_ITEM['/a/d/e'])
},
item_reqs=set(),
),
'/a/d/e':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesDirectory(path='a/d/e'),
PATH_TO_ITEM['/a/d/e'])
},
item_reqs={
ItemReq(require_directory('a/d/e'),
PATH_TO_ITEM['/a/d/e/G'])
},
),
'/a/d/e/G':
ItemReqsProvs(
item_provs={
ItemProv(ProvidesFile(path='a/d/e/G'),
PATH_TO_ITEM['/a/d/e/G'])
},
item_reqs=set(),
),
}
self.assertEqual(
ValidatedReqsProvs([provides_root, *PATH_TO_ITEM.values()
]).path_to_reqs_provs, expected)
def _make_create_ops_subvolume(subvols: TempSubvolumes, path: bytes) -> Subvol:
'Exercise all the send-stream ops that can occur on a new subvolume.'
subvol = subvols.create(path)
run = subvol.run_as_root
# `cwd` is intentionally prohibited with `run_as_root`
def p(sv_path):
return subvol.path(sv_path).decode()
# Due to an odd `btrfs send` implementation detail, creating a file or
# directory emits a rename from a temporary name to the final one.
run(['mkdir', p('hello')]) # mkdir,rename
run(['mkdir', p('dir_to_remove')])
run(['touch', p('hello/world')]) # mkfile,utimes,chmod,chown
run([ # set_xattr
'setfattr',
'-n',
'user.test_attr',
'-v',
'chickens',
p('hello/'),
])
run(['mknod', p('buffered'), 'b', '1337', '31415']) # mknod
run(['chmod', 'og-r', p('buffered')]) # chmod a device
run(['mknod', p('unbuffered'), 'c', '1337', '31415'])
run(['mkfifo', p('fifo')]) # mkfifo
run([
'python3',
'-c',
(
'import os, sys, socket as s\n'
'dir, base = os.path.split(sys.argv[1])\n'
# Otherwise, we can easily get "AF_UNIX path too long"
'os.chdir(os.path.join(".", dir))\n'
'with s.socket(s.AF_UNIX, s.SOCK_STREAM) as sock:\n'
' sock.bind(base)\n' # mksock
),
p('unix_sock')
])
run(['ln', p('hello/world'), p('goodbye')]) # link
run(['ln', '-s', 'hello/world', p('bye_symlink')]) # symlink
run([ # update_extent
# 56KB was chosen so that `btrfs send` emits more than 1 write,
# specifically 48KB + 8KB.
'dd',
'if=/dev/zero',
'of=' + p('56KB_nuls'),
'bs=1024',
'count=56',
])
run([ # clone
'cp',
'--reflink=always',
p('56KB_nuls'),
p('56KB_nuls_clone'),
])
# Make a file with a 16KB hole in the middle.
run([
'dd',
'if=/dev/zero',
'of=' + p('zeros_hole_zeros'),
'bs=1024',
'count=16',
])
run(['truncate', '-s', str(32 * 1024), p('zeros_hole_zeros')])
run([
'dd',
'if=/dev/zero',
'of=' + p('zeros_hole_zeros'),
'oflag=append',
'conv=notrunc',
'bs=1024',
'count=16',
])
# A trailing hole exercises the `truncate` sendstream command.
run(['bash', '-c', 'echo hello > ' + shlex.quote(p('hello_big_hole'))])
run(['truncate', '-s', '1G', p('hello_big_hole')])
# This just serves to show that `btrfs send` ignores nested subvolumes.
# There is no mention of `nested_subvol` in the send-stream.
nested_subvol = subvols.create(p('nested_subvol'))
nested_subvol.run_as_root(['touch', nested_subvol.path('borf')])
nested_subvol.run_as_root(['mkdir', nested_subvol.path('beep')])
return subvol
def test_remove_item(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes, \
tempfile.NamedTemporaryFile() as empty_tf:
subvol = temp_subvolumes.create('remove_action')
self.assertEqual(['(Dir)', {}], render_subvol(subvol))
MakeDirsItem(
from_target='t', path_to_make='/a/b/c', into_dir='/',
).build(subvol, DUMMY_LAYER_OPTS)
for d in ['d', 'e']:
InstallFileItem(
from_target='t', source=empty_tf.name, dest=f'/a/b/c/{d}',
).build(subvol, DUMMY_LAYER_OPTS)
MakeDirsItem(
from_target='t', path_to_make='/f/g', into_dir='/',
).build(subvol, DUMMY_LAYER_OPTS)
# Checks that `rm` won't follow symlinks
SymlinkToDirItem(
from_target='t', source='/f', dest='/a/b/f_sym',
).build(subvol, DUMMY_LAYER_OPTS_BA)
for d in ['h', 'i']:
InstallFileItem(
from_target='t', source=empty_tf.name, dest=f'/f/{d}',
).build(subvol, DUMMY_LAYER_OPTS)
SymlinkToDirItem(
from_target='t', source='/f/i', dest='/f/i_sym',
).build(subvol, DUMMY_LAYER_OPTS_BA)
intact_subvol = ['(Dir)', {
'a': ['(Dir)', {
'b': ['(Dir)', {
'c': ['(Dir)', {
'd': ['(File m444)'],
'e': ['(File m444)'],
}],
'f_sym': ['(Symlink ../../f)'],
}],
}],
'f': ['(Dir)', {
'g': ['(Dir)', {}],
'h': ['(File m444)'],
'i': ['(File m444)'],
'i_sym': ['(Symlink i)'],
}],
}]
self.assertEqual(intact_subvol, render_subvol(subvol))
# We refuse to touch protected paths, even with "if_exists". If
# the paths started with 'meta', they would trip the check in
# `_make_path_normal_relative`, so we mock-protect 'xyz'.
for prot_path in ['xyz', 'xyz/potato/carrot']:
with unittest.mock.patch(
'fs_image.compiler.items.remove_path.protected_path_set',
side_effect=lambda sv: protected_path_set(sv) | {'xyz'},
), self.assertRaisesRegex(
AssertionError, f'Cannot remove protected .*{prot_path}',
):
RemovePathItem.get_phase_builder([RemovePathItem(
from_target='t',
action=RemovePathAction.if_exists,
path=prot_path,
)], DUMMY_LAYER_OPTS)(subvol)
# Check handling of non-existent paths without removing anything
remove = RemovePathItem(
from_target='t',
action=RemovePathAction.if_exists,
path='/does/not/exist',
)
self.assertEqual(PhaseOrder.REMOVE_PATHS, remove.phase_order())
RemovePathItem.get_phase_builder([remove], DUMMY_LAYER_OPTS)(subvol)
with self.assertRaisesRegex(AssertionError, 'does not exist'):
RemovePathItem.get_phase_builder([
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/does/not/exist',
),
], DUMMY_LAYER_OPTS)(subvol)
self.assertEqual(intact_subvol, render_subvol(subvol))
# Now remove most of the subvolume.
RemovePathItem.get_phase_builder([
# These 3 removes are not covered by a recursive remove.
# And we leave behind /f/i, which lets us know that neither
# `f_sym` nor `i_sym` were followed during their deletion.
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/f/i_sym',
),
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/f/h',
),
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/f/g',
),
# The next 3 items are intentionally sequenced so that if
# they were applied in the given order, they would fail.
RemovePathItem(
from_target='t',
action=RemovePathAction.if_exists,
path='/a/b/c/e',
),
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
# The surrounding items don't delete /a/b/c/d, e.g. so
# this recursive remove is still tested.
path='/a/b/',
),
RemovePathItem(
from_target='t',
action=RemovePathAction.assert_exists,
path='/a/b/c/e',
),
], DUMMY_LAYER_OPTS)(subvol)
self.assertEqual(['(Dir)', {
'a': ['(Dir)', {}],
'f': ['(Dir)', {'i': ['(File m444)']}],
}], render_subvol(subvol))
def test_install_file_command_recursive(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('d')])
with temp_dir() as td:
with open(td / 'data.txt', 'w') as df:
print('Hello', file=df)
os.mkdir(td / 'subdir')
with open(td / 'subdir/exe.sh', 'w') as ef:
print('#!/bin/sh\necho "Hello"', file=ef)
os.chmod(td / 'subdir/exe.sh', 0o100)
dir_item = _install_file_item(
from_target='t',
source={'source': td},
dest='/d/a',
)
ps = [
_InstallablePath(
td,
ProvidesDirectory(path='d/a'),
'u+rwx,og+rx',
),
_InstallablePath(
td / 'data.txt',
ProvidesFile(path='d/a/data.txt'),
'a+r',
),
_InstallablePath(
td / 'subdir',
ProvidesDirectory(path='d/a/subdir'),
'u+rwx,og+rx',
),
_InstallablePath(
td / 'subdir/exe.sh',
ProvidesFile(path='d/a/subdir/exe.sh'),
'a+rx',
),
]
self.assertEqual(sorted(ps), sorted(dir_item.paths))
self.assertEqual(td, dir_item.source)
self._check_item(dir_item, {p.provides
for p in ps},
{require_directory('d')})
# This implicitly checks that `a` precedes its contents.
dir_item.build(subvol, DUMMY_LAYER_OPTS)
self.assertEqual(
[
'(Dir)', {
'd': [
'(Dir)', {
'a': [
'(Dir)', {
'data.txt': ['(File m444 d6)'],
'subdir': [
'(Dir)', {
'exe.sh': ['(File m555 d23)'],
}
],
}
]
}
]
}
],
render_subvol(subvol),
)
def test_mount_item_file_from_host(self):
mount_config = {
'is_directory': False,
'build_source': {
'type': 'host',
'source': '/dev/null'
},
}
with self.assertRaisesRegex(AssertionError, 'must be located under'):
_mount_item_new('t', mount_config)
bad_mount_config = mount_config.copy()
bad_mount_config['runtime_source'] = bad_mount_config['build_source']
with self.assertRaisesRegex(AssertionError,
'Only `build_source` may '):
_mount_item_new('//dummy/host_mounts:t', bad_mount_config)
mount_item = _mount_item_new('//dummy/host_mounts:t', mount_config)
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('mounter')
mount_item.build(
subvol,
DUMMY_LAYER_OPTS._replace(
target_to_path={},
subvolumes_dir='unused',
))
self.assertEqual(
[
'(Dir)',
{
'lala': ['(File)'
], # An empty mountpoint for /dev/null
'meta': [
'(Dir)',
{
'private': [
'(Dir)',
{
# No `opts/artifacts_may_require_repo` here because we
# directly created the subvol instead of using an Item.
'mount': [
'(Dir)', {
'lala': [
'(Dir)', {
'MOUNT': [
'(Dir)', {
'is_directory':
['(File d2)'],
'build_source':
[
'(Dir)', {
'type':
[
'(File d5)'
],
'source':
[
f'(File d{len("/dev/null") + 1})'
],
}
],
}
]
}
]
}
],
}
]
}
],
}
],
render_subvol(subvol))
for filename, contents in (
('is_directory', '0\n'),
('build_source/type', 'host\n'),
('build_source/source', '/dev/null\n'),
):
with open(
subvol.path(
os.path.join(
'meta/private/mount/lala/MOUNT',
filename,
))) as f:
self.assertEqual(contents, f.read())
def _test_symlink_command(self, layer_opts):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('dir')])
# We need a source file to validate a SymlinkToFileItem
with tempfile.NamedTemporaryFile() as tf:
InstallFileItem(
from_target='t',
source=tf.name,
dest='/file',
).build(subvol, layer_opts)
SymlinkToDirItem(from_target='t',
source='/dir',
dest='/dir_symlink').build(subvol, layer_opts)
SymlinkToFileItem(from_target='t',
source='file',
dest='/file_symlink').build(subvol, layer_opts)
# Make a couple of absolute symlinks to test our behavior on
# linking to paths that contain those.
subvol.run_as_root([
'bash', '-c', f'''\
ln -s /file {subvol.path('abs_link_to_file').shell_quote()}
mkdir {subvol.path('my_dir').shell_quote()}
touch {subvol.path('my_dir/inner').shell_quote()}
ln -s /my_dir {subvol.path('my_dir_link').shell_quote()}
'''
])
# A simple case: we link to an absolute link.
SymlinkToFileItem(
from_target='t',
source='/abs_link_to_file',
dest='/link_to_abs_link',
).build(subvol, layer_opts)
# This link traverses a directory that is an absolute link. The
# resulting relative symlink is not traversible from outside the
# container.
SymlinkToFileItem(
from_target='t',
source='my_dir_link/inner',
dest='/dir/inner_link',
).build(subvol, layer_opts)
self.assertEqual([
'(Dir)', {
'dir': [
'(Dir)', {
'inner_link': ['(Symlink ../my_dir_link/inner)'],
}
],
'dir_symlink': ['(Symlink dir)'],
'file': ['(File m444)'],
'file_symlink': ['(Symlink file)'],
'abs_link_to_file': ['(Symlink /file)'],
'my_dir': ['(Dir)', {
'inner': ['(File)']
}],
'my_dir_link': ['(Symlink /my_dir)'],
'link_to_abs_link': ['(Symlink abs_link_to_file)'],
}
], render_subvol(subvol))
def test_tarball_command(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('tar-sv')
subvol.run_as_root(['mkdir', subvol.path('d')])
# Fail on pre-existing files
subvol.run_as_root(['touch', subvol.path('d/exists')])
with tempfile.NamedTemporaryFile() as t:
with tarfile.TarFile(t.name, 'w') as tar_obj:
tar_obj.addfile(tarfile.TarInfo('exists'))
with self.assertRaises(subprocess.CalledProcessError):
_tarball_item(t.name, '/d').build(subvol,
DUMMY_LAYER_OPTS_BA)
# Adding new files & directories works. Overwriting a
# pre-existing directory leaves the owner+mode of the original
# directory intact.
subvol.run_as_root(['mkdir', subvol.path('d/old_dir')])
subvol.run_as_root(['chown', '123:456', subvol.path('d/old_dir')])
subvol.run_as_root(['chmod', '0301', subvol.path('d/old_dir')])
subvol_root = temp_subvolumes.snapshot(subvol, 'tar-sv-root')
subvol_zst = temp_subvolumes.snapshot(subvol, 'tar-sv-zst')
with tempfile.TemporaryDirectory() as td:
tar_path = os.path.join(td, 'test.tar')
zst_path = os.path.join(td, 'test.tar.zst')
with tarfile.TarFile(tar_path, 'w') as tar_obj:
tar_obj.addfile(tarfile.TarInfo('new_file'))
new_dir = tarfile.TarInfo('new_dir')
new_dir.type = tarfile.DIRTYPE
new_dir.uid = 12
new_dir.gid = 34
tar_obj.addfile(new_dir)
old_dir = tarfile.TarInfo('old_dir')
old_dir.type = tarfile.DIRTYPE
# These will not be applied because old_dir exists
old_dir.uid = 0
old_dir.gid = 0
old_dir.mode = 0o755
tar_obj.addfile(old_dir)
subprocess.check_call(['zstd', tar_path, '-o', zst_path])
# Fail when the destination does not exist
with self.assertRaises(subprocess.CalledProcessError):
_tarball_item(tar_path,
'/no_dir').build(subvol, DUMMY_LAYER_OPTS_BA)
# Before unpacking the tarball
orig_content = [
'(Dir)', {
'd': [
'(Dir)', {
'exists': ['(File)'],
'old_dir': ['(Dir m301 o123:456)', {}],
}
]
}
]
# After unpacking `tar_path` in `/d`.
new_content = copy.deepcopy(orig_content)
new_content[1]['d'][1].update({
'new_dir': ['(Dir m644 o12:34)', {}],
'new_file': ['(File)'],
})
# After unpacking `tar_path` in `/d` with `force_root_ownership`
new_content_root = copy.deepcopy(new_content)
# The ownership of 12:34 is gone.
new_content_root[1]['d'][1]['new_dir'] = ['(Dir m644)', {}]
self.assertNotEqual(new_content, new_content_root)
# Check the subvolume content before and after unpacking
for item, (sv, before, after) in (
(
_tarball_item(tar_path, '/d/'),
(subvol, orig_content, new_content),
),
(
_tarball_item(tar_path, 'd',
force_root_ownership=True),
(subvol_root, orig_content, new_content_root),
),
(
_tarball_item(zst_path, 'd/'),
(subvol_zst, orig_content, new_content),
),
):
self.assertEqual(before, render_subvol(sv))
item.build(sv, DUMMY_LAYER_OPTS_BA)
self.assertEqual(after, render_subvol(sv))
def test_mount_item(self):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes, \
tempfile.TemporaryDirectory() as source_dir:
runtime_source = {'so': 'me', 'arbitrary': {'j': 'son'}}
mount_config = {
'is_directory': True,
'build_source': {
'type': 'layer',
'source': '//fake:path'
},
'runtime_source': runtime_source,
}
with open(os.path.join(source_dir, 'mountconfig.json'), 'w') as f:
json.dump(mount_config, f)
self._check_item(
self._make_mount_item(
mountpoint='can/haz',
target=source_dir,
mount_config=mount_config,
),
{ProvidesDoNotAccess(path='can/haz')},
{require_directory('can')},
)
# Make a subvolume that we will mount inside `mounter`
mountee = temp_subvolumes.create('moun:tee/volume')
mountee.run_as_root(['tee', mountee.path('kitteh')],
input=b'cheez')
# These sub-mounts inside `mountee` act as canaries to make sure
# that (a) `mounter` receives the sub-mounts as a consequence of
# mounting `mountee` recursively, (b) that unmounting one in
# `mounter` does not affect the original in `mountee` -- i.e.
# that rslave propagation is set up correctly, (c) that
# unmounting in `mountee` immediately affects `mounter`.
#
# In practice, our build artifacts should NEVER be mutated after
# construction (and the only un-mount is implicitly, and
# seemingly safely, performed by `btrfs subvolume delete`).
# However, ensuring that we have correct `rslave` propagation is
# a worthwhile safeguard for host mounts, where an errant
# `umount` by a user inside their repo could otherwise break
# their host.
for submount in ('submount1', 'submount2'):
mountee.run_as_root(['mkdir', mountee.path(submount)])
mountee.run_as_root([
'mount', '-o', 'bind,ro', source_dir,
mountee.path(submount)
])
self.assertTrue(
os.path.exists(mountee.path(submount +
'/mountconfig.json')))
# Make the JSON file normally in "buck-out" that refers to `mountee`
mountee_subvolumes_dir = self._write_layer_json_into(
mountee, source_dir)
# Mount <mountee> at <mounter>/meow
mounter = temp_subvolumes.caller_will_create('mount:er/volume')
root_item = FilesystemRootItem(from_target='t')
root_item.get_phase_builder([root_item], DUMMY_LAYER_OPTS)(mounter)
mount_meow = self._make_mount_item(
mountpoint='meow',
target=source_dir,
mount_config=mount_config,
)
self.assertEqual(
runtime_source,
json.loads(mount_meow.runtime_source),
)
with self.assertRaisesRegex(AssertionError, ' could not resolve '):
mount_meow.build_source.to_path(
target_to_path={},
subvolumes_dir=mountee_subvolumes_dir,
)
mount_meow.build(
mounter,
DUMMY_LAYER_OPTS._replace(
target_to_path={'//fake:path': source_dir},
subvolumes_dir=mountee_subvolumes_dir,
))
# This checks the subvolume **contents**, but not the mounts.
# Ensure the build created a mountpoint, and populated metadata.
self._check_subvol_mounts_meow(mounter)
# `mountee` was also mounted at `/meow`
with open(mounter.path('meow/kitteh')) as f:
self.assertEqual('cheez', f.read())
def check_mountee_mounter_submounts(submount_presence):
for submount, (in_mountee, in_mounter) in submount_presence:
self.assertEqual(
in_mountee,
os.path.exists(
mountee.path(submount + '/mountconfig.json')),
f'{submount}, {in_mountee}')
self.assertEqual(
in_mounter,
os.path.exists(
mounter.path('meow/' + submount +
'/mountconfig.json')),
f'{submount}, {in_mounter}')
# Both sub-mounts are accessible in both places now.
check_mountee_mounter_submounts([
('submount1', (True, True)),
('submount2', (True, True)),
])
# Unmounting `submount1` from `mountee` also affects `mounter`.
mountee.run_as_root(['umount', mountee.path('submount1')])
check_mountee_mounter_submounts([
('submount1', (False, False)),
('submount2', (True, True)),
])
# Unmounting `submount2` from `mounter` doesn't affect `mountee`.
mounter.run_as_root(['umount', mounter.path('meow/submount2')])
check_mountee_mounter_submounts([
('submount1', (False, False)),
('submount2', (True, False)),
])
# Check that we read back the `mounter` metadata, mark `/meow`
# inaccessible, and do not emit a `ProvidesFile` for `kitteh`.
self._check_item(
PhasesProvideItem(from_target='t', subvol=mounter),
{
ProvidesDirectory(path='/'),
ProvidesDoNotAccess(path='/meta'),
ProvidesDoNotAccess(path='/meow'),
},
set(),
)
# Check that we successfully clone mounts from the parent layer.
mounter_child = temp_subvolumes.caller_will_create('child/volume')
ParentLayerItem.get_phase_builder(
[ParentLayerItem(from_target='t', subvol=mounter)],
DUMMY_LAYER_OPTS,
)(mounter_child)
# The child has the same mount, and the same metadata
self._check_subvol_mounts_meow(mounter_child)
# Check that we refuse to create nested mounts.
nested_mounter = temp_subvolumes.create('nested_mounter')
nested_item = MountItem(
layer_opts=DUMMY_LAYER_OPTS,
from_target='t',
mountpoint='/whatever',
target=None,
mount_config={
'is_directory': True,
'build_source': {
'type': 'layer',
'source': '//:fake'
},
},
)
with tempfile.TemporaryDirectory() as d:
mounter_subvolumes_dir = self._write_layer_json_into(
mounter, d)
with self.assertRaisesRegex(
AssertionError,
'Refusing .* nested mount',
):
nested_item.build(
nested_mounter,
DUMMY_LAYER_OPTS._replace(
target_to_path={'//:fake': d},
subvolumes_dir=mounter_subvolumes_dir,
))
def _check_rpm_action_item(self, layer_opts):
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
subvol = temp_subvolumes.create('rpm_action')
self.assertEqual(['(Dir)', {}], render_subvol(subvol))
# The empty action is a no-op
RpmActionItem.get_phase_builder([], layer_opts)(subvol)
self.assertEqual(['(Dir)', {}], render_subvol(subvol))
# `yum-dnf-from-snapshot` needs a `/meta` directory to work
subvol.run_as_root(['mkdir', subvol.path('meta')])
self.assertEqual(
# No `opts/artifacts_may_require_repo` here because we directly
# created the subvol instead of using an Item.
['(Dir)', {'meta': ['(Dir)', {}]}], render_subvol(subvol),
)
# Specifying RPM versions is prohibited
with self.assertRaises(subprocess.CalledProcessError):
RpmActionItem.get_phase_builder(
[RpmActionItem(
from_target='m',
name='rpm-test-milk-2.71',
source=None,
action=RpmAction.install,
)],
layer_opts,
)(subvol)
# Cannot pass both `name` and `source`
with self.assertRaisesRegex(
AssertionError,
'Exactly one of `name` or `source` must be set .*',
):
RpmActionItem.get_phase_builder(
[RpmActionItem(
from_target='m',
name='rpm-test-milk',
source='rpm-test-milk',
action=RpmAction.install,
)],
layer_opts,
)(subvol)
RpmActionItem.get_phase_builder(
[
RpmActionItem(
from_target='t', name=n, action=RpmAction.install,
) for n in [
# This specific RPM contains `/bin/sh` and a
# post-install script to test `/dev/null` isolation.
'rpm-test-milk',
'rpm-test-carrot',
]
],
layer_opts,
)(subvol)
# Clean up the `dnf`, `yum` & `rpm` litter before checking the
# packages. Maybe fixme: We end up not asserting ownership /
# permissions / etc on directories like /var and /dev.
subvol.run_as_root([
'rm', '-rf',
# Annotate all paths since `sudo rm -rf` is scary.
subvol.path('var/lib/rpm'),
subvol.path('var/lib/yum'),
subvol.path('var/lib/dnf'),
subvol.path('var/log/yum.log'),
*(subvol.path('var/log/' + log) for log in [
'yum.log', 'dnf.log', 'dnf.librepo.log', 'dnf.rpm.log',
'hawkey.log',
]),
subvol.path('usr/lib/.build-id'),
subvol.path('bin/sh'),
])
if self._YUM_DNF == YumDnf.dnf:
subvol.run_as_root([
'rmdir', subvol.path('etc/dnf/modules.d'),
subvol.path('etc/dnf'), subvol.path('etc'),
])
# The way that `RpmActionItem` nspawns into build_appliance must
# gurantee that `/var/cache/{dnf,yum}` is empty. The next two
# lines test that the cache directory is empty because `rmdir`
# fails if it is not. It is important that the cache of built
# images be empty, to avoid unnecessarily increasing the
# distributed image size.
rm_cmd = ['rmdir'] if (
layer_opts.build_appliance
and not layer_opts.preserve_yum_dnf_cache
) else ['rm', '-rf']
subvol.run_as_root(rm_cmd + [
subvol.path(f'var/cache/{self._YUM_DNF.value}')
])
subvol.run_as_root([
'rmdir',
subvol.path('dev'), # made by yum_dnf_from_snapshot.py
subvol.path('meta'),
subvol.path('var/cache'),
subvol.path('var/lib'),
subvol.path('var/log'),
subvol.path('var/tmp'),
subvol.path('var'),
subvol.path('usr/lib'),
subvol.path('bin'),
])
self.assertEqual(['(Dir)', {
'rpm_test': ['(Dir)', {
'carrot.txt': ['(File d13)'],
'milk.txt': ['(File d12)'],
'post.txt': ['(File d6)'],
}],
'usr': ['(Dir)', {}],
}], render_subvol(subvol))
def test_compile(self):
# First, test compilation with no parent layer.
expected_calls = self._expected_run_as_root_calls()
self.assertGreater( # Sanity check: at least one command per item
len(expected_calls),
len(si.ID_TO_ITEM),
)
self._assert_equal_call_sets(
expected_calls,
self._compiler_run_as_root_calls(
parent_feature_json=[],
parent_dep=[],
),
)
# Now, add an empty parent layer
with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
parent = temp_subvolumes.create('parent')
# Manually add/remove some commands from the "expected" set to
# accommodate the fact that we have a parent subvolume.
subvol_path = f'{_SUBVOLS_DIR}/{_FAKE_SUBVOL}'.encode()
# Our unittest.mock.call objects are (args, kwargs) pairs.
expected_calls_with_parent = [
c for c in expected_calls if c not in [
(
(['btrfs', 'subvolume', 'create', subvol_path], ),
{
'_subvol_exists': False
},
),
((['chmod', '0755', subvol_path], ), {}),
((['chown', 'root:root', subvol_path], ), {}),
]
] + [
(
(['test', '!', '-e', subvol_path], ),
{
'_subvol_exists': False
},
),
(
([
'btrfs',
'subvolume',
'snapshot',
parent.path(),
subvol_path,
], ),
{
'_subvol_exists': False
},
),
]
self.assertEqual( # We should've removed 3, and added 2 commands
len(expected_calls_with_parent) + 1,
len(expected_calls),
)
with mock_layer_dir_access(self, parent.path()) as parent_dir:
with open(parent_dir / 'feature.json', 'w') as out_f:
json.dump(
{
'parent_layer': [{
'subvol': {
'__BUCK_LAYER_TARGET': '//fake:parent'
},
}],
'target':
'//ignored:target',
}, out_f)
self._assert_equal_call_sets(
expected_calls_with_parent,
self._compiler_run_as_root_calls(
parent_feature_json=[
'--child-feature-json=' +
f'{parent_dir / "feature.json"}',
],
parent_dep=['//fake:parent',
parent_dir.decode()],
),
)
