def _temp_resource_subvol(self, name: str): parent_sv = find_built_subvol(load_location(__package__, name)) with TempSubvolumes(sys.argv[0]) as temp_subvols: # Cannot use `.snapshot()` since that doesn't handle mounts. child_sv = temp_subvols.caller_will_create(name) ParentLayerItem.get_phase_builder([ ParentLayerItem(from_target='t', subvol=parent_sv), ], DUMMY_LAYER_OPTS)(child_sv) yield child_sv
def test_clone_nonexistent_source(self): ci = self._clone_item('/no_such_path', '/none_such') self.assertEqual('none_such', ci.dest) with self.assertRaises(subprocess.CalledProcessError): self._check_item(ci, set(), {require_directory('/')}) with TempSubvolumes(sys.argv[0]) as temp_subvols: subvol = temp_subvols.create('test_clone_nonexistent_source') with self.assertRaises(subprocess.CalledProcessError): ci.build(subvol, DUMMY_LAYER_OPTS)
def test_install_file_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes, \ tempfile.NamedTemporaryFile() as empty_tf: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) _install_file_item( from_target='t', source={ 'source': empty_tf.name }, dest='/d/empty', ).build(subvol, DUMMY_LAYER_OPTS) self.assertEqual( ['(Dir)', { 'd': ['(Dir)', { 'empty': ['(File m444)'] }] }], render_subvol(subvol), ) # Fail to write to a nonexistent dir with self.assertRaises(subprocess.CalledProcessError): _install_file_item( from_target='t', source={ 'source': empty_tf.name }, dest='/no_dir/empty', ).build(subvol, DUMMY_LAYER_OPTS) # Running a second copy to the same destination. This just # overwrites the previous file, because we have a build-time # check for this, and a run-time check would add overhead. _install_file_item( from_target='t', source={ 'source': empty_tf.name }, dest='/d/empty', # A non-default mode & owner shows that the file was # overwritten, and also exercises HasStatOptions. mode='u+rw', user_group='12:34', ).build(subvol, DUMMY_LAYER_OPTS) self.assertEqual( ['(Dir)', { 'd': ['(Dir)', { 'empty': ['(File m600 o12:34)'] }] }], render_subvol(subvol), )
def test_receive_sendstream(self): item = ReceiveSendstreamItem( from_target='t', source=Path(__file__).dirname() / 'create_ops.sendstream', ) self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order()) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: new_subvol_name = 'differs_from_create_ops' subvol = temp_subvolumes.caller_will_create(new_subvol_name) item.get_phase_builder([item], DUMMY_LAYER_OPTS)(subvol) self.assertEqual( render_demo_subvols(create_ops=new_subvol_name), render_sendstream(subvol.mark_readonly_and_get_sendstream()), )
def _test_make_dirs_command(self, layer_opts): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) MakeDirsItem( from_target='t', path_to_make='/a/b/', into_dir='/d', user_group='77:88', mode='u+rx', ).build(subvol, layer_opts) self.assertEqual([ '(Dir)', { 'd': [ '(Dir)', { 'a': [ '(Dir m500 o77:88)', { 'b': ['(Dir m500 o77:88)', {}], } ], } ], } ], render_subvol(subvol)) # The "should never happen" cases -- since we have build-time # checks, for simplicity/speed, our runtime clobbers permissions # of preexisting directories, and quietly creates non-existent # ones with default permissions. MakeDirsItem(from_target='t', path_to_make='a', into_dir='/no_dir', user_group='4:0').build(subvol, layer_opts) MakeDirsItem(from_target='t', path_to_make='a/new', into_dir='/d', user_group='5:0').build(subvol, layer_opts) self.assertEqual(['(Dir)', { 'd': ['(Dir)', { # permissions overwritten for this whole tree 'a': ['(Dir o5:0)', { 'b': ['(Dir o5:0)', {}], 'new': ['(Dir o5:0)', {}], }], }], 'no_dir': ['(Dir)', { # default permissions! 'a': ['(Dir o4:0)', {}], }], }], render_subvol(subvol))
def test_clone_file(self): ci = self._clone_item('/rpm_test/hello_world.tar', '/cloned_hello.tar') self.assertEqual('cloned_hello.tar', ci.dest) self._check_item( ci, {ProvidesFile(path='cloned_hello.tar')}, {require_directory('/')}, ) with TempSubvolumes(sys.argv[0]) as temp_subvols: subvol = temp_subvols.create('test_clone_file') ci.build(subvol, DUMMY_LAYER_OPTS) r = render_subvol(subvol) ino, = pop_path(r, 'cloned_hello.tar') self.assertRegex(ino, '(File m444 d[0-9]+)') self.assertEqual(['(Dir)', {}], r)
def test_clone_pre_existing_dest(self): ci = self._clone_item('/foo/bar', '/', pre_existing_dest=True) self.assertEqual('', ci.dest) self._check_item( ci, { ProvidesDirectory(path='bar'), ProvidesDirectory(path='bar/baz'), ProvidesFile(path='bar/baz/bar'), ProvidesFile(path='bar/even_more_hello_world.tar'), }, {require_directory('/')}, ) with TempSubvolumes(sys.argv[0]) as temp_subvols: subvol = temp_subvols.create('test_clone_pre_existing_dest') self._check_clone_bar(ci, subvol)
def _snapshot_subvol(src_subvol: Subvol, snapshot_into: Optional[AnyStr]) -> Iterable[Subvol]: if snapshot_into: nspawn_subvol = Subvol(snapshot_into) nspawn_subvol.snapshot(src_subvol) clone_mounts(src_subvol, nspawn_subvol) yield nspawn_subvol else: with TempSubvolumes() as tmp_subvols: # To make it easier to debug where a temporary subvolume came # from, make make its name resemble that of its source. tmp_name = os.path.normpath(src_subvol.path()) tmp_name = os.path.basename(os.path.dirname(tmp_name)) or \ os.path.basename(tmp_name) nspawn_subvol = tmp_subvols.snapshot(src_subvol, tmp_name) clone_mounts(src_subvol, nspawn_subvol) yield nspawn_subvol
def test_clone_hardlinks(self): with TempSubvolumes(sys.argv[0]) as temp_subvols: src_subvol = temp_subvols.create('test_clone_hardlinks_src') dest_subvol = temp_subvols.create('test_clone_hardlinks_dest') src_subvol.run_as_root(['touch', src_subvol.path('a')]) src_subvol.run_as_root([ 'ln', src_subvol.path('a'), src_subvol.path('b'), ]) ci = self._clone_item( '/', '/', omit_outer_dir=True, pre_existing_dest=True, subvol=src_subvol, ) self.assertEqual('', ci.dest) self._check_item( ci, { ProvidesFile(path='a'), ProvidesFile(path='b'), # This looks like a bug (there's no /meta on disk here) but # it's really just an artifact of how this path is # protected. Read: This Is Fine (TM). ProvidesDoNotAccess(path='/meta'), }, {require_directory('/')}) ci.build(dest_subvol, DUMMY_LAYER_OPTS) src_r = render_subvol(src_subvol) dest_r = render_subvol(dest_subvol) self.assertEqual(src_r, dest_r) self.assertEqual( [ '(Dir)', { # Witness that they have the same (rendered) inode # of "0" 'a': [['(File)', 0]], 'b': [['(File)', 0]], } ], dest_r)
def test_clone_demo_sendstream(self): src_subvol = layer_resource_subvol(__package__, 'create_ops') ci = self._clone_item( '/', '/', omit_outer_dir=True, pre_existing_dest=True, subvol=src_subvol, ) self.assertEqual({require_directory('/')}, set(ci.requires())) self.assertGreater(len(set(ci.provides())), 1) with TempSubvolumes(sys.argv[0]) as temp_subvols: dest_subvol = temp_subvols.create('create_ops') ci.build(dest_subvol, DUMMY_LAYER_OPTS) self.assertEqual( render_subvol(src_subvol), render_subvol(dest_subvol), )
def test_parent_layer(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: parent = temp_subvolumes.create('parent') item = ParentLayerItem(from_target='t', subvol=parent) self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order()) MakeDirsItem( from_target='t', into_dir='/', path_to_make='a/b', ).build(parent, DUMMY_LAYER_OPTS) parent_content = ['(Dir)', {'a': ['(Dir)', {'b': ['(Dir)', {}]}]}] self.assertEqual(parent_content, render_subvol(parent)) # Take a snapshot and add one more directory. child = temp_subvolumes.caller_will_create('child') item.get_phase_builder([item], DUMMY_LAYER_OPTS)(child) MakeDirsItem( from_target='t', into_dir='a', path_to_make='c', ).build(child, DUMMY_LAYER_OPTS) # The parent is unchanged. self.assertEqual(parent_content, render_subvol(parent)) child_content = copy.deepcopy(parent_content) child_content[1]['a'][1]['c'] = ['(Dir)', {}] # Since the parent lacked a /meta, the child added it. child_content[1]['meta'] = [ '(Dir)', { 'private': [ '(Dir)', { 'opts': [ '(Dir)', { 'artifacts_may_require_repo': ['(File d2)'] } ] } ] } ] self.assertEqual(child_content, render_subvol(child))
def test_rpm_action_item_auto_downgrade(self): parent_subvol = layer_resource_subvol( __package__, 'test-with-one-local-rpm', ) src_rpm = Path(__file__).dirname() / "rpm-test-cheese-1-1.rpm" with TempSubvolumes(sys.argv[0]) as temp_subvolumes: # ensure cheese2 is installed in the parent from rpm-test-cheese-2-1 assert os.path.isfile(parent_subvol.path('/rpm_test/cheese2.txt')) # make sure the RPM we are installing is older in order to # trigger the downgrade src_data = RpmMetadata.from_file(src_rpm) subvol_data = RpmMetadata.from_subvol(parent_subvol, src_data.name) assert compare_rpm_versions(src_data, subvol_data) < 0 subvol = temp_subvolumes.snapshot(parent_subvol, 'rpm_action') RpmActionItem.get_phase_builder( [ RpmActionItem( from_target='t', source=src_rpm, action=RpmAction.install, ) ], self._opts(), )(subvol) subvol.run_as_root([ 'rm', '-rf', subvol.path('dev'), subvol.path('etc'), subvol.path('meta'), subvol.path('var'), ]) self.assertEqual([ '(Dir)', { 'rpm_test': ['(Dir)', { 'cheese1.txt': ['(File d42)'], }], } ], render_subvol(subvol))
def test_clone_omit_outer_dir(self): ci = self._clone_item( '/foo/bar', '/bar', omit_outer_dir=True, pre_existing_dest=True, ) self.assertEqual('bar', ci.dest) self._check_item( ci, { ProvidesDirectory(path='bar/baz'), ProvidesFile(path='bar/baz/bar'), ProvidesFile(path='bar/even_more_hello_world.tar'), }, {require_directory('/bar')}, ) with TempSubvolumes(sys.argv[0]) as temp_subvols: subvol = temp_subvols.create('test_clone_omit_outer_dir') subvol.run_as_root(['mkdir', subvol.path('bar')]) self._check_clone_bar(ci, subvol)
def _make_mutate_ops_subvolume( subvols: TempSubvolumes, create_ops: Subvol, path: bytes, ) -> Subvol: 'Exercise the send-stream ops that are unique to snapshots.' subvol = subvols.snapshot(create_ops, path) # snapshot run = subvol.run_as_root # `cwd` is intentionally prohibited with `run_as_root` def p(sv_path): return subvol.path(sv_path).decode() run(['rm', p('hello/world')]) # unlink run(['rmdir', p('dir_to_remove/')]) # rmdir run([ # remove_xattr 'setfattr', '--remove=user.test_attr', p('hello/'), ]) # You would think this would emit a `rename`, but for files, the # sendstream instead `link`s to the new location, and unlinks the old. run(['mv', p('goodbye'), p('farewell')]) # NOT a rename, {,un}link run(['mv', p('hello/'), p('hello_renamed/')]) # yes, a rename! run( # write ['dd', 'of=' + p('hello_renamed/een')], input=b'push\n', ) # This is a no-op because `btfs send` does not support `chattr` at # present. However, it's good to have a canary so that our tests start # failing the moment it is supported -- that will remind us to update # the mock VFS. NB: The absolute path to `chattr` is a clowny hack to # work around a clowny hack, to work around clowny hacks. Don't ask. run(['/usr/bin/chattr', '+a', p('hello_renamed/een')]) # Besides files with trailing holes, one can also get `truncate` # sendstream commands in incremental sendstreams by having a snapshot # truncate relative a file relative to the parent. run(['truncate', '-s', '2', p('hello_big_hole')]) return subvol
def test_gen_dependency_graph(self): dg = DependencyGraph(PATH_TO_ITEM.values(), layer_target='t-72') self.assertEqual( _fs_root_phases(FilesystemRootItem(from_target='t-72')), list(dg.ordered_phases()), ) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') self.assertIn( tuple( dg.gen_dependency_order_items( PhasesProvideItem(from_target='t', subvol=subvol), )), { tuple(PATH_TO_ITEM[p] for p in paths) for paths in [ # A few orders are valid, don't make the test fragile. ['/a/b/c', '/a/b/c/F', '/a/d/e', '/a/d/e/G'], ['/a/b/c', '/a/d/e', '/a/b/c/F', '/a/d/e/G'], ['/a/b/c', '/a/d/e', '/a/d/e/G', '/a/b/c/F'], ] }, )
def make_demo_sendstreams(path_in_repo: bytes): with TempSubvolumes(path_in_repo) as subvols: res = {} with _populate_sendstream_dict(res.setdefault('create_ops', {})) as d: create_ops = _make_create_ops_subvolume(subvols, b'create_ops') d['sendstream'] = create_ops.mark_readonly_and_get_sendstream() with _populate_sendstream_dict(res.setdefault('mutate_ops', {})) as d: d['sendstream'] = _make_mutate_ops_subvolume( subvols, create_ops, b'mutate_ops', ).mark_readonly_and_get_sendstream( parent=create_ops, # The resulting send-stream will have `update_extent` # instead of `write`, which is one way of making sure that # `update_extent` in `parse_sendstream.py` is covered. no_data=True, ) return res
def test_cycle_detection(self): def requires_provides_directory_class(requires_dir, provides_dir): @dataclass(init=False, frozen=True) class RequiresProvidesDirectory(ImageItem): def requires(self): yield require_directory(requires_dir) def provides(self): yield ProvidesDirectory(path=provides_dir) return RequiresProvidesDirectory # `dg_ok`: dependency-sorting will work without a cycle first = FilesystemRootItem(from_target='') second = requires_provides_directory_class('/', 'a')(from_target='') third = MakeDirsItem(from_target='', into_dir='a', path_to_make='b/c') dg_ok = DependencyGraph([second, first, third], layer_target='t') self.assertEqual(_fs_root_phases(first), list(dg_ok.ordered_phases())) # `dg_bad`: changes `second` to get a cycle dg_bad = DependencyGraph([ requires_provides_directory_class('a/b', 'a')(from_target=''), first, third, ], layer_target='t') self.assertEqual(_fs_root_phases(first), list(dg_bad.ordered_phases())) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') provides_root = PhasesProvideItem(from_target='t', subvol=subvol) self.assertEqual( [second, third], list(dg_ok.gen_dependency_order_items(provides_root)), ) with self.assertRaisesRegex(AssertionError, '^Cycle in '): list(dg_bad.gen_dependency_order_items(provides_root))
def _test_rpm_action_item_install_local_setup(self): parent_subvol = layer_resource_subvol(__package__, 'test-with-no-rpm') local_rpm_path = Path(__file__).dirname() / 'rpm-test-cheese-2-1.rpm' with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.snapshot(parent_subvol, 'add_cheese') RpmActionItem.get_phase_builder( [ RpmActionItem( from_target='t', source=local_rpm_path, action=RpmAction.install, ) ], self._opts(), )(subvol) r = render_subvol(subvol) self.assertEqual(['(Dir)', { 'cheese2.txt': ['(File d45)'], }], pop_path(r, 'rpm_test')) yield r
def test_phase_order(self): class FakeRemovePaths: get_phase_builder = 'kittycat' def phase_order(self): return PhaseOrder.REMOVE_PATHS first = FilesystemRootItem(from_target='') second = FakeRemovePaths() third = MakeDirsItem(from_target='', into_dir='/', path_to_make='a/b') dg = DependencyGraph([second, first, third], layer_target='t') self.assertEqual( _fs_root_phases(first) + [ (FakeRemovePaths.get_phase_builder, (second, )), ], list(dg.ordered_phases()), ) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') self.assertEqual([third], list( dg.gen_dependency_order_items( PhasesProvideItem(from_target='t', subvol=subvol), )))
def test_paths_to_reqs_provs(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') provides_root = PhasesProvideItem(from_target='t', subvol=subvol) expected = { '/meta': ItemReqsProvs( item_provs={ ItemProv(ProvidesDoNotAccess(path='/meta'), provides_root) }, item_reqs=set(), ), '/': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='/'), provides_root) }, item_reqs={ ItemReq(require_directory('/'), PATH_TO_ITEM['/a/b/c']) }, ), '/a': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a'), PATH_TO_ITEM['/a/b/c']) }, item_reqs={ ItemReq(require_directory('a'), PATH_TO_ITEM['/a/d/e']) }, ), '/a/b': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a/b'), PATH_TO_ITEM['/a/b/c']) }, item_reqs=set(), ), '/a/b/c': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a/b/c'), PATH_TO_ITEM['/a/b/c']) }, item_reqs={ ItemReq(require_directory('a/b/c'), PATH_TO_ITEM['/a/b/c/F']) }, ), '/a/b/c/F': ItemReqsProvs( item_provs={ ItemProv(ProvidesFile(path='a/b/c/F'), PATH_TO_ITEM['/a/b/c/F']) }, item_reqs=set(), ), '/a/d': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a/d'), PATH_TO_ITEM['/a/d/e']) }, item_reqs=set(), ), '/a/d/e': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a/d/e'), PATH_TO_ITEM['/a/d/e']) }, item_reqs={ ItemReq(require_directory('a/d/e'), PATH_TO_ITEM['/a/d/e/G']) }, ), '/a/d/e/G': ItemReqsProvs( item_provs={ ItemProv(ProvidesFile(path='a/d/e/G'), PATH_TO_ITEM['/a/d/e/G']) }, item_reqs=set(), ), } self.assertEqual( ValidatedReqsProvs([provides_root, *PATH_TO_ITEM.values() ]).path_to_reqs_provs, expected)
def _make_create_ops_subvolume(subvols: TempSubvolumes, path: bytes) -> Subvol: 'Exercise all the send-stream ops that can occur on a new subvolume.' subvol = subvols.create(path) run = subvol.run_as_root # `cwd` is intentionally prohibited with `run_as_root` def p(sv_path): return subvol.path(sv_path).decode() # Due to an odd `btrfs send` implementation detail, creating a file or # directory emits a rename from a temporary name to the final one. run(['mkdir', p('hello')]) # mkdir,rename run(['mkdir', p('dir_to_remove')]) run(['touch', p('hello/world')]) # mkfile,utimes,chmod,chown run([ # set_xattr 'setfattr', '-n', 'user.test_attr', '-v', 'chickens', p('hello/'), ]) run(['mknod', p('buffered'), 'b', '1337', '31415']) # mknod run(['chmod', 'og-r', p('buffered')]) # chmod a device run(['mknod', p('unbuffered'), 'c', '1337', '31415']) run(['mkfifo', p('fifo')]) # mkfifo run([ 'python3', '-c', ( 'import os, sys, socket as s\n' 'dir, base = os.path.split(sys.argv[1])\n' # Otherwise, we can easily get "AF_UNIX path too long" 'os.chdir(os.path.join(".", dir))\n' 'with s.socket(s.AF_UNIX, s.SOCK_STREAM) as sock:\n' ' sock.bind(base)\n' # mksock ), p('unix_sock') ]) run(['ln', p('hello/world'), p('goodbye')]) # link run(['ln', '-s', 'hello/world', p('bye_symlink')]) # symlink run([ # update_extent # 56KB was chosen so that `btrfs send` emits more than 1 write, # specifically 48KB + 8KB. 'dd', 'if=/dev/zero', 'of=' + p('56KB_nuls'), 'bs=1024', 'count=56', ]) run([ # clone 'cp', '--reflink=always', p('56KB_nuls'), p('56KB_nuls_clone'), ]) # Make a file with a 16KB hole in the middle. run([ 'dd', 'if=/dev/zero', 'of=' + p('zeros_hole_zeros'), 'bs=1024', 'count=16', ]) run(['truncate', '-s', str(32 * 1024), p('zeros_hole_zeros')]) run([ 'dd', 'if=/dev/zero', 'of=' + p('zeros_hole_zeros'), 'oflag=append', 'conv=notrunc', 'bs=1024', 'count=16', ]) # A trailing hole exercises the `truncate` sendstream command. run(['bash', '-c', 'echo hello > ' + shlex.quote(p('hello_big_hole'))]) run(['truncate', '-s', '1G', p('hello_big_hole')]) # This just serves to show that `btrfs send` ignores nested subvolumes. # There is no mention of `nested_subvol` in the send-stream. nested_subvol = subvols.create(p('nested_subvol')) nested_subvol.run_as_root(['touch', nested_subvol.path('borf')]) nested_subvol.run_as_root(['mkdir', nested_subvol.path('beep')]) return subvol
def test_remove_item(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes, \ tempfile.NamedTemporaryFile() as empty_tf: subvol = temp_subvolumes.create('remove_action') self.assertEqual(['(Dir)', {}], render_subvol(subvol)) MakeDirsItem( from_target='t', path_to_make='/a/b/c', into_dir='/', ).build(subvol, DUMMY_LAYER_OPTS) for d in ['d', 'e']: InstallFileItem( from_target='t', source=empty_tf.name, dest=f'/a/b/c/{d}', ).build(subvol, DUMMY_LAYER_OPTS) MakeDirsItem( from_target='t', path_to_make='/f/g', into_dir='/', ).build(subvol, DUMMY_LAYER_OPTS) # Checks that `rm` won't follow symlinks SymlinkToDirItem( from_target='t', source='/f', dest='/a/b/f_sym', ).build(subvol, DUMMY_LAYER_OPTS_BA) for d in ['h', 'i']: InstallFileItem( from_target='t', source=empty_tf.name, dest=f'/f/{d}', ).build(subvol, DUMMY_LAYER_OPTS) SymlinkToDirItem( from_target='t', source='/f/i', dest='/f/i_sym', ).build(subvol, DUMMY_LAYER_OPTS_BA) intact_subvol = ['(Dir)', { 'a': ['(Dir)', { 'b': ['(Dir)', { 'c': ['(Dir)', { 'd': ['(File m444)'], 'e': ['(File m444)'], }], 'f_sym': ['(Symlink ../../f)'], }], }], 'f': ['(Dir)', { 'g': ['(Dir)', {}], 'h': ['(File m444)'], 'i': ['(File m444)'], 'i_sym': ['(Symlink i)'], }], }] self.assertEqual(intact_subvol, render_subvol(subvol)) # We refuse to touch protected paths, even with "if_exists". If # the paths started with 'meta', they would trip the check in # `_make_path_normal_relative`, so we mock-protect 'xyz'. for prot_path in ['xyz', 'xyz/potato/carrot']: with unittest.mock.patch( 'fs_image.compiler.items.remove_path.protected_path_set', side_effect=lambda sv: protected_path_set(sv) | {'xyz'}, ), self.assertRaisesRegex( AssertionError, f'Cannot remove protected .*{prot_path}', ): RemovePathItem.get_phase_builder([RemovePathItem( from_target='t', action=RemovePathAction.if_exists, path=prot_path, )], DUMMY_LAYER_OPTS)(subvol) # Check handling of non-existent paths without removing anything remove = RemovePathItem( from_target='t', action=RemovePathAction.if_exists, path='/does/not/exist', ) self.assertEqual(PhaseOrder.REMOVE_PATHS, remove.phase_order()) RemovePathItem.get_phase_builder([remove], DUMMY_LAYER_OPTS)(subvol) with self.assertRaisesRegex(AssertionError, 'does not exist'): RemovePathItem.get_phase_builder([ RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/does/not/exist', ), ], DUMMY_LAYER_OPTS)(subvol) self.assertEqual(intact_subvol, render_subvol(subvol)) # Now remove most of the subvolume. RemovePathItem.get_phase_builder([ # These 3 removes are not covered by a recursive remove. # And we leave behind /f/i, which lets us know that neither # `f_sym` nor `i_sym` were followed during their deletion. RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/f/i_sym', ), RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/f/h', ), RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/f/g', ), # The next 3 items are intentionally sequenced so that if # they were applied in the given order, they would fail. RemovePathItem( from_target='t', action=RemovePathAction.if_exists, path='/a/b/c/e', ), RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, # The surrounding items don't delete /a/b/c/d, e.g. so # this recursive remove is still tested. path='/a/b/', ), RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/a/b/c/e', ), ], DUMMY_LAYER_OPTS)(subvol) self.assertEqual(['(Dir)', { 'a': ['(Dir)', {}], 'f': ['(Dir)', {'i': ['(File m444)']}], }], render_subvol(subvol))
def test_install_file_command_recursive(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) with temp_dir() as td: with open(td / 'data.txt', 'w') as df: print('Hello', file=df) os.mkdir(td / 'subdir') with open(td / 'subdir/exe.sh', 'w') as ef: print('#!/bin/sh\necho "Hello"', file=ef) os.chmod(td / 'subdir/exe.sh', 0o100) dir_item = _install_file_item( from_target='t', source={'source': td}, dest='/d/a', ) ps = [ _InstallablePath( td, ProvidesDirectory(path='d/a'), 'u+rwx,og+rx', ), _InstallablePath( td / 'data.txt', ProvidesFile(path='d/a/data.txt'), 'a+r', ), _InstallablePath( td / 'subdir', ProvidesDirectory(path='d/a/subdir'), 'u+rwx,og+rx', ), _InstallablePath( td / 'subdir/exe.sh', ProvidesFile(path='d/a/subdir/exe.sh'), 'a+rx', ), ] self.assertEqual(sorted(ps), sorted(dir_item.paths)) self.assertEqual(td, dir_item.source) self._check_item(dir_item, {p.provides for p in ps}, {require_directory('d')}) # This implicitly checks that `a` precedes its contents. dir_item.build(subvol, DUMMY_LAYER_OPTS) self.assertEqual( [ '(Dir)', { 'd': [ '(Dir)', { 'a': [ '(Dir)', { 'data.txt': ['(File m444 d6)'], 'subdir': [ '(Dir)', { 'exe.sh': ['(File m555 d23)'], } ], } ] } ] } ], render_subvol(subvol), )
def test_mount_item_file_from_host(self): mount_config = { 'is_directory': False, 'build_source': { 'type': 'host', 'source': '/dev/null' }, } with self.assertRaisesRegex(AssertionError, 'must be located under'): _mount_item_new('t', mount_config) bad_mount_config = mount_config.copy() bad_mount_config['runtime_source'] = bad_mount_config['build_source'] with self.assertRaisesRegex(AssertionError, 'Only `build_source` may '): _mount_item_new('//dummy/host_mounts:t', bad_mount_config) mount_item = _mount_item_new('//dummy/host_mounts:t', mount_config) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('mounter') mount_item.build( subvol, DUMMY_LAYER_OPTS._replace( target_to_path={}, subvolumes_dir='unused', )) self.assertEqual( [ '(Dir)', { 'lala': ['(File)' ], # An empty mountpoint for /dev/null 'meta': [ '(Dir)', { 'private': [ '(Dir)', { # No `opts/artifacts_may_require_repo` here because we # directly created the subvol instead of using an Item. 'mount': [ '(Dir)', { 'lala': [ '(Dir)', { 'MOUNT': [ '(Dir)', { 'is_directory': ['(File d2)'], 'build_source': [ '(Dir)', { 'type': [ '(File d5)' ], 'source': [ f'(File d{len("/dev/null") + 1})' ], } ], } ] } ] } ], } ] } ], } ], render_subvol(subvol)) for filename, contents in ( ('is_directory', '0\n'), ('build_source/type', 'host\n'), ('build_source/source', '/dev/null\n'), ): with open( subvol.path( os.path.join( 'meta/private/mount/lala/MOUNT', filename, ))) as f: self.assertEqual(contents, f.read())
def _test_symlink_command(self, layer_opts): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('dir')]) # We need a source file to validate a SymlinkToFileItem with tempfile.NamedTemporaryFile() as tf: InstallFileItem( from_target='t', source=tf.name, dest='/file', ).build(subvol, layer_opts) SymlinkToDirItem(from_target='t', source='/dir', dest='/dir_symlink').build(subvol, layer_opts) SymlinkToFileItem(from_target='t', source='file', dest='/file_symlink').build(subvol, layer_opts) # Make a couple of absolute symlinks to test our behavior on # linking to paths that contain those. subvol.run_as_root([ 'bash', '-c', f'''\ ln -s /file {subvol.path('abs_link_to_file').shell_quote()} mkdir {subvol.path('my_dir').shell_quote()} touch {subvol.path('my_dir/inner').shell_quote()} ln -s /my_dir {subvol.path('my_dir_link').shell_quote()} ''' ]) # A simple case: we link to an absolute link. SymlinkToFileItem( from_target='t', source='/abs_link_to_file', dest='/link_to_abs_link', ).build(subvol, layer_opts) # This link traverses a directory that is an absolute link. The # resulting relative symlink is not traversible from outside the # container. SymlinkToFileItem( from_target='t', source='my_dir_link/inner', dest='/dir/inner_link', ).build(subvol, layer_opts) self.assertEqual([ '(Dir)', { 'dir': [ '(Dir)', { 'inner_link': ['(Symlink ../my_dir_link/inner)'], } ], 'dir_symlink': ['(Symlink dir)'], 'file': ['(File m444)'], 'file_symlink': ['(Symlink file)'], 'abs_link_to_file': ['(Symlink /file)'], 'my_dir': ['(Dir)', { 'inner': ['(File)'] }], 'my_dir_link': ['(Symlink /my_dir)'], 'link_to_abs_link': ['(Symlink abs_link_to_file)'], } ], render_subvol(subvol))
def test_tarball_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) # Fail on pre-existing files subvol.run_as_root(['touch', subvol.path('d/exists')]) with tempfile.NamedTemporaryFile() as t: with tarfile.TarFile(t.name, 'w') as tar_obj: tar_obj.addfile(tarfile.TarInfo('exists')) with self.assertRaises(subprocess.CalledProcessError): _tarball_item(t.name, '/d').build(subvol, DUMMY_LAYER_OPTS_BA) # Adding new files & directories works. Overwriting a # pre-existing directory leaves the owner+mode of the original # directory intact. subvol.run_as_root(['mkdir', subvol.path('d/old_dir')]) subvol.run_as_root(['chown', '123:456', subvol.path('d/old_dir')]) subvol.run_as_root(['chmod', '0301', subvol.path('d/old_dir')]) subvol_root = temp_subvolumes.snapshot(subvol, 'tar-sv-root') subvol_zst = temp_subvolumes.snapshot(subvol, 'tar-sv-zst') with tempfile.TemporaryDirectory() as td: tar_path = os.path.join(td, 'test.tar') zst_path = os.path.join(td, 'test.tar.zst') with tarfile.TarFile(tar_path, 'w') as tar_obj: tar_obj.addfile(tarfile.TarInfo('new_file')) new_dir = tarfile.TarInfo('new_dir') new_dir.type = tarfile.DIRTYPE new_dir.uid = 12 new_dir.gid = 34 tar_obj.addfile(new_dir) old_dir = tarfile.TarInfo('old_dir') old_dir.type = tarfile.DIRTYPE # These will not be applied because old_dir exists old_dir.uid = 0 old_dir.gid = 0 old_dir.mode = 0o755 tar_obj.addfile(old_dir) subprocess.check_call(['zstd', tar_path, '-o', zst_path]) # Fail when the destination does not exist with self.assertRaises(subprocess.CalledProcessError): _tarball_item(tar_path, '/no_dir').build(subvol, DUMMY_LAYER_OPTS_BA) # Before unpacking the tarball orig_content = [ '(Dir)', { 'd': [ '(Dir)', { 'exists': ['(File)'], 'old_dir': ['(Dir m301 o123:456)', {}], } ] } ] # After unpacking `tar_path` in `/d`. new_content = copy.deepcopy(orig_content) new_content[1]['d'][1].update({ 'new_dir': ['(Dir m644 o12:34)', {}], 'new_file': ['(File)'], }) # After unpacking `tar_path` in `/d` with `force_root_ownership` new_content_root = copy.deepcopy(new_content) # The ownership of 12:34 is gone. new_content_root[1]['d'][1]['new_dir'] = ['(Dir m644)', {}] self.assertNotEqual(new_content, new_content_root) # Check the subvolume content before and after unpacking for item, (sv, before, after) in ( ( _tarball_item(tar_path, '/d/'), (subvol, orig_content, new_content), ), ( _tarball_item(tar_path, 'd', force_root_ownership=True), (subvol_root, orig_content, new_content_root), ), ( _tarball_item(zst_path, 'd/'), (subvol_zst, orig_content, new_content), ), ): self.assertEqual(before, render_subvol(sv)) item.build(sv, DUMMY_LAYER_OPTS_BA) self.assertEqual(after, render_subvol(sv))
def test_mount_item(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes, \ tempfile.TemporaryDirectory() as source_dir: runtime_source = {'so': 'me', 'arbitrary': {'j': 'son'}} mount_config = { 'is_directory': True, 'build_source': { 'type': 'layer', 'source': '//fake:path' }, 'runtime_source': runtime_source, } with open(os.path.join(source_dir, 'mountconfig.json'), 'w') as f: json.dump(mount_config, f) self._check_item( self._make_mount_item( mountpoint='can/haz', target=source_dir, mount_config=mount_config, ), {ProvidesDoNotAccess(path='can/haz')}, {require_directory('can')}, ) # Make a subvolume that we will mount inside `mounter` mountee = temp_subvolumes.create('moun:tee/volume') mountee.run_as_root(['tee', mountee.path('kitteh')], input=b'cheez') # These sub-mounts inside `mountee` act as canaries to make sure # that (a) `mounter` receives the sub-mounts as a consequence of # mounting `mountee` recursively, (b) that unmounting one in # `mounter` does not affect the original in `mountee` -- i.e. # that rslave propagation is set up correctly, (c) that # unmounting in `mountee` immediately affects `mounter`. # # In practice, our build artifacts should NEVER be mutated after # construction (and the only un-mount is implicitly, and # seemingly safely, performed by `btrfs subvolume delete`). # However, ensuring that we have correct `rslave` propagation is # a worthwhile safeguard for host mounts, where an errant # `umount` by a user inside their repo could otherwise break # their host. for submount in ('submount1', 'submount2'): mountee.run_as_root(['mkdir', mountee.path(submount)]) mountee.run_as_root([ 'mount', '-o', 'bind,ro', source_dir, mountee.path(submount) ]) self.assertTrue( os.path.exists(mountee.path(submount + '/mountconfig.json'))) # Make the JSON file normally in "buck-out" that refers to `mountee` mountee_subvolumes_dir = self._write_layer_json_into( mountee, source_dir) # Mount <mountee> at <mounter>/meow mounter = temp_subvolumes.caller_will_create('mount:er/volume') root_item = FilesystemRootItem(from_target='t') root_item.get_phase_builder([root_item], DUMMY_LAYER_OPTS)(mounter) mount_meow = self._make_mount_item( mountpoint='meow', target=source_dir, mount_config=mount_config, ) self.assertEqual( runtime_source, json.loads(mount_meow.runtime_source), ) with self.assertRaisesRegex(AssertionError, ' could not resolve '): mount_meow.build_source.to_path( target_to_path={}, subvolumes_dir=mountee_subvolumes_dir, ) mount_meow.build( mounter, DUMMY_LAYER_OPTS._replace( target_to_path={'//fake:path': source_dir}, subvolumes_dir=mountee_subvolumes_dir, )) # This checks the subvolume **contents**, but not the mounts. # Ensure the build created a mountpoint, and populated metadata. self._check_subvol_mounts_meow(mounter) # `mountee` was also mounted at `/meow` with open(mounter.path('meow/kitteh')) as f: self.assertEqual('cheez', f.read()) def check_mountee_mounter_submounts(submount_presence): for submount, (in_mountee, in_mounter) in submount_presence: self.assertEqual( in_mountee, os.path.exists( mountee.path(submount + '/mountconfig.json')), f'{submount}, {in_mountee}') self.assertEqual( in_mounter, os.path.exists( mounter.path('meow/' + submount + '/mountconfig.json')), f'{submount}, {in_mounter}') # Both sub-mounts are accessible in both places now. check_mountee_mounter_submounts([ ('submount1', (True, True)), ('submount2', (True, True)), ]) # Unmounting `submount1` from `mountee` also affects `mounter`. mountee.run_as_root(['umount', mountee.path('submount1')]) check_mountee_mounter_submounts([ ('submount1', (False, False)), ('submount2', (True, True)), ]) # Unmounting `submount2` from `mounter` doesn't affect `mountee`. mounter.run_as_root(['umount', mounter.path('meow/submount2')]) check_mountee_mounter_submounts([ ('submount1', (False, False)), ('submount2', (True, False)), ]) # Check that we read back the `mounter` metadata, mark `/meow` # inaccessible, and do not emit a `ProvidesFile` for `kitteh`. self._check_item( PhasesProvideItem(from_target='t', subvol=mounter), { ProvidesDirectory(path='/'), ProvidesDoNotAccess(path='/meta'), ProvidesDoNotAccess(path='/meow'), }, set(), ) # Check that we successfully clone mounts from the parent layer. mounter_child = temp_subvolumes.caller_will_create('child/volume') ParentLayerItem.get_phase_builder( [ParentLayerItem(from_target='t', subvol=mounter)], DUMMY_LAYER_OPTS, )(mounter_child) # The child has the same mount, and the same metadata self._check_subvol_mounts_meow(mounter_child) # Check that we refuse to create nested mounts. nested_mounter = temp_subvolumes.create('nested_mounter') nested_item = MountItem( layer_opts=DUMMY_LAYER_OPTS, from_target='t', mountpoint='/whatever', target=None, mount_config={ 'is_directory': True, 'build_source': { 'type': 'layer', 'source': '//:fake' }, }, ) with tempfile.TemporaryDirectory() as d: mounter_subvolumes_dir = self._write_layer_json_into( mounter, d) with self.assertRaisesRegex( AssertionError, 'Refusing .* nested mount', ): nested_item.build( nested_mounter, DUMMY_LAYER_OPTS._replace( target_to_path={'//:fake': d}, subvolumes_dir=mounter_subvolumes_dir, ))
def _check_rpm_action_item(self, layer_opts): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('rpm_action') self.assertEqual(['(Dir)', {}], render_subvol(subvol)) # The empty action is a no-op RpmActionItem.get_phase_builder([], layer_opts)(subvol) self.assertEqual(['(Dir)', {}], render_subvol(subvol)) # `yum-dnf-from-snapshot` needs a `/meta` directory to work subvol.run_as_root(['mkdir', subvol.path('meta')]) self.assertEqual( # No `opts/artifacts_may_require_repo` here because we directly # created the subvol instead of using an Item. ['(Dir)', {'meta': ['(Dir)', {}]}], render_subvol(subvol), ) # Specifying RPM versions is prohibited with self.assertRaises(subprocess.CalledProcessError): RpmActionItem.get_phase_builder( [RpmActionItem( from_target='m', name='rpm-test-milk-2.71', source=None, action=RpmAction.install, )], layer_opts, )(subvol) # Cannot pass both `name` and `source` with self.assertRaisesRegex( AssertionError, 'Exactly one of `name` or `source` must be set .*', ): RpmActionItem.get_phase_builder( [RpmActionItem( from_target='m', name='rpm-test-milk', source='rpm-test-milk', action=RpmAction.install, )], layer_opts, )(subvol) RpmActionItem.get_phase_builder( [ RpmActionItem( from_target='t', name=n, action=RpmAction.install, ) for n in [ # This specific RPM contains `/bin/sh` and a # post-install script to test `/dev/null` isolation. 'rpm-test-milk', 'rpm-test-carrot', ] ], layer_opts, )(subvol) # Clean up the `dnf`, `yum` & `rpm` litter before checking the # packages. Maybe fixme: We end up not asserting ownership / # permissions / etc on directories like /var and /dev. subvol.run_as_root([ 'rm', '-rf', # Annotate all paths since `sudo rm -rf` is scary. subvol.path('var/lib/rpm'), subvol.path('var/lib/yum'), subvol.path('var/lib/dnf'), subvol.path('var/log/yum.log'), *(subvol.path('var/log/' + log) for log in [ 'yum.log', 'dnf.log', 'dnf.librepo.log', 'dnf.rpm.log', 'hawkey.log', ]), subvol.path('usr/lib/.build-id'), subvol.path('bin/sh'), ]) if self._YUM_DNF == YumDnf.dnf: subvol.run_as_root([ 'rmdir', subvol.path('etc/dnf/modules.d'), subvol.path('etc/dnf'), subvol.path('etc'), ]) # The way that `RpmActionItem` nspawns into build_appliance must # gurantee that `/var/cache/{dnf,yum}` is empty. The next two # lines test that the cache directory is empty because `rmdir` # fails if it is not. It is important that the cache of built # images be empty, to avoid unnecessarily increasing the # distributed image size. rm_cmd = ['rmdir'] if ( layer_opts.build_appliance and not layer_opts.preserve_yum_dnf_cache ) else ['rm', '-rf'] subvol.run_as_root(rm_cmd + [ subvol.path(f'var/cache/{self._YUM_DNF.value}') ]) subvol.run_as_root([ 'rmdir', subvol.path('dev'), # made by yum_dnf_from_snapshot.py subvol.path('meta'), subvol.path('var/cache'), subvol.path('var/lib'), subvol.path('var/log'), subvol.path('var/tmp'), subvol.path('var'), subvol.path('usr/lib'), subvol.path('bin'), ]) self.assertEqual(['(Dir)', { 'rpm_test': ['(Dir)', { 'carrot.txt': ['(File d13)'], 'milk.txt': ['(File d12)'], 'post.txt': ['(File d6)'], }], 'usr': ['(Dir)', {}], }], render_subvol(subvol))
def test_compile(self): # First, test compilation with no parent layer. expected_calls = self._expected_run_as_root_calls() self.assertGreater( # Sanity check: at least one command per item len(expected_calls), len(si.ID_TO_ITEM), ) self._assert_equal_call_sets( expected_calls, self._compiler_run_as_root_calls( parent_feature_json=[], parent_dep=[], ), ) # Now, add an empty parent layer with TempSubvolumes(sys.argv[0]) as temp_subvolumes: parent = temp_subvolumes.create('parent') # Manually add/remove some commands from the "expected" set to # accommodate the fact that we have a parent subvolume. subvol_path = f'{_SUBVOLS_DIR}/{_FAKE_SUBVOL}'.encode() # Our unittest.mock.call objects are (args, kwargs) pairs. expected_calls_with_parent = [ c for c in expected_calls if c not in [ ( (['btrfs', 'subvolume', 'create', subvol_path], ), { '_subvol_exists': False }, ), ((['chmod', '0755', subvol_path], ), {}), ((['chown', 'root:root', subvol_path], ), {}), ] ] + [ ( (['test', '!', '-e', subvol_path], ), { '_subvol_exists': False }, ), ( ([ 'btrfs', 'subvolume', 'snapshot', parent.path(), subvol_path, ], ), { '_subvol_exists': False }, ), ] self.assertEqual( # We should've removed 3, and added 2 commands len(expected_calls_with_parent) + 1, len(expected_calls), ) with mock_layer_dir_access(self, parent.path()) as parent_dir: with open(parent_dir / 'feature.json', 'w') as out_f: json.dump( { 'parent_layer': [{ 'subvol': { '__BUCK_LAYER_TARGET': '//fake:parent' }, }], 'target': '//ignored:target', }, out_f) self._assert_equal_call_sets( expected_calls_with_parent, self._compiler_run_as_root_calls( parent_feature_json=[ '--child-feature-json=' + f'{parent_dir / "feature.json"}', ], parent_dep=['//fake:parent', parent_dir.decode()], ), )