def test_item_predecessors(self): dg = DependencyGraph(PATH_TO_ITEM.values(), layer_target='t-34') self.assertEqual( _fs_root_phases(FilesystemRootItem(from_target='t-34')), list(dg.ordered_phases()), ) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') phases_provide = PhasesProvideItem(from_target='t', subvol=subvol) ns = dg._prep_item_predecessors(phases_provide) path_to_item = {'/': phases_provide, **PATH_TO_ITEM} self.assertEqual( ns.item_to_predecessors, { path_to_item[k]: {path_to_item[v] for v in vs} for k, vs in { '/a/b/c': {'/'}, '/a/d/e': {'/a/b/c'}, '/a/b/c/F': {'/a/b/c'}, '/a/d/e/G': {'/a/d/e'}, }.items() }) self.assertEqual( ns.predecessor_to_items, { path_to_item[k]: {path_to_item[v] for v in vs} for k, vs in { '/': {'/a/b/c'}, '/a/b/c': {'/a/d/e', '/a/b/c/F'}, '/a/b/c/F': set(), '/a/d/e': {'/a/d/e/G'}, '/a/d/e/G': set(), }.items() }) self.assertEqual(ns.items_without_predecessors, {path_to_item['/']})
def test_filesystem_root(self): item = FilesystemRootItem(from_target='t') self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order()) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.caller_will_create('fs-root') item.get_phase_builder([item], DUMMY_LAYER_OPTS)(subvol) self.assertEqual( [ '(Dir)', { 'meta': [ '(Dir)', { 'private': [ '(Dir)', { 'opts': [ '(Dir)', { 'artifacts_may_require_repo': ['(File d2)'], } ], } ] } ] } ], render_subvol(subvol), )
def _make_mutate_ops_subvolume( subvols: TempSubvolumes, create_ops: Subvol, path: bytes, ) -> Subvol: 'Exercise the send-stream ops that are unique to snapshots.' subvol = subvols.snapshot(create_ops, path) # snapshot run = partial(subvol.run_as_root, cwd=subvol.path()) run(['rm', 'hello/world']) # unlink run(['rmdir', 'dir_to_remove/']) # rmdir run([ # remove_xattr 'setfattr', '--remove=user.test_attr', 'hello/', ]) # You would think this would emit a `rename`, but for files, the # sendstream instead `link`s to the new location, and unlinks the old. run(['mv', 'goodbye', 'farewell']) # NOT a rename, {,un}link run(['mv', 'hello/', 'hello_renamed/']) # yes, a rename! run( # write ['dd', 'of=hello_renamed/een'], input=b'push\n', ) # This is a no-op because `btfs send` does not support `chattr` at # present. However, it's good to have a canary so that our tests start # failing the moment it is supported -- that will remind us to update # the mock VFS. NB: The absolute path to `chattr` is a clowny hack to # work around a clowny hack, to work around clowny hacks. Don't ask. run(['/usr/bin/chattr', '+a', 'hello_renamed/een']) return subvol
def test_phase_order(self): class FakeRemovePaths: get_phase_builder = 'kittycat' def phase_order(self): return PhaseOrder.REMOVE_PATHS first = FilesystemRootItem(from_target='') second = FakeRemovePaths() third = MakeDirsItem(from_target='', into_dir='/', path_to_make='a/b') dg = DependencyGraph([second, first, third]) self.assertEqual( _fs_root_phases(first) + [ (FakeRemovePaths.get_phase_builder, (second,)), ], list(dg.ordered_phases()), ) # We had a phase other than PARENT_LAYER, so the dependency sorting # will need to inspect the resulting subvolume -- let it be empty. with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') self.assertEqual([third], list(dg.gen_dependency_order_items( subvol.path().decode(), )))
def test_install_order(self): dg = DependencyGraph(si.ID_TO_ITEM.values(), layer_target='ttt') builders_and_phases = list(dg.ordered_phases()) self.assertEqual([ ( FilesystemRootItem.get_phase_builder, (si.ID_TO_ITEM['/'], ), ), ( RpmActionItem.get_phase_builder, ( si.ID_TO_ITEM['.rpms/remove_if_exists/rpm-test-carrot'], si.ID_TO_ITEM['.rpms/remove_if_exists/rpm-test-milk'], ), ), ( RpmActionItem.get_phase_builder, ( si.ID_TO_ITEM['.rpms/install/rpm-test-mice'], si.ID_TO_ITEM['.rpms/install/rpm-test-cheese-2-1.rpm'], ), ), ( RemovePathItem.get_phase_builder, ( si.ID_TO_ITEM['.remove_if_exists/path/to/remove'], si.ID_TO_ITEM['.remove_assert_exists/path/to/remove'], si. ID_TO_ITEM['.remove_assert_exists/another/path/to/remove'], ), ), ], builders_and_phases) phase_items = [i for _, items in builders_and_phases for i in items] with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') doi = list( dg.gen_dependency_order_items( PhasesProvideItem(from_target='t', subvol=subvol), )) self.assertEqual(set(si.ID_TO_ITEM.values()), set(doi + phase_items)) self.assertEqual( len(si.ID_TO_ITEM), len(doi) + len(phase_items), msg='Duplicate items?', ) id_to_idx = { k: doi.index(v) for k, v in si.ID_TO_ITEM.items() if v not in phase_items } # The 2 mounts are not ordered in any way with respect to the # `foo/bar` tree, so any of these 3 can be the first. mount_idxs = sorted([id_to_idx['host_etc'], id_to_idx['meownt']]) if mount_idxs == [0, 1]: self.assertEqual(2, id_to_idx['foo/bar']) elif 0 in mount_idxs: self.assertEqual(1, id_to_idx['foo/bar']) else: self.assertEqual(0, id_to_idx['foo/bar']) self.assertLess(id_to_idx['foo/borf/beep'], id_to_idx['foo/borf/hello_world'])
def test_filesystem_root(self): self._check_item( FilesystemRootItem(from_target='t'), {ProvidesDirectory(path='/')}, set(), ) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.caller_will_create('fs-root') FilesystemRootItem(from_target='t').build(subvol) self.assertEqual(['(Dir)', {}], _render_subvol(subvol))
def test_install_file_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) _install_file_item( from_target='t', source={ 'source': '/dev/null' }, dest='/d/null', ).build(subvol, DUMMY_LAYER_OPTS) self.assertEqual( ['(Dir)', { 'd': ['(Dir)', { 'null': ['(File m444)'] }] }], render_subvol(subvol), ) # Fail to write to a nonexistent dir with self.assertRaises(subprocess.CalledProcessError): _install_file_item( from_target='t', source={ 'source': '/dev/null' }, dest='/no_dir/null', ).build(subvol, DUMMY_LAYER_OPTS) # Running a second copy to the same destination. This just # overwrites the previous file, because we have a build-time # check for this, and a run-time check would add overhead. _install_file_item( from_target='t', source={ 'source': '/dev/null' }, dest='/d/null', # A non-default mode & owner shows that the file was # overwritten, and also exercises HasStatOptions. mode='u+rw', user_group='12:34', ).build(subvol, DUMMY_LAYER_OPTS) self.assertEqual( ['(Dir)', { 'd': ['(Dir)', { 'null': ['(File m600 o12:34)'] }] }], render_subvol(subvol), )
def test_rpm_action_item_auto_downgrade(self): parent_subvol = find_built_subvol( (Path(__file__).dirname() / 'test-with-one-local-rpm').decode()) src_rpm = Path(__file__).dirname() / "rpm-test-cheese-1-1.rpm" with TempSubvolumes(sys.argv[0]) as temp_subvolumes: # ensure cheese2 is installed in the parent from rpm-test-cheese-2-1 assert os.path.isfile( parent_subvol.path('/usr/share/rpm_test/cheese2.txt')) # make sure the RPM we are installing is older in order to # trigger the downgrade src_data = RpmMetadata.from_file(src_rpm) subvol_data = RpmMetadata.from_subvol(parent_subvol, src_data.name) assert compare_rpm_versions(src_data, subvol_data) < 0 subvol = temp_subvolumes.snapshot(parent_subvol, 'rpm_action') RpmActionItem.get_phase_builder( [ RpmActionItem( from_target='t', source=src_rpm, action=RpmAction.install, ) ], DUMMY_LAYER_OPTS._replace( yum_from_snapshot=Path(__file__).dirname() / 'yum-from-test-snapshot', ), )(subvol) subvol.run_as_root([ 'rm', '-rf', subvol.path('dev'), subvol.path('meta'), subvol.path('var'), ]) self.assertEqual([ '(Dir)', { 'usr': [ '(Dir)', { 'share': [ '(Dir)', { 'rpm_test': [ '(Dir)', { 'cheese1.txt': ['(File d36)'], } ], } ], } ], } ], render_subvol(subvol))
def test_copy_file_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) CopyFileItem( # `dest` has a rsync-convention trailing / from_target='t', source='/dev/null', dest='/d/', ).build(subvol) self.assertEqual( ['(Dir)', { 'd': ['(Dir)', { 'null': ['(File m755)'] }] }], _render_subvol(subvol), ) # Fail to write to a nonexistent dir with self.assertRaises(subprocess.CalledProcessError): CopyFileItem( from_target='t', source='/dev/null', dest='/no_dir/', ).build(subvol) # Running a second copy to the same destination. This just # overwrites the previous file, because we have a build-time # check for this, and a run-time check would add overhead. CopyFileItem( # Test this works without the rsync-covnvention /, too from_target='t', source='/dev/null', dest='/d/null', # A non-default mode & owner shows that the file was # overwritten, and also exercises HasStatOptions. mode='u+rw', user='******', group='34', ).build(subvol) self.assertEqual( ['(Dir)', { 'd': ['(Dir)', { 'null': ['(File m600 o12:34)'] }] }], _render_subvol(subvol), )
def test_make_dirs_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) MakeDirsItem( from_target='t', path_to_make='/a/b/', into_dir='/d', user='******', group='88', mode='u+rx', ).build(subvol) self.assertEqual([ '(Dir)', { 'd': [ '(Dir)', { 'a': [ '(Dir m500 o77:88)', { 'b': ['(Dir m500 o77:88)', {}], } ], } ], } ], _render_subvol(subvol)) # The "should never happen" cases -- since we have build-time # checks, for simplicity/speed, our runtime clobbers permissions # of preexisting directories, and quietly creates non-existent # ones with default permissions. MakeDirsItem(from_target='t', path_to_make='a', into_dir='/no_dir', user='******').build(subvol) MakeDirsItem(from_target='t', path_to_make='a/new', into_dir='/d', user='******').build(subvol) self.assertEqual(['(Dir)', { 'd': ['(Dir)', { # permissions overwritten for this whole tree 'a': ['(Dir o5:0)', { 'b': ['(Dir o5:0)', {}], 'new': ['(Dir o5:0)', {}], }], }], 'no_dir': ['(Dir)', { # default permissions! 'a': ['(Dir o4:0)', {}], }], }], _render_subvol(subvol))
def test_receive_sendstream(self): item = ReceiveSendstreamItem( from_target='t', source=Path(__file__).dirname() / 'create_ops.sendstream', ) self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order()) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: new_subvol_name = 'differs_from_create_ops' subvol = temp_subvolumes.caller_will_create(new_subvol_name) item.get_phase_builder([item], DUMMY_LAYER_OPTS)(subvol) self.assertEqual( render_demo_subvols(create_ops=new_subvol_name), render_sendstream(subvol.mark_readonly_and_get_sendstream()), )
def _snapshot_subvol(src_subvol, snapshot_into): if snapshot_into: nspawn_subvol = Subvol(snapshot_into) nspawn_subvol.snapshot(src_subvol) clone_mounts(src_subvol, nspawn_subvol) yield nspawn_subvol else: with TempSubvolumes() as tmp_subvols: # To make it easier to debug where a temporary subvolume came # from, make make its name resemble that of its source. tmp_name = os.path.normpath(src_subvol.path()) tmp_name = os.path.basename(os.path.dirname(tmp_name)) or \ os.path.basename(tmp_name) nspawn_subvol = tmp_subvols.snapshot(src_subvol, tmp_name) clone_mounts(src_subvol, nspawn_subvol) yield nspawn_subvol
def make_demo_sendstreams(path_in_repo: bytes): with TempSubvolumes(path_in_repo) as subvols: res = {} with _populate_sendstream_dict(res.setdefault('create_ops', {})) as d: create_ops = _make_create_ops_subvolume(subvols, b'create_ops') d['sendstream'] = create_ops.mark_readonly_and_get_sendstream( no_data=True, ) with _populate_sendstream_dict(res.setdefault('mutate_ops', {})) as d: d['sendstream'] = _make_mutate_ops_subvolume( subvols, create_ops, b'mutate_ops', ).mark_readonly_and_get_sendstream(parent=create_ops) return res
def test_package_image_as_squashfs(self): with self._package_image( self._sibling_path('create_ops.layer'), 'squashfs', ) as out_path, TempSubvolumes(sys.argv[0]) as temp_subvolumes, \ tempfile.NamedTemporaryFile() as temp_sendstream: subvol = temp_subvolumes.create('subvol') with Unshare([Namespace.MOUNT, Namespace.PID]) as unshare, \ tempfile.TemporaryDirectory() as mount_dir: subprocess.check_call(nsenter_as_root( unshare, 'mount', '-t', 'squashfs', '-o', 'loop', out_path, mount_dir, )) # `unsquashfs` would have been cleaner than `mount` + # `rsync`, and faster too, but unfortunately it corrupts # device nodes as of v4.3. subprocess.check_call(nsenter_as_root( unshare, 'rsync', '--archive', '--hard-links', '--sparse', '--xattrs', mount_dir + '/', subvol.path(), )) with subvol.mark_readonly_and_write_sendstream_to_file( temp_sendstream ): pass original_render = _render_sendstream_path( self._sibling_path('create_ops-original.sendstream'), ) # SquashFS does not preserve the original's cloned extents of # zeros, nor the zero-hole-zero patter. In all cases, it # (efficiently) transmutes the whole file into 1 sparse hole. self.assertEqual(original_render[1].pop('56KB_nuls'), [ '(File d57344(create_ops@56KB_nuls_clone:0+49152@0/' + 'create_ops@56KB_nuls_clone:49152+8192@49152))' ]) original_render[1]['56KB_nuls'] = ['(File h57344)'] self.assertEqual(original_render[1].pop('56KB_nuls_clone'), [ '(File d57344(create_ops@56KB_nuls:0+49152@0/' + 'create_ops@56KB_nuls:49152+8192@49152))' ]) original_render[1]['56KB_nuls_clone'] = ['(File h57344)'] self.assertEqual(original_render[1].pop('zeros_hole_zeros'), [ '(File d16384h16384d16384)' ]) original_render[1]['zeros_hole_zeros'] = ['(File h49152)'] self.assertEqual( original_render, _render_sendstream_path(temp_sendstream.name), )
def test_parent_layer(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: parent = temp_subvolumes.create('parent') item = ParentLayerItem(from_target='t', subvol=parent) self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order()) MakeDirsItem( from_target='t', into_dir='/', path_to_make='a/b', ).build(parent, DUMMY_LAYER_OPTS) parent_content = ['(Dir)', {'a': ['(Dir)', {'b': ['(Dir)', {}]}]}] self.assertEqual(parent_content, render_subvol(parent)) # Take a snapshot and add one more directory. child = temp_subvolumes.caller_will_create('child') item.get_phase_builder([item], DUMMY_LAYER_OPTS)(child) MakeDirsItem( from_target='t', into_dir='a', path_to_make='c', ).build(child, DUMMY_LAYER_OPTS) # The parent is unchanged. self.assertEqual(parent_content, render_subvol(parent)) child_content = copy.deepcopy(parent_content) child_content[1]['a'][1]['c'] = ['(Dir)', {}] # Since the parent lacked a /meta, the child added it. child_content[1]['meta'] = [ '(Dir)', { 'private': [ '(Dir)', { 'opts': [ '(Dir)', { 'artifacts_may_require_repo': ['(File d2)'] } ] } ] } ] self.assertEqual(child_content, render_subvol(child))
def test_rpm_build_item(self): parent_subvol = find_built_subvol( (Path(__file__).dirname() / 'toy-rpmbuild-setup').decode()) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: assert os.path.isfile( parent_subvol.path('/rpmbuild/SOURCES/toy_src_file')) assert os.path.isfile( parent_subvol.path('/rpmbuild/SPECS/specfile.spec')) subvol = temp_subvolumes.snapshot(parent_subvol, 'rpm_build') item = RpmBuildItem(from_target='t', rpmbuild_dir='/rpmbuild') RpmBuildItem.get_phase_builder( [item], DUMMY_LAYER_OPTS, )(subvol) self.assertEqual(item.phase_order(), PhaseOrder.RPM_BUILD) assert os.path.isfile(subvol.path('/rpmbuild/RPMS/toy.rpm'))
def _make_mutate_ops_subvolume( subvols: TempSubvolumes, create_ops: Subvol, path: bytes, ) -> Subvol: 'Exercise the send-stream ops that are unique to snapshots.' subvol = subvols.snapshot(create_ops, path) # snapshot run = subvol.run_as_root # `cwd` is intentionally prohibited with `run_as_root` def p(sv_path): return subvol.path(sv_path).decode() run(['rm', p('hello/world')]) # unlink run(['rmdir', p('dir_to_remove/')]) # rmdir run([ # remove_xattr 'setfattr', '--remove=user.test_attr', p('hello/'), ]) # You would think this would emit a `rename`, but for files, the # sendstream instead `link`s to the new location, and unlinks the old. run(['mv', p('goodbye'), p('farewell')]) # NOT a rename, {,un}link run(['mv', p('hello/'), p('hello_renamed/')]) # yes, a rename! run( # write ['dd', 'of=' + p('hello_renamed/een')], input=b'push\n', ) # This is a no-op because `btfs send` does not support `chattr` at # present. However, it's good to have a canary so that our tests start # failing the moment it is supported -- that will remind us to update # the mock VFS. NB: The absolute path to `chattr` is a clowny hack to # work around a clowny hack, to work around clowny hacks. Don't ask. run(['/usr/bin/chattr', '+a', p('hello_renamed/een')]) # Besides files with trailing holes, one can also get `truncate` # sendstream commands in incremental sendstreams by having a snapshot # truncate relative a file relative to the parent. run(['truncate', '-s', '2', p('hello_big_hole')]) return subvol
def test_symlink_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('dir')]) # We need a source file to validate a SymlinkToFileItem InstallFileItem( from_target='t', source='/dev/null', dest='/file', ).build(subvol, DUMMY_LAYER_OPTS) SymlinkToDirItem( from_target='t', source='/dir', dest='/dir_symlink' ).build(subvol, DUMMY_LAYER_OPTS) SymlinkToFileItem( from_target='t', source='file', dest='/file_symlink' ).build(subvol, DUMMY_LAYER_OPTS) self.assertEqual(['(Dir)', { 'dir': ['(Dir)', {}], 'dir_symlink': ['(Symlink /dir)'], 'file': ['(File m444)'], 'file_symlink': ['(Symlink /file)'], }], render_subvol(subvol))
def test_gen_dependency_graph(self): dg = DependencyGraph(PATH_TO_ITEM.values(), layer_target='t-72') self.assertEqual( _fs_root_phases(FilesystemRootItem(from_target='t-72')), list(dg.ordered_phases()), ) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') self.assertIn( tuple( dg.gen_dependency_order_items( PhasesProvideItem(from_target='t', subvol=subvol), )), { tuple(PATH_TO_ITEM[p] for p in paths) for paths in [ # A few orders are valid, don't make the test fragile. ['/a/b/c', '/a/b/c/F', '/a/d/e', '/a/d/e/G'], ['/a/b/c', '/a/d/e', '/a/b/c/F', '/a/d/e/G'], ['/a/b/c', '/a/d/e', '/a/d/e/G', '/a/b/c/F'], ] }, )
def make_demo_sendstreams(path_in_repo: bytes): with TempSubvolumes(path_in_repo) as subvols: res = {} with _populate_sendstream_dict(res.setdefault('create_ops', {})) as d: create_ops = _make_create_ops_subvolume(subvols, b'create_ops') d['sendstream'] = create_ops.mark_readonly_and_get_sendstream() with _populate_sendstream_dict(res.setdefault('mutate_ops', {})) as d: d['sendstream'] = _make_mutate_ops_subvolume( subvols, create_ops, b'mutate_ops', ).mark_readonly_and_get_sendstream( parent=create_ops, # The resulting send-stream will have `update_extent` # instead of `write`, which is one way of making sure that # `update_extent` in `parse_sendstream.py` is covered. no_data=True, ) return res
def test_phases_provide(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: parent = temp_subvolumes.create('parent') # Permit _populate_temp_filesystem to make writes. parent.run_as_root([ 'chown', '--no-dereference', f'{os.geteuid()}:{os.getegid()}', parent.path(), ]) populate_temp_filesystem(parent.path().decode()) for create_meta in [False, True]: # Check that we properly handle ignoring a /meta if it's present if create_meta: parent.run_as_root(['mkdir', parent.path('meta')]) self._check_item( PhasesProvideItem(from_target='t', subvol=parent), temp_filesystem_provides() | { ProvidesDirectory(path='/'), ProvidesDoNotAccess(path='/meta'), }, set(), )
def test_cycle_detection(self): def requires_provides_directory_class(requires_dir, provides_dir): class RequiresProvidesDirectory(metaclass=ImageItem): def requires(self): yield require_directory(requires_dir) def provides(self): yield ProvidesDirectory(path=provides_dir) return RequiresProvidesDirectory # `dg_ok`: dependency-sorting will work without a cycle first = FilesystemRootItem(from_target='') second = requires_provides_directory_class('/', 'a')(from_target='') third = MakeDirsItem(from_target='', into_dir='a', path_to_make='b/c') dg_ok = DependencyGraph([second, first, third], layer_target='t') self.assertEqual(_fs_root_phases(first), list(dg_ok.ordered_phases())) # `dg_bad`: changes `second` to get a cycle dg_bad = DependencyGraph([ requires_provides_directory_class('a/b', 'a')(from_target=''), first, third, ], layer_target='t') self.assertEqual(_fs_root_phases(first), list(dg_bad.ordered_phases())) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') provides_root = PhasesProvideItem(from_target='t', subvol=subvol) self.assertEqual( [second, third], list(dg_ok.gen_dependency_order_items(provides_root)), ) with self.assertRaisesRegex(AssertionError, '^Cycle in '): list(dg_bad.gen_dependency_order_items(provides_root))
def test_parent_layer(self): # First, get reasonable coverage of enumerating files and # directories without a real btrfs subvol. with self._temp_filesystem() as parent_path: self._check_item( ParentLayerItem(from_target='t', path=parent_path), self._temp_filesystem_provides() | { ProvidesDirectory(path='/'), }, set(), ) # Now exercise actually making a btrfs snapshot. with TempSubvolumes(sys.argv[0]) as temp_subvolumes: parent = temp_subvolumes.create('parent') MakeDirsItem( from_target='t', into_dir='/', path_to_make='a/b', ).build(parent) parent_content = ['(Dir)', {'a': ['(Dir)', {'b': ['(Dir)', {}]}]}] self.assertEqual(parent_content, _render_subvol(parent)) # Take a snapshot and add one more directory. child = temp_subvolumes.caller_will_create('child') ParentLayerItem(from_target='t', path=parent.path()).build(child) MakeDirsItem( from_target='t', into_dir='a', path_to_make='c', ).build(child) # The parent is unchanged. self.assertEqual(parent_content, _render_subvol(parent)) child_content = copy.deepcopy(parent_content) child_content[1]['a'][1]['c'] = ['(Dir)', {}] self.assertEqual(child_content, _render_subvol(child))
def test_phase_order(self): class FakeRemovePaths: get_phase_builder = 'kittycat' def phase_order(self): return PhaseOrder.REMOVE_PATHS first = FilesystemRootItem(from_target='') second = FakeRemovePaths() third = MakeDirsItem(from_target='', into_dir='/', path_to_make='a/b') dg = DependencyGraph([second, first, third], layer_target='t') self.assertEqual( _fs_root_phases(first) + [ (FakeRemovePaths.get_phase_builder, (second, )), ], list(dg.ordered_phases()), ) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') self.assertEqual([third], list( dg.gen_dependency_order_items( PhasesProvideItem(from_target='t', subvol=subvol), )))
def test_remove_item(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('remove_action') self.assertEqual(['(Dir)', {}], render_subvol(subvol)) MakeDirsItem( from_target='t', path_to_make='/a/b/c', into_dir='/', ).build(subvol, DUMMY_LAYER_OPTS) for d in ['d', 'e']: InstallFileItem( from_target='t', source='/dev/null', dest=f'/a/b/c/{d}', ).build(subvol, DUMMY_LAYER_OPTS) MakeDirsItem( from_target='t', path_to_make='/f/g', into_dir='/', ).build(subvol, DUMMY_LAYER_OPTS) # Checks that `rm` won't follow symlinks SymlinkToDirItem( from_target='t', source='/f', dest='/a/b/f_sym', ).build(subvol, DUMMY_LAYER_OPTS) for d in ['h', 'i']: InstallFileItem( from_target='t', source='/dev/null', dest=f'/f/{d}', ).build(subvol, DUMMY_LAYER_OPTS) SymlinkToDirItem( from_target='t', source='/f/i', dest='/f/i_sym', ).build(subvol, DUMMY_LAYER_OPTS) intact_subvol = [ '(Dir)', { 'a': [ '(Dir)', { 'b': [ '(Dir)', { 'c': [ '(Dir)', { 'd': ['(File m444)'], 'e': ['(File m444)'], } ], 'f_sym': ['(Symlink /f)'], } ], } ], 'f': [ '(Dir)', { 'g': ['(Dir)', {}], 'h': ['(File m444)'], 'i': ['(File m444)'], 'i_sym': ['(Symlink /f/i)'], } ], } ] self.assertEqual(intact_subvol, render_subvol(subvol)) # We refuse to touch protected paths, even with "if_exists". If # the paths started with 'meta', they would trip the check in # `_make_path_normal_relative`, so we mock-protect 'xyz'. for prot_path in ['xyz', 'xyz/potato/carrot']: with unittest.mock.patch( 'fs_image.compiler.items.remove_path.protected_path_set', side_effect=lambda sv: protected_path_set(sv) | {'xyz'}, ), self.assertRaisesRegex( AssertionError, f'Cannot remove protected .*{prot_path}', ): RemovePathItem.get_phase_builder([ RemovePathItem( from_target='t', action=RemovePathAction.if_exists, path=prot_path, ) ], DUMMY_LAYER_OPTS)(subvol) # Check handling of non-existent paths without removing anything remove = RemovePathItem( from_target='t', action=RemovePathAction.if_exists, path='/does/not/exist', ) self.assertEqual(PhaseOrder.REMOVE_PATHS, remove.phase_order()) RemovePathItem.get_phase_builder([remove], DUMMY_LAYER_OPTS)(subvol) with self.assertRaisesRegex(AssertionError, 'does not exist'): RemovePathItem.get_phase_builder([ RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/does/not/exist', ), ], DUMMY_LAYER_OPTS)(subvol) self.assertEqual(intact_subvol, render_subvol(subvol)) # Now remove most of the subvolume. RemovePathItem.get_phase_builder( [ # These 3 removes are not covered by a recursive remove. # And we leave behind /f/i, which lets us know that neither # `f_sym` nor `i_sym` were followed during their deletion. RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/f/i_sym', ), RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/f/h', ), RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/f/g', ), # The next 3 items are intentionally sequenced so that if # they were applied in the given order, they would fail. RemovePathItem( from_target='t', action=RemovePathAction.if_exists, path='/a/b/c/e', ), RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, # The surrounding items don't delete /a/b/c/d, e.g. so # this recursive remove is still tested. path='/a/b/', ), RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/a/b/c/e', ), ], DUMMY_LAYER_OPTS)(subvol) self.assertEqual([ '(Dir)', { 'a': ['(Dir)', {}], 'f': ['(Dir)', { 'i': ['(File m444)'] }], } ], render_subvol(subvol))
def _make_create_ops_subvolume(subvols: TempSubvolumes, path: bytes) -> Subvol: 'Exercise all the send-stream ops that can occur on a new subvolume.' subvol = subvols.create(path) run = subvol.run_as_root # `cwd` is intentionally prohibited with `run_as_root` def p(sv_path): return subvol.path(sv_path).decode() # Due to an odd `btrfs send` implementation detail, creating a file or # directory emits a rename from a temporary name to the final one. run(['mkdir', p('hello')]) # mkdir,rename run(['mkdir', p('dir_to_remove')]) run(['touch', p('hello/world')]) # mkfile,utimes,chmod,chown run([ # set_xattr 'setfattr', '-n', 'user.test_attr', '-v', 'chickens', p('hello/'), ]) run(['mknod', p('buffered'), 'b', '1337', '31415']) # mknod run(['chmod', 'og-r', p('buffered')]) # chmod a device run(['mknod', p('unbuffered'), 'c', '1337', '31415']) run(['mkfifo', p('fifo')]) # mkfifo run([ 'python3', '-c', ( 'import os, sys, socket as s\n' 'dir, base = os.path.split(sys.argv[1])\n' # Otherwise, we can easily get "AF_UNIX path too long" 'os.chdir(os.path.join(".", dir))\n' 'with s.socket(s.AF_UNIX, s.SOCK_STREAM) as sock:\n' ' sock.bind(base)\n' # mksock ), p('unix_sock') ]) run(['ln', p('hello/world'), p('goodbye')]) # link run(['ln', '-s', 'hello/world', p('bye_symlink')]) # symlink run([ # update_extent # 56KB was chosen so that `btrfs send` emits more than 1 write, # specifically 48KB + 8KB. 'dd', 'if=/dev/zero', 'of=' + p('56KB_nuls'), 'bs=1024', 'count=56', ]) run([ # clone 'cp', '--reflink=always', p('56KB_nuls'), p('56KB_nuls_clone'), ]) # Make a file with a 16KB hole in the middle. run([ 'dd', 'if=/dev/zero', 'of=' + p('zeros_hole_zeros'), 'bs=1024', 'count=16', ]) run(['truncate', '-s', str(32 * 1024), p('zeros_hole_zeros')]) run([ 'dd', 'if=/dev/zero', 'of=' + p('zeros_hole_zeros'), 'oflag=append', 'conv=notrunc', 'bs=1024', 'count=16', ]) # A trailing hole exercises the `truncate` sendstream command. run(['bash', '-c', 'echo hello > ' + shlex.quote(p('hello_big_hole'))]) run(['truncate', '-s', '1G', p('hello_big_hole')]) # This just serves to show that `btrfs send` ignores nested subvolumes. # There is no mention of `nested_subvol` in the send-stream. nested_subvol = subvols.create(p('nested_subvol')) nested_subvol.run_as_root(['touch', nested_subvol.path('borf')]) nested_subvol.run_as_root(['mkdir', nested_subvol.path('beep')]) return subvol
def _make_create_ops_subvolume(subvols: TempSubvolumes, path: bytes) -> Subvol: 'Exercise all the send-stream ops that can occur on a new subvolume.' subvol = subvols.create(path) run = partial(subvol.run_as_root, cwd=subvol.path()) # Due to an odd `btrfs send` implementation detail, creating a file or # directory emits a rename from a temporary name to the final one. run(['mkdir', 'hello']) # mkdir,rename run(['mkdir', 'dir_to_remove']) run(['touch', 'hello/world']) # mkfile,utimes,chmod,chown run([ # set_xattr 'setfattr', '-n', 'user.test_attr', '-v', 'chickens', 'hello/', ]) run(['mknod', 'buffered', 'b', '1337', '31415']) # mknod run(['chmod', 'og-r', 'buffered']) # chmod a device run(['mknod', 'unbuffered', 'c', '1337', '31415']) run(['mkfifo', 'fifo']) # mkfifo run([ 'python3', '-c', ( 'import socket as s\n' 'with s.socket(s.AF_UNIX, s.SOCK_STREAM) as sock:\n' ' sock.bind("unix_sock")\n' # mksock ) ]) run(['ln', 'hello/world', 'goodbye']) # link run(['ln', '-s', 'hello/world', 'bye_symlink']) # symlink run([ # update_extent # 56KB was chosen so that `btrfs send` emits more than 1 write, # specifically 48KB + 8KB. 'dd', 'if=/dev/zero', 'of=56KB_nuls', 'bs=1024', 'count=56', ]) run([ # clone 'cp', '--reflink=always', '56KB_nuls', '56KB_nuls_clone', ]) # Make a file with a 16KB hole in the middle. run(['dd', 'if=/dev/zero', 'of=zeros_hole_zeros', 'bs=1024', 'count=16']) run(['truncate', '-s', str(32 * 1024), 'zeros_hole_zeros']) run([ 'dd', 'if=/dev/zero', 'of=zeros_hole_zeros', 'oflag=append', 'conv=notrunc', 'bs=1024', 'count=16', ]) # This just serves to show that `btrfs send` ignores nested subvolumes. # There is no mention of `nested_subvol` in the send-stream. nested_subvol = subvols.create(subvol.path(b'nested_subvol')) run2 = partial(nested_subvol.run_as_root, cwd=nested_subvol.path()) run2(['touch', 'borf']) run2(['mkdir', 'beep']) return subvol
def test_multi_rpm_action(self): # This works in @mode/opt since this binary is baked into the XAR yum = os.path.join(os.path.dirname(__file__), 'yum-from-test-snapshot') mice = MultiRpmAction.new( frozenset(['rpm-test-mice']), RpmActionType.install, yum, ) carrot = MultiRpmAction.new( frozenset(['rpm-test-carrot']), RpmActionType.install, yum, ) buggy_mice = mice._replace(yum_from_snapshot='bug') with self.assertRaises(AssertionError): buggy_mice.union(carrot) action = mice.union(carrot) self.assertEqual({'rpm-test-mice', 'rpm-test-carrot'}, action.rpms) self.assertEqual(carrot, action._replace(rpms={'rpm-test-carrot'})) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('rpm_action') self.assertEqual(['(Dir)', {}], _render_subvol(subvol)) # The empty action is a no-op MultiRpmAction.new( frozenset(), RpmActionType.install, None, ).build(subvol) self.assertEqual(['(Dir)', {}], _render_subvol(subvol)) # Specifying RPM versions is prohibited with self.assertRaises(subprocess.CalledProcessError): MultiRpmAction.new( frozenset(['rpm-test-mice-2']), RpmActionType.install, yum, ).build(subvol) action.build(subvol) # Clean up the `yum` & `rpm` litter before checking the packages. subvol.run_as_root([ 'rm', '-rf', # Annotate all paths since `sudo rm -rf` is scary. subvol.path('var/cache/yum'), subvol.path('var/lib/yum'), subvol.path('var/lib/rpm'), subvol.path('var/log/yum.log'), ]) subvol.run_as_root([ 'rmdir', 'var/cache', 'var/lib', 'var/log', 'var', ], cwd=subvol.path()) self.assertEqual([ '(Dir)', { 'usr': [ '(Dir)', { 'share': [ '(Dir)', { 'rpm_test': [ '(Dir)', { 'carrot.txt': ['(File d13)'], 'mice.txt': ['(File d11)'], } ], } ], } ], } ], _render_subvol(subvol))
def test_tarball_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) # Fail on pre-existing files subvol.run_as_root(['touch', subvol.path('d/exists')]) with tempfile.NamedTemporaryFile() as t: with tarfile.TarFile(t.name, 'w') as tar_obj: tar_obj.addfile(tarfile.TarInfo('exists')) with self.assertRaises(subprocess.CalledProcessError): TarballItem( from_target='t', into_dir='/d', tarball=t.name, ).build(subvol) # Adding new files & directories works. Overwriting a # pre-existing directory leaves the owner+mode of the original # directory intact. subvol.run_as_root(['mkdir', subvol.path('d/old_dir')]) subvol.run_as_root(['chown', '123:456', subvol.path('d/old_dir')]) subvol.run_as_root(['chmod', '0301', subvol.path('d/old_dir')]) with tempfile.NamedTemporaryFile() as t: with tarfile.TarFile(t.name, 'w') as tar_obj: tar_obj.addfile(tarfile.TarInfo('new_file')) new_dir = tarfile.TarInfo('new_dir') new_dir.type = tarfile.DIRTYPE tar_obj.addfile(new_dir) old_dir = tarfile.TarInfo('old_dir') old_dir.type = tarfile.DIRTYPE # These will not be applied because old_dir exists old_dir.uid = 0 old_dir.gid = 0 old_dir.mode = 0o755 tar_obj.addfile(old_dir) # Fail when the destination does not exist with self.assertRaises(subprocess.CalledProcessError): TarballItem( from_target='t', into_dir='/no_dir', tarball=t.name, ).build(subvol) # Check the subvolume content before and after unpacking content = [ '(Dir)', { 'd': [ '(Dir)', { 'exists': ['(File)'], 'old_dir': ['(Dir m301 o123:456)', {}], } ] } ] self.assertEqual(content, _render_subvol(subvol)) TarballItem( from_target='t', into_dir='/d', tarball=t.name, ).build(subvol) content[1]['d'][1].update({ 'new_dir': ['(Dir m644)', {}], 'new_file': ['(File)'], }) self.assertEqual(content, _render_subvol(subvol))
def test_paths_to_reqs_provs(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') provides_root = PhasesProvideItem(from_target='t', subvol=subvol) expected = { '/meta': ItemReqsProvs( item_provs={ ItemProv(ProvidesDoNotAccess(path='/meta'), provides_root) }, item_reqs=set(), ), '/': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='/'), provides_root) }, item_reqs={ ItemReq(require_directory('/'), PATH_TO_ITEM['/a/b/c']) }, ), '/a': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a'), PATH_TO_ITEM['/a/b/c']) }, item_reqs={ ItemReq(require_directory('a'), PATH_TO_ITEM['/a/d/e']) }, ), '/a/b': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a/b'), PATH_TO_ITEM['/a/b/c']) }, item_reqs=set(), ), '/a/b/c': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a/b/c'), PATH_TO_ITEM['/a/b/c']) }, item_reqs={ ItemReq(require_directory('a/b/c'), PATH_TO_ITEM['/a/b/c/F']) }, ), '/a/b/c/F': ItemReqsProvs( item_provs={ ItemProv(ProvidesFile(path='a/b/c/F'), PATH_TO_ITEM['/a/b/c/F']) }, item_reqs=set(), ), '/a/d': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a/d'), PATH_TO_ITEM['/a/d/e']) }, item_reqs=set(), ), '/a/d/e': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a/d/e'), PATH_TO_ITEM['/a/d/e']) }, item_reqs={ ItemReq(require_directory('a/d/e'), PATH_TO_ITEM['/a/d/e/G']) }, ), '/a/d/e/G': ItemReqsProvs( item_provs={ ItemProv(ProvidesFile(path='a/d/e/G'), PATH_TO_ITEM['/a/d/e/G']) }, item_reqs=set(), ), } self.assertEqual( ValidatedReqsProvs([provides_root, *PATH_TO_ITEM.values() ]).path_to_reqs_provs, expected)