def test_item_predecessors(self): dg = DependencyGraph(PATH_TO_ITEM.values(), layer_target='t-34') self.assertEqual( _fs_root_phases(FilesystemRootItem(from_target='t-34')), list(dg.ordered_phases()), ) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') phases_provide = PhasesProvideItem(from_target='t', subvol=subvol) ns = dg._prep_item_predecessors(phases_provide) path_to_item = {'/': phases_provide, **PATH_TO_ITEM} self.assertEqual( ns.item_to_predecessors, { path_to_item[k]: {path_to_item[v] for v in vs} for k, vs in { '/a/b/c': {'/'}, '/a/d/e': {'/a/b/c'}, '/a/b/c/F': {'/a/b/c'}, '/a/d/e/G': {'/a/d/e'}, }.items() }) self.assertEqual( ns.predecessor_to_items, { path_to_item[k]: {path_to_item[v] for v in vs} for k, vs in { '/': {'/a/b/c'}, '/a/b/c': {'/a/d/e', '/a/b/c/F'}, '/a/b/c/F': set(), '/a/d/e': {'/a/d/e/G'}, '/a/d/e/G': set(), }.items() }) self.assertEqual(ns.items_without_predecessors, {path_to_item['/']})
def test_filesystem_root(self): item = FilesystemRootItem(from_target='t') self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order()) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.caller_will_create('fs-root') item.get_phase_builder([item], DUMMY_LAYER_OPTS)(subvol) self.assertEqual( [ '(Dir)', { 'meta': [ '(Dir)', { 'private': [ '(Dir)', { 'opts': [ '(Dir)', { 'artifacts_may_require_repo': ['(File d2)'], } ], } ] } ] } ], render_subvol(subvol), )
def test_phase_order(self): class FakeRemovePaths: get_phase_builder = 'kittycat' def phase_order(self): return PhaseOrder.REMOVE_PATHS first = FilesystemRootItem(from_target='') second = FakeRemovePaths() third = MakeDirsItem(from_target='', into_dir='/', path_to_make='a/b') dg = DependencyGraph([second, first, third]) self.assertEqual( _fs_root_phases(first) + [ (FakeRemovePaths.get_phase_builder, (second,)), ], list(dg.ordered_phases()), ) # We had a phase other than PARENT_LAYER, so the dependency sorting # will need to inspect the resulting subvolume -- let it be empty. with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') self.assertEqual([third], list(dg.gen_dependency_order_items( subvol.path().decode(), )))
def test_install_order(self): dg = DependencyGraph(si.ID_TO_ITEM.values(), layer_target='ttt') builders_and_phases = list(dg.ordered_phases()) self.assertEqual([ ( FilesystemRootItem.get_phase_builder, (si.ID_TO_ITEM['/'], ), ), ( RpmActionItem.get_phase_builder, ( si.ID_TO_ITEM['.rpms/remove_if_exists/rpm-test-carrot'], si.ID_TO_ITEM['.rpms/remove_if_exists/rpm-test-milk'], ), ), ( RpmActionItem.get_phase_builder, ( si.ID_TO_ITEM['.rpms/install/rpm-test-mice'], si.ID_TO_ITEM['.rpms/install/rpm-test-cheese-2-1.rpm'], ), ), ( RemovePathItem.get_phase_builder, ( si.ID_TO_ITEM['.remove_if_exists/path/to/remove'], si.ID_TO_ITEM['.remove_assert_exists/path/to/remove'], si. ID_TO_ITEM['.remove_assert_exists/another/path/to/remove'], ), ), ], builders_and_phases) phase_items = [i for _, items in builders_and_phases for i in items] with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') doi = list( dg.gen_dependency_order_items( PhasesProvideItem(from_target='t', subvol=subvol), )) self.assertEqual(set(si.ID_TO_ITEM.values()), set(doi + phase_items)) self.assertEqual( len(si.ID_TO_ITEM), len(doi) + len(phase_items), msg='Duplicate items?', ) id_to_idx = { k: doi.index(v) for k, v in si.ID_TO_ITEM.items() if v not in phase_items } # The 2 mounts are not ordered in any way with respect to the # `foo/bar` tree, so any of these 3 can be the first. mount_idxs = sorted([id_to_idx['host_etc'], id_to_idx['meownt']]) if mount_idxs == [0, 1]: self.assertEqual(2, id_to_idx['foo/bar']) elif 0 in mount_idxs: self.assertEqual(1, id_to_idx['foo/bar']) else: self.assertEqual(0, id_to_idx['foo/bar']) self.assertLess(id_to_idx['foo/borf/beep'], id_to_idx['foo/borf/hello_world'])
def test_filesystem_root(self): self._check_item( FilesystemRootItem(from_target='t'), {ProvidesDirectory(path='/')}, set(), ) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.caller_will_create('fs-root') FilesystemRootItem(from_target='t').build(subvol) self.assertEqual(['(Dir)', {}], _render_subvol(subvol))
def test_install_file_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) _install_file_item( from_target='t', source={ 'source': '/dev/null' }, dest='/d/null', ).build(subvol, DUMMY_LAYER_OPTS) self.assertEqual( ['(Dir)', { 'd': ['(Dir)', { 'null': ['(File m444)'] }] }], render_subvol(subvol), ) # Fail to write to a nonexistent dir with self.assertRaises(subprocess.CalledProcessError): _install_file_item( from_target='t', source={ 'source': '/dev/null' }, dest='/no_dir/null', ).build(subvol, DUMMY_LAYER_OPTS) # Running a second copy to the same destination. This just # overwrites the previous file, because we have a build-time # check for this, and a run-time check would add overhead. _install_file_item( from_target='t', source={ 'source': '/dev/null' }, dest='/d/null', # A non-default mode & owner shows that the file was # overwritten, and also exercises HasStatOptions. mode='u+rw', user_group='12:34', ).build(subvol, DUMMY_LAYER_OPTS) self.assertEqual( ['(Dir)', { 'd': ['(Dir)', { 'null': ['(File m600 o12:34)'] }] }], render_subvol(subvol), )
def test_rpm_action_item_auto_downgrade(self): parent_subvol = find_built_subvol( (Path(__file__).dirname() / 'test-with-one-local-rpm').decode()) src_rpm = Path(__file__).dirname() / "rpm-test-cheese-1-1.rpm" with TempSubvolumes(sys.argv[0]) as temp_subvolumes: # ensure cheese2 is installed in the parent from rpm-test-cheese-2-1 assert os.path.isfile( parent_subvol.path('/usr/share/rpm_test/cheese2.txt')) # make sure the RPM we are installing is older in order to # trigger the downgrade src_data = RpmMetadata.from_file(src_rpm) subvol_data = RpmMetadata.from_subvol(parent_subvol, src_data.name) assert compare_rpm_versions(src_data, subvol_data) < 0 subvol = temp_subvolumes.snapshot(parent_subvol, 'rpm_action') RpmActionItem.get_phase_builder( [ RpmActionItem( from_target='t', source=src_rpm, action=RpmAction.install, ) ], DUMMY_LAYER_OPTS._replace( yum_from_snapshot=Path(__file__).dirname() / 'yum-from-test-snapshot', ), )(subvol) subvol.run_as_root([ 'rm', '-rf', subvol.path('dev'), subvol.path('meta'), subvol.path('var'), ]) self.assertEqual([ '(Dir)', { 'usr': [ '(Dir)', { 'share': [ '(Dir)', { 'rpm_test': [ '(Dir)', { 'cheese1.txt': ['(File d36)'], } ], } ], } ], } ], render_subvol(subvol))
def test_make_dirs_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) MakeDirsItem( from_target='t', path_to_make='/a/b/', into_dir='/d', user='******', group='88', mode='u+rx', ).build(subvol) self.assertEqual([ '(Dir)', { 'd': [ '(Dir)', { 'a': [ '(Dir m500 o77:88)', { 'b': ['(Dir m500 o77:88)', {}], } ], } ], } ], _render_subvol(subvol)) # The "should never happen" cases -- since we have build-time # checks, for simplicity/speed, our runtime clobbers permissions # of preexisting directories, and quietly creates non-existent # ones with default permissions. MakeDirsItem(from_target='t', path_to_make='a', into_dir='/no_dir', user='******').build(subvol) MakeDirsItem(from_target='t', path_to_make='a/new', into_dir='/d', user='******').build(subvol) self.assertEqual(['(Dir)', { 'd': ['(Dir)', { # permissions overwritten for this whole tree 'a': ['(Dir o5:0)', { 'b': ['(Dir o5:0)', {}], 'new': ['(Dir o5:0)', {}], }], }], 'no_dir': ['(Dir)', { # default permissions! 'a': ['(Dir o4:0)', {}], }], }], _render_subvol(subvol))
def test_receive_sendstream(self): item = ReceiveSendstreamItem( from_target='t', source=Path(__file__).dirname() / 'create_ops.sendstream', ) self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order()) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: new_subvol_name = 'differs_from_create_ops' subvol = temp_subvolumes.caller_will_create(new_subvol_name) item.get_phase_builder([item], DUMMY_LAYER_OPTS)(subvol) self.assertEqual( render_demo_subvols(create_ops=new_subvol_name), render_sendstream(subvol.mark_readonly_and_get_sendstream()), )
def test_copy_file_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) CopyFileItem( # `dest` has a rsync-convention trailing / from_target='t', source='/dev/null', dest='/d/', ).build(subvol) self.assertEqual( ['(Dir)', { 'd': ['(Dir)', { 'null': ['(File m755)'] }] }], _render_subvol(subvol), ) # Fail to write to a nonexistent dir with self.assertRaises(subprocess.CalledProcessError): CopyFileItem( from_target='t', source='/dev/null', dest='/no_dir/', ).build(subvol) # Running a second copy to the same destination. This just # overwrites the previous file, because we have a build-time # check for this, and a run-time check would add overhead. CopyFileItem( # Test this works without the rsync-covnvention /, too from_target='t', source='/dev/null', dest='/d/null', # A non-default mode & owner shows that the file was # overwritten, and also exercises HasStatOptions. mode='u+rw', user='******', group='34', ).build(subvol) self.assertEqual( ['(Dir)', { 'd': ['(Dir)', { 'null': ['(File m600 o12:34)'] }] }], _render_subvol(subvol), )
def _snapshot_subvol(src_subvol, snapshot_into): if snapshot_into: nspawn_subvol = Subvol(snapshot_into) nspawn_subvol.snapshot(src_subvol) clone_mounts(src_subvol, nspawn_subvol) yield nspawn_subvol else: with TempSubvolumes() as tmp_subvols: # To make it easier to debug where a temporary subvolume came # from, make make its name resemble that of its source. tmp_name = os.path.normpath(src_subvol.path()) tmp_name = os.path.basename(os.path.dirname(tmp_name)) or \ os.path.basename(tmp_name) nspawn_subvol = tmp_subvols.snapshot(src_subvol, tmp_name) clone_mounts(src_subvol, nspawn_subvol) yield nspawn_subvol
def test_package_image_as_squashfs(self): with self._package_image( self._sibling_path('create_ops.layer'), 'squashfs', ) as out_path, TempSubvolumes(sys.argv[0]) as temp_subvolumes, \ tempfile.NamedTemporaryFile() as temp_sendstream: subvol = temp_subvolumes.create('subvol') with Unshare([Namespace.MOUNT, Namespace.PID]) as unshare, \ tempfile.TemporaryDirectory() as mount_dir: subprocess.check_call(nsenter_as_root( unshare, 'mount', '-t', 'squashfs', '-o', 'loop', out_path, mount_dir, )) # `unsquashfs` would have been cleaner than `mount` + # `rsync`, and faster too, but unfortunately it corrupts # device nodes as of v4.3. subprocess.check_call(nsenter_as_root( unshare, 'rsync', '--archive', '--hard-links', '--sparse', '--xattrs', mount_dir + '/', subvol.path(), )) with subvol.mark_readonly_and_write_sendstream_to_file( temp_sendstream ): pass original_render = _render_sendstream_path( self._sibling_path('create_ops-original.sendstream'), ) # SquashFS does not preserve the original's cloned extents of # zeros, nor the zero-hole-zero patter. In all cases, it # (efficiently) transmutes the whole file into 1 sparse hole. self.assertEqual(original_render[1].pop('56KB_nuls'), [ '(File d57344(create_ops@56KB_nuls_clone:0+49152@0/' + 'create_ops@56KB_nuls_clone:49152+8192@49152))' ]) original_render[1]['56KB_nuls'] = ['(File h57344)'] self.assertEqual(original_render[1].pop('56KB_nuls_clone'), [ '(File d57344(create_ops@56KB_nuls:0+49152@0/' + 'create_ops@56KB_nuls:49152+8192@49152))' ]) original_render[1]['56KB_nuls_clone'] = ['(File h57344)'] self.assertEqual(original_render[1].pop('zeros_hole_zeros'), [ '(File d16384h16384d16384)' ]) original_render[1]['zeros_hole_zeros'] = ['(File h49152)'] self.assertEqual( original_render, _render_sendstream_path(temp_sendstream.name), )
def make_demo_sendstreams(path_in_repo: bytes): with TempSubvolumes(path_in_repo) as subvols: res = {} with _populate_sendstream_dict(res.setdefault('create_ops', {})) as d: create_ops = _make_create_ops_subvolume(subvols, b'create_ops') d['sendstream'] = create_ops.mark_readonly_and_get_sendstream( no_data=True, ) with _populate_sendstream_dict(res.setdefault('mutate_ops', {})) as d: d['sendstream'] = _make_mutate_ops_subvolume( subvols, create_ops, b'mutate_ops', ).mark_readonly_and_get_sendstream(parent=create_ops) return res
def test_parent_layer(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: parent = temp_subvolumes.create('parent') item = ParentLayerItem(from_target='t', subvol=parent) self.assertEqual(PhaseOrder.MAKE_SUBVOL, item.phase_order()) MakeDirsItem( from_target='t', into_dir='/', path_to_make='a/b', ).build(parent, DUMMY_LAYER_OPTS) parent_content = ['(Dir)', {'a': ['(Dir)', {'b': ['(Dir)', {}]}]}] self.assertEqual(parent_content, render_subvol(parent)) # Take a snapshot and add one more directory. child = temp_subvolumes.caller_will_create('child') item.get_phase_builder([item], DUMMY_LAYER_OPTS)(child) MakeDirsItem( from_target='t', into_dir='a', path_to_make='c', ).build(child, DUMMY_LAYER_OPTS) # The parent is unchanged. self.assertEqual(parent_content, render_subvol(parent)) child_content = copy.deepcopy(parent_content) child_content[1]['a'][1]['c'] = ['(Dir)', {}] # Since the parent lacked a /meta, the child added it. child_content[1]['meta'] = [ '(Dir)', { 'private': [ '(Dir)', { 'opts': [ '(Dir)', { 'artifacts_may_require_repo': ['(File d2)'] } ] } ] } ] self.assertEqual(child_content, render_subvol(child))
def test_rpm_build_item(self): parent_subvol = find_built_subvol( (Path(__file__).dirname() / 'toy-rpmbuild-setup').decode()) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: assert os.path.isfile( parent_subvol.path('/rpmbuild/SOURCES/toy_src_file')) assert os.path.isfile( parent_subvol.path('/rpmbuild/SPECS/specfile.spec')) subvol = temp_subvolumes.snapshot(parent_subvol, 'rpm_build') item = RpmBuildItem(from_target='t', rpmbuild_dir='/rpmbuild') RpmBuildItem.get_phase_builder( [item], DUMMY_LAYER_OPTS, )(subvol) self.assertEqual(item.phase_order(), PhaseOrder.RPM_BUILD) assert os.path.isfile(subvol.path('/rpmbuild/RPMS/toy.rpm'))
def test_gen_dependency_graph(self): dg = DependencyGraph(PATH_TO_ITEM.values(), layer_target='t-72') self.assertEqual( _fs_root_phases(FilesystemRootItem(from_target='t-72')), list(dg.ordered_phases()), ) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') self.assertIn( tuple( dg.gen_dependency_order_items( PhasesProvideItem(from_target='t', subvol=subvol), )), { tuple(PATH_TO_ITEM[p] for p in paths) for paths in [ # A few orders are valid, don't make the test fragile. ['/a/b/c', '/a/b/c/F', '/a/d/e', '/a/d/e/G'], ['/a/b/c', '/a/d/e', '/a/b/c/F', '/a/d/e/G'], ['/a/b/c', '/a/d/e', '/a/d/e/G', '/a/b/c/F'], ] }, )
def make_demo_sendstreams(path_in_repo: bytes): with TempSubvolumes(path_in_repo) as subvols: res = {} with _populate_sendstream_dict(res.setdefault('create_ops', {})) as d: create_ops = _make_create_ops_subvolume(subvols, b'create_ops') d['sendstream'] = create_ops.mark_readonly_and_get_sendstream() with _populate_sendstream_dict(res.setdefault('mutate_ops', {})) as d: d['sendstream'] = _make_mutate_ops_subvolume( subvols, create_ops, b'mutate_ops', ).mark_readonly_and_get_sendstream( parent=create_ops, # The resulting send-stream will have `update_extent` # instead of `write`, which is one way of making sure that # `update_extent` in `parse_sendstream.py` is covered. no_data=True, ) return res
def test_symlink_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('dir')]) # We need a source file to validate a SymlinkToFileItem InstallFileItem( from_target='t', source='/dev/null', dest='/file', ).build(subvol, DUMMY_LAYER_OPTS) SymlinkToDirItem( from_target='t', source='/dir', dest='/dir_symlink' ).build(subvol, DUMMY_LAYER_OPTS) SymlinkToFileItem( from_target='t', source='file', dest='/file_symlink' ).build(subvol, DUMMY_LAYER_OPTS) self.assertEqual(['(Dir)', { 'dir': ['(Dir)', {}], 'dir_symlink': ['(Symlink /dir)'], 'file': ['(File m444)'], 'file_symlink': ['(Symlink /file)'], }], render_subvol(subvol))
def test_phases_provide(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: parent = temp_subvolumes.create('parent') # Permit _populate_temp_filesystem to make writes. parent.run_as_root([ 'chown', '--no-dereference', f'{os.geteuid()}:{os.getegid()}', parent.path(), ]) populate_temp_filesystem(parent.path().decode()) for create_meta in [False, True]: # Check that we properly handle ignoring a /meta if it's present if create_meta: parent.run_as_root(['mkdir', parent.path('meta')]) self._check_item( PhasesProvideItem(from_target='t', subvol=parent), temp_filesystem_provides() | { ProvidesDirectory(path='/'), ProvidesDoNotAccess(path='/meta'), }, set(), )
def test_parent_layer(self): # First, get reasonable coverage of enumerating files and # directories without a real btrfs subvol. with self._temp_filesystem() as parent_path: self._check_item( ParentLayerItem(from_target='t', path=parent_path), self._temp_filesystem_provides() | { ProvidesDirectory(path='/'), }, set(), ) # Now exercise actually making a btrfs snapshot. with TempSubvolumes(sys.argv[0]) as temp_subvolumes: parent = temp_subvolumes.create('parent') MakeDirsItem( from_target='t', into_dir='/', path_to_make='a/b', ).build(parent) parent_content = ['(Dir)', {'a': ['(Dir)', {'b': ['(Dir)', {}]}]}] self.assertEqual(parent_content, _render_subvol(parent)) # Take a snapshot and add one more directory. child = temp_subvolumes.caller_will_create('child') ParentLayerItem(from_target='t', path=parent.path()).build(child) MakeDirsItem( from_target='t', into_dir='a', path_to_make='c', ).build(child) # The parent is unchanged. self.assertEqual(parent_content, _render_subvol(parent)) child_content = copy.deepcopy(parent_content) child_content[1]['a'][1]['c'] = ['(Dir)', {}] self.assertEqual(child_content, _render_subvol(child))
def test_phase_order(self): class FakeRemovePaths: get_phase_builder = 'kittycat' def phase_order(self): return PhaseOrder.REMOVE_PATHS first = FilesystemRootItem(from_target='') second = FakeRemovePaths() third = MakeDirsItem(from_target='', into_dir='/', path_to_make='a/b') dg = DependencyGraph([second, first, third], layer_target='t') self.assertEqual( _fs_root_phases(first) + [ (FakeRemovePaths.get_phase_builder, (second, )), ], list(dg.ordered_phases()), ) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') self.assertEqual([third], list( dg.gen_dependency_order_items( PhasesProvideItem(from_target='t', subvol=subvol), )))
def test_cycle_detection(self): def requires_provides_directory_class(requires_dir, provides_dir): class RequiresProvidesDirectory(metaclass=ImageItem): def requires(self): yield require_directory(requires_dir) def provides(self): yield ProvidesDirectory(path=provides_dir) return RequiresProvidesDirectory # `dg_ok`: dependency-sorting will work without a cycle first = FilesystemRootItem(from_target='') second = requires_provides_directory_class('/', 'a')(from_target='') third = MakeDirsItem(from_target='', into_dir='a', path_to_make='b/c') dg_ok = DependencyGraph([second, first, third], layer_target='t') self.assertEqual(_fs_root_phases(first), list(dg_ok.ordered_phases())) # `dg_bad`: changes `second` to get a cycle dg_bad = DependencyGraph([ requires_provides_directory_class('a/b', 'a')(from_target=''), first, third, ], layer_target='t') self.assertEqual(_fs_root_phases(first), list(dg_bad.ordered_phases())) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') provides_root = PhasesProvideItem(from_target='t', subvol=subvol) self.assertEqual( [second, third], list(dg_ok.gen_dependency_order_items(provides_root)), ) with self.assertRaisesRegex(AssertionError, '^Cycle in '): list(dg_bad.gen_dependency_order_items(provides_root))
def test_multi_rpm_action(self): # This works in @mode/opt since this binary is baked into the XAR yum = os.path.join(os.path.dirname(__file__), 'yum-from-test-snapshot') mice = MultiRpmAction.new( frozenset(['rpm-test-mice']), RpmActionType.install, yum, ) carrot = MultiRpmAction.new( frozenset(['rpm-test-carrot']), RpmActionType.install, yum, ) buggy_mice = mice._replace(yum_from_snapshot='bug') with self.assertRaises(AssertionError): buggy_mice.union(carrot) action = mice.union(carrot) self.assertEqual({'rpm-test-mice', 'rpm-test-carrot'}, action.rpms) self.assertEqual(carrot, action._replace(rpms={'rpm-test-carrot'})) with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('rpm_action') self.assertEqual(['(Dir)', {}], _render_subvol(subvol)) # The empty action is a no-op MultiRpmAction.new( frozenset(), RpmActionType.install, None, ).build(subvol) self.assertEqual(['(Dir)', {}], _render_subvol(subvol)) # Specifying RPM versions is prohibited with self.assertRaises(subprocess.CalledProcessError): MultiRpmAction.new( frozenset(['rpm-test-mice-2']), RpmActionType.install, yum, ).build(subvol) action.build(subvol) # Clean up the `yum` & `rpm` litter before checking the packages. subvol.run_as_root([ 'rm', '-rf', # Annotate all paths since `sudo rm -rf` is scary. subvol.path('var/cache/yum'), subvol.path('var/lib/yum'), subvol.path('var/lib/rpm'), subvol.path('var/log/yum.log'), ]) subvol.run_as_root([ 'rmdir', 'var/cache', 'var/lib', 'var/log', 'var', ], cwd=subvol.path()) self.assertEqual([ '(Dir)', { 'usr': [ '(Dir)', { 'share': [ '(Dir)', { 'rpm_test': [ '(Dir)', { 'carrot.txt': ['(File d13)'], 'mice.txt': ['(File d11)'], } ], } ], } ], } ], _render_subvol(subvol))
def test_tarball_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) # Fail on pre-existing files subvol.run_as_root(['touch', subvol.path('d/exists')]) with tempfile.NamedTemporaryFile() as t: with tarfile.TarFile(t.name, 'w') as tar_obj: tar_obj.addfile(tarfile.TarInfo('exists')) with self.assertRaises(subprocess.CalledProcessError): TarballItem( from_target='t', into_dir='/d', tarball=t.name, ).build(subvol) # Adding new files & directories works. Overwriting a # pre-existing directory leaves the owner+mode of the original # directory intact. subvol.run_as_root(['mkdir', subvol.path('d/old_dir')]) subvol.run_as_root(['chown', '123:456', subvol.path('d/old_dir')]) subvol.run_as_root(['chmod', '0301', subvol.path('d/old_dir')]) with tempfile.NamedTemporaryFile() as t: with tarfile.TarFile(t.name, 'w') as tar_obj: tar_obj.addfile(tarfile.TarInfo('new_file')) new_dir = tarfile.TarInfo('new_dir') new_dir.type = tarfile.DIRTYPE tar_obj.addfile(new_dir) old_dir = tarfile.TarInfo('old_dir') old_dir.type = tarfile.DIRTYPE # These will not be applied because old_dir exists old_dir.uid = 0 old_dir.gid = 0 old_dir.mode = 0o755 tar_obj.addfile(old_dir) # Fail when the destination does not exist with self.assertRaises(subprocess.CalledProcessError): TarballItem( from_target='t', into_dir='/no_dir', tarball=t.name, ).build(subvol) # Check the subvolume content before and after unpacking content = [ '(Dir)', { 'd': [ '(Dir)', { 'exists': ['(File)'], 'old_dir': ['(Dir m301 o123:456)', {}], } ] } ] self.assertEqual(content, _render_subvol(subvol)) TarballItem( from_target='t', into_dir='/d', tarball=t.name, ).build(subvol) content[1]['d'][1].update({ 'new_dir': ['(Dir m644)', {}], 'new_file': ['(File)'], }) self.assertEqual(content, _render_subvol(subvol))
def test_compile(self): # First, test compilation with no parent layer. expected_calls = self._expected_run_as_root_calls() self.assertGreater( # Sanity check: at least one command per item len(expected_calls), len(si.ID_TO_ITEM), ) self._assert_equal_call_sets( expected_calls, self._compiler_run_as_root_calls( parent_feature_json=[], parent_dep=[], ), ) # Now, add an empty parent layer with TempSubvolumes(sys.argv[0]) as temp_subvolumes: parent = temp_subvolumes.create('parent') # Manually add/remove some commands from the "expected" set to # accommodate the fact that we have a parent subvolume. subvol_path = f'{_SUBVOLS_DIR}/{_FAKE_SUBVOL}'.encode() # Our unittest.mock.call objects are (args, kwargs) pairs. expected_calls_with_parent = [ c for c in expected_calls if c not in [ ( (['btrfs', 'subvolume', 'create', subvol_path], ), { '_subvol_exists': False }, ), ((['chmod', '0755', subvol_path], ), {}), ((['chown', 'root:root', subvol_path], ), {}), ] ] + [ ( (['test', '!', '-e', subvol_path], ), { '_subvol_exists': False }, ), ( ([ 'btrfs', 'subvolume', 'snapshot', parent.path(), subvol_path, ], ), { '_subvol_exists': False }, ), ] self.assertEqual( # We should've removed 3, and added 2 commands len(expected_calls_with_parent) + 1, len(expected_calls), ) with mock_layer_dir_access(self, parent.path()) as parent_dir: with open(parent_dir / 'feature.json', 'w') as out_f: json.dump( { 'parent_layer': [{ 'subvol': { '__BUCK_LAYER_TARGET': '//fake:parent' }, }], 'target': '//ignored:target', }, out_f) self._assert_equal_call_sets( expected_calls_with_parent, self._compiler_run_as_root_calls( parent_feature_json=[ '--child-feature-json', (parent_dir / 'feature.json').decode(), ], parent_dep=['//fake:parent', parent_dir.decode()], ), )
def test_paths_to_reqs_provs(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('subvol') provides_root = PhasesProvideItem(from_target='t', subvol=subvol) expected = { '/meta': ItemReqsProvs( item_provs={ ItemProv(ProvidesDoNotAccess(path='/meta'), provides_root) }, item_reqs=set(), ), '/': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='/'), provides_root) }, item_reqs={ ItemReq(require_directory('/'), PATH_TO_ITEM['/a/b/c']) }, ), '/a': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a'), PATH_TO_ITEM['/a/b/c']) }, item_reqs={ ItemReq(require_directory('a'), PATH_TO_ITEM['/a/d/e']) }, ), '/a/b': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a/b'), PATH_TO_ITEM['/a/b/c']) }, item_reqs=set(), ), '/a/b/c': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a/b/c'), PATH_TO_ITEM['/a/b/c']) }, item_reqs={ ItemReq(require_directory('a/b/c'), PATH_TO_ITEM['/a/b/c/F']) }, ), '/a/b/c/F': ItemReqsProvs( item_provs={ ItemProv(ProvidesFile(path='a/b/c/F'), PATH_TO_ITEM['/a/b/c/F']) }, item_reqs=set(), ), '/a/d': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a/d'), PATH_TO_ITEM['/a/d/e']) }, item_reqs=set(), ), '/a/d/e': ItemReqsProvs( item_provs={ ItemProv(ProvidesDirectory(path='a/d/e'), PATH_TO_ITEM['/a/d/e']) }, item_reqs={ ItemReq(require_directory('a/d/e'), PATH_TO_ITEM['/a/d/e/G']) }, ), '/a/d/e/G': ItemReqsProvs( item_provs={ ItemProv(ProvidesFile(path='a/d/e/G'), PATH_TO_ITEM['/a/d/e/G']) }, item_reqs=set(), ), } self.assertEqual( ValidatedReqsProvs([provides_root, *PATH_TO_ITEM.values() ]).path_to_reqs_provs, expected)
def _test_rpm_action_item(self, layer_opts, preserve_yum_cache=False): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('rpm_action') self.assertEqual(['(Dir)', {}], render_subvol(subvol)) # The empty action is a no-op RpmActionItem.get_phase_builder([], layer_opts)(subvol) self.assertEqual(['(Dir)', {}], render_subvol(subvol)) # `yum-from-snapshot` needs a `/meta` directory to work subvol.run_as_root(['mkdir', subvol.path('meta')]) self.assertEqual( # No `opts/artifacts_may_require_repo` here because we directly # created the subvol instead of using an Item. ['(Dir)', { 'meta': ['(Dir)', {}] }], render_subvol(subvol), ) # Specifying RPM versions is prohibited with self.assertRaises(subprocess.CalledProcessError): RpmActionItem.get_phase_builder( # This could have been RpmActionItem(), but I want to # test `image_source_item` with `source=None`. [ image_source_item( RpmActionItem, exit_stack=None, layer_opts=DUMMY_LAYER_OPTS, )( from_target='m', name='rpm-test-milk-2.71', source=None, action=RpmAction.install, ) ], layer_opts, )(subvol) # Cannot pass both `name` and `source` with self.assertRaisesRegex( AssertionError, 'Exactly one of `name` or `source` must be set .*', ): RpmActionItem.get_phase_builder( [ RpmActionItem( from_target='m', name='rpm-test-milk', source='rpm-test-milk', action=RpmAction.install, ) ], layer_opts, )(subvol) RpmActionItem.get_phase_builder( [ RpmActionItem( from_target='t', name=n, action=RpmAction.install, ) for n in [ # This specific RPM contains `/bin/sh` and a # post-install script to test `/dev/null` isolation. 'rpm-test-milk', 'rpm-test-carrot', ] ] + [ RpmActionItem( from_target='t', source=Path(__file__).dirname() / "rpm-test-cheese-1-1.rpm", action=RpmAction.install, ) ], layer_opts, )(subvol) # Clean up the `yum` & `rpm` litter before checking the packages. # Maybe fixme: As a result, we end up not asserting ownership / # permissions / etc on directories like /var and /dev. subvol.run_as_root([ 'rm', '-rf', # Annotate all paths since `sudo rm -rf` is scary. subvol.path('var/lib/rpm'), subvol.path('var/lib/yum'), subvol.path('var/log/yum.log'), subvol.path('usr/lib/.build-id'), subvol.path('bin/sh'), ]) # The way that RpmActionItem invokes systemd_nspawn on # build_appliance must gurantee that /var/cache/yum is empty. # Next two lines test that the /var/cache/yum directory is empty # because rmdir fails if it is not. # It is important that the yum cache of built images be empty, to # avoid unnecessarily increasing the distributed image size. rm_cmd = ['rmdir' ] if (layer_opts.build_appliance and not preserve_yum_cache) else ['rm', '-rf'] subvol.run_as_root(rm_cmd + [subvol.path('var/cache/yum')]) subvol.run_as_root([ 'rmdir', subvol.path('dev'), # made by yum_from_snapshot.py subvol.path('meta'), subvol.path('var/cache'), subvol.path('var/lib'), subvol.path('var/log'), subvol.path('var/tmp'), subvol.path('var'), subvol.path('usr/lib'), subvol.path('bin'), ]) self.assertEqual([ '(Dir)', { 'usr': [ '(Dir)', { 'share': [ '(Dir)', { 'rpm_test': [ '(Dir)', { 'carrot.txt': ['(File d13)'], 'cheese1.txt': ['(File d36)'], 'milk.txt': ['(File d12)'], 'post.txt': ['(File d6)'], } ], } ], } ], } ], render_subvol(subvol))
def test_remove_item(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('remove_action') self.assertEqual(['(Dir)', {}], render_subvol(subvol)) MakeDirsItem( from_target='t', path_to_make='/a/b/c', into_dir='/', ).build(subvol, DUMMY_LAYER_OPTS) for d in ['d', 'e']: InstallFileItem( from_target='t', source='/dev/null', dest=f'/a/b/c/{d}', ).build(subvol, DUMMY_LAYER_OPTS) MakeDirsItem( from_target='t', path_to_make='/f/g', into_dir='/', ).build(subvol, DUMMY_LAYER_OPTS) # Checks that `rm` won't follow symlinks SymlinkToDirItem( from_target='t', source='/f', dest='/a/b/f_sym', ).build(subvol, DUMMY_LAYER_OPTS) for d in ['h', 'i']: InstallFileItem( from_target='t', source='/dev/null', dest=f'/f/{d}', ).build(subvol, DUMMY_LAYER_OPTS) SymlinkToDirItem( from_target='t', source='/f/i', dest='/f/i_sym', ).build(subvol, DUMMY_LAYER_OPTS) intact_subvol = [ '(Dir)', { 'a': [ '(Dir)', { 'b': [ '(Dir)', { 'c': [ '(Dir)', { 'd': ['(File m444)'], 'e': ['(File m444)'], } ], 'f_sym': ['(Symlink /f)'], } ], } ], 'f': [ '(Dir)', { 'g': ['(Dir)', {}], 'h': ['(File m444)'], 'i': ['(File m444)'], 'i_sym': ['(Symlink /f/i)'], } ], } ] self.assertEqual(intact_subvol, render_subvol(subvol)) # We refuse to touch protected paths, even with "if_exists". If # the paths started with 'meta', they would trip the check in # `_make_path_normal_relative`, so we mock-protect 'xyz'. for prot_path in ['xyz', 'xyz/potato/carrot']: with unittest.mock.patch( 'fs_image.compiler.items.remove_path.protected_path_set', side_effect=lambda sv: protected_path_set(sv) | {'xyz'}, ), self.assertRaisesRegex( AssertionError, f'Cannot remove protected .*{prot_path}', ): RemovePathItem.get_phase_builder([ RemovePathItem( from_target='t', action=RemovePathAction.if_exists, path=prot_path, ) ], DUMMY_LAYER_OPTS)(subvol) # Check handling of non-existent paths without removing anything remove = RemovePathItem( from_target='t', action=RemovePathAction.if_exists, path='/does/not/exist', ) self.assertEqual(PhaseOrder.REMOVE_PATHS, remove.phase_order()) RemovePathItem.get_phase_builder([remove], DUMMY_LAYER_OPTS)(subvol) with self.assertRaisesRegex(AssertionError, 'does not exist'): RemovePathItem.get_phase_builder([ RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/does/not/exist', ), ], DUMMY_LAYER_OPTS)(subvol) self.assertEqual(intact_subvol, render_subvol(subvol)) # Now remove most of the subvolume. RemovePathItem.get_phase_builder( [ # These 3 removes are not covered by a recursive remove. # And we leave behind /f/i, which lets us know that neither # `f_sym` nor `i_sym` were followed during their deletion. RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/f/i_sym', ), RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/f/h', ), RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/f/g', ), # The next 3 items are intentionally sequenced so that if # they were applied in the given order, they would fail. RemovePathItem( from_target='t', action=RemovePathAction.if_exists, path='/a/b/c/e', ), RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, # The surrounding items don't delete /a/b/c/d, e.g. so # this recursive remove is still tested. path='/a/b/', ), RemovePathItem( from_target='t', action=RemovePathAction.assert_exists, path='/a/b/c/e', ), ], DUMMY_LAYER_OPTS)(subvol) self.assertEqual([ '(Dir)', { 'a': ['(Dir)', {}], 'f': ['(Dir)', { 'i': ['(File m444)'] }], } ], render_subvol(subvol))
def test_tarball_command(self): with TempSubvolumes(sys.argv[0]) as temp_subvolumes: subvol = temp_subvolumes.create('tar-sv') subvol.run_as_root(['mkdir', subvol.path('d')]) # Fail on pre-existing files subvol.run_as_root(['touch', subvol.path('d/exists')]) with tempfile.NamedTemporaryFile() as t: with tarfile.TarFile(t.name, 'w') as tar_obj: tar_obj.addfile(tarfile.TarInfo('exists')) with self.assertRaises(subprocess.CalledProcessError): _tarball_item(t.name, '/d').build( subvol, DUMMY_LAYER_OPTS ) # Adding new files & directories works. Overwriting a # pre-existing directory leaves the owner+mode of the original # directory intact. subvol.run_as_root(['mkdir', subvol.path('d/old_dir')]) subvol.run_as_root(['chown', '123:456', subvol.path('d/old_dir')]) subvol.run_as_root(['chmod', '0301', subvol.path('d/old_dir')]) subvol_root = temp_subvolumes.snapshot(subvol, 'tar-sv-root') subvol_zst = temp_subvolumes.snapshot(subvol, 'tar-sv-zst') with tempfile.TemporaryDirectory() as td: tar_path = os.path.join(td, 'test.tar') zst_path = os.path.join(td, 'test.tar.zst') with tarfile.TarFile(tar_path, 'w') as tar_obj: tar_obj.addfile(tarfile.TarInfo('new_file')) new_dir = tarfile.TarInfo('new_dir') new_dir.type = tarfile.DIRTYPE new_dir.uid = 12 new_dir.gid = 34 tar_obj.addfile(new_dir) old_dir = tarfile.TarInfo('old_dir') old_dir.type = tarfile.DIRTYPE # These will not be applied because old_dir exists old_dir.uid = 0 old_dir.gid = 0 old_dir.mode = 0o755 tar_obj.addfile(old_dir) subprocess.check_call(['zstd', tar_path, '-o', zst_path]) # Fail when the destination does not exist with self.assertRaises(subprocess.CalledProcessError): _tarball_item(tar_path, '/no_dir').build( subvol, DUMMY_LAYER_OPTS ) # Before unpacking the tarball orig_content = ['(Dir)', {'d': ['(Dir)', { 'exists': ['(File)'], 'old_dir': ['(Dir m301 o123:456)', {}], }]}] # After unpacking `tar_path` in `/d`. new_content = copy.deepcopy(orig_content) new_content[1]['d'][1].update({ 'new_dir': ['(Dir m644 o12:34)', {}], 'new_file': ['(File)'], }) # After unpacking `tar_path` in `/d` with `force_root_ownership` new_content_root = copy.deepcopy(new_content) # The ownership of 12:34 is gone. new_content_root[1]['d'][1]['new_dir'] = ['(Dir m644)', {}] self.assertNotEqual(new_content, new_content_root) # Check the subvolume content before and after unpacking for item, (sv, before, after) in ( ( _tarball_item(tar_path, '/d/'), (subvol, orig_content, new_content), ), ( _tarball_item(tar_path, 'd', force_root_ownership=True), (subvol_root, orig_content, new_content_root), ), ( _tarball_item(zst_path, 'd/'), (subvol_zst, orig_content, new_content), ), ): self.assertEqual(before, render_subvol(sv)) item.build(sv, DUMMY_LAYER_OPTS) self.assertEqual(after, render_subvol(sv))