def test_lock_checks_group(tmpdir): """Ensure lock checks work with a self-owned, non-self-group repo.""" uid = getuid() gid = next((g for g in group_ids() if g != uid), None) if not gid: pytest.skip("user has no group with gid != uid") # self-owned, another group tmpdir.chown(uid, gid) # safe path = str(tmpdir) tmpdir.chmod(0o744) lk.check_lock_safety(path) # unsafe tmpdir.chmod(0o774) with pytest.raises(spack.error.SpackError): lk.check_lock_safety(path) # unsafe tmpdir.chmod(0o777) with pytest.raises(spack.error.SpackError): lk.check_lock_safety(path) # safe tmpdir.chmod(0o474) lk.check_lock_safety(path) # safe tmpdir.chmod(0o477) lk.check_lock_safety(path)
def test_lock_checks_user(tmpdir): """Ensure lock checks work with a self-owned, self-group repo.""" uid = getuid() if uid not in group_ids(): pytest.skip("user has no group with gid == uid") # self-owned, own group tmpdir.chown(uid, uid) # safe path = str(tmpdir) tmpdir.chmod(0o744) lk.check_lock_safety(path) # safe tmpdir.chmod(0o774) lk.check_lock_safety(path) # unsafe tmpdir.chmod(0o777) with pytest.raises(spack.error.SpackError): lk.check_lock_safety(path) # safe tmpdir.chmod(0o474) lk.check_lock_safety(path) # safe tmpdir.chmod(0o477) lk.check_lock_safety(path)
def check_stage_dir_perms(prefix, path): """Check the stage directory perms to ensure match expectations.""" # Ensure the path's subdirectories -- to `$user` -- have their parent's # perms while those from `$user` on are owned and restricted to the # user. assert path.startswith(prefix) user = getpass.getuser() prefix_status = os.stat(prefix) uid = getuid() # Obtain lists of ancestor and descendant paths of the $user node, if any. # # Skip processing prefix ancestors since no guarantee they will be in the # required group (e.g. $TEMPDIR on HPC machines). skip = prefix if prefix.endswith(os.sep) else prefix + os.sep group_paths, user_node, user_paths = partition_path( path.replace(skip, ""), user) for p in group_paths: p_status = os.stat(os.path.join(prefix, p)) assert p_status.st_gid == prefix_status.st_gid assert p_status.st_mode == prefix_status.st_mode # Add the path ending with the $user node to the user paths to ensure paths # from $user (on down) meet the ownership and permission requirements. if user_node: user_paths.insert(0, user_node) for p in user_paths: p_status = os.stat(os.path.join(prefix, p)) assert uid == p_status.st_uid assert p_status.st_mode & stat.S_IRWXU == stat.S_IRWXU
def create_stage_root(path): # type: (str) -> None """Create the stage root directory and ensure appropriate access perms.""" assert os.path.isabs(path) and len(path.strip()) > 1 err_msg = 'Cannot create stage root {0}: Access to {1} is denied' user_uid = getuid() # Obtain lists of ancestor and descendant paths of the $user node, if any. group_paths, user_node, user_paths = partition_path( path, getpass.getuser()) for p in group_paths: if not os.path.exists(p): # Ensure access controls of subdirs created above `$user` inherit # from the parent and share the group. par_stat = os.stat(os.path.dirname(p)) mkdirp(p, group=par_stat.st_gid, mode=par_stat.st_mode) p_stat = os.stat(p) if par_stat.st_gid != p_stat.st_gid: tty.warn( "Expected {0} to have group {1}, but it is {2}".format( p, par_stat.st_gid, p_stat.st_gid)) if par_stat.st_mode & p_stat.st_mode != par_stat.st_mode: tty.warn( "Expected {0} to support mode {1}, but it is {2}".format( p, par_stat.st_mode, p_stat.st_mode)) if not can_access(p): raise OSError(errno.EACCES, err_msg.format(path, p)) # Add the path ending with the $user node to the user paths to ensure paths # from $user (on down) meet the ownership and permission requirements. if user_node: user_paths.insert(0, user_node) for p in user_paths: # Ensure access controls of subdirs from `$user` on down are # restricted to the user. owner_uid = get_owner_uid(p) if user_uid != owner_uid: tty.warn( "Expected user {0} to own {1}, but it is owned by {2}".format( user_uid, p, owner_uid)) spack_src_subdir = os.path.join(path, _source_path_subdir) # When staging into a user-specified directory with `spack stage -p <PATH>`, we need # to ensure the `spack-src` subdirectory exists, as we can't rely on it being # created automatically by spack. It's not clear why this is the case for `spack # stage -p`, but since `mkdirp()` is idempotent, this should not change the behavior # for any other code paths. if not os.path.isdir(spack_src_subdir): mkdirp(spack_src_subdir, mode=stat.S_IRWXU)
def test_create_stage_root_bad_uid(self, tmpdir, monkeypatch): """ Test the code path that uses an existing user path -- whether `$user` in `$tempdir` or not -- and triggers the generation of the UID mismatch warning. This situation can happen with some `config:build_stage` settings for teams using a common service account for installing software. """ orig_stat = os.stat class MinStat: st_mode = -1 st_uid = -1 def _stat(path): p_stat = orig_stat(path) fake_stat = MinStat() fake_stat.st_mode = p_stat.st_mode return fake_stat user_dir = tmpdir.join(getpass.getuser()) user_dir.ensure(dir=True) user_path = str(user_dir) # TODO: If we could guarantee access to the monkeypatch context # function (i.e., 3.6.0 on), the call and assertion could be moved # to a with block, such as: # # with monkeypatch.context() as m: # m.setattr(os, 'stat', _stat) # spack.stage.create_stage_root(user_path) # assert os.stat(user_path).st_uid != os.getuid() monkeypatch.setattr(os, 'stat', _stat) spack.stage.create_stage_root(user_path) # The following check depends on the patched os.stat as a poor # substitute for confirming the generated warnings. assert os.stat(user_path).st_uid != getuid()
def test_write_lock_timeout_with_multiple_readers_3_1_ranges(lock_path): multiproc_test(AcquireRead(lock_path, 0, 5), AcquireRead(lock_path, 5, 5), AcquireRead(lock_path, 10, 5), TimeoutWrite(lock_path, 0, 15)) def test_write_lock_timeout_with_multiple_readers_3_2_ranges(lock_path): multiproc_test(AcquireRead(lock_path, 0, 5), AcquireRead(lock_path, 5, 5), AcquireRead(lock_path, 10, 5), TimeoutWrite(lock_path, 3, 10), TimeoutWrite(lock_path, 5, 1)) @pytest.mark.skipif(getuid() == 0, reason='user is root') def test_read_lock_on_read_only_lockfile(lock_dir, lock_path): """read-only directory, read-only lockfile.""" touch(lock_path) with read_only(lock_path, lock_dir): lock = lk.Lock(lock_path) with lk.ReadTransaction(lock): pass with pytest.raises(lk.LockROFileError): with lk.WriteTransaction(lock): pass def test_read_lock_read_only_dir_writable_lockfile(lock_dir, lock_path):
mutable_mock_repo, _platform_executables, tmpdir, monkeypatch): """The user runs 'spack external find'; the default path for storing manifest files exists but is empty. Ensure that the command does not fail. """ empty_manifest_dir = str(tmpdir.mkdir('manifest_dir')) monkeypatch.setenv('PATH', '') monkeypatch.setattr(spack.cray_manifest, 'default_path', empty_manifest_dir) external('find') @pytest.mark.skipif(sys.platform == 'win32', reason="Can't chmod on Windows") @pytest.mark.skipif(getuid() == 0, reason='user is root') def test_find_external_manifest_with_bad_permissions( mutable_config, working_env, mock_executable, mutable_mock_repo, _platform_executables, tmpdir, monkeypatch): """The user runs 'spack external find'; the default path for storing manifest files exists but with insufficient permissions. Check that the command does not fail. """ test_manifest_dir = str(tmpdir.mkdir('manifest_dir')) test_manifest_file_path = os.path.join(test_manifest_dir, 'badperms.json') touch(test_manifest_file_path) monkeypatch.setenv('PATH', '') monkeypatch.setattr(spack.cray_manifest, 'default_path', test_manifest_dir) try: os.chmod(test_manifest_file_path, 0) output = external('find')
class TestStage(object): stage_name = 'spack-test-stage' def test_setup_and_destroy_name_with_tmp(self, mock_stage_archive): archive = mock_stage_archive() with Stage(archive.url, name=self.stage_name) as stage: check_setup(stage, self.stage_name, archive) check_destroy(stage, self.stage_name) def test_setup_and_destroy_name_without_tmp(self, mock_stage_archive): archive = mock_stage_archive() with Stage(archive.url, name=self.stage_name) as stage: check_setup(stage, self.stage_name, archive) check_destroy(stage, self.stage_name) def test_setup_and_destroy_no_name_with_tmp(self, mock_stage_archive): archive = mock_stage_archive() with Stage(archive.url) as stage: check_setup(stage, None, archive) check_destroy(stage, None) def test_noexpand_stage_file(self, mock_stage_archive, mock_noexpand_resource): """When creating a stage with a nonexpanding URL, the 'archive_file' property of the stage should refer to the path of that file. """ test_noexpand_fetcher = spack.fetch_strategy.from_kwargs( url=_file_prefix + mock_noexpand_resource, expand=False) with Stage(test_noexpand_fetcher) as stage: stage.fetch() stage.expand_archive() assert os.path.exists(stage.archive_file) @pytest.mark.disable_clean_stage_check def test_composite_stage_with_noexpand_resource(self, mock_stage_archive, mock_noexpand_resource): archive = mock_stage_archive() composite_stage = StageComposite() root_stage = Stage(archive.url) composite_stage.append(root_stage) resource_dst_name = 'resource-dst-name.sh' test_resource_fetcher = spack.fetch_strategy.from_kwargs( url=_file_prefix + mock_noexpand_resource, expand=False) test_resource = Resource('test_resource', test_resource_fetcher, resource_dst_name, None) resource_stage = ResourceStage(test_resource_fetcher, root_stage, test_resource) composite_stage.append(resource_stage) composite_stage.create() composite_stage.fetch() composite_stage.expand_archive() assert composite_stage.expanded # Archive is expanded assert os.path.exists( os.path.join(composite_stage.source_path, resource_dst_name)) @pytest.mark.disable_clean_stage_check def test_composite_stage_with_expand_resource( self, composite_stage_with_expanding_resource): composite_stage, root_stage, resource_stage, mock_resource = ( composite_stage_with_expanding_resource) composite_stage.create() composite_stage.fetch() composite_stage.expand_archive() assert composite_stage.expanded # Archive is expanded for fname in mock_resource.files: file_path = os.path.join(root_stage.source_path, 'resource-dir', fname) assert os.path.exists(file_path) # Perform a little cleanup shutil.rmtree(root_stage.path) @pytest.mark.disable_clean_stage_check def test_composite_stage_with_expand_resource_default_placement( self, composite_stage_with_expanding_resource): """For a resource which refers to a compressed archive which expands to a directory, check that by default the resource is placed in the source_path of the root stage with the name of the decompressed directory. """ composite_stage, root_stage, resource_stage, mock_resource = ( composite_stage_with_expanding_resource) resource_stage.resource.placement = None composite_stage.create() composite_stage.fetch() composite_stage.expand_archive() for fname in mock_resource.files: file_path = os.path.join(root_stage.source_path, 'resource-expand', fname) assert os.path.exists(file_path) # Perform a little cleanup shutil.rmtree(root_stage.path) def test_setup_and_destroy_no_name_without_tmp(self, mock_stage_archive): archive = mock_stage_archive() with Stage(archive.url) as stage: check_setup(stage, None, archive) check_destroy(stage, None) @pytest.mark.parametrize('debug', [False, True]) def test_fetch(self, mock_stage_archive, debug): archive = mock_stage_archive() with spack.config.override('config:debug', debug): with Stage(archive.url, name=self.stage_name) as stage: stage.fetch() check_setup(stage, self.stage_name, archive) check_fetch(stage, self.stage_name) check_destroy(stage, self.stage_name) def test_no_search_if_default_succeeds(self, mock_stage_archive, failing_search_fn): archive = mock_stage_archive() stage = Stage(archive.url, name=self.stage_name, search_fn=failing_search_fn) with stage: stage.fetch() check_destroy(stage, self.stage_name) def test_no_search_mirror_only(self, failing_fetch_strategy, failing_search_fn): stage = Stage(failing_fetch_strategy, name=self.stage_name, search_fn=failing_search_fn) with stage: try: stage.fetch(mirror_only=True) except spack.fetch_strategy.FetchError: pass check_destroy(stage, self.stage_name) @pytest.mark.parametrize( "err_msg,expected", [('Fetch from fetch.test.com', 'Fetch from fetch.test.com'), (None, 'All fetchers failed')]) def test_search_if_default_fails(self, failing_fetch_strategy, search_fn, err_msg, expected): stage = Stage(failing_fetch_strategy, name=self.stage_name, search_fn=search_fn) with stage: with pytest.raises(spack.fetch_strategy.FetchError, match=expected): stage.fetch(mirror_only=False, err_msg=err_msg) check_destroy(stage, self.stage_name) assert search_fn.performed_search def test_ensure_one_stage_entry(self, mock_stage_archive): archive = mock_stage_archive() with Stage(archive.url, name=self.stage_name) as stage: stage.fetch() stage_path = get_stage_path(stage, self.stage_name) spack.fetch_strategy._ensure_one_stage_entry(stage_path) check_destroy(stage, self.stage_name) @pytest.mark.parametrize( "expected_file_list", [[], [_include_readme], [_include_extra, _include_readme], [_include_hidden, _include_readme]]) def test_expand_archive(self, expected_file_list, mock_stage_archive): archive = mock_stage_archive(expected_file_list) with Stage(archive.url, name=self.stage_name) as stage: stage.fetch() check_setup(stage, self.stage_name, archive) check_fetch(stage, self.stage_name) stage.expand_archive() check_expand_archive(stage, self.stage_name, expected_file_list) check_destroy(stage, self.stage_name) def test_expand_archive_extra_expand(self, mock_stage_archive): """Test expand with an extra expand after expand (i.e., no-op).""" archive = mock_stage_archive() with Stage(archive.url, name=self.stage_name) as stage: stage.fetch() check_setup(stage, self.stage_name, archive) check_fetch(stage, self.stage_name) stage.expand_archive() stage.fetcher.expand() check_expand_archive(stage, self.stage_name, [_include_readme]) check_destroy(stage, self.stage_name) def test_restage(self, mock_stage_archive): archive = mock_stage_archive() with Stage(archive.url, name=self.stage_name) as stage: stage.fetch() stage.expand_archive() with working_dir(stage.source_path): check_expand_archive(stage, self.stage_name, [_include_readme]) # Try to make a file in the old archive dir with open('foobar', 'w') as file: file.write("this file is to be destroyed.") assert 'foobar' in os.listdir(stage.source_path) # Make sure the file is not there after restage. stage.restage() check_fetch(stage, self.stage_name) assert 'foobar' not in os.listdir(stage.source_path) check_destroy(stage, self.stage_name) def test_no_keep_without_exceptions(self, mock_stage_archive): archive = mock_stage_archive() stage = Stage(archive.url, name=self.stage_name, keep=False) with stage: pass check_destroy(stage, self.stage_name) @pytest.mark.disable_clean_stage_check def test_keep_without_exceptions(self, mock_stage_archive): archive = mock_stage_archive() stage = Stage(archive.url, name=self.stage_name, keep=True) with stage: pass path = get_stage_path(stage, self.stage_name) assert os.path.isdir(path) @pytest.mark.disable_clean_stage_check def test_no_keep_with_exceptions(self, mock_stage_archive): class ThisMustFailHere(Exception): pass archive = mock_stage_archive() stage = Stage(archive.url, name=self.stage_name, keep=False) try: with stage: raise ThisMustFailHere() except ThisMustFailHere: path = get_stage_path(stage, self.stage_name) assert os.path.isdir(path) @pytest.mark.disable_clean_stage_check def test_keep_exceptions(self, mock_stage_archive): class ThisMustFailHere(Exception): pass archive = mock_stage_archive() stage = Stage(archive.url, name=self.stage_name, keep=True) try: with stage: raise ThisMustFailHere() except ThisMustFailHere: path = get_stage_path(stage, self.stage_name) assert os.path.isdir(path) def test_source_path_available(self, mock_stage_archive): """Ensure source path available but does not exist on instantiation.""" archive = mock_stage_archive() stage = Stage(archive.url, name=self.stage_name) source_path = stage.source_path assert source_path assert source_path.endswith(spack.stage._source_path_subdir) assert not os.path.exists(source_path) @pytest.mark.skipif(sys.platform == 'win32', reason="Not supported on Windows (yet)") @pytest.mark.skipif(getuid() == 0, reason='user is root') def test_first_accessible_path(self, tmpdir): """Test _first_accessible_path names.""" spack_dir = tmpdir.join('paths') name = str(spack_dir) files = [os.path.join(os.path.sep, 'no', 'such', 'path'), name] # Ensure the tmpdir path is returned since the user should have access path = spack.stage._first_accessible_path(files) assert path == name assert os.path.isdir(path) check_stage_dir_perms(str(tmpdir), path) # Ensure an existing path is returned spack_subdir = spack_dir.join('existing').ensure(dir=True) subdir = str(spack_subdir) path = spack.stage._first_accessible_path([subdir]) assert path == subdir # Ensure a path with a `$user` node has the right permissions # for its subdirectories. user = getpass.getuser() user_dir = spack_dir.join(user, 'has', 'paths') user_path = str(user_dir) path = spack.stage._first_accessible_path([user_path]) assert path == user_path check_stage_dir_perms(str(tmpdir), path) # Cleanup shutil.rmtree(str(name)) @pytest.mark.skipif(sys.platform == 'win32', reason="Not supported on Windows (yet)") def test_create_stage_root(self, tmpdir, no_path_access): """Test create_stage_root permissions.""" test_dir = tmpdir.join('path') test_path = str(test_dir) try: if getpass.getuser() in str(test_path).split(os.sep): # Simply ensure directory created if tmpdir includes user spack.stage.create_stage_root(test_path) assert os.path.exists(test_path) p_stat = os.stat(test_path) assert p_stat.st_mode & stat.S_IRWXU == stat.S_IRWXU else: # Ensure an OS Error is raised on created, non-user directory with pytest.raises(OSError) as exc_info: spack.stage.create_stage_root(test_path) assert exc_info.value.errno == errno.EACCES finally: try: shutil.rmtree(test_path) except OSError: pass @pytest.mark.nomockstage def test_create_stage_root_bad_uid(self, tmpdir, monkeypatch): """ Test the code path that uses an existing user path -- whether `$user` in `$tempdir` or not -- and triggers the generation of the UID mismatch warning. This situation can happen with some `config:build_stage` settings for teams using a common service account for installing software. """ orig_stat = os.stat class MinStat: st_mode = -1 st_uid = -1 def _stat(path): p_stat = orig_stat(path) fake_stat = MinStat() fake_stat.st_mode = p_stat.st_mode return fake_stat user_dir = tmpdir.join(getpass.getuser()) user_dir.ensure(dir=True) user_path = str(user_dir) # TODO: If we could guarantee access to the monkeypatch context # function (i.e., 3.6.0 on), the call and assertion could be moved # to a with block, such as: # # with monkeypatch.context() as m: # m.setattr(os, 'stat', _stat) # spack.stage.create_stage_root(user_path) # assert os.stat(user_path).st_uid != os.getuid() monkeypatch.setattr(os, 'stat', _stat) spack.stage.create_stage_root(user_path) # The following check depends on the patched os.stat as a poor # substitute for confirming the generated warnings. assert os.stat(user_path).st_uid != getuid() def test_resolve_paths(self): """Test _resolve_paths.""" assert spack.stage._resolve_paths([]) == [] # resolved path without user appends user paths = [os.path.join(os.path.sep, 'a', 'b', 'c')] user = getpass.getuser() can_paths = [os.path.join(paths[0], user)] assert spack.stage._resolve_paths(paths) == can_paths # resolved path with node including user does not append user paths = [os.path.join(os.path.sep, 'spack-{0}'.format(user), 'stage')] assert spack.stage._resolve_paths(paths) == paths tempdir = '$tempdir' can_tempdir = canonicalize_path(tempdir) user = getpass.getuser() temp_has_user = user in can_tempdir.split(os.sep) paths = [ os.path.join(tempdir, 'stage'), os.path.join(tempdir, '$user'), os.path.join(tempdir, '$user', '$user'), os.path.join(tempdir, '$user', 'stage', '$user') ] res_paths = [canonicalize_path(p) for p in paths] if temp_has_user: res_paths[1] = can_tempdir res_paths[2] = os.path.join(can_tempdir, user) res_paths[3] = os.path.join(can_tempdir, 'stage', user) else: res_paths[0] = os.path.join(res_paths[0], user) assert spack.stage._resolve_paths(paths) == res_paths @pytest.mark.skipif(sys.platform == 'win32', reason="Not supported on Windows (yet)") @pytest.mark.skipif(getuid() == 0, reason='user is root') def test_get_stage_root_bad_path(self, clear_stage_root): """Ensure an invalid stage path root raises a StageError.""" with spack.config.override('config:build_stage', '/no/such/path'): with pytest.raises(spack.stage.StageError, match="No accessible stage paths in"): spack.stage.get_stage_root() # Make sure the cached stage path values are unchanged. assert spack.stage._stage_root is None @pytest.mark.parametrize( 'path,purged', [('spack-stage-1234567890abcdef1234567890abcdef', True), ('spack-stage-anything-goes-here', True), ('stage-spack', False)]) def test_stage_purge(self, tmpdir, clear_stage_root, path, purged): """Test purging of stage directories.""" stage_dir = tmpdir.join('stage') stage_path = str(stage_dir) test_dir = stage_dir.join(path) test_dir.ensure(dir=True) test_path = str(test_dir) with spack.config.override('config:build_stage', stage_path): stage_root = spack.stage.get_stage_root() assert stage_path == stage_root spack.stage.purge() if purged: assert not os.path.exists(test_path) else: assert os.path.exists(test_path) shutil.rmtree(test_path) def test_stage_constructor_no_fetcher(self): """Ensure Stage constructor with no URL or fetch strategy fails.""" with pytest.raises(ValueError): with Stage(None): pass def test_stage_constructor_with_path(self, tmpdir): """Ensure Stage constructor with a path uses it.""" testpath = str(tmpdir) with Stage('file:///does-not-exist', path=testpath) as stage: assert stage.path == testpath def test_diystage_path_none(self): """Ensure DIYStage for path=None behaves as expected.""" with pytest.raises(ValueError): DIYStage(None) def test_diystage_path_invalid(self): """Ensure DIYStage for an invalid path behaves as expected.""" with pytest.raises(spack.stage.StagePathError): DIYStage('/path/does/not/exist') def test_diystage_path_valid(self, tmpdir): """Ensure DIYStage for a valid path behaves as expected.""" path = str(tmpdir) stage = DIYStage(path) assert stage.path == path assert stage.source_path == path # Order doesn't really matter for DIYStage since they are # basically NOOPs; however, call each since they are part # of the normal stage usage and to ensure full test coverage. stage.create() # Only sets the flag value assert stage.created stage.cache_local() # Only outputs a message stage.fetch() # Only outputs a message stage.check() # Only outputs a message stage.expand_archive() # Only outputs a message assert stage.expanded # The path/source_path does exist with pytest.raises(spack.stage.RestageError): stage.restage() stage.destroy() # A no-op assert stage.path == path # Ensure can still access attributes assert os.path.exists(stage.source_path) # Ensure path still exists def test_diystage_preserve_file(self, tmpdir): """Ensure DIYStage preserves an existing file.""" # Write a file to the temporary directory fn = tmpdir.join(_readme_fn) fn.write(_readme_contents) # Instantiate the DIYStage and ensure the above file is unchanged. path = str(tmpdir) stage = DIYStage(path) assert os.path.isdir(path) assert os.path.isfile(str(fn)) stage.create() # Only sets the flag value readmefn = str(fn) assert os.path.isfile(readmefn) with open(readmefn) as _file: _file.read() == _readme_contents