def test_clone_report_permission_issue(tdir): pdir = Path(tdir) / 'protected' pdir.mkdir() # make it read-only pdir.chmod(0o555) with chpwd(pdir): # first check the premise of the test. If we can write (strangely # mounted/crippled file system, subsequent assumptions are violated # and we can stop probe = Path('probe') try: probe.write_text('should not work') raise SkipTest except PermissionError: # we are indeed in a read-only situation pass res = clone('///', result_xfm=None, return_type='list', on_failure='ignore') assert_status('error', res) assert_result_count( res, 1, status='error', message="could not create work tree dir '%s/%s': Permission denied" % (pdir, get_datasets_topdir()))
def _test_setup_store(io_cls, io_args, store): io = io_cls(*io_args) store = Path(store) version_file = store / 'ria-layout-version' error_logs = store / 'error_logs' # invalid version raises: assert_raises(UnknownLayoutVersion, create_store, io, store, '2') # non-existing path should work: create_store(io, store, '1') assert_true(version_file.exists()) assert_true(error_logs.exists()) assert_true(error_logs.is_dir()) assert_equal([f for f in error_logs.iterdir()], []) # empty target directory should work as well: rmtree(str(store)) store.mkdir(exist_ok=False) create_store(io, store, '1') assert_true(version_file.exists()) assert_true(error_logs.exists()) assert_true(error_logs.is_dir()) assert_equal([f for f in error_logs.iterdir()], []) # re-execution also fine: create_store(io, store, '1') # but version conflict with existing target isn't: version_file.write_text("2|unknownflags\n") assert_raises(ValueError, create_store, io, store, '1')
def test_runner_cwd_encoding(path): env = os.environ.copy() # Add PWD to env so that runner will temporarily adjust it to point to cwd. env['PWD'] = os.getcwd() cwd = Path(path) / OBSCURE_FILENAME cwd.mkdir() # Running doesn't fail if cwd or env has unicode value. Runner(cwd=cwd, env=env).run( py2cmd( 'from pathlib import Path; (Path.cwd() / "foo").write_text("t")')) (cwd / 'foo').exists()
def test_clone_report_permission_issue(tdir): pdir = Path(tdir) / 'protected' pdir.mkdir() # make it read-only pdir.chmod(0o555) with chpwd(pdir): res = clone('///', result_xfm=None, return_type='list', on_failure='ignore') assert_status('error', res) assert_result_count( res, 1, status='error', message="could not create work tree dir '%s/%s': Permission denied" % (pdir, get_datasets_topdir()) )
def test_ensure_write_permission(path=None): # This is testing the usecase of write protected directories needed for # messing with an annex object tree (as done by the ORA special remote). # However, that doesn't work on Windows since we can't revoke write # permissions for the owner of a directory (at least on VFAT - may be # true for NTFS as well - don't know). # Hence, on windows/crippledFS only test on a file. dir_ = Path(path) if not on_windows and has_symlink_capability: # set up write-protected dir containing a file file_ = dir_ / "somefile" file_.write_text("whatever") dir_.chmod(dir_.stat().st_mode & ~stat.S_IWRITE) assert_raises(PermissionError, file_.unlink) # contextmanager lets us do it and restores permissions afterwards: mode_before = dir_.stat().st_mode with ensure_write_permission(dir_): file_.unlink() mode_after = dir_.stat().st_mode assert_equal(mode_before, mode_after) assert_raises(PermissionError, file_.write_text, "new file can't be " "written") assert_raises( FileNotFoundError, ensure_write_permission(dir_ / "non" / "existent").__enter__) # deletion within context doesn't let mode restoration fail: with ensure_write_permission(dir_): dir_.rmdir() dir_.mkdir() # recreate, since next block is executed unconditionally # set up write-protected file: file2 = dir_ / "protected.txt" file2.write_text("unchangeable") file2.chmod(file2.stat().st_mode & ~stat.S_IWRITE) assert_raises(PermissionError, file2.write_text, "modification") # within context we can: with ensure_write_permission(file2): file2.write_text("modification") # mode is restored afterwards: assert_raises(PermissionError, file2.write_text, "modification2")
def _store_new(self, url=None, authentication_type=None, authenticator_class=None, url_re=None, name=None, credential_name=None, credential_type=None, level='user'): """Stores a provider and credential config and reloads afterwards. Note ---- non-interactive version of `enter_new`. For now non-public, pending further refactoring Parameters ---------- level: str Where to store the config. Choices: 'user' (default), 'ds', 'site' Returns ------- Provider The stored `Provider` as reported by reload """ # We don't ask user for confirmation, so for this non-interactive # routine require everything to be explicitly specified. if any(not a for a in [ url, authentication_type, authenticator_class, url_re, name, credential_name, credential_type ]): raise ValueError("All arguments must be specified") if level not in ['user', 'ds', 'site']: raise ValueError("'level' must be one of 'user', 'ds', 'site'") providers_dir = Path(self._get_providers_dirs()[level]) if not providers_dir.exists(): providers_dir.mkdir(parents=True, exist_ok=True) filepath = providers_dir / f"{name}.cfg" cfg = self._CONFIG_TEMPLATE.format(**locals()) filepath.write_bytes(cfg.encode('utf-8')) self.reload() return self.get_provider(url)
def test_create_alias(ds_path, ria_path, clone_path): ds_path = Path(ds_path) clone_path = Path(clone_path) ds_path.mkdir() dsa = Dataset(ds_path / "a").create() res = dsa.create_sibling_ria(url="ria+file://{}".format(ria_path), name="origin", alias="ds-a") assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_(len(res), 1) ds_clone = clone(source="ria+file://{}#~ds-a".format(ria_path), path=clone_path / "a") assert_repo_status(ds_clone.path) # multiple datasets in a RIA store with different aliases work dsb = Dataset(ds_path / "b").create() res = dsb.create_sibling_ria(url="ria+file://{}".format(ria_path), name="origin", alias="ds-b") assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_(len(res), 1) ds_clone = clone(source="ria+file://{}#~ds-b".format(ria_path), path=clone_path / "b") assert_repo_status(ds_clone.path) # second dataset in a RIA store with the same alias emits a warning dsc = Dataset(ds_path / "c").create() with swallow_logs(logging.WARNING) as cml: res = dsc.create_sibling_ria(url="ria+file://{}".format(ria_path), name="origin", alias="ds-a") assert_in( "Alias 'ds-a' already exists in the RIA store, not adding an alias", cml.out) assert_result_count(res, 1, status='ok', action='create-sibling-ria') eq_(len(res), 1)
class SSHManager(object): """Keeps ssh connections to share. Serves singleton representation per connection. A custom identity file can be specified via `datalad.ssh.identityfile`. Callers are responsible for reloading `datalad.cfg` if they have changed this value since loading datalad. """ def __init__(self): self._socket_dir = None self._connections = dict() # Initialization of prev_connections is happening during initial # handling of socket_dir, so we do not define them here explicitly # to an empty list to fail if logic is violated self._prev_connections = None # and no explicit initialization in the constructor # self.assure_initialized() @property def socket_dir(self): """Return socket_dir, and if was not defined before, and also pick up all previous connections (if any) """ self.assure_initialized() return self._socket_dir def assure_initialized(self): """Assures that manager is initialized - knows socket_dir, previous connections """ if self._socket_dir is not None: return from ..config import ConfigManager cfg = ConfigManager() self._socket_dir = \ Path(cfg.obtain('datalad.locations.cache')) / 'sockets' self._socket_dir.mkdir(exist_ok=True, parents=True) try: os.chmod(str(self._socket_dir), 0o700) except OSError as exc: lgr.warning( "Failed to (re)set permissions on the %s. " "Most likely future communications would be impaired or fail. " "Original exception: %s", self._socket_dir, exc_str(exc)) try: self._prev_connections = [ p for p in self.socket_dir.iterdir() if not p.is_dir() ] except OSError as exc: self._prev_connections = [] lgr.warning( "Failed to list %s for existing sockets. " "Most likely future communications would be impaired or fail. " "Original exception: %s", self._socket_dir, exc_str(exc)) lgr.log(5, "Found %d previous connections", len(self._prev_connections)) def get_connection(self, url, use_remote_annex_bundle=True, force_ip=False): """Get a singleton, representing a shared ssh connection to `url` Parameters ---------- url: str ssh url force_ip : {False, 4, 6} Force the use of IPv4 or IPv6 addresses. Returns ------- SSHConnection """ # parse url: from datalad.support.network import RI, is_ssh if isinstance(url, RI): sshri = url else: if ':' not in url and '/' not in url: # it is just a hostname lgr.debug("Assuming %r is just a hostname for ssh connection", url) url += ':' sshri = RI(url) if not is_ssh(sshri): raise ValueError("Unsupported SSH URL: '{0}', use " "ssh://host/path or host:path syntax".format(url)) from datalad import cfg identity_file = cfg.get("datalad.ssh.identityfile") conhash = get_connection_hash( sshri.hostname, port=sshri.port, identity_file=identity_file or "", username=sshri.username, bundled=use_remote_annex_bundle, force_ip=force_ip, ) # determine control master: ctrl_path = self.socket_dir / conhash # do we know it already? if ctrl_path in self._connections: return self._connections[ctrl_path] else: c = SSHConnection(ctrl_path, sshri, identity_file=identity_file, use_remote_annex_bundle=use_remote_annex_bundle, force_ip=force_ip) self._connections[ctrl_path] = c return c def close(self, allow_fail=True, ctrl_path=None): """Closes all connections, known to this instance. Parameters ---------- allow_fail: bool, optional If True, swallow exceptions which might be thrown during connection.close, and just log them at DEBUG level ctrl_path: str, Path, or list of str or Path, optional If specified, only the path(s) provided would be considered """ if self._connections: ctrl_paths = [Path(p) for p in ensure_list(ctrl_path)] to_close = [ c for c in self._connections # don't close if connection wasn't opened by SSHManager if self._connections[c].ctrl_path not in self._prev_connections and self._connections[c].ctrl_path.exists() and (not ctrl_paths or self._connections[c].ctrl_path in ctrl_paths ) ] if to_close: lgr.debug("Closing %d SSH connections..." % len(to_close)) for cnct in to_close: f = self._connections[cnct].close if allow_fail: f() else: try: f() except Exception as exc: lgr.debug("Failed to close a connection: " "%s", exc_str(exc)) self._connections = dict()
class MultiplexSSHManager(BaseSSHManager): """Keeps ssh connections to share. Serves singleton representation per connection. A custom identity file can be specified via `datalad.ssh.identityfile`. Callers are responsible for reloading `datalad.cfg` if they have changed this value since loading datalad. """ def __init__(self): super().__init__() self._socket_dir = None self._connections = dict() # Initialization of prev_connections is happening during initial # handling of socket_dir, so we do not define them here explicitly # to an empty list to fail if logic is violated self._prev_connections = None # and no explicit initialization in the constructor # self.ensure_initialized() @property def socket_dir(self): """Return socket_dir, and if was not defined before, and also pick up all previous connections (if any) """ self.ensure_initialized() return self._socket_dir def ensure_initialized(self): """Assures that manager is initialized - knows socket_dir, previous connections """ if self._socket_dir is not None: return from datalad import cfg self._socket_dir = Path(cfg.obtain('datalad.locations.sockets')) self._socket_dir.mkdir(exist_ok=True, parents=True) try: os.chmod(str(self._socket_dir), 0o700) except OSError as exc: lgr.warning( "Failed to (re)set permissions on the %s. " "Most likely future communications would be impaired or fail. " "Original exception: %s", self._socket_dir, CapturedException(exc)) try: self._prev_connections = [ p for p in self.socket_dir.iterdir() if not p.is_dir() ] except OSError as exc: self._prev_connections = [] lgr.warning( "Failed to list %s for existing sockets. " "Most likely future communications would be impaired or fail. " "Original exception: %s", self._socket_dir, CapturedException(exc)) lgr.log(5, "Found %d previous connections", len(self._prev_connections)) assure_initialized = ensure_initialized def get_connection(self, url, use_remote_annex_bundle=None, force_ip=False): sshri, identity_file = self._prep_connection_args(url) conhash = get_connection_hash( sshri.hostname, port=sshri.port, identity_file=identity_file or "", username=sshri.username, force_ip=force_ip, ) # determine control master: ctrl_path = self.socket_dir / conhash # do we know it already? if ctrl_path in self._connections: return self._connections[ctrl_path] else: c = MultiplexSSHConnection( ctrl_path, sshri, identity_file=identity_file, use_remote_annex_bundle=use_remote_annex_bundle, force_ip=force_ip) self._connections[ctrl_path] = c return c def close(self, allow_fail=True, ctrl_path=None): """Closes all connections, known to this instance. Parameters ---------- allow_fail: bool, optional If True, swallow exceptions which might be thrown during connection.close, and just log them at DEBUG level ctrl_path: str, Path, or list of str or Path, optional If specified, only the path(s) provided would be considered """ if self._connections: ctrl_paths = [Path(p) for p in ensure_list(ctrl_path)] to_close = [ c for c in self._connections # don't close if connection wasn't opened by SSHManager if self._connections[c].ctrl_path not in self._prev_connections and self._connections[c].ctrl_path.exists() and (not ctrl_paths or self._connections[c].ctrl_path in ctrl_paths ) ] if to_close: lgr.debug("Closing %d SSH connections...", len(to_close)) for cnct in to_close: f = self._connections[cnct].close if allow_fail: f() else: try: f() except Exception as exc: ce = CapturedException(exc) lgr.debug("Failed to close a connection: " "%s", ce.message) self._connections = dict()