def test_clone_report_permission_issue(tdir):
pdir = Path(tdir) / 'protected'
pdir.mkdir()
# make it read-only
pdir.chmod(0o555)
with chpwd(pdir):
# first check the premise of the test. If we can write (strangely
# mounted/crippled file system, subsequent assumptions are violated
# and we can stop
probe = Path('probe')
try:
probe.write_text('should not work')
raise SkipTest
except PermissionError:
# we are indeed in a read-only situation
pass
res = clone('///',
result_xfm=None,
return_type='list',
on_failure='ignore')
assert_status('error', res)
assert_result_count(
res,
1,
status='error',
message="could not create work tree dir '%s/%s': Permission denied"
% (pdir, get_datasets_topdir()))
def _test_setup_store(io_cls, io_args, store):
io = io_cls(*io_args)
store = Path(store)
version_file = store / 'ria-layout-version'
error_logs = store / 'error_logs'
# invalid version raises:
assert_raises(UnknownLayoutVersion, create_store, io, store, '2')
# non-existing path should work:
create_store(io, store, '1')
assert_true(version_file.exists())
assert_true(error_logs.exists())
assert_true(error_logs.is_dir())
assert_equal([f for f in error_logs.iterdir()], [])
# empty target directory should work as well:
rmtree(str(store))
store.mkdir(exist_ok=False)
create_store(io, store, '1')
assert_true(version_file.exists())
assert_true(error_logs.exists())
assert_true(error_logs.is_dir())
assert_equal([f for f in error_logs.iterdir()], [])
# re-execution also fine:
create_store(io, store, '1')
# but version conflict with existing target isn't:
version_file.write_text("2|unknownflags\n")
assert_raises(ValueError, create_store, io, store, '1')
def test_runner_cwd_encoding(path):
env = os.environ.copy()
# Add PWD to env so that runner will temporarily adjust it to point to cwd.
env['PWD'] = os.getcwd()
cwd = Path(path) / OBSCURE_FILENAME
cwd.mkdir()
# Running doesn't fail if cwd or env has unicode value.
Runner(cwd=cwd, env=env).run(
py2cmd(
'from pathlib import Path; (Path.cwd() / "foo").write_text("t")'))
(cwd / 'foo').exists()
def test_clone_report_permission_issue(tdir):
pdir = Path(tdir) / 'protected'
pdir.mkdir()
# make it read-only
pdir.chmod(0o555)
with chpwd(pdir):
res = clone('///', result_xfm=None, return_type='list', on_failure='ignore')
assert_status('error', res)
assert_result_count(
res, 1, status='error',
message="could not create work tree dir '%s/%s': Permission denied"
% (pdir, get_datasets_topdir())
)
def test_ensure_write_permission(path=None):
# This is testing the usecase of write protected directories needed for
# messing with an annex object tree (as done by the ORA special remote).
# However, that doesn't work on Windows since we can't revoke write
# permissions for the owner of a directory (at least on VFAT - may be
# true for NTFS as well - don't know).
# Hence, on windows/crippledFS only test on a file.
dir_ = Path(path)
if not on_windows and has_symlink_capability:
# set up write-protected dir containing a file
file_ = dir_ / "somefile"
file_.write_text("whatever")
dir_.chmod(dir_.stat().st_mode & ~stat.S_IWRITE)
assert_raises(PermissionError, file_.unlink)
# contextmanager lets us do it and restores permissions afterwards:
mode_before = dir_.stat().st_mode
with ensure_write_permission(dir_):
file_.unlink()
mode_after = dir_.stat().st_mode
assert_equal(mode_before, mode_after)
assert_raises(PermissionError, file_.write_text, "new file can't be "
"written")
assert_raises(
FileNotFoundError,
ensure_write_permission(dir_ / "non" / "existent").__enter__)
# deletion within context doesn't let mode restoration fail:
with ensure_write_permission(dir_):
dir_.rmdir()
dir_.mkdir() # recreate, since next block is executed unconditionally
# set up write-protected file:
file2 = dir_ / "protected.txt"
file2.write_text("unchangeable")
file2.chmod(file2.stat().st_mode & ~stat.S_IWRITE)
assert_raises(PermissionError, file2.write_text, "modification")
# within context we can:
with ensure_write_permission(file2):
file2.write_text("modification")
# mode is restored afterwards:
assert_raises(PermissionError, file2.write_text, "modification2")
def _store_new(self,
url=None,
authentication_type=None,
authenticator_class=None,
url_re=None,
name=None,
credential_name=None,
credential_type=None,
level='user'):
"""Stores a provider and credential config and reloads afterwards.
Note
----
non-interactive version of `enter_new`.
For now non-public, pending further refactoring
Parameters
----------
level: str
Where to store the config. Choices: 'user' (default), 'ds', 'site'
Returns
-------
Provider
The stored `Provider` as reported by reload
"""
# We don't ask user for confirmation, so for this non-interactive
# routine require everything to be explicitly specified.
if any(not a for a in [
url, authentication_type, authenticator_class, url_re, name,
credential_name, credential_type
]):
raise ValueError("All arguments must be specified")
if level not in ['user', 'ds', 'site']:
raise ValueError("'level' must be one of 'user', 'ds', 'site'")
providers_dir = Path(self._get_providers_dirs()[level])
if not providers_dir.exists():
providers_dir.mkdir(parents=True, exist_ok=True)
filepath = providers_dir / f"{name}.cfg"
cfg = self._CONFIG_TEMPLATE.format(**locals())
filepath.write_bytes(cfg.encode('utf-8'))
self.reload()
return self.get_provider(url)
def test_create_alias(ds_path, ria_path, clone_path):
ds_path = Path(ds_path)
clone_path = Path(clone_path)
ds_path.mkdir()
dsa = Dataset(ds_path / "a").create()
res = dsa.create_sibling_ria(url="ria+file://{}".format(ria_path),
name="origin",
alias="ds-a")
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_(len(res), 1)
ds_clone = clone(source="ria+file://{}#~ds-a".format(ria_path),
path=clone_path / "a")
assert_repo_status(ds_clone.path)
# multiple datasets in a RIA store with different aliases work
dsb = Dataset(ds_path / "b").create()
res = dsb.create_sibling_ria(url="ria+file://{}".format(ria_path),
name="origin",
alias="ds-b")
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_(len(res), 1)
ds_clone = clone(source="ria+file://{}#~ds-b".format(ria_path),
path=clone_path / "b")
assert_repo_status(ds_clone.path)
# second dataset in a RIA store with the same alias emits a warning
dsc = Dataset(ds_path / "c").create()
with swallow_logs(logging.WARNING) as cml:
res = dsc.create_sibling_ria(url="ria+file://{}".format(ria_path),
name="origin",
alias="ds-a")
assert_in(
"Alias 'ds-a' already exists in the RIA store, not adding an alias",
cml.out)
assert_result_count(res, 1, status='ok', action='create-sibling-ria')
eq_(len(res), 1)
class SSHManager(object):
"""Keeps ssh connections to share. Serves singleton representation
per connection.
A custom identity file can be specified via `datalad.ssh.identityfile`.
Callers are responsible for reloading `datalad.cfg` if they have changed
this value since loading datalad.
"""
def __init__(self):
self._socket_dir = None
self._connections = dict()
# Initialization of prev_connections is happening during initial
# handling of socket_dir, so we do not define them here explicitly
# to an empty list to fail if logic is violated
self._prev_connections = None
# and no explicit initialization in the constructor
# self.assure_initialized()
@property
def socket_dir(self):
"""Return socket_dir, and if was not defined before,
and also pick up all previous connections (if any)
"""
self.assure_initialized()
return self._socket_dir
def assure_initialized(self):
"""Assures that manager is initialized - knows socket_dir, previous connections
"""
if self._socket_dir is not None:
return
from ..config import ConfigManager
cfg = ConfigManager()
self._socket_dir = \
Path(cfg.obtain('datalad.locations.cache')) / 'sockets'
self._socket_dir.mkdir(exist_ok=True, parents=True)
try:
os.chmod(str(self._socket_dir), 0o700)
except OSError as exc:
lgr.warning(
"Failed to (re)set permissions on the %s. "
"Most likely future communications would be impaired or fail. "
"Original exception: %s", self._socket_dir, exc_str(exc))
try:
self._prev_connections = [
p for p in self.socket_dir.iterdir() if not p.is_dir()
]
except OSError as exc:
self._prev_connections = []
lgr.warning(
"Failed to list %s for existing sockets. "
"Most likely future communications would be impaired or fail. "
"Original exception: %s", self._socket_dir, exc_str(exc))
lgr.log(5, "Found %d previous connections",
len(self._prev_connections))
def get_connection(self,
url,
use_remote_annex_bundle=True,
force_ip=False):
"""Get a singleton, representing a shared ssh connection to `url`
Parameters
----------
url: str
ssh url
force_ip : {False, 4, 6}
Force the use of IPv4 or IPv6 addresses.
Returns
-------
SSHConnection
"""
# parse url:
from datalad.support.network import RI, is_ssh
if isinstance(url, RI):
sshri = url
else:
if ':' not in url and '/' not in url:
# it is just a hostname
lgr.debug("Assuming %r is just a hostname for ssh connection",
url)
url += ':'
sshri = RI(url)
if not is_ssh(sshri):
raise ValueError("Unsupported SSH URL: '{0}', use "
"ssh://host/path or host:path syntax".format(url))
from datalad import cfg
identity_file = cfg.get("datalad.ssh.identityfile")
conhash = get_connection_hash(
sshri.hostname,
port=sshri.port,
identity_file=identity_file or "",
username=sshri.username,
bundled=use_remote_annex_bundle,
force_ip=force_ip,
)
# determine control master:
ctrl_path = self.socket_dir / conhash
# do we know it already?
if ctrl_path in self._connections:
return self._connections[ctrl_path]
else:
c = SSHConnection(ctrl_path,
sshri,
identity_file=identity_file,
use_remote_annex_bundle=use_remote_annex_bundle,
force_ip=force_ip)
self._connections[ctrl_path] = c
return c
def close(self, allow_fail=True, ctrl_path=None):
"""Closes all connections, known to this instance.
Parameters
----------
allow_fail: bool, optional
If True, swallow exceptions which might be thrown during
connection.close, and just log them at DEBUG level
ctrl_path: str, Path, or list of str or Path, optional
If specified, only the path(s) provided would be considered
"""
if self._connections:
ctrl_paths = [Path(p) for p in ensure_list(ctrl_path)]
to_close = [
c for c in self._connections
# don't close if connection wasn't opened by SSHManager
if self._connections[c].ctrl_path not in self._prev_connections
and self._connections[c].ctrl_path.exists() and
(not ctrl_paths or self._connections[c].ctrl_path in ctrl_paths
)
]
if to_close:
lgr.debug("Closing %d SSH connections..." % len(to_close))
for cnct in to_close:
f = self._connections[cnct].close
if allow_fail:
f()
else:
try:
f()
except Exception as exc:
lgr.debug("Failed to close a connection: "
"%s", exc_str(exc))
self._connections = dict()
class MultiplexSSHManager(BaseSSHManager):
"""Keeps ssh connections to share. Serves singleton representation
per connection.
A custom identity file can be specified via `datalad.ssh.identityfile`.
Callers are responsible for reloading `datalad.cfg` if they have changed
this value since loading datalad.
"""
def __init__(self):
super().__init__()
self._socket_dir = None
self._connections = dict()
# Initialization of prev_connections is happening during initial
# handling of socket_dir, so we do not define them here explicitly
# to an empty list to fail if logic is violated
self._prev_connections = None
# and no explicit initialization in the constructor
# self.ensure_initialized()
@property
def socket_dir(self):
"""Return socket_dir, and if was not defined before,
and also pick up all previous connections (if any)
"""
self.ensure_initialized()
return self._socket_dir
def ensure_initialized(self):
"""Assures that manager is initialized - knows socket_dir, previous connections
"""
if self._socket_dir is not None:
return
from datalad import cfg
self._socket_dir = Path(cfg.obtain('datalad.locations.sockets'))
self._socket_dir.mkdir(exist_ok=True, parents=True)
try:
os.chmod(str(self._socket_dir), 0o700)
except OSError as exc:
lgr.warning(
"Failed to (re)set permissions on the %s. "
"Most likely future communications would be impaired or fail. "
"Original exception: %s", self._socket_dir,
CapturedException(exc))
try:
self._prev_connections = [
p for p in self.socket_dir.iterdir() if not p.is_dir()
]
except OSError as exc:
self._prev_connections = []
lgr.warning(
"Failed to list %s for existing sockets. "
"Most likely future communications would be impaired or fail. "
"Original exception: %s", self._socket_dir,
CapturedException(exc))
lgr.log(5, "Found %d previous connections",
len(self._prev_connections))
assure_initialized = ensure_initialized
def get_connection(self,
url,
use_remote_annex_bundle=None,
force_ip=False):
sshri, identity_file = self._prep_connection_args(url)
conhash = get_connection_hash(
sshri.hostname,
port=sshri.port,
identity_file=identity_file or "",
username=sshri.username,
force_ip=force_ip,
)
# determine control master:
ctrl_path = self.socket_dir / conhash
# do we know it already?
if ctrl_path in self._connections:
return self._connections[ctrl_path]
else:
c = MultiplexSSHConnection(
ctrl_path,
sshri,
identity_file=identity_file,
use_remote_annex_bundle=use_remote_annex_bundle,
force_ip=force_ip)
self._connections[ctrl_path] = c
return c
def close(self, allow_fail=True, ctrl_path=None):
"""Closes all connections, known to this instance.
Parameters
----------
allow_fail: bool, optional
If True, swallow exceptions which might be thrown during
connection.close, and just log them at DEBUG level
ctrl_path: str, Path, or list of str or Path, optional
If specified, only the path(s) provided would be considered
"""
if self._connections:
ctrl_paths = [Path(p) for p in ensure_list(ctrl_path)]
to_close = [
c for c in self._connections
# don't close if connection wasn't opened by SSHManager
if self._connections[c].ctrl_path not in self._prev_connections
and self._connections[c].ctrl_path.exists() and
(not ctrl_paths or self._connections[c].ctrl_path in ctrl_paths
)
]
if to_close:
lgr.debug("Closing %d SSH connections...", len(to_close))
for cnct in to_close:
f = self._connections[cnct].close
if allow_fail:
f()
else:
try:
f()
except Exception as exc:
ce = CapturedException(exc)
lgr.debug("Failed to close a connection: "
"%s", ce.message)
self._connections = dict()
