def _make_github_repos(
gh, github_login, github_passwd, github_organization, rinfo, existing,
access_protocol, dryrun):
cred = UserPassword('github', 'https://github.com/login')
# determine the entity under which to create the repos
entity = _get_github_entity(
gh,
cred,
github_login,
github_passwd,
github_organization)
res = []
for ds, reponame in rinfo:
try:
access_url, existed = _make_github_repo(
gh,
entity,
reponame,
existing,
access_protocol,
dryrun)
res.append((ds, access_url, existed))
except gh.BadCredentialsException as e:
# things blew up, wipe out cred store, if anything is in it
if cred.is_known:
cred.delete()
raise e
return res
def _make_github_repos(gh, github_login, github_passwd, github_organization,
rinfo, existing, access_protocol, dryrun):
# make it per user if github_login was provided. People might want to use
# different credentials etc
cred_identity = "%s@github" % github_login if github_login else "github"
cred = UserPassword(cred_identity, 'https://github.com/login')
# determine the entity under which to create the repos
entity = _get_github_entity(gh, cred, github_login, github_passwd,
github_organization)
res = []
for ds, reponame in rinfo:
try:
access_url, existed = _make_github_repo(gh, github_login, entity,
reponame, existing,
access_protocol, dryrun)
res.append((ds, access_url, existed))
except gh.BadCredentialsException as e:
# things blew up, wipe out cred store, if anything is in it
if cred.is_known:
# to avoid surprises "who ate my creds?", warn the user
lgr.warning(
"Authentication failed, deleting stored credential %s",
cred_identity)
cred.delete()
raise e
return res
def _make_github_repos(
gh, github_user, github_passwd, github_organization, rinfo, existing,
access_protocol, dryrun):
cred = UserPassword('github', 'https://github.com/login')
# determine the entity under which to create the repos
entity = _get_github_entity(
gh,
cred,
github_user,
github_passwd,
github_organization)
res = []
for ds, reponame in rinfo:
try:
access_url, existed = _make_github_repo(
gh,
entity,
reponame,
existing,
access_protocol,
dryrun)
res.append((ds, access_url, existed))
except gh.BadCredentialsException as e:
# things blew up, wipe out cred store, if anything is in it
if cred.is_known:
cred.delete()
raise e
return res
def _make_github_repos(
gh, github_login, github_passwd, github_organization, rinfo, existing,
access_protocol, dryrun):
# make it per user if github_login was provided. People might want to use
# different credentials etc
cred_identity = "%s@github" % github_login if github_login else "github"
cred = UserPassword(cred_identity, 'https://github.com/login')
# determine the entity under which to create the repos
entity = _get_github_entity(
gh,
cred,
github_login,
github_passwd,
github_organization)
res = []
for ds, reponame in rinfo:
try:
access_url, existed = _make_github_repo(
gh,
github_login,
entity,
reponame,
existing,
access_protocol,
dryrun)
res.append((ds, access_url, existed))
except gh.BadCredentialsException as e:
# things blew up, wipe out cred store, if anything is in it
if cred.is_known:
# to avoid surprises "who ate my creds?", warn the user
lgr.warning(
"Authentication failed, deleting stored credential %s",
cred_identity
)
cred.delete()
raise e
return res
def get_credentials(allow_interactive=True):
# prefer the environment
if 'OSF_TOKEN' in environ or all(
k in environ for k in ('OSF_USERNAME', 'OSF_PASSWORD')):
return dict(
token=environ.get('OSF_TOKEN', None),
username=environ.get('OSF_USERNAME', None),
password=environ.get('OSF_PASSWORD', None),
)
# fall back on DataLad credential manager
token_auth = Token(
name='https://osf.io',
url='https://osf.io/settings/tokens',
)
up_auth = UserPassword(
name='https://osf.io',
url='https://osf.io/settings/account',
)
do_interactive = allow_interactive and ui.is_interactive()
# get auth token, from environment, or from datalad credential store
# if known-- we do not support first-time entry during a test run
token = environ.get(
'OSF_TOKEN',
token_auth().get('token', None)
if do_interactive or token_auth.is_known
else None)
username = None
password = None
if not token:
# now same for user/password if there was no token
username = environ.get(
'OSF_USERNAME',
up_auth().get('user', None)
if do_interactive or up_auth.is_known
else None)
password = environ.get(
'OSF_PASSWORD',
up_auth().get('password', None)
if do_interactive or up_auth.is_known
else None)
return dict(token=token, username=username, password=password)
def setup_credentials():
# check if anything need to be done still
if 'OSF_TOKEN' in environ or all(
k in environ for k in ('OSF_USERNAME', 'OSF_PASSWORD')):
return dict(
token=environ.get('OSF_TOKEN', None),
username=environ.get('OSF_USERNAME', None),
password=environ.get('OSF_USERNAME', None),
)
token_auth = Token(name='https://osf.io', url=None)
up_auth = UserPassword(name='https://osf.io', url=None)
# get auth token, form environment, or from datalad credential store
# if known-- we do not support first-time entry during a test run
token = environ.get(
'OSF_TOKEN',
token_auth().get('token', None) if token_auth.is_known else None)
username = None
password = None
if not token:
# now same for user/password if there was no token
username = environ.get(
'OSF_USERNAME',
up_auth().get('user', None) if up_auth.is_known else None)
password = environ.get(
'OSF_PASSWORD',
up_auth().get('password', None) if up_auth.is_known else None)
# place into environment, for now this is the only way the special remote
# can be supplied with credentials
for k, v in (('OSF_TOKEN', token), ('OSF_USERNAME', username),
('OSF_PASSWORD', password)):
if v:
environ[k] = v
return dict(token=token, username=username, password=password)
def test_datalad_credential_helper(path=None):
ds = Dataset(path).create()
# tell git to use git-credential-datalad
ds.config.add('credential.helper', 'datalad', scope='local')
ds.config.add('datalad.credentials.githelper.noninteractive',
'true',
scope='global')
from datalad.downloaders.providers import Providers
url1 = "https://datalad-test.org/some"
url2 = "https://datalad-test.org/other"
provider_name = "datalad-test.org"
# `Providers` code is old and only considers a dataset root based on PWD
# for config lookup. contextmanager below can be removed once the
# provider/credential system is redesigned.
with chpwd(ds.path):
gitcred = GitCredentialInterface(url=url1, repo=ds)
# There's nothing set up yet, helper should return empty
gitcred.fill()
eq_(gitcred['username'], '')
eq_(gitcred['password'], '')
# store new credentials
# Note, that `Providers.enter_new()` currently uses user-level config
# files for storage only. TODO: make that an option!
# To not mess with existing ones, fail if it already exists:
cfg_file = Path(Providers._get_providers_dirs()['user']) \
/ f"{provider_name}.cfg"
assert_false(cfg_file.exists())
# Make sure we clean up
from datalad.tests import _TEMP_PATHS_GENERATED
_TEMP_PATHS_GENERATED.append(str(cfg_file))
# Give credentials to git and ask it to store them:
gitcred = GitCredentialInterface(url=url1,
username="******",
password="******",
repo=ds)
gitcred.approve()
assert_true(cfg_file.exists())
providers = Providers.from_config_files()
p1 = providers.get_provider(url=url1, only_nondefault=True)
assert_is_instance(p1.credential, UserPassword)
eq_(p1.credential.get('user'), 'dl-user')
eq_(p1.credential.get('password'), 'dl-pwd')
# default regex should be host only, so matching url2, too
p2 = providers.get_provider(url=url2, only_nondefault=True)
assert_is_instance(p1.credential, UserPassword)
eq_(p1.credential.get('user'), 'dl-user')
eq_(p1.credential.get('password'), 'dl-pwd')
# git, too, should now find it for both URLs
gitcred = GitCredentialInterface(url=url1, repo=ds)
gitcred.fill()
eq_(gitcred['username'], 'dl-user')
eq_(gitcred['password'], 'dl-pwd')
gitcred = GitCredentialInterface(url=url2, repo=ds)
gitcred.fill()
eq_(gitcred['username'], 'dl-user')
eq_(gitcred['password'], 'dl-pwd')
# Rejection must not currently lead to deleting anything, since we would
# delete too broadly.
gitcred.reject()
assert_true(cfg_file.exists())
gitcred = GitCredentialInterface(url=url1, repo=ds)
gitcred.fill()
eq_(gitcred['username'], 'dl-user')
eq_(gitcred['password'], 'dl-pwd')
dlcred = UserPassword(name=provider_name)
eq_(dlcred.get('user'), 'dl-user')
eq_(dlcred.get('password'), 'dl-pwd')
def _enable_remote(ds, name, known_remotes, url, pushurl, fetch, description,
as_common_datasrc, publish_depends, publish_by_default,
annex_wanted, annex_required, annex_group,
annex_groupwanted, inherit, get_annex_info, **res_kwargs):
result_props = dict(action='enable-sibling',
path=ds.path,
type='sibling',
name=name,
**res_kwargs)
if not isinstance(ds.repo, AnnexRepo):
yield dict(result_props,
status='impossible',
message='cannot enable sibling of non-annex dataset')
return
if name is None:
yield dict(result_props,
status='error',
message='require `name` of sibling to enable')
return
# get info on special remote
sp_remotes = {
v['name']: dict(v, uuid=k)
for k, v in ds.repo.get_special_remotes().items()
}
remote_info = sp_remotes.get(name, None)
if remote_info is None:
yield dict(result_props,
status='impossible',
message=("cannot enable sibling '%s', not known", name))
return
env = None
cred = None
if remote_info.get('type', None) == 'webdav':
# a webdav special remote -> we need to supply a username and password
if not ('WEBDAV_USERNAME' in os.environ
and 'WEBDAV_PASSWORD' in os.environ):
# nothing user-supplied
# let's consult the credential store
hostname = urlparse(remote_info.get('url', '')).netloc
if not hostname:
yield dict(
result_props,
status='impossible',
message=
"cannot determine remote host, credential lookup for webdav access is not possible, and not credentials were supplied"
)
cred = UserPassword('webdav:{}'.format(hostname))
if not cred.is_known:
try:
cred.enter_new(
instructions=
"Enter credentials for authentication with WEBDAV server at {}"
.format(hostname),
user=os.environ.get('WEBDAV_USERNAME', None),
password=os.environ.get('WEBDAV_PASSWORD', None))
except KeyboardInterrupt:
# user hit Ctrl-C
yield dict(
result_props,
status='impossible',
message=
"credentials are required for sibling access, abort")
return
creds = cred()
# update the env with the two necessary variable
# we need to pass a complete env because of #1776
env = dict(os.environ,
WEBDAV_USERNAME=creds['user'],
WEBDAV_PASSWORD=creds['password'])
try:
ds.repo.enable_remote(name, env=env)
result_props['status'] = 'ok'
except AccessDeniedError as e:
# credentials are wrong, wipe them out
if cred and cred.is_known:
cred.delete()
result_props['status'] = 'error'
result_props['message'] = e.message
except AccessFailedError as e:
# some kind of connection issue
result_props['status'] = 'error'
result_props['message'] = e.message
except Exception as e:
# something unexpected
raise e
yield result_props
def __call__(method="token", reset=False):
auth = None
cred_spec = []
if method == 'token':
cred_spec = dict(token='token')
auth = Token(
name='https://osf.io',
url='https://osf.io/settings/tokens',
)
elif method == 'userpassword':
cred_spec = dict(user='******', password='******')
auth = UserPassword(
name='https://osf.io',
url='https://osf.io/settings/account',
)
else:
raise ValueError(
'Unknown authentication method: {}'.format(method))
if reset and auth.is_known:
auth.delete()
cred = {v: auth().get(k, None) for k, v in cred_spec.items()}
# now verify that the credentials work by querying the
# logged in user
osf = OSF(**cred)
try:
req = osf.session.get('https://api.osf.io/v2/users/me/')
req.raise_for_status()
except UnauthorizedException:
auth.delete()
yield dict(
action='osf_credentials',
status='error',
message='Invalid credentials',
path=None,
)
return
except Exception as e:
yield dict(
action='osf_credentials',
status='impossible',
message='Could not verify credentials, '
'please try again: {}'.format(exc_str(e)),
# needed to pacify DataLad 0.13.0 and earlier
path=None,
)
return
# if we get here auth has worked fine
# get some attributes for an informative message
attrs = req.json().get('data', {}).get('attributes', {})
yield dict(
action='osf_credentials',
status='ok',
message='authenticated{}{}{}'.format(
' as ' if any(
attrs.get(k, None)
for k in ('email', 'full_name')) else '',
attrs.get('full_name', ''), ' <{}>'.format(attrs['email'])
if attrs.get('email', None) else ''),
# needed to pacify DataLad 0.13.0 and earlier
path=None,
# report effective credentials
**cred,
)
def __call__(url,
path="{subject}/{session}/{scan}/",
project=None,
force=False,
dataset=None):
from pyxnat import Interface as XNATInterface
ds = require_dataset(dataset,
check_installed=True,
purpose='initialization')
config = ds.config
path = with_pathsep(path)
# prep for yield
res = dict(
action='xnat_init',
path=ds.path,
type='dataset',
logger=lgr,
refds=ds.path,
)
# obtain user credentials, use simplified/stripped URL as identifier
# given we don't have more knowledge than the user, do not
# give a `url` to provide hints on how to obtain credentials
parsed_url = urlparse(url)
no_proto_url = '{}{}'.format(parsed_url.netloc,
parsed_url.path).replace(' ', '')
cred = UserPassword(name=no_proto_url, url=None)()
xn = XNATInterface(server=url, **cred)
# now we make a simple request to obtain the server version
# we don't care much, but if the URL or the credentials are wrong
# we will not get to see one
try:
xnat_version = xn.version()
lgr.debug("XNAT server version is %s", xnat_version)
except Exception as e:
yield dict(
res,
status='error',
message=('Failed to access the XNAT server. Full error:\n%s',
e),
)
return
if project is None:
from datalad.ui import ui
projects = xn.select.projects().get()
ui.message('No project name specified. The following projects are '
'available on {} for user {}:'.format(
url, cred['user']))
for p in sorted(projects):
# list and prep for C&P
# TODO multi-column formatting?
ui.message(" {}".format(quote_cmdlinearg(p)))
return
# query the specified project to make sure it exists and is accessible
proj = xn.select.project(project)
try:
nsubj = len(proj.subjects().get())
except Exception as e:
yield dict(
res,
status='error',
message=(
'Failed to obtain information on project %s from XNAT. '
'Full error:\n%s', project, e),
)
return
lgr.info('XNAT reports %i subjects currently on-record for project %s',
nsubj, project)
# check if dataset already initialized
auth_dir = ds.pathobj / '.datalad' / 'providers'
if auth_dir.exists() and not force:
yield dict(
res,
status='error',
message='Dataset found already initialized, '
'use `force` to reinitialize',
)
return
# put essential configuration into the dataset
config.set('datalad.xnat.default.url',
url,
where='dataset',
reload=False)
config.set('datalad.xnat.default.project', project, where='dataset')
config.set('datalad.xnat.default.path', path, where='dataset')
ds.save(
path=ds.pathobj / '.datalad' / 'config',
to_git=True,
message="Configure default XNAT url and project",
)
# Configure XNAT access authentication
ds.run_procedure(spec='cfg_xnat_dataset')
yield dict(
res,
status='ok',
)
return
def _enable_remote(
ds, name, known_remotes, url, pushurl, fetch, description,
as_common_datasrc, publish_depends, publish_by_default,
annex_wanted, annex_required, annex_group, annex_groupwanted,
inherit, get_annex_info,
**res_kwargs):
result_props = dict(
action='enable-sibling',
path=ds.path,
type='sibling',
name=name,
**res_kwargs)
if not isinstance(ds.repo, AnnexRepo):
yield dict(
result_props,
status='impossible',
message='cannot enable sibling of non-annex dataset')
return
if name is None:
yield dict(
result_props,
status='error',
message='require `name` of sibling to enable')
return
# get info on special remote
sp_remotes = {v['name']: dict(v, uuid=k) for k, v in ds.repo.get_special_remotes().items()}
remote_info = sp_remotes.get(name, None)
if remote_info is None:
yield dict(
result_props,
status='impossible',
message=("cannot enable sibling '%s', not known", name))
return
env = None
cred = None
if remote_info.get('type', None) == 'webdav':
# a webdav special remote -> we need to supply a username and password
if not ('WEBDAV_USERNAME' in os.environ and 'WEBDAV_PASSWORD' in os.environ):
# nothing user-supplied
# let's consult the credential store
hostname = urlparse(remote_info.get('url', '')).netloc
if not hostname:
yield dict(
result_props,
status='impossible',
message="cannot determine remote host, credential lookup for webdav access is not possible, and not credentials were supplied")
cred = UserPassword('webdav:{}'.format(hostname))
if not cred.is_known:
try:
cred.enter_new(
instructions="Enter credentials for authentication with WEBDAV server at {}".format(hostname),
user=os.environ.get('WEBDAV_USERNAME', None),
password=os.environ.get('WEBDAV_PASSWORD', None))
except KeyboardInterrupt:
# user hit Ctrl-C
yield dict(
result_props,
status='impossible',
message="credentials are required for sibling access, abort")
return
creds = cred()
# update the env with the two necessary variable
# we need to pass a complete env because of #1776
env = dict(
os.environ,
WEBDAV_USERNAME=creds['user'],
WEBDAV_PASSWORD=creds['password'])
try:
ds.repo.enable_remote(name, env=env)
result_props['status'] = 'ok'
except AccessDeniedError as e:
# credentials are wrong, wipe them out
if cred and cred.is_known:
cred.delete()
result_props['status'] = 'error'
result_props['message'] = str(e)
except AccessFailedError as e:
# some kind of connection issue
result_props['status'] = 'error'
result_props['message'] = str(e)
except Exception as e:
# something unexpected
raise e
yield result_props
def __call__(subjects='list', dataset=None, ifexists=None, force=False):
from pyxnat import Interface as XNATInterface
ds = require_dataset(dataset, check_installed=True, purpose='update')
subjects = ensure_list(subjects)
# require a clean dataset
if ds.repo.dirty:
yield get_status_dict(
'update',
ds=ds,
status='impossible',
message=(
'Clean dataset required; use `datalad status` to inspect '
'unsaved changes'))
return
# prep for yield
res = dict(
action='xnat_update',
path=ds.path,
type='dataset',
logger=lgr,
refds=ds.path,
)
# obtain configured XNAT url and project name
xnat_cfg_name = ds.config.get('datalad.xnat.default-name', 'default')
cfg_section = 'datalad.xnat.{}'.format(xnat_cfg_name)
xnat_url = ds.config.get('{}.url'.format(cfg_section))
xnat_project = ds.config.get('{}.project'.format(cfg_section))
file_path = ds.config.get('{}.path'.format(cfg_section))
# obtain user credentials
parsed_url = urlparse(xnat_url)
no_proto_url = '{}{}'.format(parsed_url.netloc,
parsed_url.path).replace(' ', '')
cred = UserPassword(name=no_proto_url, url=None)()
xn = XNATInterface(server=xnat_url, **cred)
# provide subject list
if 'list' in subjects:
from datalad.ui import ui
subs = xn.select.project(xnat_project).subjects().get()
ui.message('The following subjects are available for XNAT '
'project {}:'.format(xnat_project))
for s in sorted(subs):
ui.message(" {}".format(quote_cmdlinearg(s)))
ui.message(
'Specify a specific subject(s) or "all" to download associated '
'files for.')
return
# query the specified subject(s) to make sure it exists and is accessible
if 'all' not in subjects:
from datalad.ui import ui
subs = []
for s in subjects:
sub = xn.select.project(xnat_project).subject(s)
nexp = len(sub.experiments().get())
if nexp > 0:
subs.append(s)
else:
ui.message(
'Failed to obtain information on subject {} from XNAT '
'project {}:'.format(s, xnat_project))
return
else:
# if all, get list of all subjects
subs = xn.select.project(xnat_project).subjects().get()
# parse and download one subject at a time
from datalad_xnat.parser import parse_xnat
addurl_dir = ds.pathobj / 'code' / 'addurl_files'
for sub in subs:
yield from parse_xnat(
ds,
sub=sub,
force=force,
xn=xn,
xnat_url=xnat_url,
xnat_project=xnat_project,
)
# add file urls for subject
lgr.info('Downloading files for subject %s', sub)
table = f"{addurl_dir}/{sub}_table.csv"
# this corresponds to the header field 'filename' in the csv table
filename = '{filename}'
filenameformat = f"{file_path}{filename}"
ds.addurls(
table,
'{url}',
filenameformat,
ifexists=ifexists,
save=False,
cfg_proc='xnat_dataset',
result_renderer='default',
)
ds.save(message=f"Update files for subject {sub}", recursive=True)
lgr.info(
'Files were updated for the following subjects in XNAT project %s:',
xnat_project)
for s in sorted(subs):
lgr.info(" {}".format(quote_cmdlinearg(s)))
yield dict(res, status='ok')
return
def _get_github_cred():
"""Trimmed down helper"""
return UserPassword("github", "does not matter")