def test_color_enabled():
# In the absence of NO_COLOR, follow ui.color, or ui.is_interactive if 'auto'
with patch.dict(os.environ), \
patch('datalad.support.ansi_colors.ui'):
os.environ.pop('NO_COLOR', None)
for is_interactive in (True, False):
colors.ui.is_interactive = is_interactive
with patch_config({'datalad.ui.color': 'off'}):
assert_equal(colors.color_enabled(), False)
with patch_config({'datalad.ui.color': 'on'}):
assert_equal(colors.color_enabled(), True)
with patch_config({'datalad.ui.color': 'auto'}):
assert_equal(colors.color_enabled(), is_interactive)
# In the presence of NO_COLOR, default to disable, unless ui.color is "on"
# The value of NO_COLOR should have no effect, so try true-ish and false-ish values
for NO_COLOR in ("", "1", "0"):
with patch.dict(os.environ, {'NO_COLOR': NO_COLOR}), \
patch('datalad.support.ansi_colors.ui'):
for is_interactive in (True, False):
colors.ui.is_interactive = is_interactive
with patch_config({'datalad.ui.color': 'on'}):
assert_equal(colors.color_enabled(), True)
for ui_color in ('off', 'auto'):
with patch_config({'datalad.ui.color': ui_color}):
assert_equal(colors.color_enabled(), False)
def test_probe_known_failure():
# should raise assert error if function no longer fails
with patch_config({'datalad.tests.knownfailures.probe': True}):
with assert_raises(AssertionError):
probe_known_failure(lambda: True)()
with patch_config({'datalad.tests.knownfailures.probe': False}):
ok_(probe_known_failure(lambda: True))
def test_search_outside1_install_default_ds(tdir, default_dspath):
with chpwd(tdir):
# let's mock out even actual install/search calls
with \
patch_config({'datalad.locations.default-dataset': default_dspath}), \
patch('datalad.api.install',
return_value=Dataset(default_dspath)) as mock_install, \
patch('datalad.distribution.dataset.Dataset.search',
new_callable=_mock_search):
_check_mocked_install(default_dspath, mock_install)
# now on subsequent run, we want to mock as if dataset already exists
# at central location and then do search again
from datalad.ui import ui
ui.add_responses('yes')
mock_install.reset_mock()
with patch('datalad.distribution.dataset.Dataset.is_installed',
True):
_check_mocked_install(default_dspath, mock_install)
# and what if we say "no" to install?
ui.add_responses('no')
mock_install.reset_mock()
with assert_raises(NoDatasetArgumentFound):
list(search("."))
# and if path exists and is a valid dataset and we say "no"
Dataset(default_dspath).create()
ui.add_responses('no')
mock_install.reset_mock()
with assert_raises(NoDatasetArgumentFound):
list(search("."))
def test_with_tempfile_dir_via_env_variable():
target = os.path.join(os.path.expanduser("~"), "dataladtesttmpdir")
assert_false(os.path.exists(target), "directory %s already exists." % target)
with patch_config({'datalad.tests.temp.dir': target}):
filename = _with_tempfile_decorated_dummy()
ok_startswith(filename, target)
def test_search_outside1_install_default_ds(tdir, default_dspath):
with chpwd(tdir):
# let's mock out even actual install/search calls
with \
patch_config({'datalad.locations.default-dataset': default_dspath}), \
patch('datalad.api.install',
return_value=Dataset(default_dspath)) as mock_install, \
patch('datalad.distribution.dataset.Dataset.search',
new_callable=_mock_search):
_check_mocked_install(default_dspath, mock_install)
# now on subsequent run, we want to mock as if dataset already exists
# at central location and then do search again
from datalad.ui import ui
ui.add_responses('yes')
mock_install.reset_mock()
with patch(
'datalad.distribution.dataset.Dataset.is_installed',
True):
_check_mocked_install(default_dspath, mock_install)
# and what if we say "no" to install?
ui.add_responses('no')
mock_install.reset_mock()
with assert_raises(NoDatasetArgumentFound):
list(search("."))
# and if path exists and is a valid dataset and we say "no"
Dataset(default_dspath).create()
ui.add_responses('no')
mock_install.reset_mock()
with assert_raises(NoDatasetArgumentFound):
list(search("."))
def test_skip_if_no_network():
cleaned_env = os.environ.copy()
cleaned_env.pop('DATALAD_TESTS_NONETWORK', None)
# we need to run under cleaned env to make sure we actually test in both conditions
with patch('os.environ', cleaned_env):
@skip_if_no_network
def somefunc(a1):
return a1
ok_(hasattr(somefunc, "network"))
with patch_config({'datalad.tests.nonetwork': '1'}):
assert_raises(SkipTest, somefunc, 1)
with patch.dict('os.environ', {}):
eq_(somefunc(1), 1)
# and now if used as a function, not a decorator
with patch_config({'datalad.tests.nonetwork': '1'}):
assert_raises(SkipTest, skip_if_no_network)
with patch.dict('os.environ', {}):
eq_(skip_if_no_network(), None)
def check_integration1(login,
keyring,
path,
organization=None,
kwargs={},
oauthtokens=None):
kwargs = kwargs.copy()
if organization:
kwargs['github_organization'] = organization
ds = Dataset(path).create()
config_patch = {}
if oauthtokens:
config_patch['hub.oauthtoken'] = tuple(ensure_list(oauthtokens))
# so we do not pick up local repo configuration/token
repo_name = 'test_integration1'
# ATM all the github goodness does not care about "this dataset"
# so patch the global config
with patch_config(config_patch):
# everything works just nice, no conflicts etc
res = ds.create_sibling_github(repo_name, **kwargs)
if organization:
url_fmt = 'https://{login}@github.com/{organization}/{repo_name}.git'
else:
url_fmt = 'https://github.com/{login}/{repo_name}.git'
assert_in_results(res,
path=ds.path,
url=url_fmt.format(**locals()),
preexisted=False)
# but if we rerun - should kaboom since already has this sibling:
assert_in_results(
ds.create_sibling_github(repo_name, on_failure='ignore', **kwargs),
message=('already has a configured sibling "%s"', 'github'),
status='error',
)
# but we can give it a new name, but it should kaboom since the remote one
# exists already
assert_in_results(
ds.create_sibling_github(repo_name,
name="github2",
on_failure='ignore',
**kwargs),
message=('repository "%s" already exists on Github',
'test_integration1'),
status='error',
)
# we should not leave the broken sibling behind
assert_not_in('github2', ds.repo.get_remotes())
# If we ask to reconfigure - should proceed normally
ds.create_sibling_github(repo_name, existing='reconfigure', **kwargs)
def test_search_outside1(tdir, newhome):
with chpwd(tdir):
# should fail since directory exists, but not a dataset
# should not even waste our response ;)
with patch_config({'datalad.locations.default-dataset': newhome}):
gen = search("bu", return_type='generator')
assert_is_generator(gen)
assert_raises(NoDatasetArgumentFound, next, gen)
# and if we point to some non-existing dataset
with assert_raises(ValueError):
next(search("bu", dataset=newhome))
def test_search_outside1(tdir, newhome):
with chpwd(tdir):
# should fail since directory exists, but not a dataset
# should not even waste our response ;)
with patch_config({'datalad.locations.default-dataset': newhome}):
gen = search("bu", return_type='generator')
assert_is_generator(gen)
assert_raises(NoDatasetArgumentFound, next, gen)
# and if we point to some non-existing dataset
with assert_raises(ValueError):
next(search("bu", dataset=newhome))
def test__gen_github_entity_organization():
# to test effectiveness of the fix, we need to provide some
# token which would not work
with patch_config(
{CONFIG_HUB_TOKEN_FIELD: "ed51111111111111111111111111111111111111"}):
org_cred = next(_gen_github_entity(None, 'datalad-collection-1'))
assert len(org_cred) == 2, "we return organization and credential"
org, _ = org_cred
assert org
repos = list(org.get_repos())
repos_names = [r.name for r in repos]
assert_greater(len(repos), 3) # we have a number of those
assert_in('datasets.datalad.org', repos_names)
def test_get_result_filter_arg_vs_config():
# just tests that we would be obtaining the same constraints via
# cmdline argument or via config variable. With cmdline overloading
# config
f = Interface._get_result_filter
eq_(f(_new_args()), None) # by default, no filter
for v in "success", "failure", "ok", "notneeded", "error":
cargs = f(_new_args(common_report_status=v))
assert cargs is not None
with patch_config({"datalad.runtime.report-status": v}):
ccfg = f(_new_args())
ccfg_none = f(_new_args(common_report_status="all"))
# cannot compare directly but at least could verify based on repr
print("%s -> %s" % (v, repr(cargs)))
eq_(repr(cargs), repr(ccfg))
# and if 'all' - none filter
eq_(None, ccfg_none)
# and we overload the "error" in config
with patch_config({"datalad.runtime.report-status": "error"}):
cargs_overload = f(_new_args(common_report_status=v))
eq_(repr(cargs), repr(cargs_overload))
def test_get_resolved_values():
from datalad.tests.utils import _get_resolved_flavors
flavors = ['networkish', 'local']
eq_(([] if dl_cfg.get('datalad.tests.nonetwork') else ['networkish'])
+ ['local'],
_get_resolved_flavors(flavors))
with patch_config({'datalad.tests.nonetwork': '1'}):
eq_(_get_resolved_flavors(flavors), ['local'])
# and one more to see the exception being raised if nothing to teston
@with_testrepos(flavors=['network'])
def magical():
raise AssertionError("Must not be ran")
assert_raises(SkipTest, magical)
def check_integration1(login,
keyring,
path,
organization=None,
kwargs={},
oauthtokens=None):
kwargs = kwargs.copy()
if organization:
kwargs['github_organization'] = organization
ds = Dataset(path).create()
config_patch = {}
if oauthtokens:
config_patch['hub.oauthtoken'] = tuple(ensure_list(oauthtokens))
# so we do not pick up local repo configuration/token
repo_name = 'test_integration1'
# ATM all the github goodness does not care about "this dataset"
# so patch the global config
with patch_config(config_patch):
# everything works just nice, no conflicts etc
res = ds.create_sibling_github(repo_name, **kwargs)
if organization:
url_fmt = 'https://{login}@github.com/{organization}/{repo_name}.git'
else:
url_fmt = 'https://github.com/{login}/{repo_name}.git'
eq_(res, [(ds, url_fmt.format(**locals()), False)])
# but if we rerun - should kaboom since already has this sibling:
with assert_raises(ValueError) as cme:
ds.create_sibling_github(repo_name, **kwargs)
assert_in("already has a configured sibling", str(cme.exception))
# but we can give it a new name, but it should kaboom since the remote one
# exists already
with assert_raises(ValueError) as cme:
ds.create_sibling_github(repo_name, name="github2", **kwargs)
assert_in("already exists on", str(cme.exception))
# we should not leave the broken sibling behind
assert_not_in('github2', ds.repo.get_remotes())
# If we ask to reconfigure - should proceed normally
ds.create_sibling_github(repo_name, existing='reconfigure', **kwargs)
def test_ssh_custom_identity_file():
ifile = "/tmp/dl-test-ssh-id" # Travis
if not op.exists(ifile):
raise SkipTest(
"Travis-specific '{}' identity file does not exist".format(ifile))
with patch_config({"datalad.ssh.identityfile": ifile}):
with swallow_logs(new_level=logging.DEBUG) as cml:
manager = SSHManager()
ssh = manager.get_connection('ssh://localhost')
cmd_out, _ = ssh("echo blah")
expected_socket = op.join(
str(manager.socket_dir),
get_connection_hash("localhost",
identity_file=ifile,
bundled=True))
ok_(exists(expected_socket))
manager.close()
assert_in("-i", cml.out)
assert_in(ifile, cml.out)
def test_download_docker_blob(path):
from datalad.consts import (
DATALAD_SPECIAL_REMOTE,
DATALAD_SPECIAL_REMOTES_UUIDS,
)
from datalad.customremotes.base import init_datalad_remote
with patch_config({"datalad.repo.backend": "SHA256E"}):
ds = Dataset(path).create()
ds_repo = ds.repo
init_datalad_remote(ds_repo, DATALAD_SPECIAL_REMOTE)
id_ = "f0b02e9d092d905d0d87a8455a1ae3e9bb47b4aa3dc125125ca5cd10d6441c9f"
outfile = ds_repo.pathobj / "blob"
url = "https://registry-1.docker.io/v2/library/busybox/blobs/sha256:" + id_
ds.download_url(urls=[url], path=str(outfile))
annex_info = ds.repo.get_content_annexinfo(paths=[outfile], init=None)
eq_(id_, annex_info[outfile]["keyname"])
assert_in(DATALAD_SPECIAL_REMOTES_UUIDS[DATALAD_SPECIAL_REMOTE],
ds_repo.whereis([str(outfile)])[0])
def test_skip_ssh():
with patch_config({'datalad.tests.ssh': False}):
with assert_raises(SkipTest):
skip_ssh(lambda: False)()
def test_procedure_discovery(path, super_path):
with chpwd(path):
# ^ Change directory so that we don't fail with an
# InvalidGitRepositoryError if the test is executed from a git
# worktree.
ps = run_procedure(discover=True)
# there are a few procedures coming with datalad, needs to find them
assert_true(len(ps) > 2)
# we get essential properties
_check_procedure_properties(ps)
# set up dataset with registered procedure (c&p from test_basics):
ds = Dataset(path).create(force=True)
# extra check: must not pick up cfg_yoda.sh in top directory
ds.run_procedure('cfg_yoda')
# path to a procedure which is not under any "standard" location but
# present in the dataset
code_dir_procedure_path = op.join(ds.path, 'code', 'datalad_test_proc.py')
top_dir_procedure_path = op.join(ds.path, 'cfg_yoda.sh')
# run discovery on the dataset:
ps = ds.run_procedure(discover=True)
# it should not be found magically by default
assert_not_in_results(ps, path=code_dir_procedure_path)
assert_not_in_results(ps, path=top_dir_procedure_path)
with patch_config({'datalad.locations.extra-procedures': op.join(ds.path, 'code')}):
# run discovery on the dataset:
ps = ds.run_procedure(discover=True)
# still needs to find procedures coming with datalad
assert_true(len(ps) > 3)
# and procedure under the path we specified
assert_result_count(ps, 1, path=code_dir_procedure_path)
assert_not_in_results(ps, path=top_dir_procedure_path)
# multiple extra locations
with patch_config({'datalad.locations.extra-procedures': [op.join(ds.path, 'code'), ds.path]}):
# run discovery on the dataset:
ps = ds.run_procedure(discover=True)
# still needs to find procedures coming with datalad
assert_true(len(ps) > 4)
# and procedure under the path we specified
assert_result_count(ps, 1, path=code_dir_procedure_path)
assert_result_count(ps, 1, path=top_dir_procedure_path)
# configure dataset to look for procedures in its code folder
ds.config.add(
'datalad.locations.dataset-procedures',
'code',
where='dataset')
ds.save(op.join('.datalad', 'config'))
# run discovery on the dataset:
ps = ds.run_procedure(discover=True)
# still needs to find procedures coming with datalad
assert_true(len(ps) > 2)
# we get three essential properties
_check_procedure_properties(ps)
# dataset's procedure needs to be in the results
# and only a single one
assert_result_count(ps, 1, path=code_dir_procedure_path)
# a subdir shouldn't be considered a procedure just because it's "executable"
assert_not_in_results(ps, path=op.join(ds.path, 'code', 'testdir'))
# make it a subdataset and try again:
# first we need to save the beast to make install work
ds.save()
super = Dataset(super_path).create()
super.install('sub', source=ds.path)
ps = super.run_procedure(discover=True)
# still needs to find procedures coming with datalad
assert_true(len(ps) > 2)
_check_procedure_properties(ps)
# dataset's procedure needs to be in the results
assert_in_results(ps, path=op.join(super.path, 'sub', 'code',
'datalad_test_proc.py'))
if not on_windows: # no symlinks
import os
# create a procedure which is a broken symlink, but recognizable as a
# python script:
os.symlink(op.join(super.path, 'sub', 'not_existent'),
op.join(super.path, 'sub', 'code', 'broken_link_proc.py'))
# broken symlink at procedure location, but we can't tell, whether it is
# an actual procedure without any guess on how to execute it:
os.symlink(op.join(super.path, 'sub', 'not_existent'),
op.join(super.path, 'sub', 'code', 'unknwon_broken_link'))
ps = super.run_procedure(discover=True)
# still needs to find procedures coming with datalad and the dataset
# procedure registered before
assert_true(len(ps) > 3)
assert_in_results(ps, path=op.join(super.path, 'sub', 'code',
'broken_link_proc.py'),
state='absent')
assert_in_results(
ps,
path=op.join(super.path, 'sub', 'code',
'unknwon_broken_link'),
state='absent')
def test_ria_http(lcl, storepath, url):
# create a local dataset with a subdataset
lcl = Path(lcl)
storepath = Path(storepath)
subds = Dataset(lcl / 'ds' / 'subdir' / 'subds').create(force=True)
subds.save()
ds = Dataset(lcl / 'ds').create(force=True)
ds.save(version_tag='original')
assert_repo_status(ds.path)
for d in (ds, subds):
_move2store(storepath, d)
# location of superds in store
storeds_loc = str(storepath / ds.id[:3] / ds.id[3:])
# now we should be able to clone from a ria+http url
# the super
riaclone = clone(
'ria+{}#{}'.format(url, ds.id),
lcl / 'clone',
)
# due to default configuration, clone() should automatically look for the
# subdataset in the store, too -- if not the following would fail, because
# we never configured a proper submodule URL
riaclonesub = riaclone.get(
op.join('subdir', 'subds'), get_data=False,
result_xfm='datasets', return_type='item-or-list')
# both datasets came from the store and must be set up in an identical
# fashion
for origds, cloneds in ((ds, riaclone), (subds, riaclonesub)):
eq_(origds.id, cloneds.id)
if not ds.repo.is_managed_branch():
# test logic cannot handle adjusted branches
eq_(origds.repo.get_hexsha(), cloneds.repo.get_hexsha())
ok_(cloneds.config.get('remote.origin.url').startswith(url))
eq_(cloneds.config.get('remote.origin.annex-ignore'), 'true')
eq_(cloneds.config.get('datalad.get.subdataset-source-candidate-200origin'),
'ria+%s#{id}' % url)
# now advance the source dataset
(ds.pathobj / 'newfile.txt').write_text('new')
ds.save()
ds.publish(to='store')
Runner(cwd=storeds_loc).run(['git', 'update-server-info'])
# re-clone as before
riaclone2 = clone(
'ria+{}#{}'.format(url, ds.id),
lcl / 'clone2',
)
# and now clone a specific version, here given be the tag name
riaclone_orig = clone(
'ria+{}#{}@{}'.format(url, ds.id, 'original'),
lcl / 'clone_orig',
)
if not ds.repo.is_managed_branch():
# test logic cannot handle adjusted branches
# we got the precise version we wanted
eq_(riaclone.repo.get_hexsha(), riaclone_orig.repo.get_hexsha())
# and not the latest
eq_(riaclone2.repo.get_hexsha(), ds.repo.get_hexsha())
neq_(riaclone2.repo.get_hexsha(), riaclone_orig.repo.get_hexsha())
# attempt to clone a version that doesn't exist
with swallow_logs():
with assert_raises(IncompleteResultsError) as cme:
clone('ria+{}#{}@impossible'.format(url, ds.id),
lcl / 'clone_failed')
assert_in("not found in upstream", str(cme.exception))
# lastly test if URL rewriting is in effect
# on the surface we clone from an SSH source identified by some custom
# label, no full URL, but URL rewriting setup maps it back to the
# HTTP URL used above
with patch_config({
'url.ria+{}#.insteadof'.format(url): 'ria+ssh://somelabel#'}):
cloned_by_label = clone(
'ria+ssh://somelabel#{}'.format(origds.id),
lcl / 'cloned_by_label',
)
# so we get the same setup as above, but....
eq_(origds.id, cloned_by_label.id)
if not ds.repo.is_managed_branch():
# test logic cannot handle adjusted branches
eq_(origds.repo.get_hexsha(), cloned_by_label.repo.get_hexsha())
ok_(cloned_by_label.config.get('remote.origin.url').startswith(url))
eq_(cloned_by_label.config.get('remote.origin.annex-ignore'), 'true')
# ... the clone candidates go with the label-based URL such that
# future get() requests acknowlege a (system-wide) configuration
# update
eq_(cloned_by_label.config.get('datalad.get.subdataset-source-candidate-200origin'),
'ria+ssh://somelabel#{id}')
if not has_symlink_capability():
return
# place a symlink in the store to serve as a dataset alias
(storepath / 'alias').mkdir()
(storepath / 'alias' / 'myname').symlink_to(storeds_loc)
with chpwd(lcl):
cloned_by_alias = clone('ria+{}#~{}'.format(url, 'myname'))
# still get the same data
eq_(cloned_by_alias.id, ds.id)
# more sensible default install path
eq_(cloned_by_alias.pathobj.name, 'myname')