def test_overrides():
cfg = ConfigManager()
# any sensible (and also our CI) test environment(s) should have this
assert_in('user.name', cfg)
# set
cfg.set('user.name', 'myoverride', scope='override')
assert_equal(cfg['user.name'], 'myoverride')
# unset just removes override, not entire config
cfg.unset('user.name', scope='override')
assert_in('user.name', cfg)
assert_not_equal('user.name', 'myoverride')
# add
# there is no initial increment
cfg.add('user.name', 'myoverride', scope='override')
assert_equal(cfg['user.name'], 'myoverride')
# same as with add, not a list
assert_equal(cfg['user.name'], 'myoverride')
# but then there is
cfg.add('user.name', 'myother', scope='override')
assert_equal(cfg['user.name'], ['myoverride', 'myother'])
# rename
assert_not_in('ups.name', cfg)
cfg.rename_section('user', 'ups', scope='override')
# original variable still there
assert_in('user.name', cfg)
# rename of override in effect
assert_equal(cfg['ups.name'], ['myoverride', 'myother'])
# remove entirely by section
cfg.remove_section('ups', scope='override')
from datalad.utils import Path
assert_not_in('ups.name', cfg, (
cfg._stores,
cfg.overrides,
))
def test_get_content_info_paths_empty_list(path=None):
ds = Dataset(path).create()
# Unlike None, passing any empty list as paths to get_content_info() does
# not report on all content.
assert_false(ds.repo.get_content_info(paths=[]))
assert_false(ds.repo.get_content_info(paths=[], ref="HEAD"))
# Add annex content to make sure its not reported.
(ds.pathobj / "foo").write_text("foo")
ds.save()
# Same for get_content_annexinfo()...
assert_false(ds.repo.get_content_annexinfo(paths=[]))
assert_false(ds.repo.get_content_annexinfo(paths=[], init=None))
assert_false(ds.repo.get_content_annexinfo(paths=[], ref="HEAD"))
assert_false(ds.repo.get_content_annexinfo(paths=[], ref="HEAD",
init=None))
# ... where whatever was passed for init will be returned as is.
assert_equal(
ds.repo.get_content_annexinfo(paths=[],
ref="HEAD",
init={"random": {
"entry": "a"
}}), {"random": {
"entry": "a"
}})
def _test_setup_store(io_cls, io_args, store=None):
io = io_cls(*io_args)
store = Path(store)
version_file = store / 'ria-layout-version'
error_logs = store / 'error_logs'
# invalid version raises:
assert_raises(UnknownLayoutVersion, create_store, io, store, '2')
# non-existing path should work:
create_store(io, store, '1')
assert_true(version_file.exists())
assert_true(error_logs.exists())
assert_true(error_logs.is_dir())
assert_equal([f for f in error_logs.iterdir()], [])
# empty target directory should work as well:
rmtree(str(store))
store.mkdir(exist_ok=False)
create_store(io, store, '1')
assert_true(version_file.exists())
assert_true(error_logs.exists())
assert_true(error_logs.is_dir())
assert_equal([f for f in error_logs.iterdir()], [])
# re-execution also fine:
create_store(io, store, '1')
# but version conflict with existing target isn't:
version_file.write_text("2|unknownflags\n")
assert_raises(ValueError, create_store, io, store, '1')
def test_get_metadata(path=None):
ds = Dataset(path).create(force=True)
p = MetadataExtractor(ds, [])
meta = p._get_dataset_metadata()
assert_equal(
dumps(meta, sort_keys=True, indent=2), """\
{
"author": "Jane Doe <*****@*****.**>",
"conformsto": "http://specs.frictionlessdata.io/data-packages",
"contributors": [
"Joe Bloggs <*****@*****.**> (http://www.example.com)"
],
"description": "Annual Consumer Price Index (CPI) for most countries in the world. Reference year is 2005.",
"license": "http://opendatacommons.org/licenses/pddl/",
"name": "cpi",
"shortdescription": "Annual Consumer Price Index (CPI)",
"tag": [
"CPI",
"World",
"Consumer Price Index",
"Annual Data",
"The World Bank"
],
"version": "2.0.0"
}""")
def decorated_test1(url):
# we expect a file-scheme url to a cached version of `ds_url`
expect_origin_path = cache_dir / name_in_cache
assert_equal(expect_origin_path.as_uri(), url)
origin = Dataset(expect_origin_path)
assert_true(origin.is_installed())
assert_false(origin.repo.file_has_content(str(annexed_file)))
def test_verify_ria_url():
# unsupported protocol
assert_raises(ValueError, verify_ria_url, 'ria+ftp://localhost/tmp/this',
{})
# bunch of caes that should work
cases = {
'ria+file:///tmp/this': (None, '/tmp/this'),
# no normalization
'ria+file:///tmp/this/': (None, '/tmp/this/'),
# with hosts
'ria+ssh://localhost/tmp/this': ('ssh://localhost', '/tmp/this'),
'ria+http://localhost/tmp/this': ('http://localhost', '/tmp/this'),
'ria+https://localhost/tmp/this': ('https://localhost', '/tmp/this'),
# with username
'ria+ssh://humbug@localhost/tmp/this':
('ssh://humbug@localhost', '/tmp/this'),
# with port
'ria+ssh://humbug@localhost:2222/tmp/this':
('ssh://humbug@localhost:2222', '/tmp/this'),
'ria+ssh://localhost:2200/tmp/this':
('ssh://*****:*****@localhost:8080/tmp/this':
('https://*****:*****@localhost:8080', '/tmp/this'),
# document a strange (MIH thinks undesirable), but pre-existing
# behavior an 'ssh example.com' would end up in the user HOME,
# not in '/'
'ria+ssh://example.com': ('ssh://example.com', '/')
}
for i, o in cases.items():
# we are not testing the URL rewriting here
assert_equal(o, verify_ria_url(i, {})[:2])
def test_stalling(kill=False):
import concurrent.futures
from datalad.cmd import WitlessRunner
def worker():
WitlessRunner().run(["echo", "1"])
t0 = time()
v1 = worker()
dt1 = time() - t0
t0 = time()
with concurrent.futures.ThreadPoolExecutor(1) as executor:
# print("submitting")
future = executor.submit(worker)
dt2_limit = dt1 * 5
# print("waiting for up to %.2f sec" % dt2_limit)
while not future.done():
# print("not yet")
sleep(dt1 / 3)
if time() - t0 > dt2_limit:
# does not even shutdown
# executor.shutdown(wait=False)
if kill:
# raising an exception isn't enough!
print("exceeded")
import os
import signal
os.kill(os.getpid(), signal.SIGTERM)
raise AssertionError("Future has not finished in 5x time")
v2 = future.result()
assert_equal(v1, v2)
def test_get_metadata(path1=None, path2=None):
for p in (path1, path2):
print('PATH')
ds = create(p, force=True)
ds.save()
meta = MetadataExtractor(
ds,
_get_metadatarelevant_paths(ds, []))._get_dataset_metadata()
assert_equal(
dumps(meta, sort_keys=True, indent=2),
"""\
{
"author": [
"Last1, First1",
"Last2, First2"
],
"citation": [
"10.1016/j.cub.2011.08.031"
],
"description": "Some long description.",
"formats": [
"application/matlab",
"NIFTY"
],
"name": "CRCNS.org xxx-1",
"sameas": "10.6080/K0QN64NG",
"shortdescription": "Main title",
"tag": [
"Neuroscience",
"fMRI"
],
"version": "1.0"
}""")
def test_get_metadata(path=None):
ds = Dataset(path).create(force=True)
ds.save()
meta = MetadataExtractor(ds, [])._get_dataset_metadata()
assert_equal(
dumps(meta, sort_keys=True, indent=2), """\
{
"citation": "Cool (2016)",
"conformsto": "http://docs.datalad.org/metadata.html#v0-1",
"description": "A text with arbitrary length and content that can span multiple\\nparagraphs (this is a new one)",
"fundedby": "BMBFGQ1411, NSF 1429999",
"homepage": "http://studyforrest.org",
"issuetracker": "https://github.com/psychoinformatics-de/studyforrest-data-phase2/issues",
"license": [
"CC0-1.0",
"The person who associated a work with this deed has dedicated the work to the public domain by waiving all of his or her rights to the work worldwide under copyright law, including all related and neighboring rights, to the extent allowed by law.\\nYou can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission."
],
"maintainer": [
"Mike One <*****@*****.**>",
"Anna Two <*****@*****.**>"
],
"name": "studyforrest_phase2",
"sameas": "http://dx.doi.org/10.5281/zenodo.48421",
"shortdescription": "Basic summary",
"version": "1.0.0-rc3"
}""")
def test_annexinfo_init(path=None):
ds = Dataset(path).create()
foo = ds.pathobj / "foo"
foo_cont = b"foo content"
foo.write_bytes(foo_cont)
bar = ds.pathobj / "bar"
bar.write_text(u"bar content")
ds.save()
# Custom init limits report, with original dict getting updated.
cinfo_custom_init = ds.repo.get_content_annexinfo(
init={foo: {
"bytesize": 0,
"this-is-surely-only-here": "right?"
}})
assert_not_in(bar, cinfo_custom_init)
assert_in(foo, cinfo_custom_init)
assert_equal(cinfo_custom_init[foo]["bytesize"], len(foo_cont))
assert_equal(cinfo_custom_init[foo]["this-is-surely-only-here"], "right?")
# "git" injects get_content_info() values.
cinfo_init_git = ds.repo.get_content_annexinfo(init="git")
assert_in("gitshasum", cinfo_init_git[foo])
# init=None, on the other hand, does not.
cinfo_init_none = ds.repo.get_content_annexinfo(init=None)
assert_in(foo, cinfo_init_none)
assert_in(bar, cinfo_init_none)
assert_not_in("gitshasum", cinfo_init_none[foo])
def test_color_enabled():
# In the absence of NO_COLOR, follow ui.color, or ui.is_interactive if 'auto'
with patch.dict(os.environ), \
patch('datalad.support.ansi_colors.ui'):
os.environ.pop('NO_COLOR', None)
for is_interactive in (True, False):
colors.ui.is_interactive = is_interactive
with patch_config({'datalad.ui.color': 'off'}):
assert_equal(colors.color_enabled(), False)
with patch_config({'datalad.ui.color': 'on'}):
assert_equal(colors.color_enabled(), True)
with patch_config({'datalad.ui.color': 'auto'}):
assert_equal(colors.color_enabled(), is_interactive)
# In the presence of NO_COLOR, default to disable, unless ui.color is "on"
# The value of NO_COLOR should have no effect, so try true-ish and false-ish values
for NO_COLOR in ("", "1", "0"):
with patch.dict(os.environ, {'NO_COLOR': NO_COLOR}), \
patch('datalad.support.ansi_colors.ui'):
for is_interactive in (True, False):
colors.ui.is_interactive = is_interactive
with patch_config({'datalad.ui.color': 'on'}):
assert_equal(colors.color_enabled(), True)
for ui_color in ('off', 'auto'):
with patch_config({'datalad.ui.color': ui_color}):
assert_equal(colors.color_enabled(), False)
def test_file_number_activity_detection():
# Expect an output queue that just has the process exit notification.
# empty output queue without active threads
# waits for the process and progresses the generator state
# to `_ResultGenerator.GeneratorState.process_exited`.
class TestFNADProtocol(GeneratorMixIn, NoCapture):
def __init__(self):
GeneratorMixIn.__init__(self)
NoCapture.__init__(self)
def process_exited(self):
self.send_result(3)
def connection_lost(self, exc: Optional[Exception]) -> None:
self.send_result(4)
wl_runner = WitlessRunner()
result_generator = wl_runner.run(cmd=["echo", "a"],
protocol=TestFNADProtocol)
runner = result_generator.runner
output_queue = runner.output_queue
assert len(result_generator.runner.active_file_numbers) == 1
while runner.should_continue():
runner.process_queue()
# Expect process exited and connection lost to be called.
assert_equal(result_generator.send(None), 3)
assert_equal(result_generator.send(None), 4)
assert_raises(StopIteration, result_generator.send, None)
def test_dirty(path=None):
for mode in _dirty_modes:
# does nothing without a dataset
handle_dirty_dataset(None, mode)
# placeholder, but not yet created
ds = Dataset(path)
# unknown mode
assert_raises(ValueError, handle_dirty_dataset, ds, 'MADEUP')
# not yet created is very dirty
assert_raises(RuntimeError, handle_dirty_dataset, ds, 'fail')
handle_dirty_dataset(ds, 'ignore')
assert_raises(RuntimeError, handle_dirty_dataset, ds, 'save-before')
# should yield a clean repo
ds.create()
orig_state = ds.repo.get_hexsha()
_check_all_clean(ds, orig_state)
# tainted: untracked
with open(opj(ds.path, 'something'), 'w') as f:
f.write('some')
# we don't want to auto-add untracked files by saving (anymore)
assert_raises(AssertionError, _check_auto_save, ds, orig_state)
# tainted: staged
ds.repo.add('something', git=True)
orig_state = _check_auto_save(ds, orig_state)
# tainted: submodule
# not added to super on purpose!
subds = ds.create('subds')
_check_all_clean(subds, subds.repo.get_hexsha())
assert_repo_status(ds.path)
# subdataset must be added as a submodule!
assert_equal(ds.subdatasets(result_xfm='relpaths'), ['subds'])
def test_result_filter():
# ensure baseline without filtering
assert_equal([r['somekey'] for r in TestUtils().__call__(4)], [0, 1, 2, 3])
# test two functionally equivalent ways to filter results
# 1. Constraint-based -- filter by exception
# we have a full set of AND and OR operators for this
# 2. custom filer function -- filter by boolean return value
for filt in (EnsureKeyChoice('somekey', (0, 2)), lambda x: x['somekey'] in
(0, 2)):
assert_equal([
r['somekey'] for r in TestUtils().__call__(4, result_filter=filt)
], [0, 2])
# constraint returns full dict
assert_dict_equal(TestUtils().__call__(4, result_filter=filt)[-1], {
'action': 'off',
'path': 'some',
'status': 'ok',
'somekey': 2
})
# test more sophisticated filters that actually get to see the
# API call's kwargs
def greatfilter(res, **kwargs):
assert_equal(kwargs.get('dataset', 'bob'), 'awesome')
return True
TestUtils().__call__(4, dataset='awesome', result_filter=greatfilter)
def sadfilter(res, **kwargs):
assert_equal(kwargs.get('dataset', 'bob'), None)
return True
TestUtils().__call__(4, result_filter=sadfilter)
def assert_no_trash_left_behind(self):
assert_equal(
list(
find_files(r'\.datalad..*',
self.annex.path,
exclude="config",
dirs=True)), [])
def test_batched_command():
bc = BatchedCommand(cmd=[sys.executable, "-i", "-u", "-q", "-"])
response = bc("print('a')")
assert_equal(response, "a")
response = bc("print(2 + 1)")
assert_equal(response, "3")
stderr = bc.close(return_stderr=True)
assert_is_not_none(stderr)
def test_line_splitter_unterminated():
# Expect two lines split at "x", after the second process-call
line_splitter = LineSplitter("x")
lines = line_splitter.process("first line")
assert_equal(lines, [])
lines = line_splitter.process("xsecond linex")
assert_equal(lines, ["first line", "second line"])
assert_is_none(line_splitter.finish_processing())
def test_choice():
c = ct.EnsureChoice('choice1', 'choice2', None)
# this should always work
assert_equal(c('choice1'), 'choice1')
assert_equal(c(None), None)
# this should always fail
assert_raises(ValueError, lambda: c('fail'))
assert_raises(ValueError, lambda: c('None'))
def test_format_no_errors():
json_objects = get_json_objects(1, 3)
result = _format_json_error_messages(json_objects)
assert_equal(result, "")
result = _format_json_error_messages(json_objects + json_objects)
assert_equal(result, "")
def test_none():
c = ct.EnsureNone()
# this should always work
assert_equal(c(None), None)
# instance of NoneDeprecated is also None
assert_equal(c(ct.NoneDeprecated), None)
# this should always fail
assert_raises(ValueError, lambda: c('None'))
assert_raises(ValueError, lambda: c([]))
def _check_mocked_install(default_dspath, mock_install):
gen = search(".", return_type='generator')
assert_is_generator(gen)
# we no longer do any custom path tune up from the one returned by search
# so should match what search returns
assert_equal(list(gen), [report for report in _mocked_search_results])
mock_install.assert_called_once_with(default_dspath,
source='///',
result_renderer='disabled')
def test_ancient_annex():
class _runner(object):
def run(self, cmd, *args, **kwargs):
if '--raw' in cmd:
raise CommandError
return dict(stdout="git-annex version: 0.1", stderr="")
ev = ExternalVersions()
with patch('datalad.support.external_versions._runner', _runner()):
assert_equal(ev['cmd:annex'], '0.1')
def test_setup():
# insufficient arguments
check_setup_parser([], 2)
assert_in('too few arguments', check_setup_parser(['datalad'], 2)['err'])
assert_in('.', check_setup_parser(['datalad', '--version'], 0)['out'])
parser = check_setup_parser(['datalad', 'wtf'])['parser']
# check into the guts of argparse to check that really only a single
# subparser was constructed
assert_equal(list(parser._positionals._group_actions[0].choices.keys()),
['wtf'])
def test_both():
# this should always work
c = ct.AltConstraints(
ct.Constraints(
ct.EnsureFloat(),
ct.EnsureRange(min=7.0, max=44.0)),
ct.EnsureNone())
assert_equal(c(7.0), 7.0)
assert_equal(c(None), None)
# this should always fail
assert_raises(ValueError, lambda: c(77.0))
def assert_provides_and_raises(pc, exception, target=None):
"""Helper to get all results before exception is raised"""
results = []
with assert_raises(exception):
for r in pc:
results.append(r)
# results should be sorted since we do not guarantee order
results = sorted(results)
if target is not None:
assert_equal(results, target)
return results
def check_producing_consumer(jobs):
def producer():
yield from range(3)
def consumer(i):
yield i
if isinstance(i, int):
pc.add_to_producer_queue(str(i**2))
# we auto-detect generator function producer
pc = ProducerConsumer(producer, consumer, jobs=jobs)
assert_equal(list(pc), [0, 1, 2, "0", "1", "4"])
def test_exit_code():
# will relay actual exit code on CommandError
cmd = ['datalad', 'sshrun', 'datalad-test', 'exit 42']
with assert_raises(SystemExit) as cme:
# running nosetests without -s
if isinstance(sys.stdout, StringIO): # pragma: no cover
with swallow_outputs(): # need to give smth with .fileno ;)
main(cmd)
else:
# to test both scenarios
main(cmd)
assert_equal(cme.value.code, 42)
def test_batched_close_ok():
# Expect a long wait and no timeout if the process runs longer than timeout
# seconds and the config for "datalad.runtime.stalled-external" has its
# default value.
bc = BatchedCommand(cmd=[sys.executable, "-i", "-u", "-q", "-"], timeout=2)
# Send at least one instruction to start the subprocess
response = bc("import time; print('a')")
assert_equal(response, "a")
bc.stdin_queue.put("time.sleep(.5); exit(3)\n".encode())
bc.close(return_stderr=False)
assert_true(bc.wait_timed_out is False)
assert_equal(bc.return_code, 3)
def test_no_stdin_swallow(fname=None):
# will relay actual exit code on CommandError
cmd = ['datalad', 'sshrun', 'datalad-test', 'cat']
out = WitlessRunner().run(cmd, stdin=open(fname), protocol=StdOutCapture)
assert_equal(out['stdout'].rstrip(), '123magic')
# test with -n switch now, which we could place even at the end
out = WitlessRunner().run(cmd + ['-n'],
stdin=open(fname),
protocol=StdOutCapture)
assert_equal(out['stdout'], '')
def test_global_config():
# from within tests, global config should be read from faked $HOME (see
# setup_package) or from GIT_CONFIG_GLOBAL
if 'GIT_CONFIG_GLOBAL' in os.environ.keys():
glb_cfg_file = Path(os.environ.get('GIT_CONFIG_GLOBAL'))
else:
glb_cfg_file = Path(os.path.expanduser('~')) / '.gitconfig'
assert any(
glb_cfg_file.samefile(Path(p)) for p in dl_cfg._stores['git']['files'])
assert_equal(dl_cfg.get("user.name"), "DataLad Tester")
assert_equal(dl_cfg.get("user.email"), "*****@*****.**")