def test_install_dataset_from_just_source_via_path(path=None):
# for remote urls only, the source could be given to `path`
# to allows for simplistic cmdline calls
url = "https://github.com/datalad/testrepo--basic--r1.git"
with chpwd(path, mkdir=True):
ds = install(url)
ok_startswith(ds.path, path)
ok_(ds.is_installed())
ok_(GitRepo.is_valid_repo(ds.path))
assert_repo_status(ds.path, annex=None)
assert_in('INFO.txt', ds.repo.get_indexed_files())
def test__version__():
# in released stage, version in the last CHANGELOG entry
# should correspond to the one in datalad
CHANGELOG_filename = op.join(
op.dirname(__file__), op.pardir, op.pardir, 'CHANGELOG.md')
if not op.exists(CHANGELOG_filename):
raise SkipTest("no %s found" % CHANGELOG_filename)
regex = re.compile(r'^# '
r'(?P<version>[0-9]+\.[0-9.abcrc~]+)\s+'
r'\((?P<date>.*)\)'
)
with open(CHANGELOG_filename, 'rb') as f:
for line in f:
line = line.rstrip()
if not line.startswith(b'# '):
# The first section header we hit, must be our changelog entry
continue
reg = regex.match(ensure_unicode(line))
if not reg: # first one at that level is the one
raise AssertionError(
"Following line must have matched our regex: %r" % line)
regd = reg.groupdict()
changelog_version = regd['version']
lv_changelog_version = Version(changelog_version)
# we might have a suffix - sanitize
san__version__ = __version__.rstrip('.dirty')
lv__version__ = Version(san__version__)
if '???' in regd['date'] and 'will be better than ever' in regd['codename']:
# we only have our template
# we can only assert that its version should be higher than
# the one we have now
assert_greater(lv_changelog_version, lv__version__)
else:
# should be a "release" record
assert_not_in('???', regd['date'])
ok_startswith(__version__, changelog_version)
if lv__version__ != lv_changelog_version:
# It was not tagged yet and Changelog has no new records
# (they are composed by auto upon release)
assert_greater(lv__version__, lv_changelog_version)
assert_in('+', san__version__) # we have build suffix
else:
# all is good, tagged etc
assert_equal(lv_changelog_version, lv__version__)
assert_equal(changelog_version, san__version__)
return
raise AssertionError(
"No log line matching our regex found in %s" % CHANGELOG_filename
)
def test_annexinfo_init(path=None):
ds = Dataset(path).create()
foo = ds.pathobj / "foo"
foo_cont = b"foo content"
foo.write_bytes(foo_cont)
bar = ds.pathobj / "bar"
bar.write_text(u"bar content")
ds.save()
# Custom init limits report, with original dict getting updated.
cinfo_custom_init = ds.repo.get_content_annexinfo(
init={foo: {
"bytesize": 0,
"this-is-surely-only-here": "right?"
}})
assert_not_in(bar, cinfo_custom_init)
assert_in(foo, cinfo_custom_init)
assert_equal(cinfo_custom_init[foo]["bytesize"], len(foo_cont))
assert_equal(cinfo_custom_init[foo]["this-is-surely-only-here"], "right?")
# "git" injects get_content_info() values.
cinfo_init_git = ds.repo.get_content_annexinfo(init="git")
assert_in("gitshasum", cinfo_init_git[foo])
# init=None, on the other hand, does not.
cinfo_init_none = ds.repo.get_content_annexinfo(init=None)
assert_in(foo, cinfo_init_none)
assert_in(bar, cinfo_init_none)
assert_not_in("gitshasum", cinfo_init_none[foo])
def test_update_docstring_with_parameters_no_kwds():
from datalad.support.param import Parameter
def fn(pos0):
"fn doc"
assert_not_in("3", fn.__doc__)
# Call doesn't crash when there are no keyword arguments.
update_docstring_with_parameters(
fn,
dict(pos0=Parameter(doc="pos0 param doc"),
pos1=Parameter(doc="pos1 param doc")),
add_args={"pos1": 3})
assert_in("3", fn.__doc__)
def custom_result_renderer(res, **kwargs):
# This custom result renderer gets the command's keyword arguments and
# all of the common ones too, even those not explicitly specified.
assert_in("x", kwargs)
assert_in("on_failure", kwargs)
assert_in("result_filter", kwargs)
assert_in("result_renderer", kwargs)
def test_crazy_cfg(path=None):
cfg = ConfigManager(GitRepo(opj(path, 'ds'), create=True), source='branch')
assert_in('crazy.padry', cfg)
# make sure crazy config is not read when in local mode
cfg = ConfigManager(Dataset(opj(path, 'ds')), source='local')
assert_not_in('crazy.padry', cfg)
# it will make it in in 'any' mode though
cfg = ConfigManager(Dataset(opj(path, 'ds')), source='any')
assert_in('crazy.padry', cfg)
# typos in the source mode arg will not have silent side-effects
assert_raises(ValueError,
ConfigManager,
Dataset(opj(path, 'ds')),
source='locale')
def test_init_fail_under_known_subdir(path=None):
repo = GitRepo(path).init()
repo.call_git(['add', op.join('subds', 'file_name')])
# Should fail even if we do not commit but only add to index:
with assert_raises(PathKnownToRepositoryError) as cme:
GitRepo(op.join(path, 'subds')).init()
assert_in("file_name", str(cme.value)) # we provide a list of offenders
# and after we commit - the same story
repo.call_git(['commit', '-m', "added file"])
with assert_raises(PathKnownToRepositoryError) as cme:
GitRepo(op.join(path, 'subds')).init()
# But it would succeed if we disable the checks
GitRepo(op.join(path, 'subds')).init(sanity_checks=False)
def test_surprise_subds(path=None):
# https://github.com/datalad/datalad/issues/3139
ds = create(path, force=True)
# a lonely repo without any commit
somerepo = AnnexRepo(path=op.join(path, 'd1', 'subrepo'), create=True)
# a proper subdataset
subds = create(op.join(path, 'd2', 'subds'), force=True)
# If subrepo is an adjusted branch, it would have a commit, making most of
# this test irrelevant because it is about the unborn branch edge case.
adjusted = somerepo.is_managed_branch()
# This edge case goes away with Git v2.22.0.
fixed_git = somerepo.git_version >= '2.22.0'
# save non-recursive
res = ds.save(recursive=False, on_failure='ignore')
if not adjusted and fixed_git:
# We get an appropriate error about no commit being checked out.
assert_in_results(res, action='add_submodule', status='error')
# the content of both subds and subrepo are not added to their
# respective parent as no --recursive was given
assert_repo_status(subds.path, untracked=['subfile'])
assert_repo_status(somerepo.path, untracked=['subfile'])
if adjusted or fixed_git:
if adjusted:
# adjusted branch: #datalad/3178 (that would have a commit)
modified = [subds.repo.pathobj, somerepo.pathobj]
untracked = []
else:
# Newer Git versions refuse to add a sub-repository with no commits
# checked out.
modified = [subds.repo.pathobj]
untracked = ['d1']
assert_repo_status(ds.path, modified=modified, untracked=untracked)
assert_not_in(ds.repo.pathobj / 'd1' / 'subrepo' / 'subfile',
ds.repo.get_content_info())
else:
# however, while the subdataset is added (and reported as modified
# because it content is still untracked) the subrepo
# cannot be added (it has no commit)
# worse: its untracked file add been added to the superdataset
assert_repo_status(ds.path, modified=['d2/subds'])
assert_in(ds.repo.pathobj / 'd1' / 'subrepo' / 'subfile',
ds.repo.get_content_info())
# with proper subdatasets, all evil is gone
assert_not_in(ds.repo.pathobj / 'd2' / 'subds' / 'subfile',
ds.repo.get_content_info())
def test_cmdline_example_to_rst():
# don't puke on nothing
out = fmt.cmdline_example_to_rst(SIO(''))
out.seek(0)
ok_startswith(out.read(), '.. AUTO-GENERATED')
out = fmt.cmdline_example_to_rst(SIO(''), ref='dummy')
out.seek(0)
assert_in('.. dummy:', out.read())
# full scale test
out = fmt.cmdline_example_to_rst(SIO(demo_example), ref='mydemo')
out.seek(0)
out_text = out.read()
assert_in('.. code-block:: sh', out_text)
assert_not_in('shame', out_text) # no SKIP'ed
assert_not_in('#', out_text) # no comments
def test_check():
ev = ExternalVersions()
# should be all good
ev.check('datalad')
ev.check('datalad', min_version=__version__)
with assert_raises(MissingExternalDependency):
ev.check('dataladkukaracha')
with assert_raises(MissingExternalDependency) as cme:
ev.check('dataladkukaracha', min_version="buga", msg="duga")
assert_in("duga", str(cme.value))
with assert_raises(OutdatedExternalDependency):
ev.check('datalad', min_version="10000000") # we will never get there!
def test_push_git_annex_branch_many_paths_same_data(path=None):
path = Path(path)
ds = Dataset(path / "ds").create(force=True)
ds.save()
mk_push_target(ds, "target", str(path / "target"), annex=True, bare=False)
nbytes = sum(
ds.repo.get_content_annexinfo(paths=[f])[f]["bytesize"] for f in [
ds.repo.pathobj / "f0", ds.repo.pathobj / "f3", ds.repo.pathobj /
"f4"
])
with swallow_logs(new_level=logging.DEBUG) as cml:
res = ds.push(to="target")
assert_in("{} bytes of annex data".format(nbytes), cml.out)
# 3 files point to content already covered by another file.
assert_result_count(res, 3, action="copy", type="file", status="notneeded")
def test_log_progress_noninteractive_filter():
name = "dl-test"
lgr = LoggerHelper(name).get_initialized_logger()
pbar_id = "lp_test"
with swallow_logs(new_level=logging.INFO, name=name) as cml:
log_progress(lgr.info, pbar_id, "Start", label="testing", total=3)
log_progress(lgr.info, pbar_id, "THERE0", update=1)
log_progress(lgr.info, pbar_id, "NOT", update=1,
noninteractive_level=logging.DEBUG)
log_progress(lgr.info, pbar_id, "THERE1", update=1,
noninteractive_level=logging.INFO)
log_progress(lgr.info, pbar_id, "Done")
for present in ["Start", "THERE0", "THERE1", "Done"]:
assert_in(present, cml.out)
assert_not_in("NOT", cml.out)
def test_report_absent_keys(path=None):
ds = Dataset(path).create()
# create an annexed file
testfile = ds.pathobj / 'dummy'
testfile.write_text(u'nothing')
ds.save()
# present in a full report and in a partial report
# based on worktree of HEAD ref
for ai in (ds.repo.get_content_annexinfo(eval_availability=True),
ds.repo.get_content_annexinfo(paths=['dummy'],
eval_availability=True),
ds.repo.get_content_annexinfo(ref='HEAD',
eval_availability=True),
ds.repo.get_content_annexinfo(ref='HEAD',
paths=['dummy'],
eval_availability=True)):
assert_in(testfile, ai)
assert_equal(ai[testfile]['has_content'], True)
# drop the key, not available anywhere else
ds.drop('dummy', reckless='kill')
# does not change a thing, except the key is gone
for ai in (ds.repo.get_content_annexinfo(eval_availability=True),
ds.repo.get_content_annexinfo(paths=['dummy'],
eval_availability=True),
ds.repo.get_content_annexinfo(ref='HEAD',
eval_availability=True),
ds.repo.get_content_annexinfo(ref='HEAD',
paths=['dummy'],
eval_availability=True)):
assert_in(testfile, ai)
assert_equal(ai[testfile]['has_content'], False)
# make sure files with URL keys are correctly reported:
from datalad.conftest import test_http_server
remote_file_name = 'imaremotefile.dat'
local_file_name = 'mehasurlkey'
(Path(test_http_server.path) / remote_file_name).write_text("weee")
remote_file_url = f'{test_http_server.url}/{remote_file_name}'
# we need to get a file with a URL key and check its local availability
ds.repo.call_annex(
['addurl', '--relaxed', remote_file_url, '--file', local_file_name])
ds.save("URL keys!")
# should not be there
res = ds.repo.get_file_annexinfo(local_file_name, eval_availability=True)
assert_equal(res['has_content'], False)
ds.get(local_file_name)
# should be there
res = ds.repo.get_file_annexinfo(local_file_name, eval_availability=True)
assert_equal(res['has_content'], True)
def test_push_git_annex_branch_when_no_data(path=None):
path = Path(path)
ds = Dataset(path / "a").create()
target = mk_push_target(ds,
"target",
str(path / "target"),
annex=False,
bare=True)
(ds.pathobj / "f0").write_text("0")
ds.save()
ds.push(to="target", data="nothing")
assert_in(
"git-annex", {
d["refname:strip=2"]
for d in target.for_each_ref_(fields="refname:strip=2")
})
def test_saving_prior(topdir=None):
# the problem is that we might be saving what is actually needed to be
# "created"
# we would like to place this structure into a hierarchy of two datasets
# so we create first top one
ds1 = create(topdir, force=True)
# and everything is ok, stuff is not added BUT ds1 will be considered dirty
assert_repo_status(ds1.path, untracked=['ds2'])
# And then we would like to initiate a sub1 subdataset
ds2 = create('ds2', dataset=ds1, force=True)
# But what will happen is file1.txt under ds2 would get committed first into
# ds1, and then the whole procedure actually crashes since because ds2/file1.txt
# is committed -- ds2 is already known to git and it just pukes with a bit
# confusing 'ds2' already exists in the index
assert_in('ds2', ds1.subdatasets(result_xfm='relpaths'))
def test_exif(path=None):
ds = Dataset(path).create()
ds.config.add('datalad.metadata.nativetype', 'exif', scope='branch')
copy(opj(dirname(dirname(dirname(__file__))), 'tests', 'data', 'exif.jpg'),
path)
ds.save()
assert_repo_status(ds.path)
res = ds.aggregate_metadata()
assert_status('ok', res)
res = ds.metadata('exif.jpg')
assert_result_count(res, 1)
# from this extractor
meta = res[0]['metadata']['exif']
for k, v in target.items():
eq_(meta[k], v)
assert_in('@context', meta)
def test_basic_setup():
# the import alone will verify that all default values match their
# constraints
from datalad import api
# random pick of something that should be there
assert_true(hasattr(api, 'install'))
assert_true(hasattr(api, 'create'))
# make sure all helper utilities do not pollute the namespace
# and we end up only with __...__ attributes
assert_false(
list(
filter(lambda s: s.startswith('_') and not re.match('__.*__', s),
dir(api))))
assert_in('Parameters', api.Dataset.install.__doc__)
assert_in('Parameters', api.Dataset.create.__doc__)
def test_rerun_outofdate_tree(path=None):
ds = Dataset(path).create()
input_file = op.join(path, "foo")
output_file = op.join(path, "out")
with open(input_file, "w") as f:
f.write("abc\ndef")
ds.save("foo", to_git=True)
# Create initial run.
ds.run(f'{grep_command} def foo > out')
assert_in('def', open(output_file).read())
# Change tree so that it is no longer compatible.
ds.remove("foo")
# Now rerunning should fail because foo no longer exists.
assert_in_results(
ds.rerun(revision=DEFAULT_BRANCH + "~",
result_renderer=None, on_failure="ignore"),
status="error", action="run")
def test_addurls_repindex(self=None, path=None):
ds = Dataset(path).create(force=True)
with assert_raises(IncompleteResultsError) as raised:
ds.addurls(self.json_file,
"{url}",
"{subdir}",
result_renderer='disabled')
assert_in("collided", str(raised.value))
ds.addurls(self.json_file,
"{url}",
"{subdir}-{_repindex}",
result_renderer='disabled')
for fname in ["foo-0", "bar-0", "foo-1"]:
ok_exists(op.join(ds.path, fname))
def decorated_test2(ds):
# we get a Dataset instance
assert_is_instance(ds, Dataset)
# it's a clone in a temp. location, not within the cache
assert_not_in(cache_dir, ds.pathobj.parents)
assert_result_count(ds.siblings(),
1,
type="sibling",
name=DEFAULT_REMOTE,
url=str(cache_dir / name_in_cache))
here = ds.config.get("annex.uuid")
origin = ds.config.get(f"remote.{DEFAULT_REMOTE}.annex-uuid")
where = ds.repo.whereis(str(annexed_file))
assert_in(here, where)
assert_in(origin, where)
return ds.pathobj, ds.repo.pathobj
def test_windows_incompatible_names(path=None):
ds = Dataset(path).create()
create_tree(
path, {
'imgood': 'Look what a nice name I have',
'illegal:character.txt': 'strange choice of name',
'spaceending ': 'who does these things?',
'lookmumadot.': 'why would you do this?',
'COM1.txt': 'I am a serial port',
'dirs with spaces': {
'seriously?': 'you are stupid',
'why somuch?wrongstuff.': "I gave up"
},
})
ds.repo.config.set('datalad.save.windows-compat-warning', 'error')
ds.save('.datalad/config')
res = ds.save(on_failure='ignore')
# check that none of the 6 problematic files was saved, but the good one was
assert_result_count(res, 6, status='impossible', action='save')
assert_result_count(res, 1, status='ok', action='save')
# check that the warning is emitted
ds.repo.config.set('datalad.save.windows-compat-warning', 'warning')
ds.save('.datalad/config')
with swallow_logs(new_level=logging.WARN) as cml:
ds.save()
cml.assert_logged(
"Some elements of your dataset are not compatible with Windows "
"systems. Disable this check by changing "
"datalad.save.windows-compat-warning or consider renaming the "
"following elements:")
assert_in("Elements using a reserved filename:", cml.out)
assert_in("Elements with illegal characters:", cml.out)
assert_in("Elements ending with a dot:", cml.out)
assert_in("Elements ending with a space:", cml.out)
# check that a setting of 'none' really does nothing
ds.repo.config.set('datalad.save.windows-compat-warning', 'none')
ds.save('.datalad/config')
create_tree(
path, {
'more illegal:characters?.py':
'My arch nemesis uses Windows and I will'
'destroy them! Muahahaha'
})
with swallow_logs(new_level=logging.WARN) as cml:
res = ds.save()
# we shouldn't see warnings
assert_not_in(
"Some elements of your dataset are not compatible with Windows "
"systems. Disable this check by changing "
"datalad.save.windows-compat-warning or consider renaming the "
"following elements:", cml.out)
# make sure the file is saved successfully
assert_result_count(res, 1, status='ok', action='save')
def test_direct_cfg(path1=None, path2=None):
# and if repo already exists and we have env var - we fail too
# Adding backend so we get some commit into the repo
ar = AnnexRepo(path1, create=True, backend='MD5E')
del ar; AnnexRepo._unique_instances.clear() # fight flyweight
for path in (path1, path2):
with patch.dict('os.environ', {'DATALAD_REPO_DIRECT': 'True'}):
# try to create annex repo in direct mode as see how it fails
with assert_raises(DirectModeNoLongerSupportedError) as cme:
AnnexRepo(path, create=True)
assert_in("no longer supported by DataLad", str(cme.value)) # we have generic part
assert_in("datalad.repo.direct configuration", str(cme.value)) # situation specific part
# assert not op.exists(path2) # that we didn't create it - we do!
# fixing for that would be too cumbersome since we first call GitRepo.__init__
# with create
ar = AnnexRepo(path1)
# check if we somehow didn't reset the flag
assert not ar.is_direct_mode()
def test_install_datasets_root(tdir=None):
with chpwd(tdir):
ds = install("///")
ok_(ds.is_installed())
eq_(ds.path, opj(tdir, get_datasets_topdir()))
# do it a second time:
result = install("///", result_xfm=None, return_type='list')
assert_status('notneeded', result)
eq_(YieldDatasets()(result[0]), ds)
# and a third time into an existing something, that is not a dataset:
with open(opj(tdir, 'sub', 'a_file.txt'), 'w') as f:
f.write("something")
with assert_raises(IncompleteResultsError) as cme:
install("sub", source='///')
assert_in("already exists and not empty", str(cme.value))
def test_assembling_decoder_mix_in_warning():
encoding = "utf-8"
data_bytes = "🐷🐶.".encode(encoding)
adm = AssemblingDecoderMixIn()
with patch("datalad.runner.utils.logger") as logger_mock:
result = adm.decode(1, data_bytes[0:1], encoding)
assert_equal(result, '')
del adm
assert_in(
call.warning("unprocessed data in AssemblingDecoderMixIn"),
logger_mock.mock_calls)
assert_in(
call.debug(
"unprocessed data in AssemblingDecoderMixIn:\n"
"fd: 1, data: b'\\xf0'\n"),
logger_mock.mock_calls)
def test_check_dates(path=None):
skip_if_no_module("dateutil")
ref_ts = 1218182889 # Fri, 08 Aug 2008 04:08:09 -0400
refdate = "@{}".format(ref_ts)
repo = os.path.join(path, "repo")
with set_date(ref_ts + 5000):
ar = AnnexRepo(repo)
ar.add(".")
ar.commit()
# The standard renderer outputs json.
with swallow_outputs() as cmo:
# Set level to WARNING to avoid the progress bar when
# DATALAD_TESTS_UI_BACKEND=console.
with swallow_logs(new_level=logging.WARNING):
check_dates([repo], reference_date=refdate, return_type="list")
assert_in("report", json.loads(cmo.out).keys())
# We find the newer objects.
newer = call([path], reference_date=refdate)
eq_(len(newer), 1)
ok_(newer[0]["report"]["objects"])
# There are no older objects to find.
older = call([repo], reference_date=refdate, older=True)
assert_false(older[0]["report"]["objects"])
# We can pass the date in RFC 2822 format.
assert_dict_equal(
newer[0],
call([path], reference_date="08 Aug 2008 04:08:09 -0400")[0])
# paths=None defaults to the current directory.
with chpwd(path):
assert_dict_equal(
newer[0]["report"],
call(paths=None, reference_date=refdate)[0]["report"])
# Only commit type is present when annex='none'.
newer_noannex = call([path], reference_date=refdate, annex="none")
for entry in newer_noannex[0]["report"]["objects"].values():
ok_(entry["type"] == "commit")
def test_reobtain_data(originpath=None, destpath=None):
origin = Dataset(originpath).create()
ds = install(source=originpath,
path=destpath,
result_xfm='datasets',
return_type='item-or-list')
# no harm
assert_result_count(ds.update(merge=True, reobtain_data=True),
1,
action="update",
status="ok")
# content
create_tree(origin.path, {'load.dat': 'heavy'})
origin.save(opj(origin.path, 'load.dat'))
# update does not bring data automatically
assert_result_count(ds.update(merge=True, reobtain_data=True),
1,
action="update",
status="ok")
assert_in('load.dat', ds.repo.get_annexed_files())
assert_false(ds.repo.file_has_content('load.dat'))
# now get data
ds.get('load.dat')
ok_file_has_content(opj(ds.path, 'load.dat'), 'heavy')
# new content at origin
create_tree(origin.path, {'novel': 'but boring'})
origin.save()
# update must not bring in data for new file
result = ds.update(merge=True, reobtain_data=True)
assert_in_results(result, action='get', status='notneeded')
ok_file_has_content(opj(ds.path, 'load.dat'), 'heavy')
assert_in('novel', ds.repo.get_annexed_files())
assert_false(ds.repo.file_has_content('novel'))
# modify content at origin
os.remove(opj(origin.path, 'load.dat'))
create_tree(origin.path, {'load.dat': 'light'})
origin.save()
# update must update file with existing data, but leave empty one alone
res = ds.update(merge=True, reobtain_data=True)
assert_result_count(res, 1, status='ok', type='dataset', action='update')
assert_result_count(res, 1, status='ok', type='file', action='get')
ok_file_has_content(opj(ds.path, 'load.dat'), 'light')
assert_false(ds.repo.file_has_content('novel'))
def test_bare(src=None, path=None):
# create a proper datalad dataset with all bells and whistles
ds = Dataset(src).create()
dlconfig_sha = ds.repo.call_git(['rev-parse', 'HEAD:.datalad/config'])
# can we handle a bare repo version of it?
gr = AnnexRepo.clone(src,
path,
clone_options=['--bare', '-b', DEFAULT_BRANCH])
# we had to specifically checkout the standard branch, because on crippled
# FS, HEAD will point to an adjusted branch by default, and the test logic
# below does not account for this case.
# this should just make sure the bare repo has the expected setup,
# but it should still be bare. Let's check that to be sure
assert_true(gr.bare)
# do we read the correct local config?
assert_in(gr.pathobj / 'config', gr.config._stores['git']['files'])
# do we pick up the default branch config too?
assert_in('blob:HEAD:.datalad/config',
gr.config._stores['branch']['files'])
# and track its reload stamp via its file shasum
assert_equal(
dlconfig_sha,
gr.config._stores['branch']['stats']['blob:HEAD:.datalad/config'])
# check that we can pick up the dsid from the commit branch config
assert_equal(ds.id, gr.config.get('datalad.dataset.id'))
# and it is coming from the correct source
assert_equal(ds.id,
gr.config.get_from_source('branch', 'datalad.dataset.id'))
assert_equal(None, gr.config.get_from_source('local',
'datalad.dataset.id'))
# any sensible (and also our CI) test environment(s) should have this
assert_in('user.name', gr.config)
# not set something that wasn't there
obscure_key = 'sec.reallyobscurename!@@.key'
assert_not_in(obscure_key, gr.config)
# to the local config, which is easily accessible
gr.config.set(obscure_key, 'myvalue', scope='local')
assert_equal(gr.config.get(obscure_key), 'myvalue')
# now make sure the config is where we think it is
assert_in(obscure_key.split('.')[1], (gr.pathobj / 'config').read_text())
# update committed config and check update
old_id = ds.id
ds.config.set('datalad.dataset.id', 'surprise!', scope='branch')
ds.save()
# fetch into default branch (like `update`, but for bare-repos)
gr.call_git(
['fetch', f'{DEFAULT_REMOTE}', f'{DEFAULT_BRANCH}:{DEFAULT_BRANCH}'])
# without a reload, no state change, like with non-bare repos
assert_equal(old_id,
gr.config.get_from_source('branch', 'datalad.dataset.id'))
# a non-forced reload() must be enough, because state change
# detection kicks in
gr.config.reload()
assert_equal('surprise!', gr.config.get('datalad.dataset.id'))
def test_from_env():
cfg = ConfigManager()
assert_not_in('datalad.crazy.cfg', cfg)
with patch.dict('os.environ', {'DATALAD_CRAZY_CFG': 'impossibletoguess'}):
cfg.reload()
assert_in('datalad.crazy.cfg', cfg)
assert_equal(cfg['datalad.crazy.cfg'], 'impossibletoguess')
# not in dataset-only mode
cfg = ConfigManager(Dataset('nowhere'), source='branch')
assert_not_in('datalad.crazy.cfg', cfg)
# check env trumps override
cfg = ConfigManager()
assert_not_in('datalad.crazy.override', cfg)
cfg.set('datalad.crazy.override', 'fromoverride', scope='override')
cfg.reload()
assert_equal(cfg['datalad.crazy.override'], 'fromoverride')
with patch.dict('os.environ', {'DATALAD_CRAZY_OVERRIDE': 'fromenv'}):
cfg.reload()
assert_equal(cfg['datalad.crazy.override'], 'fromenv')
def test_addurls_url_special_key_fail(self=None, path=None):
ds = Dataset(path).create(force=True)
res1 = ds.addurls(self.json_file,
"{url}",
"{_url4}/{_url_filename}",
on_failure="ignore",
result_renderer='disabled')
assert_in("Special key", res1[0]["message"])
data = self.data.copy()[:1]
data[0]["url"] = urlparse(data[0]["url"]).netloc
with patch("sys.stdin", new=StringIO(json.dumps(data))):
res2 = ds.addurls("-",
"{url}",
"{_url_basename}",
on_failure="ignore",
result_renderer='disabled')
assert_in("Special key", res2[0]["message"])
def test_push_custom_summary(path=None):
path = Path(path)
ds = Dataset(path / "ds").create()
sib = mk_push_target(ds, "sib", str(path / "sib"), bare=False, annex=False)
(sib.pathobj / "f1").write_text("f1")
sib.save()
(ds.pathobj / "f2").write_text("f2")
ds.save()
# These options are true by default and our tests usually run with a
# temporary home, but set them to be sure.
ds.config.set("advice.pushUpdateRejected", "true", scope="local")
ds.config.set("advice.pushFetchFirst", "true", scope="local")
with swallow_outputs() as cmo:
ds.push(to="sib", result_renderer="default", on_failure="ignore")
assert_in("Hints:", cmo.out)
assert_in("action summary:", cmo.out)
