def test_listdict2dictlist():
f = _listdict2dictlist
l1 = [1, 3, [1, 'a']]
assert f(l1) is l1, "we return it as is if no emb dict"
eq_(f([{1: 2}]), {1: 2}) # inside out no need for a list
# inside out, join into the list, skip entry with a list, or space
eq_(f([{1: [2, 3], 'a': 1}, {'a': 2, 'c': ''}]), {'a': [1, 2]})
def test_install_into_dataset(source=None, top_path=None):
src_ds = Dataset(source).create(result_renderer='disabled', force=True)
src_ds.save(['INFO.txt', 'test.dat'], to_git=True)
src_ds.save('test-annex.dat', to_git=False)
ds = create(top_path)
assert_repo_status(ds.path)
subds = ds.install("sub", source=source)
ok_(isdir(opj(subds.path, '.git')))
ok_(subds.is_installed())
assert_in('sub', ds.subdatasets(result_xfm='relpaths'))
# sub is clean:
assert_repo_status(subds.path, annex=None)
# top is too:
assert_repo_status(ds.path, annex=None)
ds.save(message='addsub')
# now it is:
assert_repo_status(ds.path, annex=None)
# but we could also save while installing and there should be no side-effect
# of saving any other changes if we state to not auto-save changes
# Create a dummy change
create_tree(ds.path, {'dummy.txt': 'buga'})
assert_repo_status(ds.path, untracked=['dummy.txt'])
subds_ = ds.install("sub2", source=source)
eq_(subds_.path, opj(ds.path, "sub2")) # for paranoid yoh ;)
assert_repo_status(ds.path, untracked=['dummy.txt'])
# and we should achieve the same behavior if we create a dataset
# and then decide to add it
create(_path_(top_path, 'sub3'))
assert_repo_status(ds.path, untracked=['dummy.txt', 'sub3/'])
ds.save('sub3')
assert_repo_status(ds.path, untracked=['dummy.txt'])
def test_unlock_raises(path=None, path2=None, path3=None):
# make sure, we are not within a dataset:
_cwd = getpwd()
chpwd(path)
# no dataset and no path:
assert_raises(InsufficientArgumentsError,
unlock, dataset=None, path=None)
# no dataset and path not within a dataset:
assert_raises(NoDatasetFound,
unlock, dataset=None, path=path2)
create(path=path, annex=False)
ds = Dataset(path)
# no complaints
ds.unlock()
# make it annex, but call unlock with invalid path:
(ds.pathobj / ".noannex").unlink()
AnnexRepo(path, create=True)
# One that doesn't exist.
res = ds.unlock(path="notexistent.txt", result_xfm=None,
on_failure='ignore', return_type='item-or-list')
eq_(res['message'], "path does not exist")
# And one that isn't associated with a dataset.
assert_in_results(
ds.unlock(path=path2, on_failure="ignore"),
status="error",
message=("path not underneath the reference dataset %s", ds.path))
chpwd(_cwd)
def test_assert_cwd_unchanged_not_masking_exceptions():
# Test that we are not masking out other "more important" exceptions
orig_cwd = os.getcwd()
@assert_cwd_unchanged
def do_chdir_value_error():
os.chdir(os.pardir)
raise ValueError("error exception")
with swallow_logs(new_level=logging.WARN) as cml:
with assert_raises(ValueError) as cm:
do_chdir_value_error()
# retrospect exception
eq_(orig_cwd, os.getcwd(),
"assert_cwd_unchanged didn't return us back to %s" % orig_cwd)
assert_in("Mitigating and changing back", cml.out)
# and again but allowing to chdir
@assert_cwd_unchanged(ok_to_chdir=True)
def do_chdir_value_error():
os.chdir(os.pardir)
raise ValueError("error exception")
with swallow_logs(new_level=logging.WARN) as cml:
assert_raises(ValueError, do_chdir_value_error)
eq_(orig_cwd, os.getcwd(),
"assert_cwd_unchanged didn't return us back to %s" % orig_cwd)
assert_not_in("Mitigating and changing back", cml.out)
def test_testsui():
# just one for now to test conflicting arguments
with assert_raises(ValueError):
@with_testsui(responses='some', interactive=False)
def some_func(): # pragma: no cover
pass
from datalad.ui import ui
@with_testsui(responses=['yes', "maybe so"])
def func2(x):
assert x == 1
eq_(ui.yesno("title"), True)
eq_(ui.question("title2"), "maybe so")
assert_raises(AssertionError, ui.question, "asking more than we know")
return x * 2
eq_(func2(1), 2)
@with_testsui(interactive=False)
def func3(x):
assert_false(ui.is_interactive)
return x * 3
eq_(func3(2), 6)
def test_addurls_row_missing_key_fields(self=None, path=None):
ds = Dataset(path).create(force=True)
if OLD_EXAMINEKEY and ds.repo.is_managed_branch():
raise SkipTest("Adjusted branch functionality requires "
"more recent `git annex examinekey`")
data = deepcopy(self.data)
for row in data:
if row["name"] == "b":
del row["md5sum"]
break
with patch("sys.stdin", new=StringIO(json.dumps(data))):
ds.addurls("-",
"{url}",
"{name}",
exclude_autometa="*",
key="MD5-s{size}--{md5sum}",
result_renderer='disabled')
repo = ds.repo
repo_path = ds.repo.pathobj
paths = [repo_path / x for x in "ac"]
annexinfo = repo.get_content_annexinfo(eval_availability=True)
for path in paths:
pstat = annexinfo[path]
eq_(pstat["backend"], "MD5")
assert_false(pstat["has_content"])
def test_addurls_version(self=None, path=None):
ds = Dataset(path).create(force=True)
def version_fn(url):
if url.endswith("b.dat"):
raise ValueError("Scheme error")
return url + ".v1"
with patch("datalad.local.addurls.get_versioned_url", version_fn):
with swallow_logs(new_level=logging.WARNING) as cml:
ds.addurls(self.json_file,
"{url}",
"{name}",
version_urls=True,
result_renderer='disabled')
assert_in("b.dat", str(cml.out))
names = ["a", "c"]
for fname in names:
ok_exists(os.path.join(path, fname))
whereis = ds.repo.whereis(names, output="full")
for fname, info in whereis.items():
eq_(info[WEB_SPECIAL_REMOTE_UUID]['urls'],
["{}udir/{}.dat.v1".format(self.url, fname)])
def test_blocking_thread_exit():
read_queue = queue.Queue()
(read_descriptor, write_descriptor) = os.pipe()
read_file = os.fdopen(read_descriptor, "rb")
read_thread = ReadThread(
identifier="test thread",
user_info=read_descriptor,
source=read_file,
destination_queue=read_queue,
signal_queues=[]
)
read_thread.start()
os.write(write_descriptor, b"some data")
assert_true(read_thread.is_alive())
identifier, state, data = read_queue.get()
eq_(data, b"some data")
read_thread.request_exit()
# Check the blocking part
sleep(.3)
assert_true(read_thread.is_alive())
# Check actual exit, we will not get
# "more data" when exit was requested,
# because the thread will not attempt
# a write
os.write(write_descriptor, b"more data")
read_thread.join()
print(read_queue.queue)
assert_true(read_queue.empty())
def test_blocking_read_closing():
# Expect that a reader thread exits when os.read throws an error.
class FakeFile:
def fileno(self):
return -1
def close(self):
pass
def fake_read(*args):
raise ValueError("test exception")
read_queue = queue.Queue()
with patch("datalad.runner.runnerthreads.os.read") as read:
read.side_effect = fake_read
read_thread = ReadThread(
identifier="test thread",
user_info=None,
source=FakeFile(),
destination_queue=None,
signal_queues=[read_queue])
read_thread.start()
read_thread.join()
identifier, state, data = read_queue.get()
eq_(data, None)
def test_runnin_on_empty(path=None):
# empty repo
repo = AnnexRepo(path, create=True)
# just wrap with a dataset
ds = Dataset(path)
# and run status ... should be good and do nothing
eq_([], ds.status(result_renderer='disabled'))
def test_merge_follow_parentds_subdataset_adjusted_warning(path=None):
path = Path(path)
ds_src = Dataset(path / "source").create()
ds_src_subds = ds_src.create("subds")
ds_clone = install(source=ds_src.path,
path=path / "clone",
recursive=True,
result_xfm="datasets")
ds_clone_subds = Dataset(ds_clone.pathobj / "subds")
maybe_adjust_repo(ds_clone_subds.repo)
# Note: Were we to save ds_clone here, we would get a merge conflict in the
# top repo for the submodule (even if using 'git annex sync' rather than
# 'git merge').
ds_src_subds.repo.call_git(["checkout", DEFAULT_BRANCH + "^0"])
(ds_src_subds.pathobj / "foo").write_text("foo content")
ds_src.save(recursive=True)
assert_repo_status(ds_src.path)
assert_in_results(ds_clone.update(merge=True,
recursive=True,
follow="parentds",
on_failure="ignore"),
status="impossible",
path=ds_clone_subds.path,
action="update")
eq_(ds_clone.repo.get_hexsha(), ds_src.repo.get_hexsha())
def test_multiple_entry_points():
""" check that generic indexer is called if multiple indexers exist for the same name """
class MockedEntryPoint(EntryPoint):
def __init__(self):
self.name = 'MockedEntryPoint'
self.dist = 'Mock Distribution 1.1'
def load(self, *args):
return 'Loaded MockedEntryPoint'
def _mocked_iter_entry_points(group, metadata):
yield MockedEntryPoint()
yield MockedEntryPoint()
with patch('pkg_resources.iter_entry_points',
MagicMock(side_effect=_mocked_iter_entry_points)):
index = _meta2autofield_dict({
'datalad_unique_content_properties': {
'extr1': {
"prop1": "v1"
}
},
'extr1': {
'prop1': 'value'
}
})
eq_(index, {'extr1.prop1': 'value'})
def test_faulty_external_indexer():
""" check that generic indexer is called on external indexer faults """
class MockedEntryPoint(EntryPoint):
def __init__(self):
self.name = 'MockedEntryPoint'
self.dist = 'Mock Distribution 1.1'
def load(self, *args):
raise Exception('Mocked indexer error')
def _mocked_iter_entry_points(group, metadata):
yield MockedEntryPoint()
with patch('pkg_resources.iter_entry_points',
MagicMock(side_effect=_mocked_iter_entry_points)):
index = _meta2autofield_dict({
'datalad_unique_content_properties': {
'extr1': {
"prop1": "v1"
}
},
'extr1': {
'prop1': 'value'
}
})
eq_(index, {'extr1.prop1': 'value'})
def test_external_indexer():
""" check that external indexer are called """
class MockedIndexer(MetadataIndexer):
def __init__(self, metadata_format_name: str):
super().__init__(metadata_format_name)
def create_index(self, metadata):
yield from {"jubel": 1, "trubel": 2}.items()
class MockedEntryPoint(EntryPoint):
def __init__(self):
pass
def load(self, *args):
return MockedIndexer
def _mocked_iter_entry_points(group, metadata):
yield MockedEntryPoint()
with patch('pkg_resources.iter_entry_points',
MagicMock(side_effect=_mocked_iter_entry_points)):
index = _meta2autofield_dict({
'datalad_unique_content_properties': {
'extr1': {
"prop1": "v1"
}
},
'extr1': {
'prop1': 'value'
}
})
eq_(index, {'extr1.jubel': '1', 'extr1.trubel': '2'})
def test_rerun_fastforwardable_mutator(path=None):
ds = Dataset(path).create()
# keep direct repo accessor to speed things up
ds_repo = ds.repo
ds_repo.checkout(DEFAULT_BRANCH, options=["-b", "side"])
ds.run("echo foo >>foo")
ds_repo.checkout(DEFAULT_BRANCH)
ds_repo.merge("side", options=["-m", "Merge side", "--no-ff"])
# o c_n
# |\
# | o b_r
# |/
# o a_n
ds.rerun(since="", onto=DEFAULT_BRANCH + "^2")
# o b_R
# o b_r
# o a_n
neq_(ds_repo.get_hexsha(DEFAULT_BRANCH + "^2"), ds_repo.get_hexsha())
ds_repo.checkout(DEFAULT_BRANCH)
hexsha_before = ds_repo.get_hexsha()
ds.rerun(since="")
# o b_R
# o c_n
# |\
# | o b_r
# |/
# o a_n
eq_(ds_repo.get_active_branch(), DEFAULT_BRANCH)
assert_false(ds_repo.commit_exists(DEFAULT_BRANCH + "^2"))
eq_(hexsha_before, ds_repo.get_hexsha(DEFAULT_BRANCH + "^"))
def test_blocking_write_exception_catching():
# Expect that a blocking writer catches exceptions and exits gracefully.
write_queue = queue.Queue()
signal_queue = queue.Queue()
(read_descriptor, write_descriptor) = os.pipe()
write_file = os.fdopen(write_descriptor, "rb")
write_thread = WriteThread(
identifier="test thread",
user_info=write_descriptor,
source_queue=write_queue,
destination=write_file,
signal_queues=[signal_queue]
)
write_thread.start()
write_queue.put(b"some data")
data = os.read(read_descriptor, 1024)
eq_(data, b"some data")
os.close(read_descriptor)
os.close(write_descriptor)
write_queue.put(b"more data")
write_thread.join()
eq_(signal_queue.get(), (write_descriptor, IOState.ok, None))
def check_addurls_from_key(self, key_arg, expected_backend, fake_dates,
path):
ds = Dataset(path).create(force=True, fake_dates=fake_dates)
if OLD_EXAMINEKEY and ds.repo.is_managed_branch():
raise SkipTest("Adjusted branch functionality requires "
"more recent `git annex examinekey`")
ds.addurls(self.json_file,
"{url}",
"{name}",
exclude_autometa="*",
key=key_arg,
result_renderer='disabled')
repo = ds.repo
repo_path = ds.repo.pathobj
paths = [repo_path / x for x in "ac"]
annexinfo = repo.get_content_annexinfo(eval_availability=True)
for path in paths:
pstat = annexinfo[path]
eq_(pstat["backend"], expected_backend)
assert_false(pstat["has_content"])
get_res = ds.get(paths,
result_renderer='disabled',
on_failure="ignore")
assert_result_count(get_res, 2, action="get", status="ok")
def test_blocking_writer_closing_timeout_signal():
# Expect that writer or reader do not block forever on a full signal queue
write_queue = queue.Queue()
signal_queue = queue.Queue(1)
signal_queue.put("This is data")
(read_descriptor, write_descriptor) = os.pipe()
write_file = os.fdopen(write_descriptor, "rb")
write_thread = WriteThread(
identifier="test thread",
user_info=write_descriptor,
source_queue=write_queue,
destination=write_file,
signal_queues=[signal_queue]
)
write_thread.start()
write_queue.put(b"some data")
data = os.read(read_descriptor, 1024)
eq_(data, b"some data")
write_queue.put(None)
write_thread.join()
eq_(signal_queue.get(), "This is data")
def test_get_url_parts():
eq_(au.get_url_parts(""), {})
assert_dict_equal(au.get_url_parts("http://datalad.org"),
{"_url_hostname": "datalad.org"})
assert_dict_equal(
au.get_url_parts("http://datalad.org/about.html"), {
"_url_hostname": "datalad.org",
"_url0": "about.html",
"_url_basename": "about.html",
"_url_basename_root_py": "about",
"_url_basename_ext_py": ".html",
"_url_basename_root": "about",
"_url_basename_ext": ".html"
})
assert_dict_equal(au.get_url_parts("http://datalad.org/about.html"),
au.get_url_parts("http://datalad.org//about.html"))
assert_dict_equal(
au.get_url_parts("http://datalad.org/for/git-users"), {
"_url_hostname": "datalad.org",
"_url0": "for",
"_url1": "git-users",
"_url_basename": "git-users",
"_url_basename_root_py": "git-users",
"_url_basename_ext_py": "",
"_url_basename_root": "git-users",
"_url_basename_ext": ""
})
def test_rerun_run_left_mutator_right(path=None):
ds = Dataset(path).create()
# keep direct repo accessor to speed things up
ds_repo = ds.repo
ds_repo.checkout(DEFAULT_BRANCH, options=["-b", "side"])
ds.run("echo ichange >>ichange")
ds_repo.checkout(DEFAULT_BRANCH)
ds.run("echo idont >idont")
ds_repo.merge("side", options=["-m", "Merge side"])
# o d_n
# |\
# o | c_r
# | o b_r
# |/
# o a_n
hexsha_before = ds_repo.get_hexsha()
ds.rerun(since="")
# o b_R
# o d_n
# |\
# o | c_r
# | o b_r
# |/
# o a_n
eq_(ds_repo.get_hexsha(hexsha_before),
ds_repo.get_hexsha(DEFAULT_BRANCH + "^"))
def test_gh1426(origin_path=None, target_path=None):
# set up a pair of repos, one the published copy of the other
origin = Dataset(origin_path).create()
target = mk_push_target(origin,
'target',
target_path,
annex=True,
bare=False)
origin.push(to='target')
assert_repo_status(origin.path)
assert_repo_status(target.path)
eq_(origin.repo.get_hexsha(DEFAULT_BRANCH),
target.get_hexsha(DEFAULT_BRANCH))
# gist of #1426 is that a newly added subdataset does not cause the
# superdataset to get published
origin.create('sub')
assert_repo_status(origin.path)
neq_(origin.repo.get_hexsha(DEFAULT_BRANCH),
target.get_hexsha(DEFAULT_BRANCH))
# now push
res = origin.push(to='target')
assert_result_count(res,
1,
status='ok',
type='dataset',
path=origin.path,
action='publish',
target='target',
operations=['fast-forward'])
eq_(origin.repo.get_hexsha(DEFAULT_BRANCH),
target.get_hexsha(DEFAULT_BRANCH))
def test_rerun_mutator_left_nonrun_right(path=None):
ds = Dataset(path).create()
# keep direct repo accessor to speed things up
ds_repo = ds.repo
ds.run("echo foo >>foo")
ds_repo.checkout(DEFAULT_BRANCH + "~", options=["-b", "side"])
with open(op.join(path, "nonrun-file"), "w") as f:
f.write("blah")
ds.save()
ds_repo.checkout(DEFAULT_BRANCH)
ds_repo.merge("side", options=["-m", "Merge side"])
# o d_n
# |\
# | o c_n
# o | b_r
# |/
# o a_n
hexsha_before = ds_repo.get_hexsha()
ds.rerun(since="")
# o b_R
# o d_n
# |\
# | o c_n
# o | b_r
# |/
# o a_n
assert_false(ds_repo.commit_exists(DEFAULT_BRANCH + "^2"))
eq_(hexsha_before, ds_repo.get_hexsha(DEFAULT_BRANCH + "^"))
def test_path_and_url(path, url):
def _urlopen(url, auth=None):
req = Request(url)
if auth:
req.add_header(
"Authorization", b"Basic " + base64.standard_b64encode(
'{0}:{1}'.format(*auth).encode('utf-8')))
return urlopen(req)
# @serve_ should remove http_proxy from the os.environ if was present
if not on_windows:
assert_false('http_proxy' in os.environ)
# get the "dir-view"
dirurl = url + test_fpath.parent.as_posix()
u = _urlopen(dirurl, auth)
assert_true(u.getcode() == 200)
html = u.read()
# get the actual content
file_html = _urlopen(url + url_quote(test_fpath.as_posix()),
auth).read().decode()
# verify we got the right one
eq_(file_html, test_fpath_full.read_text())
if bs4 is None:
return
# MIH is not sure what this part below is supposed to do
# possibly some kind of internal consistency test
soup = bs4.BeautifulSoup(html, "html.parser")
href_links = [txt.get('href') for txt in soup.find_all('a')]
assert_true(len(href_links) == 1)
parsed_url = f"{dirurl}/{href_links[0]}"
u = _urlopen(parsed_url, auth)
html = u.read().decode()
eq_(html, file_html)
def test_rerun_exclude_side(path=None):
ds = Dataset(path).create()
# keep direct repo accessor to speed things up
ds_repo = ds.repo
ds_repo.checkout(DEFAULT_BRANCH, options=["-b", "side"])
ds.run("echo foo >foo")
ds_repo.checkout(DEFAULT_BRANCH)
ds.run("echo bar >bar")
ds_repo.merge("side", options=["-m", "Merge side"])
# o d_n
# |\
# o | c_r
# | o b_r
# |/
# o a_n
ds.rerun("HEAD", since=DEFAULT_BRANCH + "^2", onto="")
# o d_M
# |\
# o | c_R
# | o b_r
# |/
# o a_n
neq_(ds_repo.get_hexsha(DEFAULT_BRANCH), ds_repo.get_hexsha())
neq_(ds_repo.get_hexsha(DEFAULT_BRANCH + "^"), ds_repo.get_hexsha("HEAD^"))
eq_(ds_repo.get_hexsha(DEFAULT_BRANCH + "^2"),
ds_repo.get_hexsha("HEAD^2"))
def test_install_recursive_with_data(src=None, path=None):
_make_dataset_hierarchy(src)
# now again; with data:
res = install(path,
source=src,
recursive=True,
get_data=True,
result_filter=None,
result_xfm=None)
assert_status('ok', res)
# installed a dataset and two subdatasets, and one file with content in
# each
assert_result_count(res, 5, type='dataset', action='install')
assert_result_count(res, 2, type='file', action='get')
# we recurse top down during installation, so toplevel should appear at
# first position in returned list
eq_(res[0]['path'], path)
top_ds = YieldDatasets()(res[0])
ok_(top_ds.is_installed())
def all_have_content(repo):
ainfo = repo.get_content_annexinfo(init=None, eval_availability=True)
return all(st["has_content"] for st in ainfo.values())
if isinstance(top_ds.repo, AnnexRepo):
ok_(all_have_content(top_ds.repo))
for subds in top_ds.subdatasets(recursive=True, result_xfm='datasets'):
ok_(subds.is_installed(), "Not installed: %s" % (subds, ))
if isinstance(subds.repo, AnnexRepo):
ok_(all_have_content(subds.repo))
def test_rerun_unrelated_mutator_left_nonrun_right(path=None):
ds = Dataset(path).create()
# keep direct repo accessor to speed things up
ds_repo = ds.repo
ds.run("echo foo >>foo")
ds_repo.checkout(DEFAULT_BRANCH + "~", options=["--orphan", "side"])
ds.save(message="squashed")
ds_repo.checkout(DEFAULT_BRANCH)
ds_repo.merge("side",
options=["-m", "Merge side", "--allow-unrelated-histories"])
# o d_n
# |\
# | o c_n
# o b_r
# o a_n
hexsha_before = ds_repo.get_hexsha()
ds.rerun(since="")
# o b_R
# o d_n
# |\
# | o c_n
# o b_r
# o a_n
eq_(hexsha_before, ds_repo.get_hexsha(DEFAULT_BRANCH + "^"))
def test_install_list(path=None, top_path=None):
_mk_submodule_annex(path, fname="test-annex.dat", fcontent="whatever")
# we want to be able to install several things, if these are known
# (no 'source' allowed). Therefore first toplevel:
ds = install(top_path, source=path, recursive=False)
assert_not_in('annex.hardlink', ds.config)
ok_(ds.is_installed())
sub1 = Dataset(opj(top_path, 'subm 1'))
sub2 = Dataset(opj(top_path, '2'))
ok_(not sub1.is_installed())
ok_(not sub2.is_installed())
# fails, when `source` is passed:
assert_raises(ValueError,
ds.install,
path=['subm 1', '2'],
source='something')
# now should work:
result = ds.install(path=['subm 1', '2'], result_xfm='paths')
ok_(sub1.is_installed())
ok_(sub2.is_installed())
eq_(set(result), {sub1.path, sub2.path})
# and if we request it again via get, result should be empty
get_result = ds.get(path=['subm 1', '2'], get_data=False)
assert_status('notneeded', get_result)
def test_rerun_octopus(path=None):
ds = Dataset(path).create()
# keep direct repo accessor to speed things up
ds_repo = ds.repo
ds.run("echo foo >>foo")
with open(op.join(ds.path, "non-run"), "w") as nrfh:
nrfh.write("non-run")
ds.save()
ds_repo.checkout(DEFAULT_BRANCH + "~", options=["-b", "topic-1"])
ds.run("echo bar >bar")
ds_repo.checkout(DEFAULT_BRANCH + "~", options=["-b", "topic-2"])
ds.run("echo baz >baz")
ds_repo.checkout(DEFAULT_BRANCH)
ds_repo.call_git(["merge", "-m", "Merge octopus", "topic-1", "topic-2"])
# o-. f_M
# |\ \
# | | o e_r
# | o | d_r
# | |/
# o | c_n
# |/
# o b_r
# o a_n
ds.rerun(since="", onto="")
neq_(ds_repo.get_hexsha("HEAD^3"),
ds_repo.get_hexsha(DEFAULT_BRANCH + "^3"))
eq_(ds_repo.get_hexsha("HEAD~3"),
ds_repo.get_hexsha(DEFAULT_BRANCH + "~3"))
ds_repo.checkout(DEFAULT_BRANCH)
hexsha_before = ds_repo.get_hexsha()
ds.rerun(since="")
eq_(hexsha_before, ds_repo.get_hexsha(DEFAULT_BRANCH + "~"))
def _check_procedure_properties(ps):
"""a common check that we get three essential properties"""
eq_(
sum([
'procedure_type' in p and 'procedure_callfmt' in p and 'path' in p
for p in ps
]), len(ps))
def test_aggregate_removal(path=None):
base = Dataset(opj(path, 'origin')).create(force=True)
# force all metadata objects into the annex
with open(opj(base.path, '.datalad', '.gitattributes'), 'w') as f:
f.write(
'** annex.largefiles=nothing\nmetadata/objects/** annex.largefiles=anything\n'
)
sub = base.create('sub', force=True)
subsub = sub.create(opj('subsub'), force=True)
base.save(recursive=True)
base.aggregate_metadata(recursive=True, update_mode='all')
assert_repo_status(base.path)
res = base.metadata(get_aggregates=True)
assert_result_count(res, 3)
assert_result_count(res, 1, path=subsub.path)
# check that we only have object files that are listed in agginfo
eq_(_get_contained_objs(base), _get_referenced_objs(base))
# now delete the deepest subdataset to test cleanup of aggregated objects
# in the top-level ds
base.remove(opj('sub', 'subsub'), reckless='kill')
# now aggregation has to detect that subsub is not simply missing, but gone
# for good
base.aggregate_metadata(recursive=True, update_mode='all')
assert_repo_status(base.path)
# internally consistent state
eq_(_get_contained_objs(base), _get_referenced_objs(base))
# info on subsub was removed at all levels
res = base.metadata(get_aggregates=True)
assert_result_count(res, 0, path=subsub.path)
assert_result_count(res, 2)
res = sub.metadata(get_aggregates=True)
assert_result_count(res, 0, path=subsub.path)
assert_result_count(res, 1)
