def test_remove_recursive_2(tdir):
# fails in some cases https://github.com/datalad/datalad/issues/1573
with chpwd(tdir):
install('///labs')
with chpwd('labs'):
install('tarr/face_place')
remove(dataset='labs', recursive=True)
def test_remove_file_handle_only(path):
ds = Dataset(path).create(force=True)
ds.save()
ok_clean_git(ds.path)
# make sure there is any key
ok_(len(ds.repo.get_file_key('one')))
# both files link to the same key
eq_(ds.repo.get_file_key('one'),
ds.repo.get_file_key('two'))
rpath_one = realpath(opj(ds.path, 'one'))
eq_(rpath_one, realpath(opj(ds.path, 'two')))
path_two = opj(ds.path, 'two')
ok_(exists(path_two))
# remove one handle, should not affect the other
ds.remove('two', check=False, message="custom msg")
eq_(ds.repo.repo.head.commit.message.rstrip(), "custom msg")
eq_(rpath_one, realpath(opj(ds.path, 'one')))
ok_(exists(rpath_one))
ok_(not exists(path_two))
# remove file without specifying the dataset -- shouldn't fail
with chpwd(path):
remove('one', check=False)
ok_(not exists("one"))
# and we should be able to remove without saving
ds.remove('three', check=False, save=False)
ok_(ds.repo.dirty)
def update_dataset(zenodo_dataset, conp_dataset, token):
# To update a dataset, we don't know which files have been updated
# so we have to remove all existing files and redownload all files
# fresh from the latest version of that zenodo dataset
dataset_dir = conp_dataset["directory"]
dats_dir = os.path.join(dataset_dir, "DATS.json")
zenodo_tracker_path = os.path.join(dataset_dir,
".conp-zenodo-crawler.json")
# Remove all data and DATS.json files
for file_name in os.listdir(dataset_dir):
if file_name[0] == "." or file_name == "README.md":
continue
api.remove(os.path.join(dataset_dir, file_name), check=False)
d = api.Dataset(dataset_dir)
for bucket in zenodo_dataset["files"]:
download_file(bucket, d, dataset_dir)
# If DATS.json isn't in downloaded files, create new DATS.json
if not os.path.isfile(dats_dir):
create_new_dats(dataset_dir, dats_dir, zenodo_dataset)
# Add/update .conp-zenodo-crawler.json tracker file
create_zenodo_tracker(zenodo_tracker_path, zenodo_dataset)
# Save all changes and push to github
d.save()
d.publish(to="github")
def test_remove_file_handle_only(path):
ds = Dataset(path).create(force=True)
ds.save()
assert_repo_status(ds.path)
# make sure there is any key
ok_(len(ds.repo.get_file_key('one')))
# both files link to the same key
eq_(ds.repo.get_file_key('one'), ds.repo.get_file_key('two'))
rpath_one = (ds.pathobj / 'one').resolve()
eq_(rpath_one, (ds.pathobj / 'two').resolve())
path_two = ds.pathobj / 'two'
ok_(path_two.exists())
# remove one handle, should not affect the other
ds.remove('two', check=False, message="custom msg")
eq_(ds.repo.format_commit("%B").rstrip(), "custom msg")
eq_(rpath_one, (ds.pathobj / 'one').resolve())
ok_(rpath_one.exists())
ok_(not path_two.exists())
# remove file without specifying the dataset -- shouldn't fail
with chpwd(path):
remove('one', check=False)
ok_(not exists("one"))
# and we should be able to remove without saving
ds.remove('three', check=False, save=False)
ok_(ds.repo.dirty)
def test_remove_nowhining(path):
# when removing a dataset under a dataset (but not a subdataset)
# should not provide a meaningless message that something was not right
ds = create(path)
# just install/clone inside of it
subds_path = _path_(path, 'subds')
install(subds_path, source=path)
remove(subds_path) # should remove just fine
def test_remove_nowhining(path=None):
# when removing a dataset under a dataset (but not a subdataset)
# should not provide a meaningless message that something was not right
ds = Dataset(path).create()
# just install/clone inside of it
subds_path = ds.pathobj / 'subds'
clone(path=subds_path, source=path)
remove(dataset=subds_path) # should remove just fine
def test_remove_subdataset_nomethod(path=None):
ds = Dataset(path).create()
ds.create('subds')
with chpwd(path):
# fails due to unique state
res = remove('subds', on_failure='ignore')
assert_in_results(res,
action='uninstall',
status='error',
type='dataset')
res = remove('subds', reckless='availability', on_failure='ignore')
assert_in_results(res, action='uninstall', status='ok', type='dataset')
assert_in_results(res, action='remove', status='ok')
assert_in_results(res, action='save', status='ok')
def test_expanduser(srcpath, destpath):
src = Dataset(Path(srcpath) / 'src').create()
dest = Dataset(Path(destpath) / 'dest').create()
with chpwd(destpath), patch.dict('os.environ', {'HOME': srcpath}):
res = clone(op.join('~', 'src'),
'dest',
result_xfm=None,
return_type='list',
on_failure='ignore')
assert_result_count(res, 1)
assert_result_count(
res,
1,
action='install',
status='error',
path=dest.path,
message='target path already exists and not empty, refuse to '
'clone into target path')
# wipe out destination, and try again
assert_status('ok', remove(dataset=dest, check=False))
# now it should do it, and clone the right one
cloneds = clone(op.join('~', 'src'), 'dest')
eq_(cloneds.pathobj, Path(destpath) / 'dest')
eq_(src.id, cloneds.id)
# and it shouldn't fail when doing it again, because it detects
# the re-clone
cloneds = clone(op.join('~', 'src'), 'dest')
eq_(cloneds.pathobj, Path(destpath) / 'dest')
def test_remove_dataset_hierarchy(path):
ds = Dataset(path).create()
ds.create('deep')
ok_clean_git(ds.path)
# fail on missing --recursive because subdataset is present
assert_raises(IncompleteResultsError, ds.remove)
ok_clean_git(ds.path)
ds.remove(recursive=True)
# completely gone
ok_(not ds.is_installed())
ok_(not exists(ds.path))
# now do it again, but without a reference dataset
ds = Dataset(path).create()
ds.create('deep')
ok_clean_git(ds.path)
remove(ds.path, recursive=True)
# completely gone
ok_(not ds.is_installed())
ok_(not exists(ds.path))
def time_remove(self):
remove(self.ds.path, recursive=True)
def time_remove(self):
remove(self.ds.path, recursive=True)
def time_remove(self, tarfile_path):
remove(self.ds.path, recursive=True)
def time_remove(self, tarfile_path):
remove(self.ds.path, recursive=True)
def test_update_volatile_subds(originpath=None, otherpath=None, destpath=None):
origin = Dataset(originpath).create()
repo = origin.repo
if repo.is_managed_branch() and repo.git_annex_version <= "8.20201129":
# Fails before git-annex's fd161da2c (adjustTree: Consider submodule
# deletions, 2021-01-06).
raise SkipTest(
"On adjusted branch, test requires fix in more recent git-annex")
ds = install(source=originpath,
path=destpath,
result_xfm='datasets',
return_type='item-or-list')
# as a submodule
sname = 'subm 1'
osm1 = origin.create(sname)
assert_result_count(ds.update(), 1, status='ok', type='dataset')
# nothing without a merge, no inappropriate magic
assert_not_in(sname, ds.subdatasets(result_xfm='relpaths'))
assert_result_count(ds.update(merge=True),
1,
action='update',
status='ok',
type='dataset')
# and we should be able to do update with recursive invocation
assert_result_count(ds.update(merge=True, recursive=True),
1,
action='update',
status='ok',
type='dataset')
# known, and placeholder exists
assert_in(sname, ds.subdatasets(result_xfm='relpaths'))
ok_(exists(opj(ds.path, sname)))
# remove from origin
origin.remove(sname, reckless='availability')
assert_result_count(ds.update(merge=True),
1,
action='update',
status='ok',
type='dataset')
# gone locally, wasn't checked out
assert_not_in(sname, ds.subdatasets(result_xfm='relpaths'))
assert_false(exists(opj(ds.path, sname)))
# re-introduce at origin
osm1 = origin.create(sname)
create_tree(osm1.path, {'load.dat': 'heavy'})
origin.save(opj(osm1.path, 'load.dat'))
assert_result_count(ds.update(merge=True),
1,
action='update',
status='ok',
type='dataset')
# grab new content of uninstall subdataset, right away
ds.get(opj(ds.path, sname, 'load.dat'))
ok_file_has_content(opj(ds.path, sname, 'load.dat'), 'heavy')
# modify ds and subds at origin
create_tree(origin.path, {'mike': 'this', sname: {'probe': 'little'}})
origin.save(recursive=True)
assert_repo_status(origin.path)
# updates for both datasets should come down the pipe
assert_result_count(ds.update(merge=True, recursive=True),
2,
action='update',
status='ok',
type='dataset')
assert_repo_status(ds.path)
# now remove just-installed subdataset from origin again
origin.remove(sname, reckless='kill')
assert_not_in(sname, origin.subdatasets(result_xfm='relpaths'))
assert_in(sname, ds.subdatasets(result_xfm='relpaths'))
# merge should disconnect the installed subdataset, but leave the actual
# ex-subdataset alone
assert_result_count(ds.update(merge=True, recursive=True),
1,
action='update',
type='dataset')
assert_not_in(sname, ds.subdatasets(result_xfm='relpaths'))
ok_file_has_content(opj(ds.path, sname, 'load.dat'), 'heavy')
ok_(Dataset(opj(ds.path, sname)).is_installed())
# now remove the now disconnected subdataset for further tests
remove(dataset=op.join(ds.path, sname), reckless='kill')
assert_repo_status(ds.path)
# new separate subdataset, not within the origin dataset
otherds = Dataset(otherpath).create()
# install separate dataset as a submodule
ds.install(source=otherds.path, path='other')
create_tree(otherds.path, {'brand': 'new'})
otherds.save()
assert_repo_status(otherds.path)
# pull in changes
res = ds.update(merge=True, recursive=True)
assert_result_count(res, 2, status='ok', action='update', type='dataset')
# the next is to check for #2858
assert_repo_status(ds.path)
def test_remove_recursive_2(tdir):
# fails in some cases https://github.com/datalad/datalad/issues/1573
with chpwd(tdir):
install('///labs')
install('labs/tarr/face_place')
remove('labs', recursive=True)
def test_update_volatile_subds(originpath, otherpath, destpath):
origin = Dataset(originpath).create()
ds = install(source=originpath,
path=destpath,
result_xfm='datasets',
return_type='item-or-list')
# as a submodule
sname = 'subm 1'
osm1 = origin.create(sname)
assert_result_count(ds.update(), 1, status='ok', type='dataset')
# nothing without a merge, no inappropriate magic
assert_not_in(sname, ds.subdatasets(result_xfm='relpaths'))
assert_result_count(ds.update(merge=True),
1,
action='update',
status='ok',
type='dataset')
# and we should be able to do update with recursive invocation
assert_result_count(ds.update(merge=True, recursive=True),
1,
action='update',
status='ok',
type='dataset')
# known, and placeholder exists
assert_in(sname, ds.subdatasets(result_xfm='relpaths'))
ok_(exists(opj(ds.path, sname)))
# remove from origin
origin.remove(sname)
assert_result_count(ds.update(merge=True),
1,
action='update',
status='ok',
type='dataset')
# gone locally, wasn't checked out
assert_not_in(sname, ds.subdatasets(result_xfm='relpaths'))
assert_false(exists(opj(ds.path, sname)))
# re-introduce at origin
osm1 = origin.create(sname)
create_tree(osm1.path, {'load.dat': 'heavy'})
origin.save(opj(osm1.path, 'load.dat'))
assert_result_count(ds.update(merge=True),
1,
action='update',
status='ok',
type='dataset')
# grab new content of uninstall subdataset, right away
ds.get(opj(ds.path, sname, 'load.dat'))
ok_file_has_content(opj(ds.path, sname, 'load.dat'), 'heavy')
# modify ds and subds at origin
create_tree(origin.path, {'mike': 'this', sname: {'probe': 'little'}})
origin.save(recursive=True)
assert_repo_status(origin.path)
# updates for both datasets should come down the pipe
assert_result_count(ds.update(merge=True, recursive=True),
2,
action='update',
status='ok',
type='dataset')
assert_repo_status(ds.path)
# now remove just-installed subdataset from origin again
origin.remove(sname, check=False)
assert_not_in(sname, origin.subdatasets(result_xfm='relpaths'))
assert_in(sname, ds.subdatasets(result_xfm='relpaths'))
# merge should disconnect the installed subdataset, but leave the actual
# ex-subdataset alone
assert_result_count(ds.update(merge=True, recursive=True),
1,
action='update',
type='dataset')
assert_not_in(sname, ds.subdatasets(result_xfm='relpaths'))
ok_file_has_content(opj(ds.path, sname, 'load.dat'), 'heavy')
ok_(Dataset(opj(ds.path, sname)).is_installed())
# now remove the now disconnected subdataset for further tests
# not using a bound method, not giving a parentds, should
# not be needed to get a clean dataset
remove(op.join(ds.path, sname), check=False)
assert_repo_status(ds.path)
# new separate subdataset, not within the origin dataset
otherds = Dataset(otherpath).create()
# install separate dataset as a submodule
ds.install(source=otherds.path, path='other')
create_tree(otherds.path, {'brand': 'new'})
otherds.save()
assert_repo_status(otherds.path)
# pull in changes
res = ds.update(merge=True, recursive=True)
assert_result_count(res, 2, status='ok', action='update', type='dataset')
# the next is to check for #2858
assert_repo_status(ds.path)
def test_update_volatile_subds(originpath, otherpath, destpath):
origin = Dataset(originpath).create()
ds = install(
source=originpath, path=destpath,
result_xfm='datasets', return_type='item-or-list')
# as a submodule
sname = 'subm 1'
osm1 = origin.create(sname)
assert_result_count(ds.update(), 1, status='ok', type='dataset')
# nothing without a merge, no inappropriate magic
assert_not_in(sname, ds.subdatasets(result_xfm='relpaths'))
assert_result_count(ds.update(merge=True), 1, status='ok', type='dataset')
# and we should be able to do update with recursive invocation
assert_result_count(ds.update(merge=True, recursive=True), 1, status='ok', type='dataset')
# known, and placeholder exists
assert_in(sname, ds.subdatasets(result_xfm='relpaths'))
ok_(exists(opj(ds.path, sname)))
# remove from origin
origin.remove(sname)
assert_result_count(ds.update(merge=True), 1, status='ok', type='dataset')
# gone locally, wasn't checked out
assert_not_in(sname, ds.subdatasets(result_xfm='relpaths'))
assert_false(exists(opj(ds.path, sname)))
# re-introduce at origin
osm1 = origin.create(sname)
create_tree(osm1.path, {'load.dat': 'heavy'})
origin.save(opj(osm1.path, 'load.dat'))
assert_result_count(ds.update(merge=True), 1, status='ok', type='dataset')
# grab new content of uninstall subdataset, right away
ds.get(opj(ds.path, sname, 'load.dat'))
ok_file_has_content(opj(ds.path, sname, 'load.dat'), 'heavy')
# modify ds and subds at origin
create_tree(origin.path, {'mike': 'this', sname: {'probe': 'little'}})
origin.save(recursive=True)
ok_clean_git(origin.path)
# updates for both datasets should come down the pipe
assert_result_count(ds.update(merge=True, recursive=True),
2, status='ok', type='dataset')
ok_clean_git(ds.path)
# now remove just-installed subdataset from origin again
origin.remove(sname, check=False)
assert_not_in(sname, origin.subdatasets(result_xfm='relpaths'))
assert_in(sname, ds.subdatasets(result_xfm='relpaths'))
# merge should disconnect the installed subdataset, but leave the actual
# ex-subdataset alone
assert_result_count(ds.update(merge=True, recursive=True),
1, type='dataset')
assert_not_in(sname, ds.subdatasets(result_xfm='relpaths'))
ok_file_has_content(opj(ds.path, sname, 'load.dat'), 'heavy')
ok_(Dataset(opj(ds.path, sname)).is_installed())
# now remove the now disconnected subdataset for further tests
# not using a bound method, not giving a parentds, should
# not be needed to get a clean dataset
remove(op.join(ds.path, sname), check=False)
ok_clean_git(ds.path)
# new separate subdataset, not within the origin dataset
otherds = Dataset(otherpath).create()
# install separate dataset as a submodule
ds.install(source=otherds.path, path='other')
create_tree(otherds.path, {'brand': 'new'})
otherds.save()
ok_clean_git(otherds.path)
# pull in changes
res = ds.update(merge=True, recursive=True)
assert_result_count(
res, 2, status='ok', action='update', type='dataset')
# the next is to check for #2858
ok_clean_git(ds.path)
