def test_remove_recursive_2(tdir): # fails in some cases https://github.com/datalad/datalad/issues/1573 with chpwd(tdir): install('///labs') with chpwd('labs'): install('tarr/face_place') remove(dataset='labs', recursive=True)
def test_remove_file_handle_only(path): ds = Dataset(path).create(force=True) ds.save() ok_clean_git(ds.path) # make sure there is any key ok_(len(ds.repo.get_file_key('one'))) # both files link to the same key eq_(ds.repo.get_file_key('one'), ds.repo.get_file_key('two')) rpath_one = realpath(opj(ds.path, 'one')) eq_(rpath_one, realpath(opj(ds.path, 'two'))) path_two = opj(ds.path, 'two') ok_(exists(path_two)) # remove one handle, should not affect the other ds.remove('two', check=False, message="custom msg") eq_(ds.repo.repo.head.commit.message.rstrip(), "custom msg") eq_(rpath_one, realpath(opj(ds.path, 'one'))) ok_(exists(rpath_one)) ok_(not exists(path_two)) # remove file without specifying the dataset -- shouldn't fail with chpwd(path): remove('one', check=False) ok_(not exists("one")) # and we should be able to remove without saving ds.remove('three', check=False, save=False) ok_(ds.repo.dirty)
def update_dataset(zenodo_dataset, conp_dataset, token): # To update a dataset, we don't know which files have been updated # so we have to remove all existing files and redownload all files # fresh from the latest version of that zenodo dataset dataset_dir = conp_dataset["directory"] dats_dir = os.path.join(dataset_dir, "DATS.json") zenodo_tracker_path = os.path.join(dataset_dir, ".conp-zenodo-crawler.json") # Remove all data and DATS.json files for file_name in os.listdir(dataset_dir): if file_name[0] == "." or file_name == "README.md": continue api.remove(os.path.join(dataset_dir, file_name), check=False) d = api.Dataset(dataset_dir) for bucket in zenodo_dataset["files"]: download_file(bucket, d, dataset_dir) # If DATS.json isn't in downloaded files, create new DATS.json if not os.path.isfile(dats_dir): create_new_dats(dataset_dir, dats_dir, zenodo_dataset) # Add/update .conp-zenodo-crawler.json tracker file create_zenodo_tracker(zenodo_tracker_path, zenodo_dataset) # Save all changes and push to github d.save() d.publish(to="github")
def test_remove_file_handle_only(path): ds = Dataset(path).create(force=True) ds.save() assert_repo_status(ds.path) # make sure there is any key ok_(len(ds.repo.get_file_key('one'))) # both files link to the same key eq_(ds.repo.get_file_key('one'), ds.repo.get_file_key('two')) rpath_one = (ds.pathobj / 'one').resolve() eq_(rpath_one, (ds.pathobj / 'two').resolve()) path_two = ds.pathobj / 'two' ok_(path_two.exists()) # remove one handle, should not affect the other ds.remove('two', check=False, message="custom msg") eq_(ds.repo.format_commit("%B").rstrip(), "custom msg") eq_(rpath_one, (ds.pathobj / 'one').resolve()) ok_(rpath_one.exists()) ok_(not path_two.exists()) # remove file without specifying the dataset -- shouldn't fail with chpwd(path): remove('one', check=False) ok_(not exists("one")) # and we should be able to remove without saving ds.remove('three', check=False, save=False) ok_(ds.repo.dirty)
def test_remove_nowhining(path): # when removing a dataset under a dataset (but not a subdataset) # should not provide a meaningless message that something was not right ds = create(path) # just install/clone inside of it subds_path = _path_(path, 'subds') install(subds_path, source=path) remove(subds_path) # should remove just fine
def test_remove_nowhining(path=None): # when removing a dataset under a dataset (but not a subdataset) # should not provide a meaningless message that something was not right ds = Dataset(path).create() # just install/clone inside of it subds_path = ds.pathobj / 'subds' clone(path=subds_path, source=path) remove(dataset=subds_path) # should remove just fine
def test_remove_subdataset_nomethod(path=None): ds = Dataset(path).create() ds.create('subds') with chpwd(path): # fails due to unique state res = remove('subds', on_failure='ignore') assert_in_results(res, action='uninstall', status='error', type='dataset') res = remove('subds', reckless='availability', on_failure='ignore') assert_in_results(res, action='uninstall', status='ok', type='dataset') assert_in_results(res, action='remove', status='ok') assert_in_results(res, action='save', status='ok')
def test_expanduser(srcpath, destpath): src = Dataset(Path(srcpath) / 'src').create() dest = Dataset(Path(destpath) / 'dest').create() with chpwd(destpath), patch.dict('os.environ', {'HOME': srcpath}): res = clone(op.join('~', 'src'), 'dest', result_xfm=None, return_type='list', on_failure='ignore') assert_result_count(res, 1) assert_result_count( res, 1, action='install', status='error', path=dest.path, message='target path already exists and not empty, refuse to ' 'clone into target path') # wipe out destination, and try again assert_status('ok', remove(dataset=dest, check=False)) # now it should do it, and clone the right one cloneds = clone(op.join('~', 'src'), 'dest') eq_(cloneds.pathobj, Path(destpath) / 'dest') eq_(src.id, cloneds.id) # and it shouldn't fail when doing it again, because it detects # the re-clone cloneds = clone(op.join('~', 'src'), 'dest') eq_(cloneds.pathobj, Path(destpath) / 'dest')
def test_remove_dataset_hierarchy(path): ds = Dataset(path).create() ds.create('deep') ok_clean_git(ds.path) # fail on missing --recursive because subdataset is present assert_raises(IncompleteResultsError, ds.remove) ok_clean_git(ds.path) ds.remove(recursive=True) # completely gone ok_(not ds.is_installed()) ok_(not exists(ds.path)) # now do it again, but without a reference dataset ds = Dataset(path).create() ds.create('deep') ok_clean_git(ds.path) remove(ds.path, recursive=True) # completely gone ok_(not ds.is_installed()) ok_(not exists(ds.path))
def time_remove(self): remove(self.ds.path, recursive=True)
def time_remove(self, tarfile_path): remove(self.ds.path, recursive=True)
def test_update_volatile_subds(originpath=None, otherpath=None, destpath=None): origin = Dataset(originpath).create() repo = origin.repo if repo.is_managed_branch() and repo.git_annex_version <= "8.20201129": # Fails before git-annex's fd161da2c (adjustTree: Consider submodule # deletions, 2021-01-06). raise SkipTest( "On adjusted branch, test requires fix in more recent git-annex") ds = install(source=originpath, path=destpath, result_xfm='datasets', return_type='item-or-list') # as a submodule sname = 'subm 1' osm1 = origin.create(sname) assert_result_count(ds.update(), 1, status='ok', type='dataset') # nothing without a merge, no inappropriate magic assert_not_in(sname, ds.subdatasets(result_xfm='relpaths')) assert_result_count(ds.update(merge=True), 1, action='update', status='ok', type='dataset') # and we should be able to do update with recursive invocation assert_result_count(ds.update(merge=True, recursive=True), 1, action='update', status='ok', type='dataset') # known, and placeholder exists assert_in(sname, ds.subdatasets(result_xfm='relpaths')) ok_(exists(opj(ds.path, sname))) # remove from origin origin.remove(sname, reckless='availability') assert_result_count(ds.update(merge=True), 1, action='update', status='ok', type='dataset') # gone locally, wasn't checked out assert_not_in(sname, ds.subdatasets(result_xfm='relpaths')) assert_false(exists(opj(ds.path, sname))) # re-introduce at origin osm1 = origin.create(sname) create_tree(osm1.path, {'load.dat': 'heavy'}) origin.save(opj(osm1.path, 'load.dat')) assert_result_count(ds.update(merge=True), 1, action='update', status='ok', type='dataset') # grab new content of uninstall subdataset, right away ds.get(opj(ds.path, sname, 'load.dat')) ok_file_has_content(opj(ds.path, sname, 'load.dat'), 'heavy') # modify ds and subds at origin create_tree(origin.path, {'mike': 'this', sname: {'probe': 'little'}}) origin.save(recursive=True) assert_repo_status(origin.path) # updates for both datasets should come down the pipe assert_result_count(ds.update(merge=True, recursive=True), 2, action='update', status='ok', type='dataset') assert_repo_status(ds.path) # now remove just-installed subdataset from origin again origin.remove(sname, reckless='kill') assert_not_in(sname, origin.subdatasets(result_xfm='relpaths')) assert_in(sname, ds.subdatasets(result_xfm='relpaths')) # merge should disconnect the installed subdataset, but leave the actual # ex-subdataset alone assert_result_count(ds.update(merge=True, recursive=True), 1, action='update', type='dataset') assert_not_in(sname, ds.subdatasets(result_xfm='relpaths')) ok_file_has_content(opj(ds.path, sname, 'load.dat'), 'heavy') ok_(Dataset(opj(ds.path, sname)).is_installed()) # now remove the now disconnected subdataset for further tests remove(dataset=op.join(ds.path, sname), reckless='kill') assert_repo_status(ds.path) # new separate subdataset, not within the origin dataset otherds = Dataset(otherpath).create() # install separate dataset as a submodule ds.install(source=otherds.path, path='other') create_tree(otherds.path, {'brand': 'new'}) otherds.save() assert_repo_status(otherds.path) # pull in changes res = ds.update(merge=True, recursive=True) assert_result_count(res, 2, status='ok', action='update', type='dataset') # the next is to check for #2858 assert_repo_status(ds.path)
def test_remove_recursive_2(tdir): # fails in some cases https://github.com/datalad/datalad/issues/1573 with chpwd(tdir): install('///labs') install('labs/tarr/face_place') remove('labs', recursive=True)
def test_update_volatile_subds(originpath, otherpath, destpath): origin = Dataset(originpath).create() ds = install(source=originpath, path=destpath, result_xfm='datasets', return_type='item-or-list') # as a submodule sname = 'subm 1' osm1 = origin.create(sname) assert_result_count(ds.update(), 1, status='ok', type='dataset') # nothing without a merge, no inappropriate magic assert_not_in(sname, ds.subdatasets(result_xfm='relpaths')) assert_result_count(ds.update(merge=True), 1, action='update', status='ok', type='dataset') # and we should be able to do update with recursive invocation assert_result_count(ds.update(merge=True, recursive=True), 1, action='update', status='ok', type='dataset') # known, and placeholder exists assert_in(sname, ds.subdatasets(result_xfm='relpaths')) ok_(exists(opj(ds.path, sname))) # remove from origin origin.remove(sname) assert_result_count(ds.update(merge=True), 1, action='update', status='ok', type='dataset') # gone locally, wasn't checked out assert_not_in(sname, ds.subdatasets(result_xfm='relpaths')) assert_false(exists(opj(ds.path, sname))) # re-introduce at origin osm1 = origin.create(sname) create_tree(osm1.path, {'load.dat': 'heavy'}) origin.save(opj(osm1.path, 'load.dat')) assert_result_count(ds.update(merge=True), 1, action='update', status='ok', type='dataset') # grab new content of uninstall subdataset, right away ds.get(opj(ds.path, sname, 'load.dat')) ok_file_has_content(opj(ds.path, sname, 'load.dat'), 'heavy') # modify ds and subds at origin create_tree(origin.path, {'mike': 'this', sname: {'probe': 'little'}}) origin.save(recursive=True) assert_repo_status(origin.path) # updates for both datasets should come down the pipe assert_result_count(ds.update(merge=True, recursive=True), 2, action='update', status='ok', type='dataset') assert_repo_status(ds.path) # now remove just-installed subdataset from origin again origin.remove(sname, check=False) assert_not_in(sname, origin.subdatasets(result_xfm='relpaths')) assert_in(sname, ds.subdatasets(result_xfm='relpaths')) # merge should disconnect the installed subdataset, but leave the actual # ex-subdataset alone assert_result_count(ds.update(merge=True, recursive=True), 1, action='update', type='dataset') assert_not_in(sname, ds.subdatasets(result_xfm='relpaths')) ok_file_has_content(opj(ds.path, sname, 'load.dat'), 'heavy') ok_(Dataset(opj(ds.path, sname)).is_installed()) # now remove the now disconnected subdataset for further tests # not using a bound method, not giving a parentds, should # not be needed to get a clean dataset remove(op.join(ds.path, sname), check=False) assert_repo_status(ds.path) # new separate subdataset, not within the origin dataset otherds = Dataset(otherpath).create() # install separate dataset as a submodule ds.install(source=otherds.path, path='other') create_tree(otherds.path, {'brand': 'new'}) otherds.save() assert_repo_status(otherds.path) # pull in changes res = ds.update(merge=True, recursive=True) assert_result_count(res, 2, status='ok', action='update', type='dataset') # the next is to check for #2858 assert_repo_status(ds.path)
def test_update_volatile_subds(originpath, otherpath, destpath): origin = Dataset(originpath).create() ds = install( source=originpath, path=destpath, result_xfm='datasets', return_type='item-or-list') # as a submodule sname = 'subm 1' osm1 = origin.create(sname) assert_result_count(ds.update(), 1, status='ok', type='dataset') # nothing without a merge, no inappropriate magic assert_not_in(sname, ds.subdatasets(result_xfm='relpaths')) assert_result_count(ds.update(merge=True), 1, status='ok', type='dataset') # and we should be able to do update with recursive invocation assert_result_count(ds.update(merge=True, recursive=True), 1, status='ok', type='dataset') # known, and placeholder exists assert_in(sname, ds.subdatasets(result_xfm='relpaths')) ok_(exists(opj(ds.path, sname))) # remove from origin origin.remove(sname) assert_result_count(ds.update(merge=True), 1, status='ok', type='dataset') # gone locally, wasn't checked out assert_not_in(sname, ds.subdatasets(result_xfm='relpaths')) assert_false(exists(opj(ds.path, sname))) # re-introduce at origin osm1 = origin.create(sname) create_tree(osm1.path, {'load.dat': 'heavy'}) origin.save(opj(osm1.path, 'load.dat')) assert_result_count(ds.update(merge=True), 1, status='ok', type='dataset') # grab new content of uninstall subdataset, right away ds.get(opj(ds.path, sname, 'load.dat')) ok_file_has_content(opj(ds.path, sname, 'load.dat'), 'heavy') # modify ds and subds at origin create_tree(origin.path, {'mike': 'this', sname: {'probe': 'little'}}) origin.save(recursive=True) ok_clean_git(origin.path) # updates for both datasets should come down the pipe assert_result_count(ds.update(merge=True, recursive=True), 2, status='ok', type='dataset') ok_clean_git(ds.path) # now remove just-installed subdataset from origin again origin.remove(sname, check=False) assert_not_in(sname, origin.subdatasets(result_xfm='relpaths')) assert_in(sname, ds.subdatasets(result_xfm='relpaths')) # merge should disconnect the installed subdataset, but leave the actual # ex-subdataset alone assert_result_count(ds.update(merge=True, recursive=True), 1, type='dataset') assert_not_in(sname, ds.subdatasets(result_xfm='relpaths')) ok_file_has_content(opj(ds.path, sname, 'load.dat'), 'heavy') ok_(Dataset(opj(ds.path, sname)).is_installed()) # now remove the now disconnected subdataset for further tests # not using a bound method, not giving a parentds, should # not be needed to get a clean dataset remove(op.join(ds.path, sname), check=False) ok_clean_git(ds.path) # new separate subdataset, not within the origin dataset otherds = Dataset(otherpath).create() # install separate dataset as a submodule ds.install(source=otherds.path, path='other') create_tree(otherds.path, {'brand': 'new'}) otherds.save() ok_clean_git(otherds.path) # pull in changes res = ds.update(merge=True, recursive=True) assert_result_count( res, 2, status='ok', action='update', type='dataset') # the next is to check for #2858 ok_clean_git(ds.path)