def update_home(force=False): """Update an existing DataLad or S3 home.""" if not force and not TF_CACHED: print("""\ TemplateFlow was not cached (TEMPLATEFLOW_HOME=%s), \ a fresh initialization was done.""" % TF_HOME) return False if TF_USE_DATALAD: from datalad.api import update print("Updating TemplateFlow's HOME using DataLad ...") try: update(str(TF_HOME), recursive=True, merge=True) except Exception as e: warn("""Error updating TemplateFlow's home directory (using DataLad): %s""" % str(e)) return True # This is an S3 type of installation from zipfile import ZipFile with ZipFile(resource_filename('templateflow', 'conf/templateflow-skel.zip'), 'r') as zipref: allfiles = zipref.namelist() current_files = [s.relative_to(TF_HOME) for s in TF_HOME.glob('**/*')] existing = sorted(set(['%s/' % s.parent for s in current_files])) + \ [str(s) for s in current_files] newfiles = sorted(set(allfiles) - set(existing)) if newfiles: print("Updating TemplateFlow's HOME using S3. " "Adding: \n%s" % "\n".join(newfiles)) zipref.extractall(str(TF_HOME), members=newfiles) return True print("TemplateFlow's HOME directory (S3 type) was up-to-date.") return False
def _update_datalad(): from datalad.api import update print("Updating TEMPLATEFLOW_HOME using DataLad ...") try: update(str(TF_HOME), recursive=True, merge=True) except Exception as e: warn(f"Error updating TemplateFlow's home directory (using DataLad): {e}") return True
def setup_home(force=False): """Initialize/update TF's home if necessary.""" if not force and not TF_CACHED: print(f"""\ TemplateFlow was not cached (TEMPLATEFLOW_HOME={TF_HOME}), \ a fresh initialization was done.""") return False return update(local=True, overwrite=False)
def test_update_simple(origin, src_path, dst_path): # prepare src source = install(src_path, source=origin, recursive=True) # forget we cloned it (provide no 'origin' anymore), which should lead to # setting tracking branch to target: source.repo.remove_remote("origin") # dataset without sibling will not need updates assert_status('notneeded', source.update()) # deprecation message doesn't ruin things assert_status('notneeded', source.update(fetch_all=True)) # but error if unknown sibling is given assert_status('impossible', source.update(sibling='funky', on_failure='ignore')) # get a clone to update later on: dest = install(dst_path, source=src_path, recursive=True) # test setup done; # assert all fine assert_repo_status(dst_path) assert_repo_status(src_path) # update yields nothing => up-to-date assert_status('ok', dest.update()) assert_repo_status(dst_path) # modify origin: with open(opj(src_path, "update.txt"), "w") as f: f.write("Additional content") source.save(path="update.txt", message="Added update.txt") assert_repo_status(src_path) # update without `merge` only fetches: assert_status('ok', dest.update()) # modification is not known to active branch: assert_not_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch())) # modification is known to branch origin/<default branch> assert_in("update.txt", dest.repo.get_files("origin/" + DEFAULT_BRANCH)) # merge: assert_status('ok', dest.update(merge=True)) # modification is now known to active branch: assert_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch())) # it's known to annex, but has no content yet: dest.repo.get_file_key("update.txt") # raises if unknown eq_([False], dest.repo.file_has_content(["update.txt"])) # check subdataset path constraints, baseline (parent + 2 subds) assert_result_count(dest.update(recursive=True), 3, status='ok', type='dataset') # no recursion and invalid path still updates the parent res = dest.update(path='whatever') assert_result_count(res, 1, status='ok', type='dataset') assert_result_count(res, 1, status='ok', path=dest.path) # invalid path with recursion also does res = dest.update(recursive=True, path='whatever') assert_result_count(res, 1, status='ok', type='dataset') assert_result_count(res, 1, status='ok', path=dest.path) # valid path and no recursion only updates the parent res = dest.update(path='subm 1') assert_result_count(res, 1, status='ok', type='dataset') assert_result_count(res, 1, status='ok', path=dest.path) # valid path and recursion updates matching res = dest.update(recursive=True, path='subm 1') assert_result_count(res, 2, status='ok', type='dataset') assert_result_count(res, 1, status='ok', path=dest.path) assert_result_count(res, 1, status='ok', path=str(dest.pathobj / 'subm 1')) # additional invalid path doesn't hurt res = dest.update(recursive=True, path=['subm 1', 'mike']) assert_result_count(res, 2, status='ok', type='dataset') # full match res = dest.update(recursive=True, path=['subm 1', '2']) assert_result_count(res, 3, status='ok', type='dataset') # test that update doesn't crash if we specify only a single path (submod) to # operate on with chpwd(dest.path): # in 0.11.x it would be a single result since "pwd" dataset is not # considered, and would be relative path (as specified). # In 0.12.0 - it would include implicit pwd dataset, and paths would be absolute res_update = update(path=['subm 1'], recursive=True) assert_result_count(res_update, 2) for p in dest.path, str(dest.pathobj / 'subm 1'): assert_in_results(res_update, path=p, action='update', status='ok', type='dataset') # and with merge we would also try to save (but there would be no changes) res_merge = update(path=['subm 1'], recursive=True, merge=True) assert_result_count(res_merge, 2, action='update') # 2 of "updates" really. assert_in_results(res_merge, action='update', status='ok', type='dataset') assert_in_results(res_merge, action='save', status='notneeded', type='dataset') # smoke-test if recursive update doesn't fail if submodule is removed # and that we can run it from within a dataset without providing it # explicitly assert_result_count(dest.remove('subm 1'), 1, status='ok', action='remove', path=opj(dest.path, 'subm 1')) with chpwd(dest.path): assert_result_count(update(recursive=True), 2, status='ok', type='dataset') assert_result_count(dest.update(merge=True, recursive=True), 2, action='update', status='ok', type='dataset') # and now test recursive update with merging in differences create_tree(opj(source.path, '2'), {'load.dat': 'heavy'}) source.save(opj('2', 'load.dat'), message="saving changes within subm2", recursive=True) assert_result_count(dest.update(merge=True, recursive=True), 2, action='update', status='ok', type='dataset') # and now we can get new file dest.get('2/load.dat') ok_file_has_content(opj(dest.path, '2', 'load.dat'), 'heavy')
def test_update_simple(origin, src_path, dst_path): # prepare src source = install(src_path, source=origin, recursive=True) # forget we cloned it (provide no 'origin' anymore), which should lead to # setting tracking branch to target: source.repo.remove_remote("origin") # get a clone to update later on: dest = install(dst_path, source=src_path, recursive=True) # test setup done; # assert all fine ok_clean_git(dst_path) ok_clean_git(src_path) # update yields nothing => up-to-date assert_status('ok', dest.update()) ok_clean_git(dst_path) # modify origin: with open(opj(src_path, "update.txt"), "w") as f: f.write("Additional content") source.add(path="update.txt") source.save("Added update.txt") ok_clean_git(src_path) # update without `merge` only fetches: assert_status('ok', dest.update()) # modification is not known to active branch: assert_not_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch())) # modification is known to branch origin/master assert_in("update.txt", dest.repo.get_files("origin/master")) # merge: assert_status('ok', dest.update(merge=True)) # modification is now known to active branch: assert_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch())) # it's known to annex, but has no content yet: dest.repo.get_file_key("update.txt") # raises if unknown eq_([False], dest.repo.file_has_content(["update.txt"])) # smoke-test if recursive update doesn't fail if submodule is removed # and that we can run it from within a dataset without providing it # explicitly assert_result_count(dest.remove('subm 1'), 1, status='ok', action='remove', path=opj(dest.path, 'subm 1')) with chpwd(dest.path): assert_result_count(update(recursive=True), 2, status='ok', type='dataset') assert_result_count(dest.update(merge=True, recursive=True), 2, status='ok', type='dataset') # and now test recursive update with merging in differences create_tree(opj(source.path, '2'), {'load.dat': 'heavy'}) source.add(opj('2', 'load.dat'), message="saving changes within subm2", recursive=True) assert_result_count(dest.update(merge=True, recursive=True), 2, status='ok', type='dataset') # and now we can get new file dest.get('2/load.dat') ok_file_has_content(opj(dest.path, '2', 'load.dat'), 'heavy')
def test_update_simple(origin, src_path, dst_path): # prepare src source = install(src_path, source=origin, recursive=True) # forget we cloned it (provide no 'origin' anymore), which should lead to # setting tracking branch to target: source.repo.remove_remote("origin") # dataset without sibling will not need updates assert_status('notneeded', source.update()) # deprecation message doesn't ruin things assert_status('notneeded', source.update(fetch_all=True)) # but error if unknown sibling is given assert_status('impossible', source.update(sibling='funky', on_failure='ignore')) # get a clone to update later on: dest = install(dst_path, source=src_path, recursive=True) # test setup done; # assert all fine ok_clean_git(dst_path) ok_clean_git(src_path) # update yields nothing => up-to-date assert_status('ok', dest.update()) ok_clean_git(dst_path) # modify origin: with open(opj(src_path, "update.txt"), "w") as f: f.write("Additional content") source.save(path="update.txt", message="Added update.txt") ok_clean_git(src_path) # fail when asked to update a non-dataset assert_status( 'impossible', source.update("update.txt", on_failure='ignore')) # fail when asked to update a something non-existent assert_status( 'impossible', source.update("nothere", on_failure='ignore')) # update without `merge` only fetches: assert_status('ok', dest.update()) # modification is not known to active branch: assert_not_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch())) # modification is known to branch origin/master assert_in("update.txt", dest.repo.get_files("origin/master")) # merge: assert_status('ok', dest.update(merge=True)) # modification is now known to active branch: assert_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch())) # it's known to annex, but has no content yet: dest.repo.get_file_key("update.txt") # raises if unknown eq_([False], dest.repo.file_has_content(["update.txt"])) # smoke-test if recursive update doesn't fail if submodule is removed # and that we can run it from within a dataset without providing it # explicitly assert_result_count( dest.remove('subm 1'), 1, status='ok', action='remove', path=opj(dest.path, 'subm 1')) with chpwd(dest.path): assert_result_count( update(recursive=True), 2, status='ok', type='dataset') assert_result_count( dest.update(merge=True, recursive=True), 2, status='ok', type='dataset') # and now test recursive update with merging in differences create_tree(opj(source.path, '2'), {'load.dat': 'heavy'}) source.save(opj('2', 'load.dat'), message="saving changes within subm2", recursive=True) assert_result_count( dest.update(merge=True, recursive=True), 2, status='ok', type='dataset') # and now we can get new file dest.get('2/load.dat') ok_file_has_content(opj(dest.path, '2', 'load.dat'), 'heavy')
def test_update_simple(origin=None, src_path=None, dst_path=None): ca = dict(result_renderer='disabled') # a remote dataset with a subdataset underneath origds = Dataset(origin).create(**ca) # naming is weird, but a legacy artifact _ = origds.create('subm 1', **ca) _ = origds.create('2', **ca) # prepare src source = install(src_path, source=origin, recursive=True) # forget we cloned it by removing remote, which should lead to # setting tracking branch to target: source.repo.remove_remote(DEFAULT_REMOTE) # also forget the declared absolute location of the submodules, and turn them # relative to this/a clone for sub in source.subdatasets(result_xfm=lambda x: x['gitmodule_name']): source.subdatasets(path=sub, set_property=[('url', './{}'.format(sub))]) # dataset without sibling will not need updates assert_status('notneeded', source.update()) # deprecation message doesn't ruin things assert_status('notneeded', source.update(fetch_all=True)) # but error if unknown sibling is given assert_status('impossible', source.update(sibling='funky', on_failure='ignore')) # get a clone to update later on: dest = install(dst_path, source=src_path, recursive=True) # test setup done; # assert all fine assert_repo_status(dst_path) assert_repo_status(src_path) # update yields nothing => up-to-date assert_status('ok', dest.update()) assert_repo_status(dst_path) # modify remote: with open(opj(src_path, "update.txt"), "w") as f: f.write("Additional content") source.save(path="update.txt", message="Added update.txt") assert_repo_status(src_path) # update without `merge` only fetches: assert_status('ok', dest.update()) # modification is not known to active branch: assert_not_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch())) # modification is known to branch <default remote>/<default branch> assert_in("update.txt", dest.repo.get_files(DEFAULT_REMOTE + "/" + DEFAULT_BRANCH)) # merge: assert_status('ok', dest.update(merge=True)) # modification is now known to active branch: assert_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch())) # it's known to annex, but has no content yet: annexprops = dest.repo.get_file_annexinfo("update.txt", eval_availability=True) annexprops['key'] # blows if unknown eq_(False, annexprops['has_content']) # check subdataset path constraints, baseline (parent + 2 subds) assert_result_count(dest.update(recursive=True), 3, status='ok', type='dataset') # no recursion and invalid path still updates the parent res = dest.update(path='whatever') assert_result_count(res, 1, status='ok', type='dataset') assert_result_count(res, 1, status='ok', path=dest.path) # invalid path with recursion also does res = dest.update(recursive=True, path='whatever') assert_result_count(res, 1, status='ok', type='dataset') assert_result_count(res, 1, status='ok', path=dest.path) # valid path and no recursion only updates the parent res = dest.update(path='subm 1') assert_result_count(res, 1, status='ok', type='dataset') assert_result_count(res, 1, status='ok', path=dest.path) # valid path and recursion updates matching res = dest.update(recursive=True, path='subm 1') assert_result_count(res, 2, status='ok', type='dataset') assert_result_count(res, 1, status='ok', path=dest.path) assert_result_count(res, 1, status='ok', path=str(dest.pathobj / 'subm 1')) # additional invalid path doesn't hurt res = dest.update(recursive=True, path=['subm 1', 'mike']) assert_result_count(res, 2, status='ok', type='dataset') # full match res = dest.update(recursive=True, path=['subm 1', '2']) assert_result_count(res, 3, status='ok', type='dataset') # test that update doesn't crash if we specify only a single path (submod) to # operate on with chpwd(dest.path): # in 0.11.x it would be a single result since "pwd" dataset is not # considered, and would be relative path (as specified). # In 0.12.0 - it would include implicit pwd dataset, and paths would be absolute res_update = update(path=['subm 1'], recursive=True) assert_result_count(res_update, 2) for p in dest.path, str(dest.pathobj / 'subm 1'): assert_in_results(res_update, path=p, action='update', status='ok', type='dataset') # and with merge we would also try to save (but there would be no changes) res_merge = update(path=['subm 1'], recursive=True, merge=True) assert_result_count(res_merge, 2, action='update') # 2 of "updates" really. assert_in_results(res_merge, action='update', status='ok', type='dataset') assert_in_results(res_merge, action='save', status='notneeded', type='dataset') # smoke-test if recursive update doesn't fail if submodule is removed # and that we can run it from within a dataset without providing it # explicitly assert_result_count(dest.remove('subm 1'), 1, status='ok', action='remove', path=opj(dest.path, 'subm 1')) with chpwd(dest.path): assert_result_count(update(recursive=True), 2, status='ok', type='dataset') assert_result_count(dest.update(merge=True, recursive=True), 2, action='update', status='ok', type='dataset') # and now test recursive update with merging in differences create_tree(opj(source.path, '2'), {'load.dat': 'heavy'}) source.save(opj('2', 'load.dat'), message="saving changes within subm2", recursive=True) assert_result_count(dest.update(merge=True, recursive=True), 2, action='update', status='ok', type='dataset') # and now we can get new file dest.get(opj('2', 'load.dat')) ok_file_has_content(opj(dest.path, '2', 'load.dat'), 'heavy')