def update_home(force=False):
"""Update an existing DataLad or S3 home."""
if not force and not TF_CACHED:
print("""\
TemplateFlow was not cached (TEMPLATEFLOW_HOME=%s), \
a fresh initialization was done.""" % TF_HOME)
return False
if TF_USE_DATALAD:
from datalad.api import update
print("Updating TemplateFlow's HOME using DataLad ...")
try:
update(str(TF_HOME), recursive=True, merge=True)
except Exception as e:
warn("""Error updating TemplateFlow's home directory (using DataLad):
%s""" % str(e))
return True
# This is an S3 type of installation
from zipfile import ZipFile
with ZipFile(resource_filename('templateflow',
'conf/templateflow-skel.zip'), 'r') as zipref:
allfiles = zipref.namelist()
current_files = [s.relative_to(TF_HOME) for s in TF_HOME.glob('**/*')]
existing = sorted(set(['%s/' % s.parent for s in current_files])) + \
[str(s) for s in current_files]
newfiles = sorted(set(allfiles) - set(existing))
if newfiles:
print("Updating TemplateFlow's HOME using S3. "
"Adding: \n%s" % "\n".join(newfiles))
zipref.extractall(str(TF_HOME), members=newfiles)
return True
print("TemplateFlow's HOME directory (S3 type) was up-to-date.")
return False
def _update_datalad():
from datalad.api import update
print("Updating TEMPLATEFLOW_HOME using DataLad ...")
try:
update(str(TF_HOME), recursive=True, merge=True)
except Exception as e:
warn(f"Error updating TemplateFlow's home directory (using DataLad): {e}")
return True
def setup_home(force=False):
"""Initialize/update TF's home if necessary."""
if not force and not TF_CACHED:
print(f"""\
TemplateFlow was not cached (TEMPLATEFLOW_HOME={TF_HOME}), \
a fresh initialization was done.""")
return False
return update(local=True, overwrite=False)
def test_update_simple(origin, src_path, dst_path):
# prepare src
source = install(src_path, source=origin, recursive=True)
# forget we cloned it (provide no 'origin' anymore), which should lead to
# setting tracking branch to target:
source.repo.remove_remote("origin")
# dataset without sibling will not need updates
assert_status('notneeded', source.update())
# deprecation message doesn't ruin things
assert_status('notneeded', source.update(fetch_all=True))
# but error if unknown sibling is given
assert_status('impossible',
source.update(sibling='funky', on_failure='ignore'))
# get a clone to update later on:
dest = install(dst_path, source=src_path, recursive=True)
# test setup done;
# assert all fine
assert_repo_status(dst_path)
assert_repo_status(src_path)
# update yields nothing => up-to-date
assert_status('ok', dest.update())
assert_repo_status(dst_path)
# modify origin:
with open(opj(src_path, "update.txt"), "w") as f:
f.write("Additional content")
source.save(path="update.txt", message="Added update.txt")
assert_repo_status(src_path)
# update without `merge` only fetches:
assert_status('ok', dest.update())
# modification is not known to active branch:
assert_not_in("update.txt",
dest.repo.get_files(dest.repo.get_active_branch()))
# modification is known to branch origin/<default branch>
assert_in("update.txt", dest.repo.get_files("origin/" + DEFAULT_BRANCH))
# merge:
assert_status('ok', dest.update(merge=True))
# modification is now known to active branch:
assert_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch()))
# it's known to annex, but has no content yet:
dest.repo.get_file_key("update.txt") # raises if unknown
eq_([False], dest.repo.file_has_content(["update.txt"]))
# check subdataset path constraints, baseline (parent + 2 subds)
assert_result_count(dest.update(recursive=True),
3,
status='ok',
type='dataset')
# no recursion and invalid path still updates the parent
res = dest.update(path='whatever')
assert_result_count(res, 1, status='ok', type='dataset')
assert_result_count(res, 1, status='ok', path=dest.path)
# invalid path with recursion also does
res = dest.update(recursive=True, path='whatever')
assert_result_count(res, 1, status='ok', type='dataset')
assert_result_count(res, 1, status='ok', path=dest.path)
# valid path and no recursion only updates the parent
res = dest.update(path='subm 1')
assert_result_count(res, 1, status='ok', type='dataset')
assert_result_count(res, 1, status='ok', path=dest.path)
# valid path and recursion updates matching
res = dest.update(recursive=True, path='subm 1')
assert_result_count(res, 2, status='ok', type='dataset')
assert_result_count(res, 1, status='ok', path=dest.path)
assert_result_count(res, 1, status='ok', path=str(dest.pathobj / 'subm 1'))
# additional invalid path doesn't hurt
res = dest.update(recursive=True, path=['subm 1', 'mike'])
assert_result_count(res, 2, status='ok', type='dataset')
# full match
res = dest.update(recursive=True, path=['subm 1', '2'])
assert_result_count(res, 3, status='ok', type='dataset')
# test that update doesn't crash if we specify only a single path (submod) to
# operate on
with chpwd(dest.path):
# in 0.11.x it would be a single result since "pwd" dataset is not
# considered, and would be relative path (as specified).
# In 0.12.0 - it would include implicit pwd dataset, and paths would be absolute
res_update = update(path=['subm 1'], recursive=True)
assert_result_count(res_update, 2)
for p in dest.path, str(dest.pathobj / 'subm 1'):
assert_in_results(res_update,
path=p,
action='update',
status='ok',
type='dataset')
# and with merge we would also try to save (but there would be no changes)
res_merge = update(path=['subm 1'], recursive=True, merge=True)
assert_result_count(res_merge, 2, action='update')
# 2 of "updates" really.
assert_in_results(res_merge,
action='update',
status='ok',
type='dataset')
assert_in_results(res_merge,
action='save',
status='notneeded',
type='dataset')
# smoke-test if recursive update doesn't fail if submodule is removed
# and that we can run it from within a dataset without providing it
# explicitly
assert_result_count(dest.remove('subm 1'),
1,
status='ok',
action='remove',
path=opj(dest.path, 'subm 1'))
with chpwd(dest.path):
assert_result_count(update(recursive=True),
2,
status='ok',
type='dataset')
assert_result_count(dest.update(merge=True, recursive=True),
2,
action='update',
status='ok',
type='dataset')
# and now test recursive update with merging in differences
create_tree(opj(source.path, '2'), {'load.dat': 'heavy'})
source.save(opj('2', 'load.dat'),
message="saving changes within subm2",
recursive=True)
assert_result_count(dest.update(merge=True, recursive=True),
2,
action='update',
status='ok',
type='dataset')
# and now we can get new file
dest.get('2/load.dat')
ok_file_has_content(opj(dest.path, '2', 'load.dat'), 'heavy')
def test_update_simple(origin, src_path, dst_path):
# prepare src
source = install(src_path, source=origin, recursive=True)
# forget we cloned it (provide no 'origin' anymore), which should lead to
# setting tracking branch to target:
source.repo.remove_remote("origin")
# get a clone to update later on:
dest = install(dst_path, source=src_path, recursive=True)
# test setup done;
# assert all fine
ok_clean_git(dst_path)
ok_clean_git(src_path)
# update yields nothing => up-to-date
assert_status('ok', dest.update())
ok_clean_git(dst_path)
# modify origin:
with open(opj(src_path, "update.txt"), "w") as f:
f.write("Additional content")
source.add(path="update.txt")
source.save("Added update.txt")
ok_clean_git(src_path)
# update without `merge` only fetches:
assert_status('ok', dest.update())
# modification is not known to active branch:
assert_not_in("update.txt",
dest.repo.get_files(dest.repo.get_active_branch()))
# modification is known to branch origin/master
assert_in("update.txt", dest.repo.get_files("origin/master"))
# merge:
assert_status('ok', dest.update(merge=True))
# modification is now known to active branch:
assert_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch()))
# it's known to annex, but has no content yet:
dest.repo.get_file_key("update.txt") # raises if unknown
eq_([False], dest.repo.file_has_content(["update.txt"]))
# smoke-test if recursive update doesn't fail if submodule is removed
# and that we can run it from within a dataset without providing it
# explicitly
assert_result_count(dest.remove('subm 1'),
1,
status='ok',
action='remove',
path=opj(dest.path, 'subm 1'))
with chpwd(dest.path):
assert_result_count(update(recursive=True),
2,
status='ok',
type='dataset')
assert_result_count(dest.update(merge=True, recursive=True),
2,
status='ok',
type='dataset')
# and now test recursive update with merging in differences
create_tree(opj(source.path, '2'), {'load.dat': 'heavy'})
source.add(opj('2', 'load.dat'),
message="saving changes within subm2",
recursive=True)
assert_result_count(dest.update(merge=True, recursive=True),
2,
status='ok',
type='dataset')
# and now we can get new file
dest.get('2/load.dat')
ok_file_has_content(opj(dest.path, '2', 'load.dat'), 'heavy')
def test_update_simple(origin, src_path, dst_path):
# prepare src
source = install(src_path, source=origin, recursive=True)
# forget we cloned it (provide no 'origin' anymore), which should lead to
# setting tracking branch to target:
source.repo.remove_remote("origin")
# dataset without sibling will not need updates
assert_status('notneeded', source.update())
# deprecation message doesn't ruin things
assert_status('notneeded', source.update(fetch_all=True))
# but error if unknown sibling is given
assert_status('impossible', source.update(sibling='funky', on_failure='ignore'))
# get a clone to update later on:
dest = install(dst_path, source=src_path, recursive=True)
# test setup done;
# assert all fine
ok_clean_git(dst_path)
ok_clean_git(src_path)
# update yields nothing => up-to-date
assert_status('ok', dest.update())
ok_clean_git(dst_path)
# modify origin:
with open(opj(src_path, "update.txt"), "w") as f:
f.write("Additional content")
source.save(path="update.txt", message="Added update.txt")
ok_clean_git(src_path)
# fail when asked to update a non-dataset
assert_status(
'impossible',
source.update("update.txt", on_failure='ignore'))
# fail when asked to update a something non-existent
assert_status(
'impossible',
source.update("nothere", on_failure='ignore'))
# update without `merge` only fetches:
assert_status('ok', dest.update())
# modification is not known to active branch:
assert_not_in("update.txt",
dest.repo.get_files(dest.repo.get_active_branch()))
# modification is known to branch origin/master
assert_in("update.txt", dest.repo.get_files("origin/master"))
# merge:
assert_status('ok', dest.update(merge=True))
# modification is now known to active branch:
assert_in("update.txt",
dest.repo.get_files(dest.repo.get_active_branch()))
# it's known to annex, but has no content yet:
dest.repo.get_file_key("update.txt") # raises if unknown
eq_([False], dest.repo.file_has_content(["update.txt"]))
# smoke-test if recursive update doesn't fail if submodule is removed
# and that we can run it from within a dataset without providing it
# explicitly
assert_result_count(
dest.remove('subm 1'), 1,
status='ok', action='remove', path=opj(dest.path, 'subm 1'))
with chpwd(dest.path):
assert_result_count(
update(recursive=True), 2,
status='ok', type='dataset')
assert_result_count(
dest.update(merge=True, recursive=True), 2,
status='ok', type='dataset')
# and now test recursive update with merging in differences
create_tree(opj(source.path, '2'), {'load.dat': 'heavy'})
source.save(opj('2', 'load.dat'),
message="saving changes within subm2",
recursive=True)
assert_result_count(
dest.update(merge=True, recursive=True), 2,
status='ok', type='dataset')
# and now we can get new file
dest.get('2/load.dat')
ok_file_has_content(opj(dest.path, '2', 'load.dat'), 'heavy')
def test_update_simple(origin=None, src_path=None, dst_path=None):
ca = dict(result_renderer='disabled')
# a remote dataset with a subdataset underneath
origds = Dataset(origin).create(**ca)
# naming is weird, but a legacy artifact
_ = origds.create('subm 1', **ca)
_ = origds.create('2', **ca)
# prepare src
source = install(src_path, source=origin, recursive=True)
# forget we cloned it by removing remote, which should lead to
# setting tracking branch to target:
source.repo.remove_remote(DEFAULT_REMOTE)
# also forget the declared absolute location of the submodules, and turn them
# relative to this/a clone
for sub in source.subdatasets(result_xfm=lambda x: x['gitmodule_name']):
source.subdatasets(path=sub,
set_property=[('url', './{}'.format(sub))])
# dataset without sibling will not need updates
assert_status('notneeded', source.update())
# deprecation message doesn't ruin things
assert_status('notneeded', source.update(fetch_all=True))
# but error if unknown sibling is given
assert_status('impossible',
source.update(sibling='funky', on_failure='ignore'))
# get a clone to update later on:
dest = install(dst_path, source=src_path, recursive=True)
# test setup done;
# assert all fine
assert_repo_status(dst_path)
assert_repo_status(src_path)
# update yields nothing => up-to-date
assert_status('ok', dest.update())
assert_repo_status(dst_path)
# modify remote:
with open(opj(src_path, "update.txt"), "w") as f:
f.write("Additional content")
source.save(path="update.txt", message="Added update.txt")
assert_repo_status(src_path)
# update without `merge` only fetches:
assert_status('ok', dest.update())
# modification is not known to active branch:
assert_not_in("update.txt",
dest.repo.get_files(dest.repo.get_active_branch()))
# modification is known to branch <default remote>/<default branch>
assert_in("update.txt",
dest.repo.get_files(DEFAULT_REMOTE + "/" + DEFAULT_BRANCH))
# merge:
assert_status('ok', dest.update(merge=True))
# modification is now known to active branch:
assert_in("update.txt", dest.repo.get_files(dest.repo.get_active_branch()))
# it's known to annex, but has no content yet:
annexprops = dest.repo.get_file_annexinfo("update.txt",
eval_availability=True)
annexprops['key'] # blows if unknown
eq_(False, annexprops['has_content'])
# check subdataset path constraints, baseline (parent + 2 subds)
assert_result_count(dest.update(recursive=True),
3,
status='ok',
type='dataset')
# no recursion and invalid path still updates the parent
res = dest.update(path='whatever')
assert_result_count(res, 1, status='ok', type='dataset')
assert_result_count(res, 1, status='ok', path=dest.path)
# invalid path with recursion also does
res = dest.update(recursive=True, path='whatever')
assert_result_count(res, 1, status='ok', type='dataset')
assert_result_count(res, 1, status='ok', path=dest.path)
# valid path and no recursion only updates the parent
res = dest.update(path='subm 1')
assert_result_count(res, 1, status='ok', type='dataset')
assert_result_count(res, 1, status='ok', path=dest.path)
# valid path and recursion updates matching
res = dest.update(recursive=True, path='subm 1')
assert_result_count(res, 2, status='ok', type='dataset')
assert_result_count(res, 1, status='ok', path=dest.path)
assert_result_count(res, 1, status='ok', path=str(dest.pathobj / 'subm 1'))
# additional invalid path doesn't hurt
res = dest.update(recursive=True, path=['subm 1', 'mike'])
assert_result_count(res, 2, status='ok', type='dataset')
# full match
res = dest.update(recursive=True, path=['subm 1', '2'])
assert_result_count(res, 3, status='ok', type='dataset')
# test that update doesn't crash if we specify only a single path (submod) to
# operate on
with chpwd(dest.path):
# in 0.11.x it would be a single result since "pwd" dataset is not
# considered, and would be relative path (as specified).
# In 0.12.0 - it would include implicit pwd dataset, and paths would be absolute
res_update = update(path=['subm 1'], recursive=True)
assert_result_count(res_update, 2)
for p in dest.path, str(dest.pathobj / 'subm 1'):
assert_in_results(res_update,
path=p,
action='update',
status='ok',
type='dataset')
# and with merge we would also try to save (but there would be no changes)
res_merge = update(path=['subm 1'], recursive=True, merge=True)
assert_result_count(res_merge, 2, action='update')
# 2 of "updates" really.
assert_in_results(res_merge,
action='update',
status='ok',
type='dataset')
assert_in_results(res_merge,
action='save',
status='notneeded',
type='dataset')
# smoke-test if recursive update doesn't fail if submodule is removed
# and that we can run it from within a dataset without providing it
# explicitly
assert_result_count(dest.remove('subm 1'),
1,
status='ok',
action='remove',
path=opj(dest.path, 'subm 1'))
with chpwd(dest.path):
assert_result_count(update(recursive=True),
2,
status='ok',
type='dataset')
assert_result_count(dest.update(merge=True, recursive=True),
2,
action='update',
status='ok',
type='dataset')
# and now test recursive update with merging in differences
create_tree(opj(source.path, '2'), {'load.dat': 'heavy'})
source.save(opj('2', 'load.dat'),
message="saving changes within subm2",
recursive=True)
assert_result_count(dest.update(merge=True, recursive=True),
2,
action='update',
status='ok',
type='dataset')
# and now we can get new file
dest.get(opj('2', 'load.dat'))
ok_file_has_content(opj(dest.path, '2', 'load.dat'), 'heavy')