annex = AnnexRepo('.', create=False, init=False)
# enable datalad special remote
urls_to_register = defaultdict(list) # key: urls
try:
annex.call_annex(["enableremote", "datalad"])
# go through each and see where urls aren't yet under web
# seems might have also --in=datalad to restrict
w = annex.whereis([], options=['--all'], output='full')
lgr.info("Got %d entries", len(w))
for k, rec in tqdm(w.items()):
datalad_urls = get_remote_urls(rec, 'datalad')
web_urls = set(get_remote_urls(rec, 'web'))
for url in datalad_urls:
if url not in web_urls:
if 'openneuro.s3' in url or 'openfmri.s3' in url:
urls_to_register[k].append(url)
else:
lgr.warning("Found unexpected url %s" % url)
finally:
# disable datalad special remote
annex.remove_remote("datalad") # need to disable it first
lgr.info("Got %d entries which could get new urls", len(urls_to_register))
for k, urls in tqdm(urls_to_register.items()):
for url in urls:
annex.call_annex(
["registerurl", '-c', 'annex.alwayscommit=false', k, url])
# to cause annex to commit all the changes
annex.call_annex(["merge"])
annex.gc(allow_background=False)
annex._annex_custom_command([], ["git", "annex", "enableremote", "datalad"])
# go through each and see where urls aren't yet under web
# seems might have also --in=datalad to restrict
w = annex.whereis([], options=['--all'], output='full')
lgr.info("Got %d entries", len(w))
for k, rec in tqdm(w.items()):
datalad_urls = get_remote_urls(rec, 'datalad')
web_urls = set(get_remote_urls(rec, 'web'))
for url in datalad_urls:
if url not in web_urls:
if 'openneuro.s3' in url or 'openfmri.s3' in url:
urls_to_register[k].append(url)
else:
lgr.warning("Found unexpected url %s" % url)
finally:
# disable datalad special remote
annex.remove_remote("datalad") # need to disable it first
lgr.info(
"Got %d entries which could get new urls",
len(urls_to_register)
)
for k, urls in tqdm(urls_to_register.items()):
for url in urls:
annex._annex_custom_command(
[],
["git", "annex", "registerurl", '-c', 'annex.alwayscommit=false', k, url]
)
# to cause annex to commit all the changes
annex._annex_custom_command([], ["git", "annex", "merge"])
annex.gc(allow_background=False)
