annex = AnnexRepo('.', create=False, init=False) # enable datalad special remote urls_to_register = defaultdict(list) # key: urls try: annex.call_annex(["enableremote", "datalad"]) # go through each and see where urls aren't yet under web # seems might have also --in=datalad to restrict w = annex.whereis([], options=['--all'], output='full') lgr.info("Got %d entries", len(w)) for k, rec in tqdm(w.items()): datalad_urls = get_remote_urls(rec, 'datalad') web_urls = set(get_remote_urls(rec, 'web')) for url in datalad_urls: if url not in web_urls: if 'openneuro.s3' in url or 'openfmri.s3' in url: urls_to_register[k].append(url) else: lgr.warning("Found unexpected url %s" % url) finally: # disable datalad special remote annex.remove_remote("datalad") # need to disable it first lgr.info("Got %d entries which could get new urls", len(urls_to_register)) for k, urls in tqdm(urls_to_register.items()): for url in urls: annex.call_annex( ["registerurl", '-c', 'annex.alwayscommit=false', k, url]) # to cause annex to commit all the changes annex.call_annex(["merge"]) annex.gc(allow_background=False)
annex._annex_custom_command([], ["git", "annex", "enableremote", "datalad"]) # go through each and see where urls aren't yet under web # seems might have also --in=datalad to restrict w = annex.whereis([], options=['--all'], output='full') lgr.info("Got %d entries", len(w)) for k, rec in tqdm(w.items()): datalad_urls = get_remote_urls(rec, 'datalad') web_urls = set(get_remote_urls(rec, 'web')) for url in datalad_urls: if url not in web_urls: if 'openneuro.s3' in url or 'openfmri.s3' in url: urls_to_register[k].append(url) else: lgr.warning("Found unexpected url %s" % url) finally: # disable datalad special remote annex.remove_remote("datalad") # need to disable it first lgr.info( "Got %d entries which could get new urls", len(urls_to_register) ) for k, urls in tqdm(urls_to_register.items()): for url in urls: annex._annex_custom_command( [], ["git", "annex", "registerurl", '-c', 'annex.alwayscommit=false', k, url] ) # to cause annex to commit all the changes annex._annex_custom_command([], ["git", "annex", "merge"]) annex.gc(allow_background=False)