def download_bids(
client, to_download, root_path,
folders_to_download=['anat', 'dwi', 'func', 'fmap', 'perf'],
attachments=None, dry_run=True, name='bids_dataset'
):
if dry_run:
logger.info("Preparing output directory tree...")
else:
logger.info("Downloading files...")
root_path = "/".join([root_path, name])
Path(root_path).mkdir()
# handle dataset description
if to_download['dataset_description']:
description = to_download['dataset_description'][0]
path = "/".join([root_path, description['name']])
if dry_run:
Path(path).touch()
else:
download_sidecar(description['data'], path, remove_bids=False)
# write bids ignore
if not any(x['name'] == '.bidsignore' for x in to_download['project']):
# write bids ignore
path = "/".join([root_path, ".bidsignore"])
ignored_modalities = ['perf/', 'qsm/', '**/fmap/*.bvec', '**/fmap/*.bval']
if dry_run:
Path(path).touch()
else:
with open(path, 'w') as bidsignore:
bidsignore.writelines('\n'.join(ignored_modalities))
# deal with project level files
# Project's subject data
for fi in to_download['project']:
if fi['BIDS'] is not None:
output_path = get_nested(fi, 'BIDS', 'Path')
path = str(Path(output_path, fi['name']))
if not Path(output_path).exists():
os.makedirs(Path(output_path).resolve(), exist_ok=True)
if dry_run:
Path(path).touch()
else:
container = client.get(fi['data'])
container.download_file(fi['name'], path)
# deal with subject level files
for fi in to_download['subject']:
if attachments:
if fi['name'] not in attachments:
continue
if fi['BIDS'] is not None:
output_path = get_nested(fi, 'BIDS', 'Path')
path = str(Path(output_path, fi['name']))
if not Path(output_path).exists():
os.makedirs(Path(output_path).resolve(), exist_ok=True)
if dry_run:
Path(path).touch()
else:
container = client.get(fi['data'])
container.download_file(fi['name'], path)
for fi in to_download['session']:
if attachments:
if not any([re.search(att, fi['name']) for att in attachments]):
continue
if fi['BIDS'] is not None:
output_path = Path(root_path, get_nested(fi, 'BIDS', 'Path'))
path = str(Path(output_path, fi['name']))
if not Path(output_path).exists():
os.makedirs(Path(output_path).resolve(), exist_ok=True)
if dry_run:
Path(path).touch()
else:
container = client.get(fi['data'])
container.download_file(fi['name'], path)
# deal with acquisition level files
for fi in to_download['acquisition']:
project_path = get_nested(fi, 'BIDS', 'Path')
folder = get_nested(fi, 'BIDS', 'Folder')
ignore = get_nested(fi, 'BIDS', 'ignore')
if project_path and folder in folders_to_download and not ignore:
# only download files with sidecars
if 'sidecar' in fi:
fname = get_nested(fi, 'BIDS', 'Filename')
extensions = ['nii.gz', 'bval', 'bvec']
sidecar_name = fname
for x in extensions:
sidecar_name = sidecar_name.replace(x, 'json')
download_path = '/'.join([root_path, project_path])
file_path = '/'.join([download_path, fname])
if Path(file_path).exists():
logger.error("Found conflicting file paths:")
logger.error(file_path)
logger.error("Cleaning up...")
shutil.rmtree(root_path)
raise FileExistsError
sidecar_path = '/'.join([download_path, sidecar_name])
acq = client.get(fi['data'])
if not os.path.exists(download_path):
os.makedirs(download_path)
if dry_run:
Path(file_path).touch()
Path(sidecar_path).touch()
else:
acq.download_file(fi['name'], file_path)
download_sidecar(fi['sidecar'], sidecar_path, remove_bids=True)
#exception: it may be an events tsv
elif any(x in fi['name'] for x in ['bval', 'bvec', 'tsv']):
fname = get_nested(fi, 'BIDS', 'Filename')
download_path = '/'.join([root_path, project_path])
file_path = '/'.join([download_path, fname])
acq = client.get(fi['data'])
if not os.path.exists(download_path):
os.makedirs(download_path)
if dry_run:
Path(file_path).touch()
Path(sidecar_path).touch()
else:
acq.download_file(fi['name'], file_path)
#check_tasks(root_path)
logger.info("Done!")
print_directory_tree(root_path)
if not os.path.exists(path):
os.makedirs(path)
if os.path.isfile("/".join([path, fname])):
logger.info("Overwriting current T2w image...")
os.remove("/".join([path, fname]))
t2_acq.download_file(nifti.name, "/".join([path, fname]))
sidecar_name = fname.replace('.nii.gz', '.json')
if os.path.isfile(path + sidecar_name):
os.remove(path + sidecar_name)
export.download_sidecar(sidecar, "/".join([path, sidecar_name]))
if t1_acq or t2_acq:
logger.info("Final directory tree with additional anatomical files:")
print_directory_tree("/flywheel/v0/output/bids_dataset")
if not dry_run:
pass
# tidy up
# output_dir = "/flywheel/v0/output"
# os.system("zip -r {}_BIDSexport.zip output/*".format(destination['id']))
# os.system("mv *.zip output")
# to_remove = os.listdir(output_dir)
# to_remove = ["{}/{}".format(output_dir, x) for x in to_remove if ".zip" not in x]
# for x in to_remove:
# if os.path.isfile(x):
# os.remove(x)
# else:
# shutil.rmtree(x)
def download_bids(client,
to_download,
root_path,
folders_to_download=['anat', 'dwi', 'func', 'fmap'],
dry_run=True,
name='bids_dataset'):
if dry_run:
logger.info("Preparing output directory tree...")
else:
logger.info("Downloading files...")
root_path = "/".join([root_path, name])
Path(root_path).mkdir()
# handle dataset description
if to_download['dataset_description']:
description = to_download['dataset_description'][0]
path = "/".join([root_path, description['name']])
if dry_run:
Path(path).touch()
else:
download_sidecar(description['data'], path, remove_bids=False)
# write bids ignore
if not any(x['name'] == '.bidsignore' for x in to_download['project']):
# write bids ignore
path = "/".join([root_path, ".bidsignore"])
ignored_modalities = [
'asl/', 'qsm/', '**/fmap/*.bvec', '**/fmap/*.bval'
]
if dry_run:
Path(path).touch()
else:
with open(path, 'w') as bidsignore:
bidsignore.writelines('\n'.join(ignored_modalities))
# deal with project level files
# Project's subject data
for fi in to_download['project']:
project_path = get_nested(fi, 'BIDS', 'Path')
folder = get_nested(fi, 'BIDS', 'Folder')
ignore = get_nested(fi, 'BIDS', 'ignore')
if project_path \
and folder in folders_to_download \
and not ignore \
and any(fi['name'] == 'participants.tsv' or fi['name'] == 'participants.json'):
proj = client.get(fi['data'])
#download_path = get_metadata(fi, ['BIDS', 'Path'])
#if download_path:
# print('/'.join([root_path, download_path, fi['name']]))
#proj.download_file(fi['name'], file_path)
#download_sidecar(fi['sidecar'], sidecar_path, remove_bids=True)
# deal with session level files
# NOT YET IMPLEMENTED
for fi in to_download['session']:
pass
#download_path = get_metadata(fi, ['BIDS', 'Path'])
#if download_path:
# print('/'.join([root_path, download_path, fi['name']]))
# deal with acquisition level files
for fi in to_download['acquisition']:
project_path = get_nested(fi, 'BIDS', 'Path')
folder = get_nested(fi, 'BIDS', 'Folder')
ignore = get_nested(fi, 'BIDS', 'ignore')
if project_path and folder in folders_to_download and not ignore:
# only download files with sidecars
if 'sidecar' in fi:
fname = get_nested(fi, 'BIDS', 'Filename')
extensions = ['nii.gz', 'bval', 'bvec']
sidecar_name = fname
for x in extensions:
sidecar_name = sidecar_name.replace(x, 'json')
download_path = '/'.join([root_path, project_path])
file_path = '/'.join([download_path, fname])
sidecar_path = '/'.join([download_path, sidecar_name])
acq = client.get(fi['data'])
if not os.path.exists(download_path):
os.makedirs(download_path)
if dry_run:
Path(file_path).touch()
Path(sidecar_path).touch()
else:
acq.download_file(fi['name'], file_path)
download_sidecar(fi['sidecar'],
sidecar_path,
remove_bids=True)
#exception: it may be an events tsv
elif any(x in fi['name'] for x in ['bval', 'bvec', 'tsv']):
fname = get_nested(fi, 'BIDS', 'Filename')
download_path = '/'.join([root_path, project_path])
file_path = '/'.join([download_path, fname])
acq = client.get(fi['data'])
if not os.path.exists(download_path):
os.makedirs(download_path)
if dry_run:
Path(file_path).touch()
Path(sidecar_path).touch()
else:
acq.download_file(fi['name'], file_path)
#check_tasks(root_path)
logger.info("Done!")
print_directory_tree(root_path)