def medeface(bidsdir: str, pattern: str, maskpattern: str, subjects: list,
force: bool, output: str, cluster: bool, nativespec: str,
kwargs: dict):
"""
:param bidsdir: The bids-directory with the (multi-echo) subject data
:param pattern: Globlike search pattern (relative to the subject/session folder) to select the echo-images that need to be defaced, e.g. 'anat/*_T1w*'
:param maskpattern: Globlike search pattern (relative to the subject/session folder) to select the images from which the defacemask is computed, e.g. 'anat/*_part-mag_*_T2starw*'. If not given then 'pattern' is used
:param subjects: List of sub-# identifiers to be processed (the sub- prefix can be left out). If not specified then all sub-folders in the bidsfolder will be processed
:param force: If True then images will be processed, regardless if images have already been defaced (i.e. if {"Defaced": True} in the json sidecar file)
:param output: Determines where the defaced images are saved. It can be the name of a BIDS datatype folder, such as 'anat', or of the derivatives folder, i.e. 'derivatives'. If output is left empty then the original images are replaced by the defaced images
:param cluster: Flag to submit the deface jobs to the high-performance compute (HPC) cluster
:param nativespec: DRMAA native specifications for submitting deface jobs to the HPC cluster
:param kwargs: Additional arguments (in dict/json-style) that are passed to pydeface. See examples for usage
:return:
"""
# Input checking
bidsdir = Path(bidsdir).resolve()
if not maskpattern:
maskpattern = pattern
# Start logging
bidscoin.setup_logging(bidsdir / 'code' / 'bidscoin' / 'deface.log')
LOGGER.info('')
LOGGER.info('------------ START multi-echo deface ----------')
LOGGER.info(
f">>> medeface bidsfolder={bidsdir} pattern={pattern} subjects={subjects} output={output}"
f" cluster={cluster} nativespec={nativespec} {kwargs}")
# Get the list of subjects
if not subjects:
subjects = bidscoin.lsdirs(bidsdir, 'sub-*')
if not subjects:
LOGGER.warning(f"No subjects found in: {bidsdir/'sub-*'}")
else:
subjects = [
'sub-' + subject.replace('sub-', '') for subject in subjects
] # Make sure there is a "sub-" prefix
subjects = [
bidsdir / subject for subject in subjects
if (bidsdir / subject).is_dir()
]
# Prepare the HPC pydeface job submission
with drmaa.Session() as pbatch:
if cluster:
jt = pbatch.createJobTemplate()
jt.jobEnvironment = os.environ
jt.remoteCommand = shutil.which('pydeface')
jt.nativeSpecification = nativespec
jt.joinFiles = True
# Loop over bids subject/session-directories to first get all the echo-combined deface masks
for n, subject in enumerate(subjects, 1):
sessions = bidscoin.lsdirs(subject, 'ses-*')
if not sessions:
sessions = [subject]
for session in sessions:
LOGGER.info('--------------------------------------')
LOGGER.info(f"Processing ({n}/{len(subjects)}): {session}")
datasource = bids.DataSource(session / 'dum.my',
subprefix='sub-',
sesprefix='ses-')
subid, sesid = datasource.subid_sesid()
# Read the echo-images that will be combined to compute the deface mask
echofiles = sorted([
match for match in session.glob(maskpattern)
if '.nii' in match.suffixes
])
if not echofiles:
LOGGER.info(
f'No mask files found for: {session}/{maskpattern}')
continue
# Check the json "Defaced" field to see if it has already been defaced
if not force:
with echofiles[0].with_suffix('').with_suffix(
'.json').open('r') as fid:
jsondata = json.load(fid)
if jsondata.get('Defaced'):
LOGGER.info(
f"Skipping already defaced images: {[str(echofile) for echofile in echofiles]}"
)
continue
LOGGER.info(
f'Loading mask files: {[str(echofile) for echofile in echofiles]}'
)
echos = [nib.load(echofile) for echofile in echofiles]
# Create a temporary echo-combined image
tmpfile = session / 'tmp_echocombined_deface.nii'
combined = nib.Nifti1Image(
np.mean([echo.get_fdata() for echo in echos], axis=0),
echos[0].affine, echos[0].header)
combined.to_filename(tmpfile)
# Deface the echo-combined image
LOGGER.info(
f"Creating a deface-mask from the echo-combined image: {tmpfile}"
)
if cluster:
jt.args = [
str(tmpfile), '--outfile',
str(tmpfile), '--force'
] + [
item for pair in [[f"--{key}", val]
for key, val in kwargs.items()]
for item in pair
]
jt.jobName = f"pydeface_{subid}_{sesid}"
jobid = pbatch.runJob(jt)
LOGGER.info(
f"Your deface job has been submitted with ID: {jobid}")
else:
pdu.deface_image(str(tmpfile),
str(tmpfile),
force=True,
forcecleanup=True,
**kwargs)
if cluster:
LOGGER.info('Waiting for the deface jobs to finish...')
pbatch.synchronize(jobIds=[pbatch.JOB_IDS_SESSION_ALL],
timeout=pbatch.TIMEOUT_WAIT_FOREVER,
dispose=True)
pbatch.deleteJobTemplate(jt)
# Loop again over bids subject/session-directories to apply the deface masks and write meta-data
with logging_redirect_tqdm():
for n, subject in enumerate(
tqdm(subjects, unit='subject', leave=False), 1):
sessions = bidscoin.lsdirs(subject, 'ses-*')
if not sessions:
sessions = [subject]
for session in sessions:
LOGGER.info('--------------------------------------')
LOGGER.info(f"Processing ({n}/{len(subjects)}): {session}")
datasource = bids.DataSource(session / 'dum.my',
subprefix='sub-',
sesprefix='ses-')
subid, sesid = datasource.subid_sesid()
# Read the temporary defacemask
tmpfile = session / 'tmp_echocombined_deface.nii'
if not tmpfile.is_file():
LOGGER.info(f'No {tmpfile} file found')
continue
defacemask = nib.load(tmpfile).get_fdata(
) != 0 # The original defacemask is saved in a temporary folder so it may be deleted -> use the defaced image to infer the mask
tmpfile.unlink()
# Process the echo-images that need to be defaced
for echofile in sorted([
match for match in session.glob(pattern)
if '.nii' in match.suffixes
]):
# Construct the output filename and relative path name (used in BIDS)
echofile_rel = echofile.relative_to(session).as_posix()
if not output:
outputfile = echofile
outputfile_rel = echofile_rel
elif output == 'derivatives':
outputfile = bidsdir / 'derivatives' / 'deface' / subid / sesid / echofile.parent.name / echofile.name
outputfile_rel = outputfile.relative_to(
bidsdir).as_posix()
else:
outputfile = session / output / echofile.name
outputfile_rel = outputfile.relative_to(
session).as_posix()
outputfile.parent.mkdir(parents=True, exist_ok=True)
# Apply the defacemask
LOGGER.info(
f'Applying deface mask on: {echofile} -> {outputfile_rel}'
)
echoimg = nib.load(echofile)
outputimg = nib.Nifti1Image(
echoimg.get_fdata() * defacemask, echoimg.affine,
echoimg.header)
outputimg.to_filename(outputfile)
# Overwrite or add a json sidecar-file
inputjson = echofile.with_suffix('').with_suffix('.json')
outputjson = outputfile.with_suffix('').with_suffix(
'.json')
if inputjson.is_file() and inputjson != outputjson:
if outputjson.is_file():
LOGGER.info(
f"Overwriting the json sidecar-file: {outputjson}"
)
outputjson.unlink()
else:
LOGGER.info(
f"Adding a json sidecar-file: {outputjson}")
shutil.copyfile(inputjson, outputjson)
# Add a custom "Defaced" field to the json sidecar-file
with outputjson.open('r') as output_fid:
data = json.load(output_fid)
data['Defaced'] = True
with outputjson.open('w') as output_fid:
json.dump(data, output_fid, indent=4)
# Update the IntendedFor fields in the fieldmap sidecar-files NB: IntendedFor must be relative to the subject folder
if output and output != 'derivatives' and (
session / 'fmap').is_dir():
for fmap in (session / 'fmap').glob('*.json'):
with fmap.open('r') as fmap_fid:
fmap_data = json.load(fmap_fid)
intendedfor = fmap_data['IntendedFor']
if isinstance(intendedfor, str):
intendedfor = [intendedfor]
if (Path(sesid) /
echofile_rel).as_posix() in intendedfor:
LOGGER.info(
f"Updating 'IntendedFor' to {Path(sesid)/outputfile_rel} in {fmap}"
)
fmap_data['IntendedFor'] = intendedfor + [
(Path(sesid) / outputfile_rel).as_posix()
]
with fmap.open('w') as fmap_fid:
json.dump(fmap_data, fmap_fid, indent=4)
# Update the scans.tsv file
if (bidsdir / '.bidsignore').is_file():
bidsignore = (bidsdir /
'.bidsignore').read_text().splitlines()
else:
bidsignore = []
bidsignore.append('derivatives/')
scans_tsv = session / f"{subid}{bids.add_prefix('_',sesid)}_scans.tsv"
if output and output + '/' not in bidsignore and scans_tsv.is_file(
):
LOGGER.info(f"Adding {outputfile_rel} to {scans_tsv}")
scans_table = pd.read_csv(scans_tsv,
sep='\t',
index_col='filename')
scans_table.loc[outputfile_rel] = scans_table.loc[
echofile_rel]
scans_table.sort_values(by=['acq_time', 'filename'],
inplace=True)
scans_table.to_csv(scans_tsv,
sep='\t',
encoding='utf-8')
LOGGER.info('-------------- FINISHED! -------------')
LOGGER.info('')
def bidscoiner(rawfolder: str,
bidsfolder: str,
subjects: list = (),
force: bool = False,
participants: bool = False,
bidsmapfile: str = 'bidsmap.yaml') -> None:
"""
Main function that processes all the subjects and session in the sourcefolder and uses the
bidsmap.yaml file in bidsfolder/code/bidscoin to cast the data into the BIDS folder.
:param rawfolder: The root folder-name of the sub/ses/data/file tree containing the source data files
:param bidsfolder: The name of the BIDS root folder
:param subjects: List of selected subjects / participants (i.e. sub-# names / folders) to be processed (the sub- prefix can be removed). Otherwise all subjects in the sourcefolder will be selected
:param force: If True, subjects will be processed, regardless of existing folders in the bidsfolder. Otherwise existing folders will be skipped
:param participants: If True, subjects in particpants.tsv will not be processed (this could be used e.g. to protect these subjects from being reprocessed), also when force=True
:param bidsmapfile: The name of the bidsmap YAML-file. If the bidsmap pathname is relative (i.e. no "/" in the name) then it is assumed to be located in bidsfolder/code/bidscoin
:return: Nothing
"""
# Input checking & defaults
rawfolder = Path(rawfolder).resolve()
bidsfolder = Path(bidsfolder).resolve()
bidsmapfile = Path(bidsmapfile)
# Start logging
bidscoin.setup_logging(bidsfolder / 'code' / 'bidscoin' / 'bidscoiner.log')
LOGGER.info('')
LOGGER.info(
f"-------------- START BIDScoiner {localversion}: BIDS {bidscoin.bidsversion()} ------------"
)
LOGGER.info(
f">>> bidscoiner sourcefolder={rawfolder} bidsfolder={bidsfolder} subjects={subjects} force={force} participants={participants} bidsmap={bidsmapfile}"
)
# Create a code/bidscoin subfolder
(bidsfolder / 'code' / 'bidscoin').mkdir(parents=True, exist_ok=True)
# Create a dataset description file if it does not exist
dataset_file = bidsfolder / 'dataset_description.json'
generatedby = [{
"Name": "BIDScoin",
"Version": localversion,
"CodeURL": "https://github.com/Donders-Institute/bidscoin"
}]
if not dataset_file.is_file():
LOGGER.info(f"Creating dataset description file: {dataset_file}")
dataset_description = {
"Name":
"REQUIRED. Name of the dataset",
"GeneratedBy":
generatedby,
"BIDSVersion":
str(bidscoin.bidsversion()),
"DatasetType":
"raw",
"License":
"RECOMMENDED. The license for the dataset. The use of license name abbreviations is RECOMMENDED for specifying a license. The corresponding full license text MAY be specified in an additional LICENSE file",
"Authors": [
"OPTIONAL. List of individuals who contributed to the creation/curation of the dataset"
],
"Acknowledgements":
"OPTIONAL. Text acknowledging contributions of individuals or institutions beyond those listed in Authors or Funding",
"HowToAcknowledge":
"OPTIONAL. Instructions how researchers using this dataset should acknowledge the original authors. This field can also be used to define a publication that should be cited in publications that use the dataset",
"Funding":
["OPTIONAL. List of sources of funding (grant numbers)"],
"EthicsApprovals": [
"OPTIONAL. List of ethics committee approvals of the research protocols and/or protocol identifiers"
],
"ReferencesAndLinks": [
"OPTIONAL. List of references to publication that contain information on the dataset, or links",
"https://github.com/Donders-Institute/bidscoin"
],
"DatasetDOI":
"OPTIONAL. The Document Object Identifier of the dataset (not the corresponding paper)"
}
else:
with dataset_file.open('r') as fid:
dataset_description = json.load(fid)
if 'BIDScoin' not in [
generatedby_['Name']
for generatedby_ in dataset_description.get('GeneratedBy', [])
]:
LOGGER.info(f"Adding {generatedby} to {dataset_file}")
dataset_description['GeneratedBy'] = dataset_description.get(
'GeneratedBy', []) + generatedby
with dataset_file.open('w') as fid:
json.dump(dataset_description, fid, indent=4)
# Create a README file if it does not exist
readme_file = bidsfolder / 'README'
if not readme_file.is_file():
LOGGER.info(f"Creating README file: {readme_file}")
readme_file.write_text(
f"A free form text ( README ) describing the dataset in more details that SHOULD be provided\n\n"
f"The raw BIDS data was created using BIDScoin {localversion}\n"
f"All provenance information and settings can be found in ./code/bidscoin\n"
f"For more information see: https://github.com/Donders-Institute/bidscoin\n"
)
# Get the bidsmap heuristics from the bidsmap YAML-file
bidsmap, _ = bids.load_bidsmap(bidsmapfile,
bidsfolder / 'code' / 'bidscoin')
dataformats = [
dataformat for dataformat in bidsmap
if dataformat and dataformat not in ('Options', 'PlugIns')
] # Handle legacy bidsmaps (-> 'PlugIns')
if not bidsmap:
LOGGER.error(
f"No bidsmap file found in {bidsfolder}. Please run the bidsmapper first and/or use the correct bidsfolder"
)
return
# Load the data conversion plugins
plugins = [
bidscoin.import_plugin(plugin, ('bidscoiner_plugin', ))
for plugin, options in bidsmap['Options']['plugins'].items()
]
plugins = [plugin for plugin in plugins
if plugin] # Filter the empty items from the list
if not plugins:
LOGGER.warning(
f"The plugins listed in your bidsmap['Options'] did not have a usable `bidscoiner_plugin` function, nothing to do"
)
LOGGER.info('-------------- FINISHED! ------------')
LOGGER.info('')
return
# Append options to the .bidsignore file
bidsignore_items = [
item.strip()
for item in bidsmap['Options']['bidscoin']['bidsignore'].split(';')
]
bidsignore_file = bidsfolder / '.bidsignore'
if bidsignore_items:
LOGGER.info(f"Writing {bidsignore_items} entries to {bidsignore_file}")
if bidsignore_file.is_file():
bidsignore_items += bidsignore_file.read_text().splitlines()
with bidsignore_file.open('w') as bidsignore:
for item in set(bidsignore_items):
bidsignore.write(item + '\n')
# Get the table & dictionary of the subjects that have been processed
participants_tsv = bidsfolder / 'participants.tsv'
if participants_tsv.is_file():
participants_table = pd.read_csv(participants_tsv, sep='\t')
participants_table.set_index(['participant_id'],
verify_integrity=True,
inplace=True)
else:
participants_table = pd.DataFrame()
participants_table.index.name = 'participant_id'
# Get the list of subjects
subprefix = bidsmap['Options']['bidscoin']['subprefix'].replace('*', '')
sesprefix = bidsmap['Options']['bidscoin']['sesprefix'].replace('*', '')
if not subjects:
subjects = bidscoin.lsdirs(
rawfolder, (subprefix if subprefix != '*' else '') + '*')
if not subjects:
LOGGER.warning(f"No subjects found in: {rawfolder/subprefix}*")
else:
subjects = [
rawfolder / (subprefix + re.sub(f"^{subprefix}", '', subject))
for subject in subjects
] # Make sure there is a sub-prefix
# Loop over all subjects and sessions and convert them using the bidsmap entries
with logging_redirect_tqdm():
for n, subject in enumerate(
tqdm(subjects, unit='subject', leave=False), 1):
LOGGER.info(
f"------------------- Subject {n}/{len(subjects)} -------------------"
)
if participants and subject.name in list(participants_table.index):
LOGGER.info(
f"Skipping subject: {subject} ({n}/{len(subjects)})")
continue
if not subject.is_dir():
LOGGER.warning(
f"The '{subject}' subject folder does not exist")
continue
sessions = bidscoin.lsdirs(
subject, (sesprefix if sesprefix != '*' else '') + '*')
if not sessions or (subject / 'DICOMDIR').is_file():
sessions = [subject]
for session in sessions:
# Unpack the data in a temporary folder if it is tarballed/zipped and/or contains a DICOMDIR file
sesfolders, unpacked = bids.unpack(session)
for sesfolder in sesfolders:
# Check if we should skip the session-folder
datasource = bids.get_datasource(
sesfolder, bidsmap['Options']['plugins'])
if not datasource.dataformat:
LOGGER.info(
f"No coinable datasources found in '{sesfolder}'")
continue
subid = bidsmap[datasource.dataformat]['subject']
sesid = bidsmap[datasource.dataformat]['session']
subid, sesid = datasource.subid_sesid(
subid, sesid if sesid else '')
bidssession = bidsfolder / subid / sesid # TODO: Support DICOMDIR with multiple subjects (as in PYDICOMDIR)
if not force and bidssession.is_dir():
datatypes = []
for dataformat in dataformats:
for datatype in bidscoin.lsdirs(
bidssession
): # See what datatypes we already have in the bids session-folder
if datatype.iterdir(
) and bidsmap[dataformat].get(
datatype.name
): # See if we are going to add data for this datatype
datatypes.append(datatype.name)
if datatypes:
LOGGER.info(
f"Skipping processed session: {bidssession} already has {datatypes} data (you can carefully use the -f option to overrule)"
)
continue
LOGGER.info(f"Coining datasources in: {sesfolder}")
if bidssession.is_dir():
LOGGER.warning(
f"Existing BIDS output-directory found, which may result in duplicate data (with increased run-index). Make sure {bidssession} was cleaned-up from old data before (re)running the bidscoiner"
)
bidssession.mkdir(parents=True, exist_ok=True)
# Run the bidscoiner plugins
for module in plugins:
LOGGER.info(
f"Executing plugin: {Path(module.__file__).name}")
module.bidscoiner_plugin(sesfolder, bidsmap,
bidssession)
# Add the special fieldmap metadata (IntendedFor, B0FieldIdentifier, TE, etc)
addmetadata(bidssession, subid, sesid)
# Clean-up the temporary unpacked data
if unpacked:
shutil.rmtree(sesfolder)
# Re-read the participants_table and store the collected personals in the json sidecar-file
if participants_tsv.is_file():
participants_table = pd.read_csv(participants_tsv, sep='\t')
participants_table.set_index(['participant_id'],
verify_integrity=True,
inplace=True)
participants_json = participants_tsv.with_suffix('.json')
participants_dict = {}
if participants_json.is_file():
with participants_json.open('r') as json_fid:
participants_dict = json.load(json_fid)
if not participants_dict.get('participant_id'):
participants_dict['participant_id'] = {
'Description': 'Unique participant identifier'
}
if not participants_dict.get(
'session_id') and 'session_id' in participants_table.columns:
participants_dict['session_id'] = {'Description': 'Session identifier'}
newkey = False
for col in participants_table.columns:
if col not in participants_dict:
newkey = True
participants_dict[col] = dict(
LongName='Long (unabbreviated) name of the column',
Description='Description of the the column',
Levels=dict(
Key=
'Value (This is for categorical variables: a dictionary of possible values (keys) and their descriptions (values))'
),
Units=
'Measurement units. [<prefix symbol>]<unit symbol> format following the SI standard is RECOMMENDED'
)
# Write the collected data to the participant files
if newkey:
LOGGER.info(f"Writing subject meta data to: {participants_json}")
with participants_json.open('w') as json_fid:
json.dump(participants_dict, json_fid, indent=4)
LOGGER.info('-------------- FINISHED! ------------')
LOGGER.info('')
bidscoin.reporterrors()
def bidsmapper(rawfolder: str, bidsfolder: str, bidsmapfile: str, templatefile: str, plugins: list, subprefix: str, sesprefix: str, store: bool=False, noedit: bool=False, force: bool=False) -> None:
"""
Main function that processes all the subjects and session in the sourcefolder
and that generates a maximally filled-in bidsmap.yaml file in bidsfolder/code/bidscoin.
Folders in sourcefolder are assumed to contain a single dataset.
:param rawfolder: The root folder-name of the sub/ses/data/file tree containing the source data files
:param bidsfolder: The name of the BIDS root folder
:param bidsmapfile: The name of the bidsmap YAML-file
:param templatefile: The name of the bidsmap template YAML-file
:param plugins: Optional list of plugins that should be used (overrules the list in the study/template bidsmaps)
:param subprefix: The prefix common for all source subject-folders
:param sesprefix: The prefix common for all source session-folders
:param store: If True, the provenance samples will be stored
:param noedit: The bidseditor will not be launched if True
:param force: If True, the previous bidsmap and logfiles will be deleted
:return:
"""
# Input checking
rawfolder = Path(rawfolder).resolve()
bidsfolder = Path(bidsfolder).resolve()
bidsmapfile = Path(bidsmapfile)
templatefile = Path(templatefile)
bidscoinfolder = bidsfolder/'code'/'bidscoin'
# Start logging
if force:
(bidscoinfolder/'bidsmapper.log').unlink(missing_ok=True)
bidscoin.setup_logging(bidscoinfolder/'bidsmapper.log')
LOGGER.info('')
LOGGER.info('-------------- START BIDSmapper ------------')
LOGGER.info(f">>> bidsmapper sourcefolder={rawfolder} bidsfolder={bidsfolder} bidsmap={bidsmapfile} "
f"template={templatefile} plugins={plugins} subprefix={subprefix} sesprefix={sesprefix} store={store} force={force}")
# Get the heuristics for filling the new bidsmap
bidsmap_old, bidsmapfile = bids.load_bidsmap(bidsmapfile, bidscoinfolder, plugins)
template, _ = bids.load_bidsmap(templatefile, bidscoinfolder, plugins)
# Create the new bidsmap as a copy / bidsmap skeleton with no datatype entries (i.e. bidsmap with empty lists)
if force:
bidsmapfile.unlink(missing_ok=True)
bidsmap_old = {}
if bidsmap_old:
bidsmap_new = copy.deepcopy(bidsmap_old)
else:
bidsmap_new = copy.deepcopy(template)
template['Options'] = bidsmap_new['Options'] # Always use the options of the new bidsmap
bidscoindatatypes = bidsmap_new['Options']['bidscoin'].get('datatypes',[])
unknowndatatypes = bidsmap_new['Options']['bidscoin'].get('unknowntypes',[])
ignoredatatypes = bidsmap_new['Options']['bidscoin'].get('ignoretypes',[])
for dataformat in bidsmap_new:
if dataformat in ('Options','PlugIns'): continue # Handle legacy bidsmaps (-> 'PlugIns')
for datatype in bidscoindatatypes + unknowndatatypes + ignoredatatypes:
if bidsmap_new[dataformat].get(datatype):
bidsmap_new[dataformat][datatype] = None
# Store/retrieve the empty or user-defined sub-/ses-prefix
subprefix, sesprefix = setprefix(bidsmap_new, subprefix, sesprefix)
# Start with an empty skeleton if we didn't have an old bidsmap
if not bidsmap_old:
bidsmap_old = copy.deepcopy(bidsmap_new)
bidsmapfile = bidscoinfolder/'bidsmap.yaml'
# Import the data scanning plugins
plugins = [bidscoin.import_plugin(plugin, ('bidsmapper_plugin',)) for plugin in bidsmap_new['Options']['plugins']]
plugins = [plugin for plugin in plugins if plugin] # Filter the empty items from the list
if not plugins:
LOGGER.warning(f"The plugins listed in your bidsmap['Options'] did not have a usable `bidsmapper_plugin` function, nothing to do")
LOGGER.info('-------------- FINISHED! ------------')
LOGGER.info('')
return
# Loop over all subjects and sessions and built up the bidsmap entries
subjects = bidscoin.lsdirs(rawfolder, (subprefix if subprefix!='*' else '') + '*')
if not subjects:
LOGGER.warning(f'No subjects found in: {rawfolder/subprefix}*')
with logging_redirect_tqdm():
for n, subject in enumerate(tqdm(subjects, unit='subject', leave=False), 1):
sessions = bidscoin.lsdirs(subject, (sesprefix if sesprefix!='*' else '') + '*')
if not sessions or (subject/'DICOMDIR').is_file():
sessions = [subject]
for session in sessions:
LOGGER.info(f"Mapping: {session} (subject {n}/{len(subjects)})")
# Unpack the data in a temporary folder if it is tarballed/zipped and/or contains a DICOMDIR file
sesfolders, unpacked = bids.unpack(session)
for sesfolder in sesfolders:
if store:
store = {'source': sesfolder.parent.parent.parent.parent if unpacked else rawfolder.parent, 'target': bidscoinfolder/'provenance'}
else:
store = {}
# Run the bidsmapper plugins
for module in plugins:
LOGGER.info(f"Executing plugin: {Path(module.__file__).name} -> {sesfolder}")
module.bidsmapper_plugin(sesfolder, bidsmap_new, bidsmap_old, template, store)
# Clean-up the temporary unpacked data
if unpacked:
shutil.rmtree(sesfolder)
# Save the new study bidsmap in the bidscoinfolder or launch the bidseditor UI_MainWindow
if noedit:
bids.save_bidsmap(bidsmapfile, bidsmap_new)
else:
LOGGER.info('Opening the bidseditor')
app = QApplication(sys.argv)
app.setApplicationName(f"{bidsmapfile} - BIDS editor {localversion}")
mainwin = bidseditor.MainWindow(bidsfolder, bidsmap_new, template)
mainwin.show()
messagebox = QMessageBox(mainwin)
messagebox.setText(f"The bidsmapper has finished scanning {rawfolder}\n\n"
f"Please carefully check all the different BIDS output names "
f"and BIDScoin options and (re)edit them to your needs.\n\n"
f"You can always redo this step later by re-running the "
f"bidsmapper or by just running the bidseditor tool\n\n"
f"{versionmessage}")
messagebox.setWindowTitle('About the BIDS-mapping workflow')
messagebox.setIconPixmap(QtGui.QPixmap(str(bidseditor.BIDSCOIN_LOGO)).scaled(150, 150, QtCore.Qt.KeepAspectRatio, QtCore.Qt.SmoothTransformation))
messagebox.setWindowFlags(messagebox.windowFlags() & ~QtCore.Qt.WindowMinMaxButtonsHint)
messagebox.show()
app.exec()
LOGGER.info('-------------- FINISHED! -------------------')
LOGGER.info('')
bidscoin.reporterrors()
def echocombine(bidsdir: str, pattern: str, subjects: list, output: str, algorithm: str, weights: list, force: bool=False):
"""
:param bidsdir: The bids-directory with the (multi-echo) subject data
:param pattern: Globlike recursive search pattern (relative to the subject/session folder) to select the first echo of the images that need to be combined, e.g. '*task-*echo-1*'
:param subjects: List of sub-# identifiers to be processed (the sub- prefix can be left out). If not specified then all sub-folders in the bidsfolder will be processed
:param output: Determines where the output is saved. It can be the name of a BIDS datatype folder, such as 'func', or of the derivatives folder, i.e. 'derivatives'. If output = [the name of the input datatype folder] then the original echo images are replaced by one combined image. If output is left empty then the combined image is saved in the input datatype folder and the original echo images are moved to the {unknowndatatype} folder
:param algorithm: Combination algorithm, either 'PAID', 'TE' or 'average'
:param weights: Weights for each echo
:param force: Boolean to overwrite existing ME target files
:return:
"""
# Input checking
bidsdir = Path(bidsdir).resolve()
# Start logging
bidscoin.setup_logging(bidsdir/'code'/'bidscoin'/'echocombine.log')
LOGGER.info('')
LOGGER.info(f"--------- START echocombine ---------")
LOGGER.info(f">>> echocombine bidsfolder={bidsdir} pattern={pattern} subjects={subjects} output={output}"
f" algorithm={algorithm} weights={weights}")
# Get the list of subjects
if not subjects:
subjects = bidscoin.lsdirs(bidsdir, 'sub-*')
if not subjects:
LOGGER.warning(f"No subjects found in: {bidsdir/'sub-*'}")
else:
subjects = ['sub-' + subject.replace('sub-', '') for subject in subjects] # Make sure there is a "sub-" prefix
subjects = [bidsdir/subject for subject in subjects if (bidsdir/subject).is_dir()]
# Loop over bids subject/session-directories
with logging_redirect_tqdm():
for n, subject in enumerate(tqdm(subjects, unit='subject', leave=False), 1):
sessions = bidscoin.lsdirs(subject, 'ses-*')
if not sessions:
sessions = [subject]
for session in sessions:
LOGGER.info('-------------------------------------')
LOGGER.info(f"Combining echos for ({n}/{len(subjects)}): {session}")
subid, sesid = bids.DataSource(session/'dum.my', subprefix='sub-', sesprefix='ses-').subid_sesid()
# Search for multi-echo matches
for match in sorted([match for match in session.rglob(pattern) if '.nii' in match.suffixes]):
# Check if it is normal/BIDS multi-echo data or that the echo-number is appended to the acquisition label (as done in BIDScoin)
if '_echo-' in match.name:
echonr = bids.get_bidsvalue(match, 'echo')
mepattern = bids.get_bidsvalue(match, 'echo', '*') # The pattern that selects all echos
cename = match.name.replace(f"_echo-{echonr}", '') # The combined-echo output filename
elif '_acq-' in match.name and bids.get_bidsvalue(match, 'acq').split('e')[-1].isnumeric():
acq, echonr = bids.get_bidsvalue(match, 'acq').rsplit('e',1)
mepattern = bids.get_bidsvalue(match, 'acq', acq + 'e*') # The pattern that selects all echos
cename = match.name.replace(f"_acq-{acq}e{echonr}", f"_acq-{acq}") # The combined-echo output filename
LOGGER.info(f"No 'echo' key-value pair found in the filename, using the 'acq-{acq}e{echonr}' pair instead (BIDScoin-style)")
else:
LOGGER.warning(f"No 'echo' encoding found in the filename, skipping: {match}")
continue
echos = sorted(match.parent.glob(mepattern.name))
newechos = [echo.parents[1]/unknowndatatype/echo.name for echo in echos]
if len(echos) == 1:
LOGGER.warning(f"Only one echo image found, nothing to do for: {match}")
continue
# Construct the combined-echo output filename and check if that file already exists
datatype = match.parent.name
if not output:
cefile = session/datatype/cename
elif output == 'derivatives':
cefile = bidsdir/'derivatives'/'multiecho'/subid/sesid/datatype/cename
else:
cefile = session/output/cename
cefile.parent.mkdir(parents=True, exist_ok=True)
if cefile.is_file() and not force:
LOGGER.warning(f"Outputfile {cefile} already exists, skipping: {match}")
continue
# Combine the multi-echo images
me.me_combine(mepattern, cefile, algorithm, weights, saveweights=False)
# (Re)move the original multi-echo images
if not output:
for echo, newecho in zip(echos, newechos):
LOGGER.info(f"Moving original echo image: {echo} -> {newecho}")
newecho.parent.mkdir(parents=True, exist_ok=True)
echo.replace(newecho)
echo.with_suffix('').with_suffix('.json').replace(newecho.with_suffix('').with_suffix('.json'))
elif output == datatype:
for echo in echos:
LOGGER.info(f"Removing original echo image: {echo}")
echo.unlink()
echo.with_suffix('').with_suffix('.json').unlink()
# Construct the path names relative to the session folder (as in the scans.tsv file)
oldechos_rel = [echo.relative_to(session).as_posix() for echo in echos]
newechos_rel = [echo.relative_to(session).as_posix() for echo in echos + newechos if echo.is_file()]
if output == 'derivatives':
cefile_rel = '' # A remote folder cannot be specified as IntendedFor :-(
else:
cefile_rel = cefile.relative_to(session).as_posix()
# Update the IntendedFor fields of the fieldmaps (i.e. remove the old echos, add the echo-combined image and, optionally, the new echos)
if output != 'derivatives' and (session/'fmap').is_dir():
for fmap in (session/'fmap').glob('*.json'):
with fmap.open('r') as fmap_fid:
metadata = json.load(fmap_fid)
intendedfor = metadata.get('IntendedFor', [])
if isinstance(intendedfor, str):
intendedfor = [intendedfor]
if sesid: # NB: IntendedFor is relative to the subject folder
intendedfor = [file.split(sesid+'/',1)[1] for file in intendedfor]
if oldechos_rel[0] in intendedfor:
LOGGER.info(f"Updating 'IntendedFor' in {fmap}")
relfiles = [file for file in intendedfor if file not in oldechos_rel] + newechos_rel + [cefile_rel]
metadata['IntendedFor'] = [(Path(sesid)/relfile).as_posix() for relfile in relfiles]
with fmap.open('w') as fmap_fid:
json.dump(metadata, fmap_fid, indent=4)
# Update the scans.tsv file
if (bidsdir/'.bidsignore').is_file():
bidsignore = (bidsdir/'.bidsignore').read_text().splitlines()
else:
bidsignore = [unknowndatatype + '/']
scans_tsv = session/f"{subid}{bids.add_prefix('_', sesid)}_scans.tsv"
if scans_tsv.is_file():
scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename')
if oldechos_rel[0] in scans_table.index:
scans_table.loc['oldrow'] = scans_table.loc[oldechos_rel[0]]
elif 'acq_time' in scans_table:
with cefile.with_suffix('').with_suffix('.json').open('r') as fid:
metadata = json.load(fid)
date = scans_table.iloc[0]['acq_time'].split('T')[0]
scans_table.loc['oldrow', 'acq_time'] = f"{date}T{metadata.get('AcquisitionTime')}"
else:
scans_table.loc['oldrow'] = None
if output+'/' not in bidsignore + ['derivatives/'] and cefile.parent.name in bids.bidsdatatypes:
LOGGER.info(f"Adding '{cefile_rel}' to '{scans_tsv}'")
scans_table.loc[cefile_rel] = scans_table.loc['oldrow']
for echo in oldechos_rel + newechos_rel:
if echo in scans_table.index and not (session/echo).is_file():
LOGGER.info(f"Removing '{echo}' from '{scans_tsv}'")
scans_table.drop(echo, inplace=True)
elif echo not in scans_table.index and (session/echo).is_file() and echo.split('/')[0] in bids.bidsdatatypes:
LOGGER.info(f"Adding '{echo}' to '{scans_tsv}'")
scans_table.loc[echo] = scans_table.loc['oldrow'] # NB: Assuming that the echo-rows are all identical
scans_table.drop('oldrow', inplace=True)
scans_table.sort_values(by=['acq_time','filename'], inplace=True)
scans_table.replace('','n/a').to_csv(scans_tsv, sep='\t', encoding='utf-8', na_rep='n/a')
for scan in scans_table.index:
if not (session/scan).is_file():
LOGGER.warning(f"Found non-existent file '{scan}' in '{scans_tsv}'")
LOGGER.info('-------------- FINISHED! -------------')
LOGGER.info('')
def bidsparticipants(rawfolder: str, bidsfolder: str, keys: str, bidsmapfile: str='bidsmap.yaml', dryrun: bool=False) -> None:
"""
Main function that processes all the subjects and session in the sourcefolder to (re)generate the particpants.tsv file in the BIDS folder.
:param rawfolder: The root folder-name of the sub/ses/data/file tree containing the source data files
:param bidsfolder: The name of the BIDS root folder
:param keys: The keys that are extracted fro mthe source data when populating the participants.tsv file
:param bidsmapfile: The name of the bidsmap YAML-file. If the bidsmap pathname is relative (i.e. no "/" in the name) then it is assumed to be located in bidsfolder/code/bidscoin
:param dryrun: Boolean to just display the participants info
:return: Nothing
"""
# Input checking & defaults
rawfolder = Path(rawfolder).resolve()
bidsfolder = Path(bidsfolder).resolve()
# Start logging
if dryrun:
bidscoin.setup_logging()
else:
bidscoin.setup_logging(bidsfolder/'code'/'bidscoin'/'bidsparticipants.log')
LOGGER.info('')
LOGGER.info(f"-------------- START bidsparticipants {bidscoin.version()} ------------")
LOGGER.info(f">>> bidsparticipants sourcefolder={rawfolder} bidsfolder={bidsfolder} bidsmap={bidsmapfile}")
# Get the bidsmap sub-/ses-prefix from the bidsmap YAML-file
bidsmap,_ = bids.load_bidsmap(Path(bidsmapfile), bidsfolder/'code'/'bidscoin')
subprefix = bidsmap['Options']['bidscoin']['subprefix']
sesprefix = bidsmap['Options']['bidscoin']['sesprefix']
# Get the table & dictionary of the subjects that have been processed
participants_tsv = bidsfolder/'participants.tsv'
participants_json = participants_tsv.with_suffix('.json')
if participants_tsv.is_file():
participants_table = pd.read_csv(participants_tsv, sep='\t')
participants_table.set_index(['participant_id'], verify_integrity=True, inplace=True)
else:
participants_table = pd.DataFrame()
participants_table.index.name = 'participant_id'
if participants_json.is_file():
with participants_json.open('r') as json_fid:
participants_dict = json.load(json_fid)
else:
participants_dict = {'participant_id': {'Description': 'Unique participant identifier'}}
# Get the list of subjects
subjects = bidscoin.lsdirs(bidsfolder, 'sub-*')
if not subjects:
LOGGER.warning(f"No subjects found in: {bidsfolder}")
# Remove obsolete participants from the participants table
for participant in participants_table.index:
if participant not in [sub.name for sub in subjects]:
participants_table = participants_table.drop(participant)
# Loop over all subjects in the bids-folder and add them to the participants table
with logging_redirect_tqdm():
for n, subject in enumerate(tqdm(subjects, unit='subject', leave=False), 1):
LOGGER.info(f"------------------- Subject {n}/{len(subjects)} -------------------")
personals = dict()
subject = rawfolder/subject.name.replace('sub-', subprefix.replace('*','')) # TODO: This assumes e.g. that the subject-ids in the rawfolder did not contain BIDS-invalid characters (such as '_')
sessions = bidscoin.lsdirs(subject, (sesprefix if sesprefix!='*' else '') + '*')
if not subject.is_dir():
LOGGER.error(f"Could not find source-folder: {subject}")
continue
if not sessions:
sessions = [subject]
for session in sessions:
# Only take data from the first session -> BIDS specification
subid, sesid = bids.DataSource(session/'dum.my', subprefix='sub-', sesprefix='ses-').subid_sesid()
if sesprefix and sesid and 'session_id' not in personals:
personals['session_id'] = sesid
participants_dict['session_id'] = {'Description': 'Session identifier'}
# Unpack the data in a temporary folder if it is tarballed/zipped and/or contains a DICOMDIR file
sesfolders, unpacked = bids.unpack(session)
for sesfolder in sesfolders:
# Update / append the personal source data
LOGGER.info(f"Scanning session: {sesfolder}")
success = scanpersonals(bidsmap, sesfolder, personals)
# Clean-up the temporary unpacked data
if unpacked:
shutil.rmtree(sesfolder)
if success:
break
# Store the collected personals in the participant_table. TODO: Check that only values that are consistent over sessions go in the participants.tsv file, otherwise put them in a sessions.tsv file
for key in keys:
if key not in participants_dict:
participants_dict[key] = dict(LongName = 'Long (unabbreviated) name of the column',
Description = 'Description of the the column',
Levels = dict(Key='Value (This is for categorical variables: a dictionary of possible values (keys) and their descriptions (values))'),
Units = 'Measurement units. [<prefix symbol>]<unit symbol> format following the SI standard is RECOMMENDED')
participants_table.loc[subid, key] = personals.get(key)
# Write the collected data to the participant files
LOGGER.info(f"Writing subject data to: {participants_tsv}")
if not dryrun:
participants_table.replace('','n/a').to_csv(participants_tsv, sep='\t', encoding='utf-8', na_rep='n/a')
LOGGER.info(f"Writing subject data dictionary to: {participants_json}")
if not dryrun:
with participants_json.open('w') as json_fid:
json.dump(participants_dict, json_fid, indent=4)
print(participants_table)
LOGGER.info('-------------- FINISHED! ------------')
LOGGER.info('')
bidscoin.reporterrors()