def bidstrainer(bidsfolder: str, samplefolder: str, bidsmapfile: str, pattern: str) -> None: """ Main function uses all samples in the samplefolder as training / example data to generate a maximally filled-in bidsmap_sample.yaml file. :param bidsfolder: The name of the BIDS root folder :param samplefolder: The name of the root directory of the tree containing the sample files / training data. If left empty, bidsfolder/code/bidscoin/samples is used or such an empty directory tree is created :param bidsmapfile: The name of the bidsmap YAML-file :param pattern: The regular expression pattern used in re.match(pattern, dicomfile) to select the dicom files', default='.*\\.(IMA|dcm)$') :return: """ bidsfolder = Path(bidsfolder) samplefolder = Path(samplefolder) bidsmapfile = Path(bidsmapfile) # Start logging bids.setup_logging(bidsfolder / 'code' / 'bidscoin' / 'bidstrainer.log') LOGGER.info('------------ START BIDStrainer ------------') # Get the heuristics for creating the bidsmap heuristics, _ = bids.load_bidsmap(bidsmapfile, bidsfolder / 'code' / 'bidscoin') # Input checking if not samplefolder: samplefolder = bidsfolder / 'code' / 'bidscoin' / 'samples' if not samplefolder.is_dir(): LOGGER.info( f"Creating an empty samples directory tree: {samplefolder}") for modality in bids.bidsmodalities + (bids.ignoremodality, bids.unknownmodality): for run in heuristics['DICOM'][modality]: if not run['bids']['suffix']: run['bids']['suffix'] = '' (samplefolder / modality / run['bids']['suffix']).mkdir( parents=True, exist_ok=True) LOGGER.info( 'Fill the directory tree with example DICOM files and re-run bidstrainer.py' ) return # Create a copy / bidsmap skeleton with no modality entries (i.e. bidsmap with empty lists) bidsmap = copy.deepcopy(heuristics) for logic in ('DICOM', 'PAR', 'P7', 'Nifti', 'FileSystem'): for modality in bids.bidsmodalities: if bidsmap[logic] and modality in bidsmap[logic]: bidsmap[logic][modality] = None # Loop over all bidsmodalities and instances and built up the bidsmap entries files = samplefolder.rglob('*') samples = [ Path(dcmfile) for dcmfile in files if re.match(pattern, str(dcmfile)) ] for sample in samples: if not sample.is_file(): continue LOGGER.info(f"Parsing: {sample}") # Try to get a dicom mapping if bids.is_dicomfile(sample) and heuristics['DICOM']: bidsmap = built_dicommap(sample, bidsmap, heuristics) # Try to get a PAR/REC mapping if bids.is_parfile(sample) and heuristics['PAR']: bidsmap = built_parmap(sample, bidsmap, heuristics) # Try to get a P7 mapping if bids.is_p7file(sample) and heuristics['P7']: bidsmap = built_p7map(sample, bidsmap, heuristics) # Try to get a nifti mapping if bids.is_niftifile(sample) and heuristics['Nifti']: bidsmap = built_niftimap(sample, bidsmap, heuristics) # Try to get a file-system mapping if heuristics['FileSystem']: bidsmap = built_filesystemmap(sample, bidsmap, heuristics) # Try to get a plugin mapping if heuristics['PlugIn']: bidsmap = built_pluginmap(sample, bidsmap) # Create the bidsmap_sample YAML-file in bidsfolder/code/bidscoin (bidsfolder / 'code' / 'bidscoin').mkdir(parents=True, exist_ok=True) bidsmapfile = bidsfolder / 'code' / 'bidscoin' / 'bidsmap_sample.yaml' # Save the bidsmap to the bidsmap YAML-file bids.save_bidsmap(bidsmapfile, bidsmap) LOGGER.info('------------ FINISHED! ------------')
def deface(bidsdir: str, pattern: str, subjects: list, output: str, cluster: bool, nativespec: str, kwargs: dict): """ :param bidsdir: The bids-directory with the (multi-echo) subject data :param pattern: Globlike search pattern (relative to the subject/session folder) to select the images that need to be defaced, e.g. 'anat/*_T1w*' :param subjects: List of sub-# identifiers to be processed (the sub- prefix can be left out). If not specified then all sub-folders in the bidsfolder will be processed :param output: Determines where the defaced images are saved. It can be the name of a BIDS datatype folder, such as 'anat', or of the derivatives folder, i.e. 'derivatives'. If output is left empty then the original images are replaced by the defaced images :param cluster: Flag to submit the deface jobs to the high-performance compute (HPC) cluster :param nativespec: DRMAA native specifications for submitting deface jobs to the HPC cluster :param kwargs: Additional arguments (in dict/json-style) that are passed to pydeface. See examples for usage :return: """ # Input checking bidsdir = Path(bidsdir).resolve() # Start logging bids.setup_logging(bidsdir/'code'/'bidscoin'/'deface.log') LOGGER.info('') LOGGER.info('------------ START deface ------------') LOGGER.info(f">>> deface bidsfolder={bidsdir} pattern={pattern} subjects={subjects} output={output}" f" cluster={cluster} nativespec={nativespec} {kwargs}") # Get the list of subjects if not subjects: subjects = bids.lsdirs(bidsdir, 'sub-*') if not subjects: LOGGER.warning(f"No subjects found in: {bidsdir/'sub-*'}") else: subjects = ['sub-' + subject.replace('^sub-', '') for subject in subjects] # Make sure there is a "sub-" prefix subjects = [bidsdir/subject for subject in subjects if (bidsdir/subject).is_dir()] # Prepare the HPC job submission with drmaa.Session() as pbatch: if cluster: jt = pbatch.createJobTemplate() jt.jobEnvironment = os.environ jt.remoteCommand = shutil.which('pydeface') jt.nativeSpecification = nativespec jt.joinFiles = True # Loop over bids subject/session-directories for n, subject in enumerate(subjects, 1): sessions = bids.lsdirs(subject, 'ses-*') if not sessions: sessions = [subject] for session in sessions: LOGGER.info('--------------------------------------') LOGGER.info(f"Processing ({n}/{len(subjects)}): {session}") sub_id, ses_id = bids.get_subid_sesid(session/'dum.my') # Search for images that need to be defaced for match in sorted([match for match in session.glob(pattern) if '.nii' in match.suffixes]): # Construct the output filename and relative path name (used in BIDS) match_rel = str(match.relative_to(session)) if not output: outputfile = match outputfile_rel = match_rel elif output == 'derivatives': outputfile = bidsdir/'derivatives'/'deface'/sub_id/ses_id/match.parent.name/match.name outputfile_rel = str(outputfile.relative_to(bidsdir)) else: outputfile = session/output/match.name outputfile_rel = str(outputfile.relative_to(session)) outputfile.parent.mkdir(parents=True, exist_ok=True) # Deface the image LOGGER.info(f"Defacing: {match_rel} -> {outputfile_rel}") if cluster: jt.args = [str(match), '--outfile', str(outputfile), '--force'] + [item for pair in [[f"--{key}",val] for key,val in kwargs.items()] for item in pair] jt.jobName = f"pydeface_{sub_id}_{ses_id}" jobid = pbatch.runJob(jt) LOGGER.info(f"Your deface job has been submitted with ID: {jobid}") else: pdu.deface_image(str(match), str(outputfile), force=True, forcecleanup=True, **kwargs) # Overwrite or add a json sidecar-file inputjson = match.with_suffix('').with_suffix('.json') outputjson = outputfile.with_suffix('').with_suffix('.json') if inputjson.is_file() and inputjson != outputjson: if outputjson.is_file(): LOGGER.info(f"Overwriting the json sidecar-file: {outputjson}") outputjson.unlink() else: LOGGER.info(f"Adding a json sidecar-file: {outputjson}") shutil.copyfile(inputjson, outputjson) # Add a custom "Defaced" field to the json sidecar-file with outputjson.open('r') as output_fid: data = json.load(output_fid) data['Defaced'] = True with outputjson.open('w') as output_fid: json.dump(data, output_fid, indent=4) # Update the IntendedFor fields in the fieldmap sidecar-files if output and output != 'derivatives' and (session/'fmap').is_dir(): for fmap in (session/'fmap').glob('*.json'): with fmap.open('r') as fmap_fid: fmap_data = json.load(fmap_fid) intendedfor = fmap_data['IntendedFor'] if type(intendedfor)==str: intendedfor = [intendedfor] if match_rel in intendedfor: LOGGER.info(f"Updating 'IntendedFor' to {outputfile_rel} in {fmap}") fmap_data['IntendedFor'] = intendedfor + [outputfile_rel] with fmap.open('w') as fmap_fid: json.dump(fmap_data, fmap_fid, indent=4) # Update the scans.tsv file if (bidsdir/'.bidsignore').is_file(): with (bidsdir/'.bidsignore').open('r') as fid: bidsignore = fid.read().splitlines() else: bidsignore = [bids.unknowndatatype + '/'] bidsignore.append('derivatives/') scans_tsv = session/f"{sub_id}{bids.add_prefix('_',ses_id)}_scans.tsv" if output and output+'/' not in bidsignore and scans_tsv.is_file(): LOGGER.info(f"Adding {outputfile_rel} to {scans_tsv}") scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename') scans_table.loc[outputfile_rel] = scans_table.loc[match_rel] scans_table.sort_values(by=['acq_time','filename'], inplace=True) scans_table.to_csv(scans_tsv, sep='\t', encoding='utf-8') if cluster: LOGGER.info('Waiting for the deface jobs to finish...') pbatch.synchronize(jobIds=[pbatch.JOB_IDS_SESSION_ALL], timeout=pbatch.TIMEOUT_WAIT_FOREVER, dispose=True) pbatch.deleteJobTemplate(jt) LOGGER.info('-------------- FINISHED! -------------') LOGGER.info('')
def bidscoiner(rawfolder: str, bidsfolder: str, subjects: list = (), force: bool = False, participants: bool = False, bidsmapfile: str = 'bidsmap.yaml', subprefix: str = 'sub-', sesprefix: str = 'ses-') -> None: """ Main function that processes all the subjects and session in the sourcefolder and uses the bidsmap.yaml file in bidsfolder/code/bidscoin to cast the data into the BIDS folder. :param rawfolder: The root folder-name of the sub/ses/data/file tree containing the source data files :param bidsfolder: The name of the BIDS root folder :param subjects: List of selected subjects / participants (i.e. sub-# names / folders) to be processed (the sub- prefix can be removed). Otherwise all subjects in the sourcefolder will be selected :param force: If True, subjects will be processed, regardless of existing folders in the bidsfolder. Otherwise existing folders will be skipped :param participants: If True, subjects in particpants.tsv will not be processed (this could be used e.g. to protect these subjects from being reprocessed), also when force=True :param bidsmapfile: The name of the bidsmap YAML-file. If the bidsmap pathname is relative (i.e. no "/" in the name) then it is assumed to be located in bidsfolder/code/bidscoin :param subprefix: The prefix common for all source subject-folders :param sesprefix: The prefix common for all source session-folders :return: Nothing """ # Input checking & defaults rawfolder = Path(rawfolder).resolve() bidsfolder = Path(bidsfolder).resolve() bidsmapfile = Path(bidsmapfile) # Start logging bids.setup_logging(bidsfolder / 'code' / 'bidscoin' / 'bidscoiner.log') LOGGER.info('') LOGGER.info( f"-------------- START BIDScoiner {bids.version()}: BIDS {bids.bidsversion()} ------------" ) LOGGER.info( f">>> bidscoiner sourcefolder={rawfolder} bidsfolder={bidsfolder} subjects={subjects} force={force}" f" participants={participants} bidsmap={bidsmapfile} subprefix={subprefix} sesprefix={sesprefix}" ) # Create a code/bidscoin subfolder (bidsfolder / 'code' / 'bidscoin').mkdir(parents=True, exist_ok=True) # Create a dataset description file if it does not exist dataset_file = bidsfolder / 'dataset_description.json' if not dataset_file.is_file(): dataset_description = { "Name": "REQUIRED. Name of the dataset", "BIDSVersion": str(bids.bidsversion()), "DatasetType": "raw", "License": "RECOMMENDED. The license for the dataset. The use of license name abbreviations is RECOMMENDED for specifying a license. The corresponding full license text MAY be specified in an additional LICENSE file", "Authors": [ "OPTIONAL. List of individuals who contributed to the creation/curation of the dataset" ], "Acknowledgements": "OPTIONAL. Text acknowledging contributions of individuals or institutions beyond those listed in Authors or Funding", "HowToAcknowledge": "OPTIONAL. Instructions how researchers using this dataset should acknowledge the original authors. This field can also be used to define a publication that should be cited in publications that use the dataset", "Funding": ["OPTIONAL. List of sources of funding (grant numbers)"], "EthicsApprovals": [ "OPTIONAL. List of ethics committee approvals of the research protocols and/or protocol identifiers" ], "ReferencesAndLinks": [ "OPTIONAL. List of references to publication that contain information on the dataset, or links", "https://github.com/Donders-Institute/bidscoin" ], "DatasetDOI": "OPTIONAL. The Document Object Identifier of the dataset (not the corresponding paper)" } LOGGER.info(f"Creating dataset description file: {dataset_file}") with open(dataset_file, 'w') as fid: json.dump(dataset_description, fid, indent=4) # Create a README file if it does not exist readme_file = bidsfolder / 'README' if not readme_file.is_file(): LOGGER.info(f"Creating README file: {readme_file}") with open(readme_file, 'w') as fid: fid.write( f"A free form text ( README ) describing the dataset in more details that SHOULD be provided\n\n" f"The raw BIDS data was created using BIDScoin {bids.version()}\n" f"All provenance information and settings can be found in ./code/bidscoin\n" f"For more information see: https://github.com/Donders-Institute/bidscoin" ) # Get the bidsmap heuristics from the bidsmap YAML-file bidsmap, _ = bids.load_bidsmap(bidsmapfile, bidsfolder / 'code' / 'bidscoin') if not bidsmap: LOGGER.error( f"No bidsmap file found in {bidsfolder}. Please run the bidsmapper first and / or use the correct bidsfolder" ) return # Save options to the .bidsignore file bidsignore_items = [ item.strip() for item in bidsmap['Options']['bidscoin']['bidsignore'].split(';') ] LOGGER.info( f"Writing {bidsignore_items} entries to {bidsfolder}.bidsignore") with (bidsfolder / '.bidsignore').open('w') as bidsignore: for item in bidsignore_items: bidsignore.write(item + '\n') # Get the table & dictionary of the subjects that have been processed participants_tsv = bidsfolder / 'participants.tsv' participants_json = participants_tsv.with_suffix('.json') if participants_tsv.is_file(): participants_table = pd.read_csv(participants_tsv, sep='\t') participants_table.set_index(['participant_id'], verify_integrity=True, inplace=True) else: participants_table = pd.DataFrame() participants_table.index.name = 'participant_id' if participants_json.is_file(): with participants_json.open('r') as json_fid: participants_dict = json.load(json_fid) else: participants_dict = { 'participant_id': { 'Description': 'Unique participant identifier' } } # Get the list of subjects if not subjects: subjects = bids.lsdirs(rawfolder, subprefix + '*') if not subjects: LOGGER.warning(f"No subjects found in: {rawfolder/subprefix}*") else: subjects = [ subprefix + re.sub(f"^{subprefix}", '', subject) for subject in subjects ] # Make sure there is a "sub-" prefix subjects = [ rawfolder / subject for subject in subjects if (rawfolder / subject).is_dir() ] # Loop over all subjects and sessions and convert them using the bidsmap entries for n, subject in enumerate(subjects, 1): LOGGER.info( f"------------------- Subject {n}/{len(subjects)} -------------------" ) if participants and subject.name in list(participants_table.index): LOGGER.info(f"Skipping subject: {subject} ({n}/{len(subjects)})") continue personals = dict() sessions = bids.lsdirs(subject, sesprefix + '*') if not sessions: sessions = [subject] for session in sessions: # Unpack the data in a temporary folder if it is tarballed/zipped and/or contains a DICOMDIR file session, unpacked = bids.unpack(session, subprefix, sesprefix) # See what dataformat we have dataformat = bids.get_dataformat(session) if not dataformat: LOGGER.info(f"Skipping unknown session: {session}") continue # Check if we should skip the session-folder if not force: subid, sesid = bids.get_subid_sesid(session / 'dum.my', subprefix=subprefix, sesprefix=sesprefix) bidssession = bidsfolder / subid / sesid if not bidsmap[dataformat]['session']: bidssession = bidssession.parent datatypes = [] for datatype in bids.lsdirs( bidssession ): # See what datatypes we already have in the bids session-folder if datatype.glob('*') and bidsmap[dataformat].get( datatype.name ): # See if we are going to add data for this datatype datatypes.append(datatype.name) if datatypes: LOGGER.info( f"Skipping processed session: {bidssession} already has {datatypes} data (use the -f option to overrule)" ) continue LOGGER.info(f"Coining session: {session}") # Update / append the source data mapping if dataformat in ('DICOM', 'PAR'): coin_data2bids(dataformat, session, bidsmap, bidsfolder, personals, subprefix, sesprefix) # Update / append the P7 mapping if dataformat == 'P7': LOGGER.error( f"{dataformat} not (yet) supported, skipping session: {session}" ) continue # Update / append the nifti mapping if dataformat == 'Nifti': coin_nifti(session, bidsmap, bidsfolder, personals) # Update / append the file-system mapping if dataformat == 'FileSystem': coin_filesystem(session, bidsmap, bidsfolder, personals) # Update / append the plugin mapping if bidsmap['PlugIns']: coin_plugin(session, bidsmap, bidsfolder, personals) # Clean-up the temporary unpacked data if unpacked: shutil.rmtree(session) # Store the collected personals in the participant_table for key in personals: # participant_id is the index of the participants_table assert 'participant_id' in personals if key == 'participant_id': continue # TODO: Check that only values that are consistent over sessions go in the participants.tsv file, otherwise put them in a sessions.tsv file if key not in participants_dict: participants_dict[key] = dict( LongName='Long (unabbreviated) name of the column', Description='Description of the the column', Levels=dict( Key= 'Value (This is for categorical variables: a dictionary of possible values (keys) and their descriptions (values))' ), Units= 'Measurement units. [<prefix symbol>]<unit symbol> format following the SI standard is RECOMMENDED', TermURL= 'URL pointing to a formal definition of this type of data in an ontology available on the web' ) participants_table.loc[personals['participant_id'], key] = personals[key] # Write the collected data to the participant files LOGGER.info(f"Writing subject data to: {participants_tsv}") participants_table.replace('', 'n/a').to_csv(participants_tsv, sep='\t', encoding='utf-8', na_rep='n/a') LOGGER.info(f"Writing subject data dictionary to: {participants_json}") with participants_json.open('w') as json_fid: json.dump(participants_dict, json_fid, indent=4) LOGGER.info('-------------- FINISHED! ------------') LOGGER.info('') bids.reporterrors()
def sortsessions(session: Path, subprefix: str = '', sesprefix: str = '', dicomfield: str = 'SeriesDescription', rename: bool = False, ext: str = '', nosort: bool = False, pattern: str = '.*\.(IMA|dcm)$', dryrun: bool = False) -> None: """ :param session: The root folder containing the source [sub/][ses/]dicomfiles or the DICOMDIR file :param subprefix: The prefix for searching the sub folders in session :param sesprefix: The prefix for searching the ses folders in sub folder :param dicomfield: The dicomfield that is used to construct the series folder name (e.g. SeriesDescription or ProtocolName, which are both used as fallback) :param rename: Boolean to rename the DICOM files to a PatientName_SeriesNumber_SeriesDescription_AcquisitionNumber_InstanceNumber scheme :param ext: The file extension after sorting (empty value keeps original file extension) :param nosort: Boolean to skip sorting of DICOM files into SeriesNumber-SeriesDescription directories (useful in combination with -r for renaming only) :param pattern: The regular expression pattern used in re.match() to select the dicom files :param dryrun: Boolean to just display the action :return: Nothing """ # Input checking session = Path(session) # Start logging bids.setup_logging() # Do a recursive call if subprefix is given if subprefix: for subfolder in bids.lsdirs(session, subprefix + '*'): if sesprefix: sessionfolders = bids.lsdirs(subfolder, sesprefix + '*') else: sessionfolders = [subfolder] for sessionfolder in sessionfolders: sortsessions(session=sessionfolder, dicomfield=dicomfield, rename=rename, ext=ext, nosort=nosort, pattern=pattern, dryrun=dryrun) # Use the DICOMDIR file if it is there if (session / 'DICOMDIR').is_file(): dicomdir = pydicom.dcmread(str(session / 'DICOMDIR')) sessionfolder = session for patient in dicomdir.patient_records: if len(dicomdir.patient_records) > 1: sessionfolder = session / f"sub-{cleanup(patient.PatientName)}" for n, study in enumerate(patient.children, 1): # TODO: Check order if len(patient.children) > 1: sessionfolder = session / f"ses-{n:02}{cleanup(study.StudyDescription)}" # TODO: Leave out StudyDescription? Include PatientName/StudiesDescription? LOGGER.warning( f"The session index-number '{n:02}' is not necessarily meaningful: {sessionfolder}" ) dicomfiles = [ session.joinpath(*image.ReferencedFileID) for series in study.children for image in series.children ] sortsession(sessionfolder, dicomfiles, dicomfield, rename, ext, nosort, dryrun) else: dicomfiles = [ dcmfile for dcmfile in session.iterdir() if dcmfile.is_file() and re.match(pattern, str(dcmfile)) ] sortsession(session, dicomfiles, dicomfield, rename, ext, nosort, dryrun)
def deface(bidsdir: str, pattern: str, subjects: list, output: str, cluster: bool, nativespec: str, kwargs: dict): # Input checking bidsdir = Path(bidsdir) # Start logging bids.setup_logging(bidsdir / 'code' / 'bidscoin' / 'deface.log') LOGGER.info('') LOGGER.info('------------ START deface ------------') LOGGER.info( f">>> deface bidsfolder={bidsdir} pattern={pattern} subjects={subjects} output={output}" f" cluster={cluster} nativespec={nativespec} {kwargs}") # Get the list of subjects if not subjects: subjects = bids.lsdirs(bidsdir, 'sub-*') if not subjects: LOGGER.warning(f"No subjects found in: {bidsdir/'sub-*'}") else: subjects = [ 'sub-' + subject.replace('^sub-', '') for subject in subjects ] # Make sure there is a "sub-" prefix subjects = [ bidsdir / subject for subject in subjects if (bidsdir / subject).is_dir() ] # Prepare the HPC job submission with drmaa.Session() as pbatch: if cluster: jt = pbatch.createJobTemplate() jt.jobEnvironment = os.environ jt.remoteCommand = shutil.which('pydeface') jt.nativeSpecification = nativespec jt.joinFiles = True # Loop over bids subject/session-directories for n, subject in enumerate(subjects, 1): sessions = bids.lsdirs(subject, 'ses-*') if not sessions: sessions = [subject] for session in sessions: LOGGER.info('--------------------------------------') LOGGER.info(f"Processing ({n}/{len(subjects)}): {session}") sub_id, ses_id = bids.get_subid_sesid(session / 'dum.my') # Search for images that need to be defaced for match in sorted([ match for match in session.glob(pattern) if '.nii' in match.suffixes ]): # Construct the output filename and relative path name (used in BIDS) match_rel = str(match.relative_to(session)) if not output: outputfile = match outputfile_rel = match_rel elif output == 'derivatives': outputfile = bidsdir / 'derivatives' / 'deface' / sub_id / ses_id / match.parent.name / match.name outputfile_rel = str(outputfile.relative_to(bidsdir)) else: outputfile = session / output / match.name outputfile_rel = str(outputfile.relative_to(session)) outputfile.parent.mkdir(parents=True, exist_ok=True) # Deface the image LOGGER.info(f"Defacing: {match_rel} -> {outputfile_rel}") if cluster: jt.args = [ str(match), '--outfile', str(outputfile), '--force' ] + [ item for pair in [[f"--{key}", val] for key, val in kwargs.items()] for item in pair ] jt.jobName = f"pydeface_{sub_id}_{ses_id}" jobid = pbatch.runJob(jt) LOGGER.info( f"Your deface job has been submitted with ID: {jobid}" ) else: pdu.deface_image(str(match), str(outputfile), force=True, forcecleanup=True, **kwargs) # Add a json sidecar-file outputjson = outputfile.with_suffix('').with_suffix( '.json') LOGGER.info(f"Adding a json sidecar-file: {outputjson}") shutil.copyfile( match.with_suffix('').with_suffix('.json'), outputjson) # Update the IntendedFor fields in the fieldmap sidecar files if output and output != 'derivatives' and ( session / 'fmap').is_dir(): for fmap in (session / 'fmap').glob('*.json'): with fmap.open('r') as fmap_fid: fmap_data = json.load(fmap_fid) intendedfor = fmap_data['IntendedFor'] if type(intendedfor) == str: intendedfor = [intendedfor] if match_rel in intendedfor: LOGGER.info( f"Updating 'IntendedFor' to {outputfile_rel} in {fmap}" ) fmap_data['IntendedFor'] = intendedfor + [ outputfile_rel ] with fmap.open('w') as fmap_fid: json.dump(fmap_data, fmap_fid, indent=4) # Update the scans.tsv file scans_tsv = session / f"{sub_id}{bids.add_prefix('_',ses_id)}_scans.tsv" if output and output != 'derivatives' and scans_tsv.is_file( ): LOGGER.info(f"Adding {outputfile_rel} to {scans_tsv}") scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename') scans_table.loc[outputfile_rel] = scans_table.loc[ match_rel] scans_table.sort_values(by=['acq_time', 'filename'], inplace=True) scans_table.to_csv(scans_tsv, sep='\t', encoding='utf-8') if cluster: LOGGER.info('Waiting for the deface jobs to finish...') pbatch.synchronize(jobIds=[pbatch.JOB_IDS_SESSION_ALL], timeout=pbatch.TIMEOUT_WAIT_FOREVER, dispose=True) pbatch.deleteJobTemplate(jt) LOGGER.info('-------------- FINISHED! -------------') LOGGER.info('')
def echocombine(bidsdir: str, pattern: str, subjects: list, output: str, algorithm: str, weights: list, force: bool = False): """ :param bidsdir: The bids-directory with the (multi-echo) subject data :param pattern: Globlike recursive search pattern (relative to the subject/session folder) to select the first echo of the images that need to be combined, e.g. '*task-*echo-1*' :param subjects: List of sub-# identifiers to be processed (the sub- prefix can be left out). If not specified then all sub-folders in the bidsfolder will be processed :param output: Determines where the output is saved. It can be the name of a BIDS datatype folder, such as 'func', or of the derivatives folder, i.e. 'derivatives'. If output = [the name of the input datatype folder] then the original echo images are replaced by one combined image. If output is left empty then the combined image is saved in the input datatype folder and the original echo images are moved to the {bids.unknowndatatype} folder :param algorithm: Combination algorithm, either 'PAID', 'TE' or 'average' :param weights: Weights for each echo :param force: Boolean to overwrite existing ME target files :return: """ # Input checking bidsdir = Path(bidsdir).resolve() # Start logging bids.setup_logging(bidsdir / 'code' / 'bidscoin' / 'echocombine.log') LOGGER.info('') LOGGER.info(f"--------- START echocombine ---------") LOGGER.info( f">>> echocombine bidsfolder={bidsdir} pattern={pattern} subjects={subjects} output={output}" f" algorithm={algorithm} weights={weights}") if 'echo' not in pattern: LOGGER.warning( f"Missing 'echo-#' substring in glob-like search pattern, i.e. '{pattern}' does not seem to select the first echo" ) # Get the list of subjects if not subjects: subjects = bids.lsdirs(bidsdir, 'sub-*') if not subjects: LOGGER.warning(f"No subjects found in: {bidsdir/'sub-*'}") else: subjects = [ 'sub-' + subject.replace('^sub-', '') for subject in subjects ] # Make sure there is a "sub-" prefix subjects = [ bidsdir / subject for subject in subjects if (bidsdir / subject).is_dir() ] # Loop over bids subject/session-directories for n, subject in enumerate(subjects, 1): sessions = bids.lsdirs(subject, 'ses-*') if not sessions: sessions = [subject] for session in sessions: LOGGER.info('-------------------------------------') LOGGER.info( f"Combining echos for ({n}/{len(subjects)}): {session}") sub_id, ses_id = bids.get_subid_sesid(session / 'dum.my') # Search for multi-echo matches for match in sorted([ match for match in session.rglob(pattern) if '.nii' in match.suffixes ]): # Check if it is normal/BIDS multi-echo data datatype = match.parent.name echonr = bids.get_bidsvalue(match, 'echo') mepattern = bids.get_bidsvalue(match, 'echo', '*') echos = sorted(match.parent.glob(mepattern.name)) newechos = [ echo.parents[1] / bids.unknowndatatype / echo.name for echo in echos ] if not echonr: LOGGER.warning( f"No 'echo' key-value pair found in the filename, skipping: {match}" ) continue if len(echos) == 1: LOGGER.warning( f"Only one echo image found, nothing to do for: {match}" ) continue # Construct the combined-echo output filename and check if that file already exists cename = match.name.replace(f"_echo-{echonr}", '') if not output: cefile = session / datatype / cename elif output == 'derivatives': cefile = bidsdir / 'derivatives' / 'multiecho' / sub_id / ses_id / datatype / cename else: cefile = session / output / cename cefile.parent.mkdir(parents=True, exist_ok=True) if cefile.is_file() and not force: LOGGER.warning( f"Outputfile {cefile} already exists, skipping: {match}" ) continue # Combine the multi-echo images me.me_combine(mepattern, cefile, algorithm, weights, saveweights=False, logger=LOGGER.name) # (Re)move the original multi-echo images if not output: for echo, newecho in zip(echos, newechos): LOGGER.info( f"Moving original echo image: {echo} -> {newecho}") newecho.parent.mkdir(parents=True, exist_ok=True) echo.replace(newecho) echo.with_suffix('').with_suffix('.json').replace( newecho.with_suffix('').with_suffix('.json')) elif output == datatype: for echo in echos: LOGGER.info(f"Removing original echo image: {echo}") echo.unlink() echo.with_suffix('').with_suffix('.json').unlink() # Construct relative path names as they are used in BIDS echos_rel = [str(echo.relative_to(session)) for echo in echos] newechos_rel = [ str(echo.relative_to(session)) for echo in newechos ] if output != 'derivatives': cefile_rel = str(cefile.relative_to(session)) # Update the IntendedFor fields in the fieldmap sidecar files (i.e. remove the old echos, add the echo-combined image and, optionally, the new echos) if output != 'derivatives' and (session / 'fmap').is_dir(): for fmap in (session / 'fmap').glob('*.json'): with fmap.open('r') as fmap_fid: fmap_data = json.load(fmap_fid) if 'IntendedFor' in fmap_data: intendedfor = fmap_data['IntendedFor'] if type(intendedfor) == str: intendedfor = [intendedfor] if echos_rel[0] in intendedfor: LOGGER.info( f"Updating 'IntendedFor' to {cefile_rel} in {fmap}" ) if not output: intendedfor = [ file for file in intendedfor if not file in echos_rel ] + [cefile_rel] + [ newecho for newecho in newechos_rel ] elif output == datatype: intendedfor = [ file for file in intendedfor if not file in echos_rel ] + [cefile_rel] else: intendedfor = intendedfor + [cefile_rel] fmap_data['IntendedFor'] = intendedfor with fmap.open('w') as fmap_fid: json.dump(fmap_data, fmap_fid, indent=4) # Update the scans.tsv file if (bidsdir / '.bidsignore').is_file(): with (bidsdir / '.bidsignore').open('r') as fid: bidsignore = fid.read().splitlines() else: bidsignore = [bids.unknowndatatype + '/'] bidsignore.append('derivatives/') scans_tsv = session / f"{sub_id}{bids.add_prefix('_',ses_id)}_scans.tsv" if output + '/' not in bidsignore and scans_tsv.is_file(): LOGGER.info(f"Adding {cefile_rel} to {scans_tsv}") scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename') scans_table.loc[cefile_rel] = scans_table.loc[echos_rel[0]] for echo, newecho in zip(echos_rel, newechos_rel): if not output: LOGGER.info( f"Updating {echo} -> {newecho} in {scans_tsv}") scans_table.loc[newecho] = scans_table.loc[echo] scans_table.drop(echo, inplace=True) elif output == datatype: LOGGER.info(f"Removing {echo} from {scans_tsv}") scans_table.drop(echo, inplace=True) scans_table.sort_values(by=['acq_time', 'filename'], inplace=True) scans_table.to_csv(scans_tsv, sep='\t', encoding='utf-8') LOGGER.info('-------------- FINISHED! -------------') LOGGER.info('')
def bidsmapper(rawfolder: str, bidsfolder: str, bidsmapfile: str, templatefile: str, subprefix: str = 'sub-', sesprefix: str = 'ses-', interactive: bool = True) -> None: """ Main function that processes all the subjects and session in the sourcefolder and that generates a maximally filled-in bidsmap.yaml file in bidsfolder/code/bidscoin. Folders in sourcefolder are assumed to contain a single dataset. :param rawfolder: The root folder-name of the sub/ses/data/file tree containing the source data files :param bidsfolder: The name of the BIDS root folder :param bidsmapfile: The name of the bidsmap YAML-file :param templatefile: The name of the bidsmap template YAML-file :param subprefix: The prefix common for all source subject-folders :param sesprefix: The prefix common for all source session-folders :param interactive: If True, the user will be asked for help if an unknown run is encountered :return:bidsmapfile: The name of the mapped bidsmap YAML-file """ # Input checking rawfolder = Path(rawfolder) bidsfolder = Path(bidsfolder) bidsmapfile = Path(bidsmapfile) templatefile = Path(templatefile) # Start logging bids.setup_logging(bidsfolder / 'code' / 'bidscoin' / 'bidsmapper.log') LOGGER.info('') LOGGER.info('-------------- START BIDSmapper ------------') LOGGER.info( f">>> bidsmapper sourcefolder={rawfolder} bidsfolder={bidsfolder} bidsmap={bidsmapfile} " f" template={templatefile} subprefix={subprefix} sesprefix={sesprefix} interactive={interactive}" ) # Get the heuristics for filling the new bidsmap bidsmap_old, _ = bids.load_bidsmap(bidsmapfile, bidsfolder / 'code' / 'bidscoin') template, _ = bids.load_bidsmap(templatefile, bidsfolder / 'code' / 'bidscoin') # Create the new bidsmap as a copy / bidsmap skeleton with no modality entries (i.e. bidsmap with empty lists) if bidsmap_old: bidsmap_new = copy.deepcopy(bidsmap_old) else: bidsmap_new = copy.deepcopy(template) for logic in ('DICOM', 'PAR', 'P7', 'Nifti', 'FileSystem'): for modality in bids.bidsmodalities + (bids.unknownmodality, bids.ignoremodality): if bidsmap_new[logic] and modality in bidsmap_new[logic]: bidsmap_new[logic][modality] = None # Start with an empty skeleton if we didn't have an old bidsmap if not bidsmap_old: bidsmap_old = copy.deepcopy(bidsmap_new) # Start the Qt-application gui = interactive if gui: app = QApplication(sys.argv) app.setApplicationName('BIDS editor') mainwin = bidseditor.MainWindow() gui = bidseditor.Ui_MainWindow() gui.interactive = interactive gui.subprefix = subprefix gui.sesprefix = sesprefix if gui.interactive == 2: QMessageBox.information( mainwin, 'BIDS mapping workflow', f"The bidsmapper will now scan {bidsfolder} and whenever " f"it detects a new type of scan it will ask you to identify it.\n\n" f"It is important that you choose the correct BIDS modality " f"(e.g. 'anat', 'dwi' or 'func') and suffix (e.g. 'bold' or 'sbref').\n\n" f"At the end you will be shown an overview of all the " f"different scan types and BIDScoin options (as in the " f"bidseditor) that you can then (re)edit to your needs") # Loop over all subjects and sessions and built up the bidsmap entries subjects = bids.lsdirs(rawfolder, subprefix + '*') if not subjects: LOGGER.warning(f'No subjects found in: {rawfolder/subprefix}*') gui = None for n, subject in enumerate(subjects, 1): sessions = bids.lsdirs(subject, sesprefix + '*') if not sessions: sessions = [subject] for session in sessions: LOGGER.info(f"Parsing: {session} (subject {n}/{len(subjects)})") for runfolder in bids.lsdirs(session): # Update / append the dicom mapping if bidsmap_old['DICOM']: bidsmap_new = build_dicommap(runfolder, bidsmap_new, bidsmap_old, template, gui) # Update / append the PAR/REC mapping if bidsmap_old['PAR']: bidsmap_new = build_parmap(runfolder, bidsmap_new, bidsmap_old) # Update / append the P7 mapping if bidsmap_old['P7']: bidsmap_new = build_p7map(runfolder, bidsmap_new, bidsmap_old) # Update / append the nifti mapping if bidsmap_old['Nifti']: bidsmap_new = build_niftimap(runfolder, bidsmap_new, bidsmap_old) # Update / append the file-system mapping if bidsmap_old['FileSystem']: bidsmap_new = build_filesystemmap(runfolder, bidsmap_new, bidsmap_old) # Update / append the plugin mapping if bidsmap_old['PlugIns']: bidsmap_new = build_pluginmap(runfolder, bidsmap_new, bidsmap_old) # Create the bidsmap YAML-file in bidsfolder/code/bidscoin bidsmapfile = bidsfolder / 'code' / 'bidscoin' / 'bidsmap.yaml' bidsmapfile.parent.mkdir(parents=True, exist_ok=True) # Save the bidsmap to the bidsmap YAML-file bids.save_bidsmap(bidsmapfile, bidsmap_new) # (Re)launch the bidseditor UI_MainWindow if gui: QMessageBox.information( mainwin, 'BIDS mapping workflow', f"The bidsmapper has finished scanning {rawfolder}\n\n" f"Please carefully check all the different BIDS output names " f"and BIDScoin options and (re)edit them to your needs.\n\n" f"You can always redo this step later by re-running the " f"bidsmapper or by just running the bidseditor tool") LOGGER.info('Opening the bidseditor') gui.setupUi(mainwin, bidsfolder, rawfolder, bidsmapfile, bidsmap_new, copy.deepcopy(bidsmap_new), template, subprefix=subprefix, sesprefix=sesprefix) mainwin.show() app.exec() LOGGER.info('-------------- FINISHED! -------------------') LOGGER.info('') bids.reporterrors()
def bidsparticipants(rawfolder: str, bidsfolder: str, keys: str, subprefix: str = 'sub-', sesprefix: str = 'ses-', dryrun: bool = False) -> None: """ Main function that processes all the subjects and session in the sourcefolder to (re)generate the particpants.tsv file in the BIDS folder. :param rawfolder: The root folder-name of the sub/ses/data/file tree containing the source data files :param bidsfolder: The name of the BIDS root folder :param keys: The keys that are extracted fro mthe source data when populating the participants.tsv file :param subprefix: The prefix common for all source subject-folders :param sesprefix: The prefix common for all source session-folders :param dryrun: Boolean to just display the participants info :return: Nothing """ # Input checking & defaults rawfolder = Path(rawfolder).resolve() bidsfolder = Path(bidsfolder).resolve() # Start logging if dryrun: bids.setup_logging() else: bids.setup_logging(bidsfolder / 'code' / 'bidscoin' / 'bidsparticipants.log') LOGGER.info('') LOGGER.info( f"-------------- START bidsparticipants {bids.version()} ------------") LOGGER.info( f">>> bidsparticipants sourcefolder={rawfolder} bidsfolder={bidsfolder} subprefix={subprefix} sesprefix={sesprefix}" ) # Get the table & dictionary of the subjects that have been processed participants_tsv = bidsfolder / 'participants.tsv' participants_json = participants_tsv.with_suffix('.json') if participants_tsv.is_file(): participants_table = pd.read_csv(participants_tsv, sep='\t') participants_table.set_index(['participant_id'], verify_integrity=True, inplace=True) else: participants_table = pd.DataFrame() participants_table.index.name = 'participant_id' if participants_json.is_file(): with participants_json.open('r') as json_fid: participants_dict = json.load(json_fid) else: participants_dict = { 'participant_id': { 'Description': 'Unique participant identifier' } } # Get the list of subjects subjects = bids.lsdirs(bidsfolder, 'sub-*') if not subjects: LOGGER.warning(f"No subjects found in: {bidsfolder}") # Remove obsolete participants from the participants table for participant in participants_table.index: if participant not in subjects: participants_table = participants_table.drop(participant) # Loop over all subjects in the bids-folder and add them to the participants table for n, subject in enumerate(subjects, 1): LOGGER.info( f"------------------- Subject {n}/{len(subjects)} -------------------" ) personals = dict() subid, sesid = bids.get_subid_sesid(subject / 'dum.my') subject = rawfolder / subid.replace( 'sub-', subprefix ) # TODO: This assumes that the subject-ids in the rawfolder did not contain BIDS-invalid characters (such as '_') sessions = bids.lsdirs(subject, sesprefix + '*') if not subject.is_dir(): LOGGER.error(f"Could not find source-folder: {subject}") continue if not sessions: sessions = [subject] for session in sessions: # Unpack the data in a temporary folder if it is tarballed/zipped and/or contains a DICOMDIR file session, unpacked = bids.unpack(session, subprefix, sesprefix, '*') LOGGER.info(f"Scanning session: {session}") # Update / append the sourde data mapping success = scanparticipant('DICOM', session, personals, subid, sesid) # Clean-up the temporary unpacked data if unpacked: shutil.rmtree(session) if success: break # Store the collected personals in the participant_table for key in personals: # participant_id is the index of the participants_table assert 'participant_id' in personals if key == 'participant_id' or key not in keys: continue # TODO: Check that only values that are consistent over sessions go in the participants.tsv file, otherwise put them in a sessions.tsv file if key not in participants_dict: participants_dict[key] = dict( LongName='Long (unabbreviated) name of the column', Description='Description of the the column', Levels=dict( Key= 'Value (This is for categorical variables: a dictionary of possible values (keys) and their descriptions (values))' ), Units= 'Measurement units. [<prefix symbol>]<unit symbol> format following the SI standard is RECOMMENDED', TermURL= 'URL pointing to a formal definition of this type of data in an ontology available on the web' ) participants_table.loc[personals['participant_id'], key] = personals[key] # Write the collected data to the participant files LOGGER.info(f"Writing subject data to: {participants_tsv}") if not dryrun: participants_table.replace('', 'n/a').to_csv(participants_tsv, sep='\t', encoding='utf-8', na_rep='n/a') LOGGER.info(f"Writing subject data dictionary to: {participants_json}") if not dryrun: with participants_json.open('w') as json_fid: json.dump(participants_dict, json_fid, indent=4) print(participants_table) LOGGER.info('-------------- FINISHED! ------------') LOGGER.info('') bids.reporterrors()
def bidsmapper(rawfolder: str, bidsfolder: str, bidsmapfile: str, templatefile: str, subprefix: str='sub-', sesprefix: str='ses-', store: bool=False, interactive: bool=True) -> None: """ Main function that processes all the subjects and session in the sourcefolder and that generates a maximally filled-in bidsmap.yaml file in bidsfolder/code/bidscoin. Folders in sourcefolder are assumed to contain a single dataset. :param rawfolder: The root folder-name of the sub/ses/data/file tree containing the source data files :param bidsfolder: The name of the BIDS root folder :param bidsmapfile: The name of the bidsmap YAML-file :param templatefile: The name of the bidsmap template YAML-file :param subprefix: The prefix common for all source subject-folders :param sesprefix: The prefix common for all source session-folders :param store: If True, the provenance samples will be stored :param interactive: If True, the user will be asked for help if an unknown run is encountered :return:bidsmapfile: The name of the mapped bidsmap YAML-file """ # Input checking rawfolder = Path(rawfolder).resolve() bidsfolder = Path(bidsfolder).resolve() bidsmapfile = Path(bidsmapfile) templatefile = Path(templatefile) bidscoinfolder = bidsfolder/'code'/'bidscoin' # Start logging bids.setup_logging(bidscoinfolder/'bidsmapper.log') LOGGER.info('') LOGGER.info('-------------- START BIDSmapper ------------') LOGGER.info(f">>> bidsmapper sourcefolder={rawfolder} bidsfolder={bidsfolder} bidsmap={bidsmapfile} " f" template={templatefile} subprefix={subprefix} sesprefix={sesprefix} store={store} interactive={interactive}") # Get the heuristics for filling the new bidsmap bidsmap_old, _ = bids.load_bidsmap(bidsmapfile, bidscoinfolder) template, _ = bids.load_bidsmap(templatefile, bidscoinfolder) # Create the new bidsmap as a copy / bidsmap skeleton with no datatype entries (i.e. bidsmap with empty lists) if bidsmap_old: bidsmap_new = copy.deepcopy(bidsmap_old) else: bidsmap_new = copy.deepcopy(template) for logic in ('DICOM', 'PAR', 'P7', 'Nifti', 'FileSystem'): for datatype in bids.bidsdatatypes + (bids.unknowndatatype, bids.ignoredatatype): if bidsmap_new.get(logic) and datatype in bidsmap_new[logic]: bidsmap_new[logic][datatype] = None # Start with an empty skeleton if we didn't have an old bidsmap if not bidsmap_old: bidsmap_old = copy.deepcopy(bidsmap_new) # Start the Qt-application gui = interactive if gui: app = QApplication(sys.argv) app.setApplicationName(f"{bidsmapfile} - BIDS editor {bids.version()}") mainwin = bidseditor.MainWindow() gui = bidseditor.Ui_MainWindow() gui.interactive = interactive gui.subprefix = subprefix gui.sesprefix = sesprefix if gui.interactive == 2: QMessageBox.information(mainwin, 'BIDS mapping workflow', f"The bidsmapper will now scan {bidsfolder} and whenever " f"it detects a new type of scan it will ask you to identify it.\n\n" f"It is important that you choose the correct BIDS datatype " f"(e.g. 'anat', 'dwi' or 'func') and suffix (e.g. 'bold' or 'sbref').\n\n" f"At the end you will be shown an overview of all the " f"different scan types and BIDScoin options (as in the " f"bidseditor) that you can then (re)edit to your needs") # Loop over all subjects and sessions and built up the bidsmap entries dataformat = '' subjects = bids.lsdirs(rawfolder, subprefix + '*') if not subjects: LOGGER.warning(f'No subjects found in: {rawfolder/subprefix}*') gui = None for n, subject in enumerate(subjects,1): sessions = bids.lsdirs(subject, sesprefix + '*') if not sessions: sessions = [subject] for session in sessions: # Unpack the data in a temporary folder if it is tarballed/zipped and/or contains a DICOMDIR file session, unpacked = bids.unpack(session, subprefix, sesprefix) if unpacked: store = dict(source=unpacked, target=bidscoinfolder/'provenance') elif store: store = dict(source=rawfolder, target=bidscoinfolder/'provenance') else: store = dict() # Loop of the different DICOM runs (series) and collect source files sourcefiles = [] dataformat = bids.get_dataformat(session) if not dataformat: LOGGER.info(f"Skipping: {session} (subject {n}/{len(subjects)})") continue LOGGER.info(f"Parsing: {session} (subject {n}/{len(subjects)})") if dataformat=='DICOM': for sourcedir in bids.lsdirs(session): sourcefile = bids.get_dicomfile(sourcedir) if sourcefile.name: sourcefiles.append(sourcefile) if dataformat=='PAR': sourcefiles = bids.get_parfiles(session) if dataformat=='P7': sourcefiles = bids.get_p7file(session) # Update the bidsmap with the info from the source files for sourcefile in sourcefiles: bidsmap_new = build_bidsmap(dataformat, sourcefile, bidsmap_new, bidsmap_old, template, store, gui) # Update / append the nifti mapping if dataformat=='Nifti': bidsmap_new = build_niftimap(session, bidsmap_new, bidsmap_old) # Update / append the file-system mapping if dataformat=='FileSystem': bidsmap_new = build_filesystemmap(session, bidsmap_new, bidsmap_old) # Update / append the plugin mapping if bidsmap_old['PlugIns']: bidsmap_new = build_pluginmap(session, bidsmap_new, bidsmap_old) # Clean-up the temporary unpacked data if unpacked: shutil.rmtree(session) if not dataformat: LOGGER.warning('Could not determine the dataformat of the source data') # (Re)launch the bidseditor UI_MainWindow bidsmapfile = bidscoinfolder/'bidsmap.yaml' if gui: if not dataformat: QMessageBox.information(mainwin, 'BIDS mapping workflow', 'Could not determine the dataformat of the source data.\n' 'You can try running the bidseditor tool yourself') else: QMessageBox.information(mainwin, 'BIDS mapping workflow', f"The bidsmapper has finished scanning {rawfolder}\n\n" f"Please carefully check all the different BIDS output names " f"and BIDScoin options and (re)edit them to your needs.\n\n" f"You can always redo this step later by re-running the " f"bidsmapper or by just running the bidseditor tool") LOGGER.info('Opening the bidseditor') gui.setupUi(mainwin, bidsfolder, bidsmapfile, bidsmap_new, copy.deepcopy(bidsmap_new), template, dataformat, subprefix=subprefix, sesprefix=sesprefix) mainwin.show() app.exec() else: # Save the bidsmap in the bidscoinfolder bids.save_bidsmap(bidsmapfile, bidsmap_new) LOGGER.info('-------------- FINISHED! -------------------') LOGGER.info('') bids.reporterrors()