def filter_subj_bids(df_files, location, bids_ids): import clinica.iotools.bids_utils as bids # Depending of the file that needs to be open, identify and # do needed preprocessing on the column that contains the # subjects ids bids_ids = [x[8:] for x in bids_ids if "sub-ADNI" in x] if location == "ADNIMERGE.csv": df_files["RID"] = df_files["PTID"].apply( lambda x: bids.remove_space_and_symbols(x)[4:]) else: df_files["RID"] = df_files["RID"].apply(lambda x: pad_id(x)) return df_files.loc[df_files["RID"].isin([x[4:] for x in bids_ids])]
def create_adni_sessions_dict(bids_ids, clinic_specs_path, clinical_data_dir, bids_subjs_paths): """ Extract all the data required for the sessions files and organize them in a dictionary Args: bids_ids: clinic_specs_path: path to the specifications for converting the clinical data clinical_data_dir: path to the clinical data folder bids_subjs_paths: a list with the path to all the BIDS subjects Returns: """ import pandas as pd from os import path from datetime import datetime import clinica.iotools.bids_utils as bids from clinica.utils.stream import cprint # Load data sessions = pd.read_excel(clinic_specs_path, sheetname='sessions.tsv') sessions_fields = sessions['ADNI'] field_location = sessions['ADNI location'] sessions_fields_bids = sessions['BIDS CLINICA'] fields_dataset = [] previous_location = '' fields_bids = [] sessions_dict = {} for i in range(0, len(sessions_fields)): if not pd.isnull(sessions_fields[i]): fields_bids.append(sessions_fields_bids[i]) fields_dataset.append(sessions_fields[i]) sessions_df = pd.DataFrame(columns=fields_bids) for i in range(0, len(field_location)): # If the i-th field is available if (not pd.isnull(field_location[i])) and path.exists( path.join(clinical_data_dir, field_location[i].split('/')[0])): # Load the file tmp = field_location[i].split('/') location = tmp[0] if location != previous_location: previous_location = location file_to_read_path = path.join(clinical_data_dir, location) cprint('\tReading clinical data file : ' + location) file_to_read = pd.read_csv(file_to_read_path, dtype=str) for r in range(0, len(file_to_read.values)): row = file_to_read.iloc[r] # Depending of the file that needs to be open, identify and # do needed preprocessing on the column that contains the # subjects ids if location == 'ADNIMERGE.csv': id_ref = 'PTID' # What was the meaning of this line ? # subj_id = row[id_ref.decode('utf-8')] subj_id = row[id_ref] subj_id = bids.remove_space_and_symbols(subj_id) else: id_ref = 'RID' rid = str(row[id_ref]) # Fill the rid with the needed number of zero if 4 - len(rid) > 0: zeros_to_add = 4 - len(rid) subj_id = '0' * zeros_to_add + rid else: subj_id = rid # Extract the BIDS subject id related with the original # subject id subj_bids = [s for s in bids_ids if subj_id in s] if len(subj_bids) == 0: pass elif len(subj_bids) > 1: raise ( 'Error: multiple subjects found for the same RID') else: subj_bids = subj_bids[0] for j in range(0, len(sessions_fields)): # If the i-th field is available if not pd.isnull(sessions_fields[j]): # Extract only the fields related to the current file opened if location in field_location[i]: if location == 'ADAS_ADNIGO2.csv' or location == 'DXSUM_PDXCONV_ADNIALL.csv' \ or location == 'CDR.csv' or location == 'NEUROBAT.csv': if type(row['VISCODE2']) == float: continue visit_id = row['VISCODE2'] # Convert sc to bl if visit_id == 'sc': visit_id = 'bl' else: visit_id = row['VISCODE'] try: field_value = row[sessions_fields[j]] bids_field_name = sessions_fields_bids[ j] # Calculating age from ADNIMERGE if (sessions_fields[j] == "AGE") and ( visit_id != "bl"): examdate = datetime.strptime( row["EXAMDATE"], "%Y-%m-%d") examdate_bl = datetime.strptime( row["EXAMDATE_bl"], "%Y-%m-%d") delta = examdate - examdate_bl # Adding time passed since bl to patient's age in current visit field_value = round( float(field_value) + (delta.days / 365.25), 1) sessions_dict = update_sessions_dict( sessions_dict, subj_bids, visit_id, field_value, bids_field_name) except KeyError: pass # cprint('Field value ' + Fore.RED + sessions_fields[j] + Fore.RESET + # ' could not be added to sessions.tsv') else: continue # Write the sessions dictionary created in several tsv files write_adni_sessions_tsv(sessions_dict, fields_bids, bids_subjs_paths)
def generate_subject_files(subj, images, dest_dir, mod_to_update): import clinica.iotools.bids_utils as bids import clinica.iotools.converters.adni_to_bids.adni_utils as adni_utils from clinica.utils.stream import cprint import subprocess import os import shutil from os import path from glob import glob global counter alpha_id = bids.remove_space_and_symbols(subj) bids_id = 'sub-ADNI' + alpha_id # Extract the list of sessions available from the dwi paths files, removing the duplicates sess_list = images[(images['Subject_ID'] == subj)]['VISCODE'].unique() if not os.path.exists(path.join(dest_dir, bids_id)): os.mkdir(path.join(dest_dir, bids_id)) # For each session available, create the folder if doesn't exist and convert the files for ses in sess_list: with counter.get_lock(): counter.value += 1 cprint('[DWI] Processing subject ' + str(subj) + ' - session ' + ses + ', ' + str(counter.value) + ' / ' + str(len(images))) ses_bids = adni_utils.viscode_to_session(ses) bids_ses_id = 'ses-' + ses_bids bids_file_name = bids_id + '_ses-' + ses_bids ses_path = path.join(dest_dir, bids_id, bids_ses_id) if mod_to_update: if os.path.exists(path.join(ses_path, 'dwi')): shutil.rmtree(path.join(ses_path, 'dwi')) if not os.path.exists(ses_path): os.mkdir(ses_path) dwi_info = images[(images['Subject_ID'] == subj) & (images['VISCODE'] == ses)] # For the same subject, same session there could be multiple dwi with different acq label for j in range(len(dwi_info)): dwi_subj = dwi_info.iloc[j] if type(dwi_subj['Path']) != float and dwi_subj['Path'] != '': if not os.path.exists(path.join(ses_path, 'dwi')): os.mkdir(path.join(ses_path, 'dwi')) dwi_path = dwi_subj['Path'] bids_name = bids_file_name + '_acq-axial_dwi' bids_dest_dir = path.join(ses_path, 'dwi') if not os.path.exists(bids_dest_dir): os.mkdir(dest_dir) command = 'dcm2niix -b n -z y -o ' + bids_dest_dir + ' -f ' + bids_name + ' ' + dwi_path subprocess.run(command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # Removing ADC images adc_image = path.join(bids_dest_dir, bids_name + '_ADC.nii.gz') if os.path.exists(adc_image): os.remove(adc_image) # If dcm2niix didn't work use dcm2nii # print path.join(dest_dir, bids_name + '.nii.gz') if not os.path.exists( path.join(bids_dest_dir, bids_name + '.nii.gz') ) or not os.path.exists( path.join(bids_dest_dir, bids_name + '.bvec') or not os.path.exists( path.join(bids_dest_dir, bids_name + '.bval'))): cprint( '\tConversion with dcm2niix failed, trying with dcm2nii' ) # Find all the files eventually created by dcm2niix and remove them dwi_dcm2niix = glob( path.join(bids_dest_dir, bids_name + '*')) for d in dwi_dcm2niix: # print 'Removing the old', d os.remove(d) command = 'dcm2nii -a n -d n -e n -i y -g y -p n -m n -r n -x n -o %s %s' \ % (bids_dest_dir, dwi_path) subprocess.run(command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) nii_file = path.join(bids_dest_dir, subj.replace('_', '') + '.nii.gz') bvec_file = path.join(bids_dest_dir, subj.replace('_', '') + '.bvec') bval_file = path.join(bids_dest_dir, subj.replace('_', '') + '.bval') if os.path.exists(bvec_file) and os.path.exists(bval_file): os.rename( bvec_file, path.join(bids_dest_dir, bids_name + '.bvec')) os.rename( bval_file, path.join(bids_dest_dir, bids_name + '.bval')) else: cprint( 'WARNING: bvec and bval not generated by dcm2nii' + ' for subject ' + subj + ' and session ' + ses) if os.path.exists(nii_file): os.rename( nii_file, path.join(bids_dest_dir, bids_name + '.nii.gz')) else: cprint('WARNING: CONVERSION FAILED...' + ' for subject ' + subj + ' and session ' + ses)
def dwi_paths_to_bids(images, dest_dir, mod_to_update=False): """ Convert DWI images Args: images: dataframe returned by the method compute_dwi_paths dest_dir: path to the destination directory mod_to_update: if is true and an image is already existing it will overwrite the old version """ import clinica.iotools.bids_utils as bids import clinica.iotools.converters.adni_to_bids.adni_utils as adni_utils from os import path import os from glob import glob import shutil from clinica.utils.stream import cprint subjs_list = images['Subject_ID'].unique() for i in range(0, len(subjs_list)): # print '--Converting dwi for subject ', subjs_list[i], '--' alpha_id = bids.remove_space_and_symbols(subjs_list[i]) bids_id = 'sub-ADNI' + alpha_id # Extract the list of sessions available from the dwi paths files, removing the duplicates sess_list = images[( images['Subject_ID'] == subjs_list[i])]['VISCODE'].unique() if not os.path.exists(path.join(dest_dir, bids_id)): os.mkdir(path.join(dest_dir, bids_id)) # For each session available, create the folder if doesn't exist and convert the files for ses in sess_list: ses_bids = adni_utils.viscode_to_session(ses) bids_ses_id = 'ses-' + ses_bids bids_file_name = bids_id + '_ses-' + ses_bids ses_path = path.join(dest_dir, bids_id, bids_ses_id) if mod_to_update: if os.path.exists(path.join(ses_path, 'dwi')): shutil.rmtree(path.join(ses_path, 'dwi')) if not os.path.exists(ses_path): os.mkdir(ses_path) dwi_info = images[(images['Subject_ID'] == subjs_list[i]) & (images['VISCODE'] == ses)] # For the same subject, same session there could be multiple dwi with different acq label for j in range(0, len(dwi_info)): dwi_subj = dwi_info.iloc[j] # TODO For now in CLINICA we ignore Enhanced DWI. if dwi_subj['Enhanced']: continue if type(dwi_subj['Path']) != float and dwi_subj['Path'] != '': if not os.path.exists(path.join(ses_path, 'dwi')): os.mkdir(path.join(ses_path, 'dwi')) dwi_path = dwi_subj['Path'] bids_name = bids_file_name + '_acq-' + ( 'axialEnhanced' if dwi_subj['Enhanced'] else 'axial') + '_dwi' # bids.dcm_to_nii(dwi_path, path.join(ses_path, 'dwi'), bids_name) bids_dest_dir = path.join(ses_path, 'dwi') if not os.path.exists(bids_dest_dir): os.mkdir(dest_dir) os.system('dcm2niix -b n -z y -o ' + bids_dest_dir + ' -f ' + bids_name + ' ' + dwi_path) # If dcm2niix didn't work use dcm2nii # print path.join(dest_dir, bids_name + '.nii.gz') if not os.path.exists( path.join(bids_dest_dir, bids_name + '.nii.gz') ) or not os.path.exists( path.join(bids_dest_dir, bids_name + '.bvec') or not os.path.exists( path.join(bids_dest_dir, bids_name + '.bval'))): cprint( '\nConversion with dcm2niix failed, trying with dcm2nii' ) # Find all the files eventually created by dcm2niix and remove them dwi_dcm2niix = glob( path.join(bids_dest_dir, bids_name + '*')) for d in dwi_dcm2niix: # print 'Removing the old', d os.remove(d) os.system( 'dcm2nii -a n -d n -e n -i y -g y -p n -m n -r n -x n -o ' + bids_dest_dir + ' ' + dwi_path) nii_file = path.join( bids_dest_dir, subjs_list[i].replace('_', '') + '.nii.gz') bvec_file = path.join( bids_dest_dir, subjs_list[i].replace('_', '') + '.bvec') bval_file = path.join( bids_dest_dir, subjs_list[i].replace('_', '') + '.bval') if os.path.exists(bvec_file) and os.path.exists( bval_file): os.rename( bvec_file, path.join(bids_dest_dir, bids_name + '.bvec')) os.rename( bval_file, path.join(bids_dest_dir, bids_name + '.bval')) else: cprint( 'WARNING: bvec and bval not generated by dcm2nii' ) if os.path.exists(nii_file): os.rename( nii_file, path.join(bids_dest_dir, bids_name + '.nii.gz')) else: cprint('WARNING: CONVERSION FAILED...')
def create_adni_sessions_dict(bids_ids, clinic_specs_path, clinical_data_dir, bids_subjs_paths): """Extract all the data required for the sessions files and organize them in a dictionary. Args: bids_ids: clinic_specs_path: path to the specifications for converting the clinical data clinical_data_dir: path to the clinical data folder bids_subjs_paths: a list with the path to all the BIDS subjects """ from datetime import datetime from os import path import pandas as pd import clinica.iotools.bids_utils as bids from clinica.utils.stream import cprint # Load data sessions = pd.read_excel(clinic_specs_path, sheet_name="sessions.tsv") sessions_fields = sessions["ADNI"] field_location = sessions["ADNI location"] sessions_fields_bids = sessions["BIDS CLINICA"] fields_dataset = [] previous_location = "" fields_bids = [] sessions_dict = {} for i in range(0, len(sessions_fields)): if not pd.isnull(sessions_fields[i]): fields_bids.append(sessions_fields_bids[i]) fields_dataset.append(sessions_fields[i]) sessions_df = pd.DataFrame(columns=fields_bids) for i in range(0, len(field_location)): # If the i-th field is available if (not pd.isnull(field_location[i])) and path.exists( path.join(clinical_data_dir, field_location[i].split("/")[0])): # Load the file tmp = field_location[i].split("/") location = tmp[0] if location != previous_location: previous_location = location file_to_read_path = path.join(clinical_data_dir, location) cprint(f"\tReading clinical data file: {location}") file_to_read = pd.read_csv(file_to_read_path, dtype=str) for r in range(0, len(file_to_read.values)): row = file_to_read.iloc[r] # Depending of the file that needs to be open, identify and # do needed preprocessing on the column that contains the # subjects ids if location == "ADNIMERGE.csv": id_ref = "PTID" # What was the meaning of this line ? # subj_id = row[id_ref.decode('utf-8')] subj_id = row[id_ref] subj_id = bids.remove_space_and_symbols(subj_id) else: id_ref = "RID" rid = str(row[id_ref]) # Fill the rid with the needed number of zero if 4 - len(rid) > 0: zeros_to_add = 4 - len(rid) subj_id = "0" * zeros_to_add + rid else: subj_id = rid # Extract the BIDS subject id related with the original # subject id subj_bids = [s for s in bids_ids if subj_id in s] if len(subj_bids) == 0: pass elif len(subj_bids) > 1: raise ( "Error: multiple subjects found for the same RID") else: subj_bids = subj_bids[0] for j in range(0, len(sessions_fields)): # If the i-th field is available if not pd.isnull(sessions_fields[j]): # Extract only the fields related to the current file opened if location in field_location[i]: if location in [ "ADAS_ADNIGO2.csv", "DXSUM_PDXCONV_ADNIALL.csv", "CDR.csv", "NEUROBAT.csv", "GDSCALE.csv", "MODHACH.csv", "MOCA.csv", "NPIQ.csv", "MEDHIST.csv", "VITALS.csv", "UWNPSYCHSUM_03_07_19.csv", "ECOGPT.csv", "ECOGSP.csv", "FCI.csv", "CCI.csv", "NPIQ.csv", "NPI.csv", ]: if (pd.isnull(row["VISCODE2"]) or row["VISCODE2"] == "f"): continue visit_id = row["VISCODE2"] # Convert sc to bl if visit_id == "sc": visit_id = "bl" elif location in [ "BHR_EVERYDAY_COGNITION.csv", "BHR_BASELINE_QUESTIONNAIRE.csv", "BHR_LONGITUDINAL_QUESTIONNAIRE.csv", ]: visit_id = row["Timepoint"] else: visit_id = row["VISCODE"] try: field_value = row[sessions_fields[j]] bids_field_name = sessions_fields_bids[ j] # Calculating age from ADNIMERGE if (sessions_fields[j] == "AGE") and ( visit_id != "bl"): examdate = datetime.strptime( row["EXAMDATE"], "%Y-%m-%d") examdate_bl = datetime.strptime( row["EXAMDATE_bl"], "%Y-%m-%d") delta = examdate - examdate_bl # Adding time passed since bl to patient's age in current visit field_value = round( float(field_value) + (delta.days / 365.25), 1, ) sessions_dict = update_sessions_dict( sessions_dict, subj_bids, visit_id, field_value, bids_field_name, ) except KeyError: pass # cprint('Field value ' + Fore.RED + sessions_fields[j] + Fore.RESET + # ' could not be added to sessions.tsv') else: continue # Write the sessions dictionary created in several tsv files write_adni_sessions_tsv(sessions_dict, fields_bids, bids_subjs_paths)
def create_adni_sessions_dict(bids_ids, clinic_specs_path, clinical_data_dir, bids_subjs_paths): """ Extract all the data required for the sessions files and organize them in a dictionary Args: bids_ids: clinic_specs_path: path to the specifications for converting the clinical data clinical_data_dir: path to the clinical data folder bids_subjs_paths: a list with the path to all the BIDS subjects Returns: """ import pandas as pd from os import path import clinica.iotools.bids_utils as bids # Load data sessions = pd.read_excel(clinic_specs_path, sheetname='sessions.tsv') sessions_fields = sessions['ADNI'] field_location = sessions['ADNI location'] sessions_fields_bids = sessions['BIDS CLINICA'] fields_dataset = [] previous_location = '' fields_bids = [] sessions_dict = {} for i in range(0, len(sessions_fields)): if not pd.isnull(sessions_fields[i]): fields_bids.append(sessions_fields_bids[i]) fields_dataset.append(sessions_fields[i]) sessions_df = pd.DataFrame(columns=fields_bids) for i in range(0, len(field_location)): # If the i-th field is available if not pd.isnull(field_location[i]): # Load the file tmp = field_location[i].split('/') location = tmp[0] if location != previous_location: previous_location = location file_to_read_path = path.join(clinical_data_dir, location) file_to_read = pd.read_csv(file_to_read_path, dtype=str) for r in range(0, len(file_to_read.values)): row = file_to_read.iloc[r] # Depending of the file that needs to be open, identify and do needed preprocessing on the column # that contains the subjects ids if location == 'ADNIMERGE.csv': id_ref = 'PTID' subj_id = row[id_ref.decode('utf-8')] subj_id = bids.remove_space_and_symbols(subj_id) else: id_ref = 'RID' rid = str(row[id_ref.decode('utf-8')]) # Fill the rid with the needed number of zero if 4 - len(rid) > 0: zeros_to_add = 4 - len(rid) subj_id = '0' * zeros_to_add + rid else: subj_id = rid # Extract the BIDS subject id related with the original subject id subj_bids = [s for s in bids_ids if subj_id in s] if len(subj_bids) == 0: pass elif len(subj_bids) > 1: raise 'Error: multiple subjects found for the same RID' else: subj_bids = subj_bids[0] for i in range(0, len(sessions_fields)): # If the i-th field is available if not pd.isnull(sessions_fields[i]): # Extract only the fields related to the current file opened if location in field_location[i]: if location == 'ADAS_ADNIGO2.csv' or location == 'DXSUM_PDXCONV_ADNIALL.csv' or location == 'CDR.csv' or location == 'NEUROBAT.csv': if type(row['VISCODE2']) == float: continue visit_id = row['VISCODE2'] # Convert sc to bl if visit_id == 'sc': visit_id = 'bl' else: visit_id = row['VISCODE'] field_value = row[sessions_fields[i]] bids_field_name = sessions_fields_bids[i] sessions_dict = update_sessions_dict( sessions_dict, subj_bids, visit_id, field_value, bids_field_name) else: continue # Write the sessions dictionary created in several tsv files write_adni_sessions_tsv(sessions_dict, fields_bids, bids_subjs_paths)
def generate_subject_files(subj, images, dest_dir, mod_to_update): import clinica.iotools.bids_utils as bids import clinica.iotools.converters.adni_to_bids.adni_utils as adni_utils from clinica.utils.stream import cprint import subprocess import os import shutil from os import path from glob import glob alpha_id = bids.remove_space_and_symbols(subj) bids_id = 'sub-ADNI' + alpha_id # Extract the list of sessions available from the flair paths files, removing the duplicates sess_list = images[(images['Subject_ID'] == subj)]['VISCODE'].unique() if not os.path.exists(path.join(dest_dir, bids_id)): os.mkdir(path.join(dest_dir, bids_id)) # For each session available, create the folder if doesn't exist and convert the files for ses in sess_list: with counter.get_lock(): counter.value += 1 cprint('[FLAIR] Processing subject ' + str(subj) + ' - session ' + ses + ', ' + str(counter.value) + ' / ' + str(len(images))) ses_bids = adni_utils.viscode_to_session(ses) bids_ses_id = 'ses-' + ses_bids bids_file_name = bids_id + '_ses-' + ses_bids ses_path = path.join(dest_dir, bids_id, bids_ses_id) if mod_to_update: if os.path.exists(path.join(ses_path, 'FLAIR')): shutil.rmtree(path.join(ses_path, 'FLAIR')) if not os.path.exists(ses_path): os.mkdir(ses_path) flair_info = images[(images['Subject_ID'] == subj) & (images['VISCODE'] == ses)] # For the same subject, same session there could be multiple flar with different acq label for j in range(len(flair_info)): flair_subj = flair_info.iloc[j] # TODO For now in CLINICA we ignore Enhanced FLAIR. if flair_subj['Enhanced']: continue if type(flair_subj['Path']) != float and flair_subj['Path'] != '': if not os.path.exists(path.join(ses_path, 'FLAIR')): os.mkdir(path.join(ses_path, 'FLAIR')) flair_path = flair_subj['Path'] bids_name = bids_file_name + '_FLAIR' # bids.dcm_to_nii(dwi_path, path.join(ses_path, 'dwi'), bids_name) bids_dest_dir = path.join(ses_path, 'FLAIR') if not os.path.exists(bids_dest_dir): os.mkdir(dest_dir) command = 'dcm2niix -b y -z y -o ' + bids_dest_dir + ' -f ' + bids_name + ' ' + flair_path subprocess.run(command, shell=True, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) # If dcm2niix didn't work use dcm2nii # print path.join(dest_dir, bids_name + '.nii.gz') if not os.path.exists( path.join(bids_dest_dir, bids_name + '.nii.gz')): cprint( '\tConversion with dcm2niix failed, trying with dcm2nii' ) # Find all the files eventually created by dcm2niix and remove them flair_dcm2niix = glob( path.join(bids_dest_dir, bids_name + '*')) for d in flair_dcm2niix: os.remove(d) command = 'dcm2nii -a n -d n -e n -i y -g y -p n -m n -r n -x n -o ' + bids_dest_dir + ' ' + flair_path subprocess.run(command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) nii_file = path.join(bids_dest_dir, subj.replace('_', '') + '.nii.gz') if os.path.exists(nii_file): os.rename( nii_file, path.join(bids_dest_dir, bids_name + '.nii.gz')) else: cprint('WARNING: CONVERSION FAILED...')