def consistencyCheck(ref_csv, outputBshellFile=None, outPutResolutionFile=None): try: ref_imgs, _ = read_imgs_masks(ref_csv) except: ref_imgs = read_imgs(ref_csv) if isfile(outputBshellFile) and isfile(outPutResolutionFile): ref_bvals = read_bvals(outputBshellFile) ref_res = np.load(outPutResolutionFile) else: ref_bshell_img = ref_imgs[0] print(f'Using {ref_bshell_img} to determine b-shells') inPrefix = abspath(ref_bshell_img).split('.nii')[0] ref_bvals = findBShells(inPrefix + '.bval', outputBshellFile) ref_res = load(ref_bshell_img).header['pixdim'][1:4] np.save(outPutResolutionFile, ref_res) print('b-shells are', ref_bvals) print('\nSite', ref_csv, '\n') print('Checking consistency of b-shells among subjects') check_bshells(ref_imgs, ref_bvals) print('spatial resolution is', ref_res) print('Checking consistency of spatial resolution among subjects') check_resolution(ref_imgs, ref_res)
def verifyNshmForAll(csvFile, N_shm): for imgPath in read_imgs_masks(csvFile)[0]: directory = dirname(imgPath) prefix = basename(imgPath).split('.nii')[0] bvalFile = pjoin(directory, prefix + '.bval') verifyNshm(N_shm, bvalFile)
def printStat(ref_mean, csvFile): print('mean FA over IIT_mean_FA_skeleton.nii.gz for all cases: ') imgs, _ = read_imgs_masks(csvFile) for i, imgPath in enumerate(imgs): print(basename(imgPath), ref_mean[i]) print('') print('mean meanFA: ', np.mean(ref_mean)) print('std meanFA: ', np.std(ref_mean)) print('')
def generate_csv(imgs, site_means, outPrefix): try: imgs, _ = read_imgs_masks(imgs) except: pass stat = {'subject': [basename(f) for f in imgs], 'meanFA': site_means} df = pd.DataFrame(stat) statFile = outPrefix + '_stat.csv' df.to_csv(statFile, index=False)
def separateAllBshells(ref_csv, ref_bvals_file, ncpu=4, outPrefix=None): ref_bvals = read_bvals(ref_bvals_file) try: imgs, masks = read_imgs_masks(ref_csv) except: imgs = read_imgs(ref_csv) masks = None pool = Pool(int(ncpu)) for imgPath in imgs: pool.apply_async(separateBshells, kwds={ 'imgPath': imgPath, 'ref_bvals': ref_bvals }, error_callback=RAISE) pool.close() pool.join() if outPrefix: outPrefix = abspath(outPrefix) else: return for bval in ref_bvals: f = open(f'{outPrefix}_b{int(bval)}.csv', 'w') if masks: for imgPath, maskPath in zip(imgs, masks): inPrefix = abspath(imgPath).split('.nii')[0] bPrefix = inPrefix + f'_b{int(bval)}' f.write(f'{bPrefix}.nii.gz,{maskPath}\n') else: for imgPath in imgs: inPrefix = abspath(imgPath).split('.nii')[0] bPrefix = inPrefix + f'_b{int(bval)}' f.write(f'{bPrefix}.nii.gz\n') f.close()
def main(self): self.template = str(self.template) self.labelMap = str(self.labelMap) imgs, masks = read_imgs_masks(self.imagelist) if int(self.N_proc) == -1: self.N_proc = psutil.cpu_count() pool = multiprocessing.Pool(int(self.N_proc)) for imgPath, maskPath in zip(imgs, masks): imgPath = imgPath inPrefix = imgPath.split('.')[0] bvalFile = None bvecFile = None if imgPath.endswith('.nii') or imgPath.endswith('.nii.gz'): bvalFile = inPrefix + '.bval' bvecFile = inPrefix + '.bvec' out_dir = join(dirname(imgPath), self.qcDir) if isdir(out_dir): # force re-run rmtree(out_dir) mkdir(out_dir) pool.apply_async(func=dwi_quality_wrapper, args=(imgPath, maskPath, bvalFile, bvecFile, self.mk_low_high, self.fa_low_high, self.md_low_high, out_dir, self.name, self.template, self.labelMap, self.lut._path if self.lut else None)) pool.close() pool.join() summarize_csvs(imgs, self.labelMap, self.lut._path if self.lut else None, self.qcDir, self.name, self.out_csv)
def joinAllBshells(tar_csv, ref_bvals_file, separatedPrefix=None, ncpu=4): ref_bvals = read_bvals(ref_bvals_file) if tar_csv: try: imgs, _ = read_imgs_masks(tar_csv) except: imgs = read_imgs(tar_csv) pool = Pool(int(ncpu)) for imgPath in imgs: pool.apply_async(joinBshells, kwds=({ 'imgPath': imgPath, 'ref_bvals': ref_bvals, 'sep_prefix': separatedPrefix }), error_callback=RAISE) pool.close() pool.join()
def main(self): self.templatePath = abspath(self.templatePath) self.N_shm = int(self.N_shm) self.N_proc = int(self.N_proc) if self.N_proc == -1: self.N_proc = N_CPU if self.target_csv.endswith('.modified'): self.tar_unproc_csv = str(self.target_csv).split('.modified')[0] else: self.tar_unproc_csv = str(self.target_csv) # check appropriateness of N_shm if self.N_shm != -1 and (self.N_shm < 2 or self.N_shm > 8): raise ValueError('2<= --nshm <=8') # determine N_shm in default mode during template creation if self.N_shm == -1 and self.create: if self.ref_csv: ref_nshm_img = read_imgs_masks(self.ref_csv)[0][0] elif self.target_csv: ref_nshm_img = read_imgs_masks(self.target_csv)[0][0] directory = dirname(ref_nshm_img) prefix = basename(ref_nshm_img).split('.nii')[0] bvalFile = pjoin(directory, prefix + '.bval') self.N_shm, _ = determineNshm(bvalFile) # automatic determination of N_shm during data harmonization is limited by N_shm used during template creation # Scale_L{i}.nii.gz of <= {N_shm during template creation} are present only elif self.N_shm == -1 and self.process: for i in range(0, 8, 2): if isfile( pjoin(self.templatePath, f'Scale_L{i}_b{self.bshell_b}.nii.gz')): self.N_shm = i else: break # verify validity of provided/determined N_shm for all subjects if self.ref_csv: verifyNshmForAll(self.ref_csv, self.N_shm) if self.target_csv: verifyNshmForAll(self.target_csv, self.N_shm) # write config file to temporary directory configFile = f'/tmp/harm_config_{os.getpid()}.ini' with open(configFile, 'w') as f: f.write('[DEFAULT]\n') f.write(f'N_shm = {self.N_shm}\n') f.write(f'N_proc = {self.N_proc}\n') f.write(f'N_zero = {self.N_zero}\n') f.write(f'resample = {self.resample if self.resample else 0}\n') f.write(f'bvalMap = {self.bvalMap if self.bvalMap else 0}\n') f.write(f'bshell_b = {self.bshell_b}\n') f.write(f'denoise = {1 if self.denoise else 0}\n') f.write(f'travelHeads = {1 if self.travelHeads else 0}\n') f.write(f'debug = {1 if self.debug else 0}\n') f.write(f'verbose = {1 if self.verbose else 0}\n') f.write('diffusionMeasures = {}\n'.format( (',').join(self.diffusionMeasures))) self.sanityCheck() if self.create: self.createTemplate() if self.process: self.harmonizeData() if self.create and self.process and self.debug: self.post_debug() os.remove(configFile)
def main(): parser = argparse.ArgumentParser(description='''Warps diffusion measures (FA, MD, GFA) to template space and then to MNI space. Finally, calculates mean FA over IITmean_FA_skeleton.nii.gz''') parser.add_argument('-i', '--input', type=str, required=True, help='a .txt/.csv file having one column for FA imgs, ' 'or two columns for (img,mask) pair, the latter list is what you used in/obtained from harmonization.py. ' 'See pnlbwh/dMRIharmonization documentation for more details') parser.add_argument('-s', '--site', type= str, required=True, help='site name for locating template FA and mask in tempalte directory') parser.add_argument('-t', '--template', type=str, required=True, help='template directory where Mean_{site}_FA.nii.gz and {site}_Mask.nii.gz is located') parser.add_argument('--bshell_b', required=True, help='bvalue of the bshell') parser.add_argument('--ncpu', help='number of cpus to use', default= '4') args = parser.parse_args() imgList=abspath(args.input) siteName=args.site templatePath=abspath(args.template) bshell_b= int(args.bshell_b) N_proc= int(args.ncpu) # read FA image list try: imgs, _ = read_imgs_masks(imgList) print('imgs,masks list is provided. FA images are assumed to be directoryOfImg/dti/ImgPrefix_FA.nii.gz, make sure they are there') faImgs= [] for imgPath in imgs: directory = dirname(imgPath) prefix = basename(imgPath).split('.nii')[0] faImg= pjoin(directory, 'dti', prefix+ '_FA.nii.gz') if not isfile(faImg): raise FileNotFoundError(f'{faImg} not found. Did you run \"--create --debug\" and \"--process --debug\" before?') faImgs.append(faImg) except: faImgs= read_imgs(imgList) print('FA image list is provided.') # register and obtain *_InMNI_FA.nii.gz mniFAimgs= sub2tmp2mni(templatePath, siteName, faImgs, bshell_b, N_proc) # save statistics for future statFile= os.path.join(self.templatePath, 'meanFAstat.txt') f= open(statFile,'a') stdout= sys.stdout sys.stdout= f print(datetime.now().strftime('%c'),'\n') print('b-shell', bshell_b, '\n') # pass *_InMNI_FA.nii.gz list to analyzeStat site_means= analyzeStat(mniFAimgs) print(f'{siteName} site: ') printStat(site_means, mniFAimgs) f.close() sys.stdout= stdout # print statistics on console print('') with open(statFile) as f: print(f.read()) print('\nThe statistics are also saved in ', statFile)
def main(): parser = argparse.ArgumentParser( description='''Warps diffusion measures (FA, MD, GFA) to template space and then to MNI space. Finally, calculates mean FA over IITmean_FA_skeleton.nii.gz''' ) parser.add_argument( '-i', '--input', type=str, required=True, help= 'a .txt/.csv file that you used in/obtained from harmonization.py having two columns for (img,mask) pair. ' 'See documentation for more details') parser.add_argument( '-s', '--site', type=str, required=True, help='site name for locating template FA and mask in template directory' ) parser.add_argument( '-t', '--template', type=str, required=True, help= 'template directory where Mean_{site}_FA.nii.gz and {site}_Mask.nii.gz is located' ) parser.add_argument('--ncpu', help='number of cpus to use', default='4') args = parser.parse_args() imgList = abspath(args.input) siteName = args.site templatePath = abspath(args.template) N_proc = int(args.ncpu) # read FA image list try: imgs, _ = read_imgs_masks(imgList) print( '(Img,Mask) list is provided. FA images are assumed to be directoryOfImg/dti/ImgPrefix_FA.nii.gz, make sure they are there\n' ) faImgs = [] for imgPath in imgs: directory = dirname(imgPath) prefix = basename(imgPath).split('.nii')[0] faImg = pjoin(directory, 'dti', prefix + '_FA.nii.gz') if not isfile(faImg): raise FileNotFoundError( f'{faImg} not found. Did you run \"--create --debug\" and \"--process --debug\" before?' ) faImgs.append(faImg) except: faImgs = read_imgs(imgList) print('FA image list is provided.') # register and obtain *_InMNI_FA.nii.gz mniFAimgs = sub2tmp2mni(templatePath, siteName, faImgs, N_proc) # target harmonized if imgList.endswith('.modified.harmonized'): header = siteName + '_after' # reference elif imgList.endswith('.modified'): header = siteName # target unprocessed else: header = siteName + '_before' # FIXME user FA image list will use the header {siteName+'_before'}, which is not correct all the time # as shown in the above block: # reference should use {siteName} while harmonized target should use {siteName+'_after'} # impact of this discrepancy is minor since we deprecated use of FA image list outPrefix = pjoin(templatePath, header) print('\n\nComputing statistics\n\n') print(f'{siteName} site: ') site_means = analyzeStat(mniFAimgs) generate_csv(faImgs, site_means, outPrefix) # save statistics for future statFile = pjoin(templatePath, 'meanFAstat.csv') with open(statFile, 'a') as f: f.write(datetime.now().strftime('%m/%d/%y %H:%M') + ',mean meanFA,std meanFA\n') f.write(f'{header},{np.mean(site_means)},{np.std(site_means)}\n') # print an empty line so future results, if appended, are visually separate f.write('\n') # print statistics on console print('\n\nPrinting statistics\n\n') with open(statFile) as f: print(f.read()) # generate demonstrative plots ebar = harm_plot([site_means], [header], outPrefix) print( f'\nDetailed statistics, summary results, and demonstrative plots are saved in:\n\n{outPrefix}_stat.csv' f'\n{statFile}\n{ebar}\n')
def main(self): self.sanityCheck() self.templatePath= abspath(self.templatePath) self.N_shm= int(self.N_shm) self.N_proc= int(self.N_proc) if self.N_proc==-1: self.N_proc= N_CPU if self.ref_csv: self.ref_unproc_csv= self.ref_csv.strip('.modified') self.tar_unproc_csv= self.target_csv.strip('.modified') if not self.stats: # check appropriateness of N_shm if self.N_shm!=-1 and (self.N_shm<2 or self.N_shm>8): raise ValueError('2<= --nshm <=8') # determine N_shm in default mode during template creation if self.N_shm==-1 and self.create: if self.ref_csv: ref_nshm_img = read_imgs_masks(self.ref_csv)[0][0] elif self.target_csv: ref_nshm_img = read_imgs_masks(self.target_csv)[0][0] directory= dirname(ref_nshm_img) prefix= basename(ref_nshm_img).split('.nii')[0] bvalFile= pjoin(directory, prefix+'.bval') self.N_shm, _= determineNshm(bvalFile) # automatic determination of N_shm during data harmonization is limited by N_shm used during template creation # Scale_L{i}.nii.gz of <= {N_shm during template creation} are present only elif self.N_shm==-1 and self.process: for i in range(0,8,2): if isfile(pjoin(self.templatePath, f'Scale_L{i}.nii.gz')): self.N_shm= i else: break # verify validity of provided/determined N_shm for all subjects # single-shell-ness is verified inside verifyNshmForAll if self.ref_csv: verifyNshmForAll(self.ref_csv, self.N_shm) if self.target_csv: verifyNshmForAll(self.target_csv, self.N_shm) # copy provided config file to temporary directory configFile= pjoin(gettempdir(),f'harm_config_{getpid()}.ini') with open(configFile,'w') as f: f.write('[DEFAULT]\n') f.write(f'N_shm = {self.N_shm}\n') f.write(f'N_proc = {self.N_proc}\n') f.write(f'N_zero = {self.N_zero}\n') f.write(f'resample = {self.resample if self.resample else 0}\n') f.write(f'bvalMap = {self.bvalMap if self.bvalMap else 0}\n') f.write(f'denoise = {1 if self.denoise else 0}\n') f.write(f'travelHeads = {1 if self.travelHeads else 0}\n') f.write(f'debug = {1 if self.debug else 0}\n') f.write(f'force = {1 if self.force else 0}\n') f.write('diffusionMeasures = {}\n'.format((',').join(self.diffusionMeasures))) if self.create: self.createTemplate() import fileinput for line in fileinput.input(configFile, inplace=True): if 'force' in line: print('force = 0') else: print(line) self.force= False if self.process: self.harmonizeData() if self.create and self.process and self.debug: self.post_debug() if self.stats: if (self.create or self.process or self.debug): raise AttributeError('--stats option is for recomputing site statistics exclusively') else: self.showStat() remove(configFile)
def harmonizeData(self): from reconstSignal import reconst, approx from preprocess import dti_harm, common_processing, preprocessing # check the templatePath if not exists(self.templatePath): raise NotADirectoryError(f'{self.templatePath} does not exist') else: if not listdir(self.templatePath): raise ValueError(f'{self.templatePath} is empty') # fit spherical harmonics on reference site if self.debug and self.ref_csv: check_csv(self.ref_unproc_csv, self.force) refImgs, refMasks= read_imgs_masks(self.ref_unproc_csv) res= [] pool = multiprocessing.Pool(self.N_proc) for imgPath, maskPath in zip(refImgs, refMasks): res.append(pool.apply_async(func=preprocessing, args=(imgPath, maskPath))) attributes = [r.get() for r in res] pool.close() pool.join() for i in range(len(refImgs)): refImgs[i] = attributes[i][0] refMasks[i] = attributes[i][1] pool = multiprocessing.Pool(self.N_proc) for imgPath, maskPath in zip(refImgs, refMasks): pool.apply_async(func= approx, args=(imgPath,maskPath,)) pool.close() pool.join() # go through each file listed in csv, check their existence, create dti and harm directories check_csv(self.target_csv, self.force) targetImgs, targetMasks= common_processing(self.tar_unproc_csv) # reconstSignal steps ------------------------------------------------------------------------------------------ moving= pjoin(self.templatePath, f'Mean_{self.target}_FA.nii.gz') if not self.target_csv.endswith('.modified'): self.target_csv += '.modified' self.harm_csv= self.target_csv+'.harmonized' fh= open(self.harm_csv, 'w') pool = multiprocessing.Pool(self.N_proc) res= [] for imgPath, maskPath in zip(targetImgs, targetMasks): res.append(pool.apply_async(func= reconst, args= (imgPath, maskPath, moving, self.templatePath,))) for r in res: harmImg, harmMask= r.get() fh.write(harmImg + ',' + harmMask + '\n') pool.close() pool.join() fh.close() if self.debug: harmImgs, harmMasks= read_imgs_masks(self.harm_csv) pool = multiprocessing.Pool(self.N_proc) for imgPath,maskPath in zip(harmImgs,harmMasks): pool.apply_async(func= dti_harm, args= (imgPath,maskPath,)) pool.close() pool.join() print('\n\nHarmonization completed\n\n')