def common_processing(caselist):
imgs, masks = read_caselist(caselist)
res = []
pool = multiprocessing.Pool(N_proc)
for imgPath, maskPath in zip(imgs, masks):
res.append(
pool.apply_async(func=preprocessing, args=(imgPath, maskPath)))
attributes = [r.get() for r in res]
pool.close()
pool.join()
f = open(caselist + '.modified', 'w')
for i in range(len(imgs)):
imgs[i] = attributes[i][0]
masks[i] = attributes[i][1]
f.write(f'{imgs[i]},{masks[i]}\n')
f.close()
# the following imgs, masks is for diagnosing MemoryError i.e. computing rish w/o preprocessing
# to diagnose, comment all the above and uncomment the following
# imgs, masks = read_caselist(caselist+'.modified')
# experimentally found ncpu=4 to be memroy optimal
pool = multiprocessing.Pool(4)
for imgPath, maskPath in zip(imgs, masks):
pool.apply_async(func=dti_harm, args=(imgPath, maskPath))
pool.close()
pool.join()
return (imgs, masks)
def sub2tmp2mni(templatePath,
siteName,
caselist,
ref=False,
tar_unproc=False,
tar_harm=False):
# obtain the transform
moving = pjoin(templatePath, f'Mean_{siteName}_FA_b{bshell_b}.nii.gz')
outPrefix = pjoin(templatePath, f'TemplateToMNI_{siteName}')
warp2mni = outPrefix + '1Warp.nii.gz'
trans2mni = outPrefix + '0GenericAffine.mat'
# check existence of transforms created with _b{bmax}
if not exists(warp2mni):
antsReg(mniTmp, None, moving, outPrefix)
imgs, _ = read_caselist(caselist)
pool = multiprocessing.Pool(N_proc)
for imgPath in imgs:
if ref:
pool.apply_async(func=register_reference,
args=(
imgPath,
warp2mni,
trans2mni,
templatePath,
))
elif tar_unproc:
pool.apply_async(func=register_target,
args=(
imgPath,
templatePath,
))
elif tar_harm:
pool.apply_async(func=register_harmonized,
args=(
imgPath,
warp2mni,
trans2mni,
templatePath,
siteName,
))
pool.close()
pool.join()
def common_processing(caselist):
imgs, masks = read_caselist(caselist)
# compute dti_harm of unprocessed data
pool = multiprocessing.Pool(N_proc)
for imgPath, maskPath in zip(imgs, masks):
pool.apply_async(func=dti_harm, args=(imgPath, maskPath))
pool.close()
pool.join()
try:
copyfile(caselist, caselist + '.modified')
except SameFileError:
pass
# data is not manipulated in multi-shell-dMRIharmonization i.e. bvalMapped, resampled, nor denoised
# this block may be uncommented in a future design
# preprocess data
# res=[]
# pool = multiprocessing.Pool(N_proc)
# for imgPath,maskPath in zip(imgs,masks):
# res.append(pool.apply_async(func= preprocessing, args= (imgPath,maskPath)))
#
# attributes= [r.get() for r in res]
#
# pool.close()
# pool.join()
#
# f = open(caselist + '.modified', 'w')
# for i in range(len(imgs)):
# imgs[i] = attributes[i][0]
# masks[i] = attributes[i][1]
# f.close()
#
#
# # compute dti_harm of preprocessed data
# pool = multiprocessing.Pool(N_proc)
# for imgPath,maskPath in zip(imgs,masks):
# pool.apply_async(func= dti_harm, args= (imgPath,maskPath))
# pool.close()
# pool.join()
#
#
# if debug:
# #TODO compute dti_harm for all intermediate data _denoised, _denoised_bmapped, _bmapped
# pass
return (imgs, masks)
def harmonizeData(self):
from reconstSignal import reconst
# check the templatePath
if not exists(self.templatePath):
raise NotADirectoryError(f'{self.templatePath} does not exist')
else:
if not os.listdir(self.templatePath):
raise ValueError(f'{self.templatePath} is empty')
# go through each file listed in csv, check their existence, create dti and harm directories
check_csv(self.target_csv, self.force)
# target data is not manipulated in multi-shell-dMRIharmonization i.e. bvalMapped, resampled, nor denoised
# this block may be uncommented in a future design
# from preprocess import dti_harm
# if self.debug:
# # calcuate diffusion measures of target site before any processing so we are able to compare
# # with the ones after harmonization
# imgs, masks= read_caselist(self.tar_unproc_csv)
# pool = multiprocessing.Pool(self.N_proc)
# for imgPath, maskPath in zip(imgs, masks):
# imgPath= convertedPath(imgPath)
# maskPath= convertedPath(maskPath)
# pool.apply_async(func= dti_harm, args= ((imgPath, maskPath, )))
#
# pool.close()
# pool.join()
# reconstSignal steps ------------------------------------------------------------------------------------------
# read target image list
moving = pjoin(self.templatePath,
f'Mean_{self.target}_FA_b{self.bshell_b}.nii.gz')
imgs, masks = read_caselist(self.target_csv)
preFlag = 1 # omit preprocessing of target data again
if self.target_csv.endswith('.modified'):
preFlag = 0
else:
# this file will be used later for debugging
self.target_csv += '.modified'
fm = open(self.target_csv, 'w')
self.harm_csv = self.target_csv + '.harmonized'
fh = open(self.harm_csv, 'w')
pool = multiprocessing.Pool(self.N_proc)
res = []
for imgPath, maskPath in zip(imgs, masks):
res.append(
pool.apply_async(func=reconst,
args=(
imgPath,
maskPath,
moving,
self.templatePath,
preFlag,
)))
for r in res:
imgPath, maskPath, harmImg, harmMask = r.get()
if preFlag:
fm.write(imgPath + ',' + maskPath + '\n')
fh.write(harmImg + ',' + harmMask + '\n')
pool.close()
pool.join()
# loop for debugging
# res= []
# for imgPath, maskPath in zip(imgs, masks):
# res.append(reconst(imgPath, maskPath, moving, self.templatePath, preFlag))
#
# for r in res:
# imgPath, maskPath, harmImg, harmMask= r
#
# if preFlag:
# fm.write(imgPath + ',' + maskPath + '\n')
# fh.write(harmImg + ',' + harmMask + '\n')
if preFlag:
fm.close()
fh.close()
print('\n\nHarmonization completed\n\n')
def main(self):
self.templatePath= abspath(self.templatePath)
self.N_shm= int(self.N_shm)
self.N_proc= int(self.N_proc)
if self.N_proc==-1:
self.N_proc= N_CPU
if self.ref_csv:
self.ref_unproc_csv= self.ref_csv.strip('.modified')
self.tar_unproc_csv= self.target_csv.strip('.modified')
# check appropriateness of N_shm
if self.N_shm!=-1 and (self.N_shm<2 or self.N_shm>8):
raise ValueError('2<= --nshm <=8')
# determine N_shm in default mode during template creation
if self.N_shm==-1 and self.create:
if self.ref_csv:
ref_nshm_img = read_caselist(self.ref_csv)[0][0]
elif self.target_csv:
ref_nshm_img = read_caselist(self.target_csv)[0][0]
directory= dirname(ref_nshm_img)
prefix= basename(ref_nshm_img).split('.nii')[0]
bvalFile= pjoin(directory, prefix+'.bval')
self.N_shm, _= determineNshm(bvalFile)
# automatic determination of N_shm during data harmonization is limited by N_shm used during template creation
# Scale_L{i}.nii.gz of <= {N_shm during template creation} are present only
elif self.N_shm==-1 and self.process:
for i in range(0,8,2):
if isfile(pjoin(self.templatePath, f'Scale_L{i}_b{self.bshell_b}.nii.gz')):
self.N_shm= i
else:
break
# verify validity of provided/determined N_shm for all subjects
if self.ref_csv:
verifyNshmForAll(self.ref_csv, self.N_shm)
if self.target_csv:
verifyNshmForAll(self.target_csv, self.N_shm)
# write config file to temporary directory
configFile= f'/tmp/harm_config_{getpid()}.ini'
with open(configFile,'w') as f:
f.write('[DEFAULT]\n')
f.write(f'N_shm = {self.N_shm}\n')
f.write(f'N_proc = {self.N_proc}\n')
f.write(f'N_zero = {self.N_zero}\n')
f.write(f'resample = {self.resample if self.resample else 0}\n')
f.write(f'bvalMap = {self.bvalMap if self.bvalMap else 0}\n')
f.write(f'bshell_b = {self.bshell_b}\n')
f.write(f'denoise = {1 if self.denoise else 0}\n')
f.write(f'travelHeads = {1 if self.travelHeads else 0}\n')
f.write(f'debug = {1 if self.debug else 0}\n')
f.write(f'force = {1 if self.force else 0}\n')
f.write(f'verbose = {1 if self.verbose else 0}\n')
f.write('diffusionMeasures = {}\n'.format((',').join(self.diffusionMeasures)))
self.sanityCheck()
if self.create:
self.createTemplate()
import fileinput
for line in fileinput.input(configFile, inplace=True):
if 'force' in line:
print('force = 0')
else:
print(line)
self.force= False
if self.process:
self.harmonizeData()
if self.create and self.process and self.debug:
self.post_debug()
remove(configFile)
def harmonizeData(self):
from reconstSignal import reconst, approx
from preprocess import dti_harm, preprocessing, common_processing
# check the templatePath
if not exists(self.templatePath):
raise NotADirectoryError(f'{self.templatePath} does not exist')
else:
if not listdir(self.templatePath):
raise ValueError(f'{self.templatePath} is empty')
# fit spherical harmonics on reference site
if self.debug and self.ref_csv:
check_csv(self.ref_unproc_csv, self.force)
refImgs, refMasks= read_caselist(self.ref_unproc_csv)
# reference data is not manipulated in multi-shell-dMRIharmonization i.e. bvalMapped, resampled, nor denoised
# this block may be uncommented in a future design
# res= []
# pool = multiprocessing.Pool(self.N_proc)
# for imgPath, maskPath in zip(refImgs, refMasks):
# res.append(pool.apply_async(func=preprocessing, args=(imgPath, maskPath)))
#
# attributes = [r.get() for r in res]
#
# pool.close()
# pool.join()
#
# for i in range(len(refImgs)):
# refImgs[i] = attributes[i][0]
# refMasks[i] = attributes[i][1]
pool = multiprocessing.Pool(self.N_proc)
for imgPath, maskPath in zip(refImgs, refMasks):
pool.apply_async(func= approx, args=(imgPath,maskPath,))
pool.close()
pool.join()
# go through each file listed in csv, check their existence, create dti and harm directories
check_csv(self.target_csv, self.force)
targetImgs, targetMasks= common_processing(self.tar_unproc_csv)
# reconstSignal steps ------------------------------------------------------------------------------------------
# read target image list
moving= pjoin(self.templatePath, f'Mean_{self.target}_FA_b{self.bshell_b}.nii.gz')
if not self.target_csv.endswith('.modified'):
self.target_csv += '.modified'
self.harm_csv= self.target_csv+'.harmonized'
fh= open(self.harm_csv, 'w')
pool = multiprocessing.Pool(self.N_proc)
res= []
for imgPath, maskPath in zip(targetImgs, targetMasks):
res.append(pool.apply_async(func= reconst, args= (imgPath, maskPath, moving, self.templatePath,)))
for r in res:
harmImg, harmMask= r.get()
fh.write(harmImg + ',' + harmMask + '\n')
pool.close()
pool.join()
# loop for debugging
# res= []
# for imgPath, maskPath in zip(imgs, masks):
# res.append(reconst(imgPath, maskPath, moving, self.templatePath))
#
# for r in res:
# harmImg, harmMask= r
# fh.write(harmImg + ',' + harmMask + '\n')
fh.close()
if self.debug:
harmImgs, harmMasks= read_caselist(self.harm_csv)
pool = multiprocessing.Pool(self.N_proc)
for imgPath,maskPath in zip(harmImgs,harmMasks):
pool.apply_async(func= dti_harm, args= (imgPath,maskPath,))
pool.close()
pool.join()
print('\n\nHarmonization completed\n\n')
pjoin(templatePath,
basename(maskPath).split('.nii')[0] + 'Warped.nii.gz'))
'''
# warping the rish features
for i in range(0, N_shm+1, 2):
applyXform(pjoin(directory, 'harm', f'{prefix}_L{i}.nii.gz'),
pjoin(templatePath, 'template0.nii.gz'),
warp, trans,
pjoin(templatePath, f'{prefix}_WarpedL{i}.nii.gz'))
# warping the diffusion measures
for dm in diffusionMeasures:
applyXform(pjoin(directory, 'dti', f'{prefix}_{dm}.nii.gz'),
pjoin(templatePath, 'template0.nii.gz'),
warp, trans,
pjoin(templatePath, f'{prefix}_Warped{dm}.nii.gz'))
'''
if __name__ == '__main__':
templatePath = '/data/pnl/HarmonizationProject/abcd/site21/site21_cluster/retest_multi/template_April4'
img_list = '/data/pnl/HarmonizationProject/abcd/site21/site21_cluster/retest_multi/target_b1000.csv.modified'
imgs, masks = read_caselist(img_list)
for imgPath, maskPath in zip(imgs, masks):
warp_bands(imgPath, maskPath, templatePath)