def savegifti(arr, filename, giftiobj, intent=2005): ''' savegifti(arr, filename, giftiobj, intent=2005):: save 1D or 2D <array> into a full gifti filename using <filename>. We used the affine and header information from <giftiobj>. Note that the file name should have '.gii'. nibabel cannot read '.gii.gz' file This is different from readgifti, we must supply an reference <giftiobj> to save the data <intent> is a list of integers with intent numbers for each column of arr. Typically we can treat it as a normal morph data, so default is 2005. For multiple columns in <arr>, you should input a list such as [2005, 2005, 2005]. For full list intent code. check the website below: https://nifti.nimh.nih.gov/nifti-1/documentation/nifti1fields/nifti1fields_pages/group__NIFTI1__INTENT__CODES.html/document_view 20190409 RZ created it ''' from nibabel import save from nibabel.gifti import GiftiDataArray from numpy import ndarray from RZutilpy.system import makedirs, Path from nibabel.gifti.giftiio import write as giftiwrite # check input assert isinstance( arr, ndarray) and (1 <= arr.ndim <= 2), 'Please input 1d or 2d an ndarray!' # make the dir if it does not exist filename = Path(filename) if ~isinstance(filename, Path) else filename # add .gii suffix if not supplied filename = Path(filename.str + '.gii') if filename.suffix != '.gii' else filename makedirs(filename) # note here we add a os.sep # make darrays for each column if arr.ndim == 1: giftiobj.darrays[0].data = arr giftiobj.darrays[0].intent = intent else: assert len(intent) == arr.shape[ 1], 'data column number not equal to intent number!' darray_list = [giftiobj.darrays[0] for _ in range(arr.shape[1])] for i in range(arr.shape[1]): darray_list[i].data = arr[:, i] darray_list[i].intent = intent[i] giftiobj.darrays = darray_list save(giftiobj, filename.str)
def alignT2toT1alt(subjectid, wantmi=True): ''' alignT2toT1alt(subjectid, wantmi=True): <subjectid> is like 'C0001' <wantmi> is whether to use mutual information as the metric Load T2 volume from T2average.nii.gz. Register to the FreeSurfer T1.nii.gz volume. To perform the registration, we use flirt, using a rigid-body transformation and sinc interpolation. The output is a file called T2alignedtoT1.nii.gz written to the FreeSurfer mri directory. See code for assumptions. ''' from RZutilpy.cvnpy import cvnpath from RZutilpy.system import unix_wrapper from RZutilpy.imageprocess import makeimagestack3dfiles from RZutilpy.system import Path dir0 = (Path(cvnpath('anatomicals')) / subjectid).str fsdir = (Path(cvnpath('freesurfer')) / subjectid).str pp0 = (Path(cvnpath('ppresult')) / subjectid).str # find T2 NIFTI t2nifti = (Path(dir0) / 'T2average.nii.gz').str # find T1 NIFTI t1nifti = (Path(fsdir) / 'mri' / 'T1.nii.gz').str # define output file t2tot1nifti = (Path(fsdir) / 'mri' / 'T2alignedtoT1.nii.gz').str # call flirt to perform the alignment if wantmi: extrastr = ['-cost', 'mutualinfo', '-searchcost', 'mutualinfo'] else: extrastr = [] cmd = ['flirt', '-v',\ '-in', t2nifti,\ '-ref', t1nifti,\ '-out', t2tot1nifti,\ ' -interp', 'sinc',\ '-dof', '6',\ ] + extrastr # note that we add extrastr here # do it unix_wrapper(cmd) # inspect the results makeimagestack3dfiles(t1nifti, (Path(pp0) / 'T1T2alinment' / 'T1').str, skips=(5, 5, 5), k=[1, 1, 0], wantnorm=1, addborder=1) makeimagestack3dfiles(t2tot1nifti, (Path(pp0) / 'T1T2alinment' / 'T2').str, skips=(5, 5, 5), k=[1, 1, 0], wantnorm=1, addborder=1)
def loadmath5py(filename): """ Similar to loadmat function, except that this function import h5py package to read v7.3 mat file can only supply the <filename>, <filename> can be a string or a Path object Return, example see loadmat function To do: 1. consider to selective read in? """ from h5py import File import numpy as np from RZutilpy.system import Path filename = Path(filename) if ~isinstance(filename, Path) else filename print('read the file' + filename.str) Fileobject = File(filename.str, 'r') keys = [] values = [] mat_contend = {} for k, v in Fileobject.items(): keys.append(k) values.append(v) mat_contend[k] = np.array(v) #del keys[0:3] # remove first 3 information keys. #del values[0:3] # delete first 3 values return mat_contend, keys, values
def savepkl(filename, varsdict): ''' savepkl(filename, varsnamelist): Equivalent function as save in matlab. Save function save multiple variables into a .pkl file. This function save several variables into a pickle file. <filename>: a string 'test.pkl', or 'test' with .pkl extension, filename of the pickle file, variable should be saved as 'filename.pkl' <varsdict>: a dict to save # to do create the folder if it does not exists ''' from dill import dump from RZutilpy.system import Path filename = Path(filename) if ~isinstance(filename, Path) else filename # open a file f = open(f'{filename.strnosuffix}.pkl', 'wb') dump(varsdict, f) f.close()
def getmultifilename(pattern, N): ''' getmultifilename(patterns, N) create multiple file names into a list <filenames>. <filenames> are labeled by numbers Useful when saving multiple files,e.g., multiple images <pattern>: a file pattern, e.g., 'image%02d' <N>: is : (1) int, 100. (2) a int array, like [1,2,3,4,5] Example: files = rz.rzio.getmultifilename('image%02d', N=10) ''' from RZutilpy.system import Path from numpy import arange, ndarray pattern = Path(pattern).str # replace '~' to home directory if isinstance(N, int): N = arange(N) + 1 assert isinstance(N, ndarray), 'Input N is wrong, double check' try: filenames = [(pattern % i) for i in N] except ValueError: raise ValueError('The pattern seems wrong...check it') return filenames
def savejson(filename, varsdict): ''' def savejson(filename, varsdict): <filename>: a string 'test.json', or 'test', can have full path <varsdict>: a dict to save ''' from json import dump from RZutilpy.system import Path, makedirs filename = Path(filename) if filename.suffix != 'json': filename = Path(filename.str + '.json') makedirs(filename) # open a file f = open('{}.json'.format(filename.strnosuffix), 'w') dump(varsdict, f, indent=4) f.close()
def getsampleimage(imgnum): ''' Get a sample image for image processing <imgnum>: int ''' from cv2 import imread from RZutilpy.system import Path from RZutilpy import imageprocess # get the image path pic = Path(imageprocess.__file__) pic = pic.parent / 'imageprocessutils' / f'getsampleimage{imgnum}.png' return imread(pic.str)
def jupyter2pdf(jupyterFile, outputDir=None): ''' convert jupyter2pdf using pdfkit. <jupyterFile>: a string or a Path-lib object <outputDir>: outputDir, default:cwd ''' from RZutilpy.system import Path, makedirs import pdfkit jupyterFile = Path(jupyterFile) outputDir = Path.cwd() if outputDir is None else Path(outputDir) makedirs(outputDir) # create the dir if no exists name = jupyterFile.pstem !jupyter nbconvert --to html {name} htmlFile = Path(jupyterFile.strnosuffix+'.html') pdfkit.from_file(htmlFile.str, (outputDir/(name+'.pdf')).str) !rm -f {name}.html
def splitafniname(filename): ''' Extract afni dataset prefix. Afni dataset typically contains '+', we extract the name before '+' file. Assume only one '+' exists in the filename ''' assert isinstance(filename, str), 'filename should be a string' import re from RZutilpy.system import Path p = re.compile(r'(.*)\+') matchgroup = p.match(filename) assert matchgroup is not None, 'no match!' fullprefix = matchgroup.group(1) # strip path information shortprefix = Path(fullprefix).pstem return shortprefix, fullprefix
def splitniftiname(file): ''' get nift file names. Assume nifti files is xxx.nii.gz or xxx.nii, we want to extract xxx <file> is a string of nifti file. We return ('xxx', '.nii') or ('xxx', '.nii.gz') ''' from RZutilpy.system import Path suffix = Path(file).suffixes if suffix[-1] == '.gz': #.nii.gz format return file[:-7], '.nii.gz' elif suffix[-1] == '.nii': return file[:-4], '.nii' else: return None
def loadjson(filename, varname=None): ''' loadjson(filename, varname=None) load jsonfile ''' from json import load from RZutilpy.system import Path # convert to path-like object filename = Path(filename) if not isinstance(filename, Path) else filename # open a file with open(filename.str) as f: data = load(f) return data[varname] if varname is not None else data
def loadmat(filename, **kwargs): """ loadmat(filename, **kwargs): load .mat file, a wrapper of scipy.io.loadmat. This function is mainly used to load in matlab matfile, we remove some redundant information, i.e. _header_ and only keep valid saved data. Args: <filename>: a string of filename to read, e.g., 'read' **kwargs: kwargs for scipy.io.loadmat function, can check it using help(scipy.io.loadmat) Returns: a tuple (mat_contend, ) <mat_contend>: a dict that scipy.io.loadmat produces to load the matfile <keys>: a list of keys in <mat_contend> <values>: a list of values in <mat_contend> Example: mat,keys,values = loadmat('data01.mat') History and Notes: 20180720 <filename> can accept Path object 20170802 RZ add more input 20170326 RZ created it """ import scipy.io as spio from RZutilpy.system import Path # convert str to path-like object filename = Path(filename) if not isinstance(filename, Path) else filename print('read in' + filename.str + '...') mat_contend = spio.loadmat(filename.str, **kwargs) keys = list(mat_contend.keys()) # we read keys as a list values = list(mat_contend.values()) del keys[0:3] # remove first 3 information keys. del values[0:3] # delete first 3 values return mat_contend, keys, values
def loadpkl(filename, varname=None): ''' loadpkl(filename): Equivalent function as load in matlab. Return a dict load from pkl file 20180814 add only load one varname 20180714 <filename> accept path-like obj, use dill replace pickle ''' from dill import load from RZutilpy.system import Path # convert to path-like object filename = Path(filename) if not isinstance(filename, Path) else filename # open a file f = open(filename.str, 'rb') data = load(f) f.close return data[varname] if varname is not None else data
def savenifti(arr, filename, niftiobj=None, affine=None, header=None): ''' savenifti(arr, filename, niftiobj=None, affine=None, header=None): save 3D or 4D <array> into a full nifti filename using <filename>. We used the affine and header information from <niftiobj>. Note that the file name can have either the ext as '.nii' or '.nii.gz'. nibabel can take care of it. If <affine> or <header> is supplied, they will overwrite information from the <niftiobj> 20180622 RZ add support for affine and header ''' from nibabel import save, Nifti1Image from numpy import ndarray from RZutilpy.system import makedirs, Path # check input assert isinstance( arr, ndarray) and (3 <= arr.ndim <= 4), 'Please input 3d or 4d an ndarray!' # make the dir if it does not exist filename = Path(filename) if ~isinstance(filename, Path) else filename makedirs(filename) # note here we add a os.sep if affine is None or header is None: assert isinstance( niftiobj, Nifti1Image), 'affine or header is none, you must supply niftiobj' # get affine and header information affine = niftiobj.affine.copy() if affine is None else affine.copy() header = niftiobj.header.copy() if header is None else header.copy() save(Nifti1Image(arr, affine, header), filename.str)
def savemat(filename, dict, exist_ok=False, **kwargs): """ savemat(filename, **kwargs): load .mat file, a wrapper of scipy.io.savemat. This function is mainly used to save a dict into matlab matfile. Args: <filename>: a string of filename to read, e.g., 'read' **kwargs: kwargs for scipy.io.loadmat function, can check it using help(scipy.io.loadmat) Returns: a tuple (mat_contend, ) <mat_contend>: a dict that scipy.io.loadmat produces to load the matfile <keys>: a list of keys in <mat_contend> <values>: a list of values in <mat_contend> Example: savemat('data01.mat', '') History and Notes: 201901220 """ from scipy.io import savemat from RZutilpy.system import Path,makedirs # convert str to path-like object filename = Path(filename) if not isinstance(filename, Path) else filename try: savemat(filename.str, **kwargs) return True except: raise IOError('can not save the file!')
def convertxfm(xfm, prefix=None): ''' convert an affine transform between lps+ space and ras+ space Note that AFNI uses lps+ space, but FSL, Freesurfer, nibabel use ras+ space. This function is thus useful to convert and xfm obtained from on space to another nibabel team is trying to make an uniform platform for convert xfm obtained from differet software. It seems that <xfm>: can be (1). ndarray, 12 or 16 elements, can be (12,),(16,)(3,4)(4,4) format (2). a string file name, use np.loadtxt to load xfm from the file <prefix>: if not None, save the result xfm to the txt file. ''' from numpy import ndarray, hstack, loadtxt, savetxt from RZutilpy.system import Path if isinstance(xfm, ndarray): assert xfm.size == 12 or xfm.size == 16, 'xfm matrix should be 12 or 16 elements' elif isinstance(xfm, str): # read xfm from file xfm = loadtxt(xfm) assert xfm.size == 12 or xfm.size == 16, 'xfm matrix should be 12 or 16 elements' xfm = xfm.flatten() xfm = hstack((xfm, [0, 0, 0, 1])) if xfm.size == 12 else xfm xfm[[2, 3, 6, 7, 8, 9]] = -1 * xfm[[2, 3, 6, 7, 8, 9]] xfm = xfm.reshape(4, 4) if prefix is not None: savetxt(Path(prefix).strnosuffix + '.txt', xfm) return xfm
def runfreesurfer2(subjectid,extraflags=''): ''' def runfreesurfer2(subjectid,extraflags): <subjectid> is like 'C0001' <extraflags> (optional) is a string with extra flags to pass to recon-all. Default: '' This is part 2/2 for pushing anatomical data through FreeSurfer. see code for assumptions. ''' from RZutilpy.cvnpy import cvnpath, writemgz, fstoint from RZutilpy.system import makedirs,Path from RZutilpy.rzio import savepkl, loadpkl import nibabel.freesurfer.io as fsio import nibabel as nib import re from numpy import stack from sklearn.neighbors import NearestNeighbors # calc dir0 = (Path(cvnpath('anatomicals')) / subjectid).str fsdir = (Path(cvnpath('freesurfer')) / subjectid).str # make subject anatomical directory makedirs(dir0) # convert some miscellaneous files # convert .thickness files to ASCII # no need for python since nibabel can directly read the file # unix_wrapper('mris_convert -c {0}/surf/lh.thickness {0}/surf/lh.white {0}/surf/lh.thickness.asc'.format(str(fsdir))) # unix_wrapper('mris_convert -c {0}/surf/rh.thickness {0}/surf/rh.white {0}/surf/rh.thickness.asc'.format(str(fsdir))) # # convert .curv files to ASCII # unix_wrapper('mris_convert -c {0}/surf/lh.curv {0}/surf/lh.white {0}/surf/lh.curv.asc'.format(str(fsdir))) # unix_wrapper('mris_convert -c {0}/surf/rh.curv {0}/surf/rh.white {0}/surf/rh.curv.asc'.format(str(fsdir))) #### make mid-gray surface unix_wrapper('mris_expand -thickness {0}/surf/lh.white 0.5 {0}/surf/lh.graymid'.format(fsdir)) unix_wrapper('mris_expand -thickness {0}/surf/rh.white 0.5 {0}/surf/rh.graymid'.format(fsdir)) #### consolidate mid-gray surface stuff into a .mat file for hemi in ['lh' 'rh']: # read .graymid surface vertices,faces = fsio.read_geometry((Path(fsdir) / 'surf'/ f'{hemi}.graymid').str) # construct vertices (4 x V), becareful here, numpy and matlab index might be different!!! vertices = vertices.T + np.array([128, 129, 128]).reshape(-1,1) vertices = np.vstack((vertices, np.ones(vertices.shape[1]).reshape(1,-1))) # construct faces (F x 3) faces = faces[:,[0, 2, 1]] # necessary to convert freesurfer to matlab # load auxiliary info (V x 1) thickness = fsio.read_morph_data((Path(fsdir) / 'surf' / f'{hemi}.thickness').str) curvature = fsio.read_morph_data((Path(fsdir) / 'surf' / f'{hemi}.curv').str) # get freesurfer labels (fslabels is V x 1) fslabels, _, _ = fsio.read_annot((Path(fsdir) / 'label' / f'{hemi}.aparc.annot').str) # save savepkl((Path(cvnpath('anatomicals')) / subjectid / f'{hemi}midgray.pkl'.format(hemi)).str, {'vertices':vertices, 'faces':faces, 'thickness':thickness, \ 'curvature':curvature, 'fslabels': fslabels}) #### calculate gray-matter information if isempty(regexp(extraflags,'hires')): # load ribbon ribmgz = nib.load((Path(fsdir)/ 'mri' / 'ribbon.mgz').str) rib = fstoint(ribmgz.get_data()) # load coordinates of surface vertices coord0 = stack(\ (loadpkl(Path((cvnpath('anatomicals')) / subjectid / 'lhmidgray.mat').str)['vertices'],\ loadpkl((Path(cvnpath('anatomicals')) / subjectid / 'rhmidgray.mat').str)['vertices']),\ axis=1) #### use nearestNeighour, need to double check this nbrs = NearestNeighbors(1, metric='l2') nbrs.fit(coord0.T) dist, mnix = nbrs.kneighbors(rib, 1) # do I need to reshape dist and mnix? # compute distances to vertices [i.e. create a volume where gray matter voxels have certain informative values] #[dist,mnix] = surfaceslice2(ismember(rib,[3 42]),coord0, 3, 4) # NOTICE HARD-CODED VALUES HERE #### # save # 1-mm volume with, for each gray matter voxel, distance to closest vertex (of mid-gray surface) nib.save(inttofs(dist), (Path(fsdir) / 'mri'/ 'ribbonsurfdist.mgz').str, ribmgz) # 1-mm volume with, for each gray matter voxel, index of closest vertex (of mid-gray surface) nib.save(inttofs(mnix),(Path(fsdir) / 'mri'/ 'ribbonsurfindex.mgz').str, ribmgz) #### calculate transfer functions # calc [tfunFSSSlh,tfunFSSSrh,tfunSSFSlh,tfunSSFSrh] = \ calctransferfunctions((Path(cvnpath('freesurfer')).joinpath('fsaverage', 'surf','lh.sphere.reg')).str, \ (Path(cvnpath('freesurfer')).joinpath('fsaverage', 'surf','rh.sphere.reg')).str, \ (Path(cvnpath('freesurfer')).joinpath(subjectid, 'surf','lh.sphere.reg')).str, \ (Path(cvnpath('freesurfer')).joinpath(subjectid, 'surf','rh.sphere.reg')).str) # save savepkl((Path(cvnpath('anatomicals')) / subjectid /'tfun.mat').str,\ {'tfunFSSSlh': tfunFSSSlh,\ 'tfunFSSSrh': tfunFSSSrh,\ 'tfunSSFSlh': tfunSSFSlh\ 'tfunSSFSrh': tfunSSFSrh})
def t1warp(t1, outputDir=None, template=None, maskvol=None, skullstrip_kw=[], affine_kw=[], affonly=False): ''' Nonlinear (or linear warping) warping t1 file to a template. In afni_proc, they use @SSwarper by default. For a standard human T1 file, @SSwarper produce similar warping files as this script. But this script might be useful for like unusual cases, for example, Monkey data Input: <t1>: t1 file, a string or a pathlib object <outputDir>: output directory, we first copy t1 file into this outputDir. if None, use cwd. <template>: the template t1 file,a string or a pathlib object. if None, we use afni MNI152_2009_template_SSW.nii.gz file <maskvol>: mask volume, a str or a pathlib object. You can supply an maskvol file here so the skullstrip step will be skipped. This is useful when you want to manually edit the mask volume and supply it to skullstrip <skullstrip_kw>, <affine_kw>: lists, extra options supply to '3dSkullstrip' and 'align_epi_anat.py' commands. This is useful for like monkey data. Default: [] (empty list) <affonly>: boolean (default: False), only need affine transform, no nonlinear warping. Output: this will produce several files in <outputDir>, they are generated in order **.nii.gz: copy of t1 file **_ss.nii.gz: skullstriped(ss) T1 **_ssmask.nii.gz: skullstriped(ss) brain mask **_ssiu.nii.gz: intenstivity unifize(iu) t1 after ss **_ssiu_shft: shift to center of mass of the template after iu **_ssiu_shft_aff: affine tranformed T1 in the template space **_ssiu_shft_aff_mat.aff12.1D: affine transformation matrix **_ssui_shft_aff_matINV.aff12.1D: inverse affine transformation **_ssiu_shft_aff_nl.nii.gz: nonlinear warp to the template in the template space WARP.nii.gz: nonlinear warp file Note: 1. Here we do three transform, shift-affine-nlwarp. The later two can be concatenated but it is not recommended to concatenate all three because combining shift into affine and nlwarp will make the "3dNwarpapply" take a lot of memory and very long time (and might fail). If you want to inversely warp an atlas from template space into native T1 space. You can do like # first do inverse (affine+nlwarp) cmd = ['3dNwarpApply', '-prefix', f'{template.strnosuffix}_nl2t1.nii.gz', \ '-nwarp', f'inv({(outputDir/"WARP.nii.gz").str} {t1.strnosuffix}_ssiu_shft_aff2NMT_mat.aff12.1D)', \ '-source', template.str, '-master', f'{t1.strnosuffix}_ssiu_shft.nii.gz', '-overwrite'] unix_wrapper(cmd) # then inverse the shift # Note that here do not use 3dAllineate as it will resample the data one more time # Use 3drefit, it will not resample data cmd = f'3drefit -duporigin {t1.strnosuffix}_ssiu.nii.gz {template.strnosuffix}_nl2t1.nii.gz}' unix_wrapper(cmd) 2. Be very CAREFUL about the order when concatenating affine and nlwarp! To do: 1. automatically perform inversed warp atlas?? History: 20190413 RZ create ''' from RZutilpy.system import Path, unix_wrapper, makedirs t1 = Path(t1) if template: template = Path(template) if ~isinstance(template, Path) else template else: # default template is mni template afnipath = unix_wrapper(f'which afni') # get the afni install Path template = Path(afnipath).parent / 'MNI152_2009_template_SSW.nii.gz' outputDir = Path.cwd() if outputDir is None else Path(outputDir) makedirs(outputDir) # create outputdir if not exist unix_wrapper(f'cp {t1} {outputDir}') # copy t1 file to outputdir t1 = outputDir / f'{t1.name}' # switch to new t1 # ======= step 1, skull strip ===================== if maskvol is None: # make brain mask, this step typically is good for a human brain, but tricky for a monkey brain cmd = ['3dSkullStrip', \ '-input', t1.str, \ '-prefix', f'{t1.strnosuffix}_ssmask.nii.gz', \ '-mask_vol'] cmd = cmd + skullstrip_kw unix_wrapper(cmd) maskvol = Path(f'{t1.strnosuffix}_ssmask.nii.gz') # skullstrip-based on mask cmd=['3dcalc', '-a', t1.str, '-b', maskvol.str, \ '-expr', "a*step(b)", \ '-prefix', f'{t1.strnosuffix}_ss.nii.gz'] unix_wrapper(cmd) # ===== step 2, # intensity unifize =========== cmd = f'3dUnifize -prefix {t1.strnosuffix}_ssiu.nii.gz {t1.strnosuffix}_ss.nii.gz' unix_wrapper(cmd) # ====== step 3, affine transform to NMT template ================== # first we can align center of mass cmd = f'@Align_Centers -base {template.str} -dset {t1.strnosuffix}_ssiu.nii.gz -cm' unix_wrapper( cmd) # this step will generate {t1.strnosuffix}_ssiu_shft.nii.gz shftfile = Path.cwd() / f'{t1.pstem}_ssiu_shft.1D' unix_wrapper(f'mv {shftfile.str} {outputDir.str}') # This step will generate {t1.strnosuffix}_ssiu_shft.nii.gz # This step will also generate a 1D transformation file under CWD not OUTPUTDIR # we have to copy this 1D file into <outputDir> # do affine alignment, note that data will be resampled on NMT grid cmd = ['align_epi_anat.py', '-dset1', f'{t1.strnosuffix}_ssiu_shft.nii.gz', \ '-dset2', template.str, \ '-master_dset1', template.str,\ '-suffix', '_aff.nii.gz',\ '-dset1to2', '-dset1_strip','None','-dset2_strip','None','-overwrite',\ '-output_dir', outputDir.str] cmd = cmd + affine_kw unix_wrapper(cmd) # change transformation file name unix_wrapper( f'mv {t1.strnosuffix}_ssiu_shft_aff.nii.gz_mat.aff12.1D {t1.strnosuffix}_ssiu_shft_aff_mat.aff12.1D' ) # calc the inverse affine transformation cmd = f'cat_matvec -ONELINE {t1.strnosuffix}_ssiu_shft_aff_mat.aff12.1D -I > {t1.strnosuffix}_ssiu_shft_aff_matINV.aff12.1D' unix_wrapper(cmd) # ================= step 4, nonlinear registration ============== # note that this step should be run after completing affine transformation # Also here, input dataset is the anat dataset but put on the base grid, unix_wrapper(f'rm -rf {(outputDir/"awpy*").str}') # use superhard, to get the best alignment, but it take a long time cmd = ['auto_warp.py', '-base', template.str, '-skip_affine', 'yes', \ '-input', f'{t1.strnosuffix}_ssiu_shft_aff.nii.gz', '-overwrite', \ '-output_dir', (outputDir/'awpy').str, '-qw_opts','-iwarp','-superhard'] unix_wrapper(cmd) # copy and delete redundant files unix_wrapper( f'cp {(outputDir/"awpy"/"anat.un.qw_WARP.nii").str} {(outputDir/"WARP.nii").str}' ) unix_wrapper(f'gzip -f {(outputDir/"WARP.nii").str}' ) # compress nonlinear warp file unix_wrapper(f'rm -rf {(outputDir/"awpy")}') # apply transform warp t1 to the template space cmd = ['3dNwarpApply', '-prefix', f'{t1.strnosuffix}_ssiu_shft_aff_nl.nii.gz', \ '-nwarp', f'{(outputDir/"WARP.nii.gz").str} {t1.strnosuffix}_ssiu_shft_aff_mat.aff12.1D', \ '-source', f'{t1.strnosuffix}_ssiu_shft.nii.gz', '-master', template.str, '-overwrite'] unix_wrapper(cmd)
def collectT1s(subjectid, dataloc, gradfile=None, str0='T1w', wantskip=True): ''' def collectT1s(subjectid,dataloc,gradfile,str0,wantskip): <subjectid> a str, like 'C0001' <dataloc> is a scan directory like '/home/stone-ext1/fmridata/20151014-ST001-wynn,subject1' or a list of scan directories <gradfile> is gradunwarp's scanner or coeff file (e.g. 'prisma'). Default is None which means do not perform gradunwarp. <str0> a str is the filename match thing. Default: 'T1w'. <wantskip> boolean, is whether to treat as pairs and use the second of each pair. (The idea is that the second of each pair might be homogeneity-corrected.) Default: True. Within the specified scan directories (in the order as given), find all of the T1 (or whatever) DICOM directories, and if <wantskip>, ignoring the 1st of each pair and keeping the 2nd of each pair. Then convert these DICOM directories to NIFTI files. If <gradfile> is specified, we additionally run fslreorient2std and gradunwarp. We return a cell vector of the final NIFTI paths, preserving the order. Note that filenames will be different depending on whether <gradfile> is used. See code for specific assumptions. we use muliprocessin.Pool for speed-ups! To do: - consider using nipype wrapper of dcm2nii?? - history: - 20180720 <dataloc> now can accept an Path object - 20180620 RZ created it based on cvncollectT1s.m ''' from RZutilpy.cvnpy import cvnpath from RZutilpy.system import makedirs, unix_wrapper, Path from RZutilpy.rzio import matchfiles import re from pathos.multiprocessing import Pool # note 1st letter uppercase from os.path import join dir0 = (Path(cvnpath('anatomicals')) / subjectid).str # cvnpath output pathlib objects # make subject anatomical directory makedirs(dir0) # massage, a single string input if ~isinstance(dataloc, list): dataloc = [dataloc] # figure out T1 DICOM directories [ASSUME THAT THERE ARE AN EVEN NUMBER OF DIRECTORIES IN EACH SCAN SESSION] t1files = [] for p in dataloc: t1files0 = matchfiles((Path(p) / 'dicom' / '*%s*'.format(str0)).str) if wantskip: assert len(t1files0) % 2 == 0 t1files0 = t1files0[1::2] # collect them up t1files = t1files + t1files0 # convert dicoms to NIFTIS and get the filenames files = [] # a list of path-like object for p in t1files: result = unix_wrapper('dcm2nii -o %s -r N -x N %s'.format(dir0, p)) temp = re.findall(r'GZip[.]*(\w+.nii.gz)', result) files.append((Path(dir0) / temp[0]).str) # perform gradunwarp if gradfile: # run Keith's fslreorient2std on each [unix_wrapper('fslreorient2std_inplace %s'.format(p)) for p in files] # then do grandunwarp # extract filename and give new files assert all([Path(p).suffixesstr == '.nii.gz' for p in files]) file0 = [ Path(p).strnosuffix for p in files ] # a list of string, remove multiple suffixes in pathlib objects newfiles = [ (Path(p).parent / (Path(p).pstem + '_gradunwarped.nii.gz')).str for p in file0 ] # a list of path-like obj # do the grandunwarp def gradunwarp(filename): # filename is a string unix_wrapper('gradunwarp -w %s_warp.nii.gz -m %s_mask.nii.gz %s.nii.gz %s_gradunwarped.nii.gz %s' \ % (str(filename),str(filename),str(filename),str(filename), gradfile)) # if __name__ == '__main__': # with Pool(12) as p: # p.imap_unordered(gradunwarp, file0) with Pool(12) as p: p.imap_unordered(gradunwarp, file0) #files = list(newfiles) return files
def surfsearchlight(surf, datafile, func, radius=3, openmp=True, mp=4,\ outprefix=None, intent=2005, verbose=False, method='3dsphere'): ''' def surfsearchlight(surf, datafile, func, radius=3, openmp=True, mp=4,\ outprefix=None, intent=2005): Perform surface-based search light analysis. Input: <surf>: a string, or path object indicate a surface file either in freesurfer format or .gii format. We read in this surface file to obtain geometry of surface vertices <datafile>: can be (1), .gii, .gii.gz surface data file, it should be in .gii format and the data should be in nVert x M matrix, nVert is the number of vertex, M columns are data (2), nVert x M matric that internally can be directly used <func>: The function object to calculate, it takes in data file and generate output. Note that func either output a single value or output a tuple for multiple results <radius>: in mm (default: 3), radius to include vertex <openmp>: boolean, whether to use parallal computing, (default=True) <mp>: how many cores to open, default:20 <outPrefix>: a string, we save to a .gii file, if you want to save to .gii file ,you must supply a .gii file for <datafile> <intent>: an int or a list of ints, intent number for each column of result array. This is necessary when saving results into a .gii file. check savegifti.py for more info <method>: (1) '3dsphere' (default), including vertex within a 3dsphere, typically run on a sphere or inflated surface (2) 'geodensic', using geodensic distance, which is more accurate but take a long time currently, this method seems problematic, I would not recommand this Output: We save a .gii file with the output 20190413 RZ add <method> 20190412 RZ created the file ''' from numpy import ndarray, vstack, hstack, array, arange, where from RZutilpy.system import unix_wrapper, Path, gettimestr from RZutilpy.mri import savegifti from numpy import ndarray, vstack, array, arange from nibabel import load from nibabel.freesurfer.io import read_geometry from time import time from sklearn.neighbors import NearestNeighbors from pathos.multiprocessing import Pool from surfdist import surfdist # first read the surf file surf = Path(surf) if ~isinstance(surf, Path) else surf if surf.suffix == '.gii': # .gii format vtrx, faces = load(surf.str).darrays[0].data, load( surf.str).darrays[1].data else: # freesurfer format vtrx, faces = read_geometry(surf.str) # and read the data file if not isinstance(datafile, ndarray) and isinstance(datafile, str): datafile = Path(datafile) assert datafile.suffix == '.gii', 'data file should be .gii format!' giftiobj = load(datafile.str) data = [i.data for i in giftiobj.darrays] data = vstack(data).T # now data is nVert x M columns data file else: data = datafile del datafile # assert same number of vertices in surface and data assert data.shape[0] == vtrx.shape[ 0], 'surface file and data have different number of vertices!' nVtrx = data.shape[0] index = range(nVtrx) # calculate neighbour if method == '3dsphere': neigh = NearestNeighbors(radius=radius, metric='euclidean', n_jobs=mp) neigh.fit(vtrx) # in this case we first fit to the x, y, z nbrs = neigh.radius_neighbors(vtrx, return_distance=False) elif method == 'geodesic': # slow... do not recommand # using surfdist def calcneighbors(i): print(i) dist = surfdist.dist_calc((vtrx, faces), index, i) return where(dist <= radius)[0] with Pool(mp) as p: # use imap, the returned results are in order #b = p.imap(calcneighbors, index, chunksize=2000) b = p.imap(calcneighbors, range(2000), chunksize=2000) nbrs = list(b) # note that nbrs is a ndarray, each element is an array since each element might have different # Define the wrapper function def runsearchlight(i): # get the index of neighbors idx = nbrs[i] # get the data of neighbors data_i = data[idx, :] if verbose: print(i) return func(data_i) # do it tstr = gettimestr('full') with Pool(mp) as p: # use imap, the returned results are in order b = p.imap(runsearchlight, arange(vtrx.shape[0]), chunksize=2000) #b = p.imap(runsearchlight, arange(2000), chunksize=2000) data2save = list(b) print(f'searchlight starts from {tstr}') print(f'searchlight ends at {gettimestr("full")}') # let's take about 1d or 2d nCol = len(data2save[0]) if isinstance(data2save[0], tuple) else 1 data2save = array(data2save) if nCol == 1 else vstack(data2save) # save the file if outprefix: assert 'giftiobj' in locals( ), 'You must input a .gii file for data if you want to save result to .gii' savegifti(data2save, outprefix, giftiobj, intent) return data2save
def matchfiles(pattern, wantsort='numName'): ''' matchfiles(pattern, wantsort='numName'): return a list of filename following a wildcard expression pattern. We use glob module here. Another option is fnmatch module. The difference is that glob will not match the files start with '.' <pattern> is (1)a string that matches zero or more files or directories (wildcards '*' okay). (2) or a list of strings of (1) NOTE THAT WE DO NOT ACCEPT PATH OBJECT, ONLY STRING!!! <wantsort> indicates the key function (e.g., {'numName', 'name', 'time'}) input for sorted function. search 'sorted' in python for more information. It can be {'numName','name', 'time'}, indicating sorting the files by name and the last-modification time. 'numName' consider the numerical number in a string Default: 'numName' If only one match pattern is supplied, we output (1) a string, if only one file is matched (2) a list of string, if multiple files are matched If multiple match patterns are supplied, we output a list of cases of output for one match pattern Note that we first convert the pattern to absolute path to ensure the output is also absolute path History: 20181231 rz add numerical string functionality 20180930 output list of string 20180714 now return all paths as path-like objects, but <pattern> still need to be string 20180626 always output absolute path 20180624 support relative path, like '../test.py' 20180621 implement multiple input patterns as a string list 20180517 add reminder when no matchfiled files 20180430 RZ adds sort arg. Original filenames returned by glob.glob is not sorted. ''' import os import re from glob import glob from RZutilpy.system import Path # first covert them to Path object if isinstance(pattern, str): pattern = [Path(pattern)] else: pattern = [Path(p) for p in pattern] # do it, we still supply str to glob function filelist = [glob(p.str) for p in pattern] # note filelist is a list of string # if len([p for p in filelist if len(p) == 0]) > 0: # no match print('Note, can not match some patterns...') if wantsort == 'name': filelist = [sorted(f, key=os.path.basename) for f in filelist] elif wantsort == 'time': filelist = [sorted(f, key=os.path.getmtime) for f in filelist] elif wantsort == 'numName': def stringSplitByNumbers(x): r = re.compile('(\d+)') l = r.split(x) return [int(y) if y.isdigit() else y for y in l] filelist = [sorted(f, key=stringSplitByNumbers) for f in filelist] else: print('filenames not sorted') if len(pattern) == 1: # only one match input return filelist[0][0] if len( filelist[0]) == 1 else [p for p in filelist[0]] else: # multiple match input return [f[0] if len(f) == 1 else [p for p in f] for f in filelist]
#### calculate transfer functions # calc [tfunFSSSlh,tfunFSSSrh,tfunSSFSlh,tfunSSFSrh] = \ calctransferfunctions((Path(cvnpath('freesurfer')).joinpath('fsaverage', 'surf','lh.sphere.reg')).str, \ (Path(cvnpath('freesurfer')).joinpath('fsaverage', 'surf','rh.sphere.reg')).str, \ (Path(cvnpath('freesurfer')).joinpath(subjectid, 'surf','lh.sphere.reg')).str, \ (Path(cvnpath('freesurfer')).joinpath(subjectid, 'surf','rh.sphere.reg')).str) # save savepkl((Path(cvnpath('anatomicals')) / subjectid /'tfun.mat').str,\ {'tfunFSSSlh': tfunFSSSlh,\ 'tfunFSSSrh': tfunFSSSrh,\ 'tfunSSFSlh': tfunSSFSlh\ 'tfunSSFSrh': tfunSSFSrh}) # write out some useful mgz files (inherited from cvnmakelayers.m) # do it for hemi in ['lh', 'rh']: # load a1 = loadpkl((Path(dir0) / f'{hemi}midgray.mat').str) a3 = fsio.read_geometry((Path(fsdir) / 'surf' / f'{hemi}.sulc').str) # write mgz writemgz(subjectid,'thickness',a1.thickness, hemi) writemgz(subjectid,'curvature',a1.curvature, hemi) writemgz(subjectid,'sulc', a3, hemi)
def cvnrunfreesurfer(subjectid, dataloc, extraflags='', scanstouse=None,t2nifti=None): ''' function cvnrunfreesurfer(subjectid,dataloc,extraflags,scanstouse,t2nifti) <subjectid> is like 'C0001' <dataloc> is: (1) the scan directory like '/home/stone-ext1/fmridata/20151014-ST001-wynn,subject1' (2) a NIFTI T1 .nii.gz file like '/home/stone-ext1/fmridata/AurelieData/Austin_3D.nii.gz' (3) or a pathlib object of (1) or (2) <extraflags> (optional) is a string with extra flags to pass to recon-all. Default: '' <scanstouse> (optional) is a vector of indices of T1 scans to use. For example, if there are 5 scans, [1 3 5] means to use the 1st, 3rd, and 5th. Default is to use all available. <t2nifti> (optional) is a NIFTI T2 .nii.gz file (str or path obj). If you specify this case, <dataloc> must be case (2). push anatomical data through FreeSurfer. see code for assumptions. history: - 2016/11/28 - major update for the new scheme with manual FS edits. ''' from RZutilpy.cvnpy import cvnpath from RZutilpy.system import makedirs, unix_wrapper, Path from RZutilpy.rzio import matchfiles # calc dir0 = (Path(cvnpath('anatomicals')) / subjectid).str fsdir = (Path(cvnpath('freesurfer')) / subjectid).str # make subject anatomical directory makedirs(dir0) # case 1 if Path(dataloc).is_dir(): # figure out T1 files [ASSUME THAT THERE ARE AN EVEN NUMBER OF DIRECTORIES] t1file = matchfiles((Path(dataloc) / 'dicom' / '*T1w*').str) assert len(t1file) % 2==0 t1file = t1file[1::2] # [hint: 2nd of each pair is the one that is homogenity-corrected] # # figure out T2 file [ASSUME THAT WE WILL MATCH TWO DIRECTORIES] # t2file = matchfiles(sprintf('%s/*T2w*',dataloc)) # assert(mod(length(t2file),2)==0) # t2file = t2file(2:2:end) # [hint: 2nd of the two is the one to use, as it is homogenity-corrected] # convert dicoms to NIFTIs for p in t1file: unix_wrapper('dcm2nii -o {} -r N -x N {}'.format(dir0, p)) # assert(0==unix(sprintf('dcm2nii -o %s -r N -x N %s',dir0,t2file))) # find the NIFTIs t1nifti = matchfiles('{}/dicom/*T1w*nii.gz'.format(dir0)) # t2nifti = matchfiles(sprintf('%s/*T2w*nii.gz',dir0)) #assert(length(t1nifti)==1) # assert(length(t2nifti)==1) #t1nifti = t1nifti{1} # t2nifti = t2nifti{1} assert t2nifti is None # case 2, dataloc is a nifti file else: assert Path(dataloc).is_file() and Path(dataloc).exists() # find the NIFTI t1nifti = matchfiles(dataloc) #assert(length(t1nifti)==1) #t1nifti = t1nifti{1} if not t2nifti: t2nifti = matchfiles(t2nifti) assert len(t2nifti)==1 t2nifti = t2nifti[0] # deal with scanstouse if not scanstouse: scanstouse = range(len(t1nifti)) # call recon-all str0 = ['-i %s '.format(str(x)) for x in t1nifti[scanstouse]] if not t2nifti: extrat2stuff = '' else: extrat2stuff = '-T2 %s -T2pial'.format(t2nifti) unix_wrapper('recon-all -s {} {} {} -all {} > {}/reconlog.txt'.format(subjectid,str0,extrat2stuff,extraflags,str(dir0))) # convert T1 to NIFTI for external use unix_wrapper('mri_convert {}/mri/T1.mgz {}/mri/T1.nii.gz'.format(str(fsdir),str(fsdir)))
def makeimagestack3dfiles(m, outputprefix=None, skips=(5, 5, 5), k=(0, 0, 0), \ cmap='gray', returnstack=False, **stack_kw): ''' makeimagestack3dfiles(m, outputprefix=None, skips=[5, 5, 5], k=[0, 0, 0], \ cmap='gray', **kwargs): Input: <m>: is a 3D matrix or a nibabel image object <outputprefix>: is a output prefix,if it is NONE, then do not write images <skips> (optional) is a sequence of numbers of slices to skip in each of the 3 dimensions. Default: [5, 5, 5]. <k> (optional) is a sequence with numbers containing the times to CCW rotate matrix rotation info. See np.rot90. Default: [0, 0, 0]. <k[i]> indicates rotate the image when writing image stack along ith dimension <cmap> is the colormap to use. Default: gray(256), any matplotlib colormap input is fine <returnstack>: boolean, whether to return the 3 imagestack. Useful to visualize and can help adjust <skips> and <k>. Default:False <stack_kw>: kwargs for makeimagestack, include <wantnorm>, <addborder> <csize>,<bordersize> Output: <imglist>: a 1x3 list containing the image matrix for 2,1,0 dimensions. note here the order is not from 0-2 dimensions. Each image is within range 0~1 float 64. We take <m> and write out three .png files, one for each slicing: <outputprefix>_view0.png <outputprefix>_view1.png <outputprefix>_view2.png The first slicing is through the first dimension with ordering [0 1 2]. The second slicing is through the second dimension with ordering [1 3 2]. The third slicing is through the third dimension with ordering [2 3 1]. Note that this is different from KK's makeimagestack3dfiles.m After slicing, rotation (if supplied) is applied within the first two dimensions using np.rot90 Example: vol = makegaussian3d([100 100 100],[.7 .3 .5],[.1 .4 .1]); makeimagestack3dfiles(vol,'test',(10, 10, 10),k=(5,5,5),wantnorm=(0, 1)) To do: 1. Fix the filepath problem History 20180621 RZ added <returnstack> input 20180502 RZ fixed the k rotdaion bug, now should be more clear. 20180412 RZ changed the default cmap to 'gray' 20180419 RZ changed the functionality of outputprefix, not saving images if None.. ''' from matplotlib.pyplot import imsave from nibabel.nifti1 import Nifti1Image as nifti from RZutilpy.imageprocess import makeimagestack from RZutilpy.system import Path, makedirs import numpy as np import os if isinstance(m, nifti): m = m.get_data() _is_writeimage = True if outputprefix is None: outputprefix = Path.cwd() _is_writeimage = False # create the folder if not exist. assert makedirs(Path(outputprefix).parent) # define permutes imglist = [] # we make the dimension to slice to the last one permutes = np.array([[1, 2, 0], [0, 2, 1], [0, 1, 2]]) for dim in range(3): temp = m if dim == 0: # note that the first element in the output image is along 1st dimension temp = temp[::skips[dim], :, :].transpose(permutes[dim, :]) elif dim == 1: temp = temp[:, ::skips[dim], :].transpose(permutes[dim, :]) elif dim == 2: temp = temp[:, :, ::skips[dim]].transpose(permutes[dim, :]) # rotate image if k[dim]: # note temp = np.rot90(temp, k=k[dim], axes=(0, 1)) # CCW rotate f = makeimagestack(temp, **stack_kw) imglist.append(f) # write the image fname = Path(outputprefix).parent / f'{Path(outputprefix).pstem}_view{dim}.png' # note that plt.imsave can automatically recognize the vmin and vmax, no # need to convert it to uint8 like in matlab if _is_writeimage: # if not, only return the images of three views imsave(fname.str, f, cmap=cmap) # return the imagestacks for visualization and debugging if returnstack: return imglist # reverse list to keep compatible the original axis
def transferatlastosurface(subjectid, fsmap, hemi, outpre, fstruncate=None,fun=None,outputdir=None): ''' def transferatlastosurface(subjectid, fsmap, hemi, outpre, fstruncate=None,fun=None,outputdir=None): <subjectid> is like 'C0001' <fsmap> is the fsaverage surface file like '/software/freesurfer/fsaveragemaps/KayDataFFA1-RH.mgz' <hemi> is 'lh' or 'rh' indicating whether the surface file is left or right hemisphere <outpre> is the prefix of the destination .mgz files to write, like 'KayDataFFA1' <fstruncate> is the name of the truncation surface in fsaverage. if None, this indicates the non-dense processing case. <fun> (optional) is a function to apply to <fsmap> after loading it. Default is to do nothing (use values as-is). <outputdir> (optional) is the directory to write the .mgz files to. Default is cvnpath('freesurfer')/<subjectid>/label/ Take the <fsmap> file, apply <fun>, and then transfer to single-subject surface space using nearest-neighbor interpolation. Values in the other hemisphere are just set to 0. We write three versions: (1) <hemi>.<outpre>.mgz - standard (non-dense) surface (2) <hemi>DENSE.<outpre>.mgz - dense surface (3) <hemi>DENSETRUNC<fstruncate>.<outpre>.mgz - dense, truncated surface Note that (2) and (3) are not written if <fstruncate> is []. NOTES FROM KEITH: cvnroimask looks for label/rh[DENSE|DENSETRUNCpt].roiname.(mgz|mgh|label|annot) For multi-label files the label names are contained in a file called label/rh[DENSE|DENSETRUNCpt].roiname.(mgz|mgh|label|annot).ctab Or possibly without the rh|lh # ============== RZ notes =============================== Notes: * check cvntransferatlastosurface.m * <outputdir> now accept both str and a path-like object * default <fstruncate>,<fun>,<outputdir> to None History: 20180714 RZ created it based on cvntransferatlastosurface.m ''' from RZutilpy.cvnpy import cvnpath, writemgz from RZutilpy.rzio import loadpkl from RZutilpy.system import Path, makedirs from numpy import zeros # internal constants fsnumv = 163842 # vertices # calc fsdir = cvnpath('freesurfer') / subjectid # input fun = (lambda x: x) if fun is None else fun outputdir = fsdir / label if outputdir is None else outputdir outputdir = Path(outputdir) if ~isinstance(outputdir, Path) else outputdir # load transfer functions a1 = loadpkl((cvnpath('anatomicals') / subjectid / 'tfun.pkl').str) # load more if fstruncate is None: a2 = loadpkl((cvnpath('anatomicals')/subjectid/'tfunDENSE.pkl').str) # load truncation indices a3 = loadpkl((fsdir/'surf'/f'{hemi}.DENSETRUNC{fstruncate}.pkl').str) # contains 'validix' ## !! ## fix here, make sure what load_mgh actually dod # load fsaverage map vals = flatten(load_mgh(fsmap)) # 1 x 163842 assert len(vals)==fsnumv ## !! ## fix here, make sure what load_mgh actually dod # apply fun and expand map into full format (1 x 2*163842) vals = hstack((zeros(fsnumv), fun(vals))) if hemi=='rh' else hstack((fun(vals), zeros(1,fsnumv))) # make destination directory if necessary makedirs(outputdir) ##### STANDARD CASE (NON-DENSE) # transfer to single subject space (using nearest neighbor interpolation) vals0 = a1.tfunFSSSrh(vals) if hemi='rh' else a1.tfunFSSSlh(vals) # write mgz writemgz(subjectid,outpre,vals0,hemi,outputdir.str) ##### DENSE CASES (DENSE ON THE SPHERE ALSO TRUNCATED VERSION) if fstruncate is not None: # transfer to single subject space (using nearest neighbor interpolation) vals0 = a2.tfunFSSSrh(vals) if hemi=='rh' else a2.tfunFSSSlh(vals) # write mgz writemgz(subjectid,outpre,vals0,hemi+'DENSE',outputdir.str) # write mgz (truncated) writemgz(subjectid,outpre,vals0[a3.validix], hemi+'DENSETRUNC'+fstruncate, outputdir.str)
def cvnpath(whichpath): ''' cvnpath(whichpath): Return the path specified in 'whichpath' Hardcode common paths here, then call this function elsewhere to maximize flexibility. Possibilities: 'code' (common cvnlab code on Dropbox) 'ppresults' (cvnlab pre-processing results on Dropbox) 'freesurfer' (FreeSurfer subjects directory) 'fmridata' (fmridata directory on stone) 'anatomicals' (anatomicals directory on stone) 'workbench' (location of HCP wb_command) eg: fsdir=sprintf('%s/%s',cvnpath('freesurfer'),subjectid) instead of hardcoding in every function Note: If you have /stone/ext1 followed by /home/stone-ext1, this ensures that you can access it from any machine, and if we ARE on stone, it will use the faster local route /stone/ext1 Note: 20180930 output string rather than Path 20180714 RZ switch to return a Path-like object 20180620 RZ create it. based on cvnpath.m ''' from RZutilpy.system import Path paths = {\ 'code':\ [ '/home/stone/generic/Dropbox/cvnlab/code', ], 'commonmatlab':\ [ '/home/stone/software/commonmatlabcode', ], 'ppresults':\ [ '/home/stone/generic/Dropbox/cvnlab/ppresults', ], 'freesurfer':\ [ '/stone/ext1/freesurfer/subjects', '/home/stone-ext1/freesurfer/subjects' ], 'fmridata':\ [ '/stone/ext1/fmridata', '/home/stone-ext1/fmridata' ], 'anatomicals':\ [ '/stone/ext1/anatomicals', '/home/stone-ext1/anatomicals' ], 'workbench':\ [ '/home/stone/software/workbench_v1.1.1/bin_rh_linux64', '/Applications/workbench/bin_macosx64' ] } # get the path and convert them to pathlib object p = [p for p in paths[whichpath] if Path(p).exists()] if p: return p[0] # we return the 1st valid path else: raise ValueError('No path found for {}'.format(whichpath))
def imagesequencetovideo(images, videoname, imagepercnt=1, videocodec=None, fps=15): ''' imagesequencetovideo(images, videoname, imagepercnt=1, videocodec=None, fps=15): write a image sequence to video. We use utility from moviepy module. We first convert input to a list, then convert all images to uint8 format. Then we create the video based on the image list Input: <images>: can be: (1). a 3d or 4d array. Assume the last dimension is time. We convert the array to uint8 type. if 3d, assuming it is gray scale, we convert it to 4d if 4d, assume the 3rd dimension is RGB. Also, for 3d and 4d, we split them to imagelist (2). a wildcard pattern for glob module, we use rz.rzio.matchfiles to read in all images. (3). a list of images, in this case each element must 3d (RGBimage) image. Note that all images should have equal size. <videoname>: output pathname, need to specify the type with the extension. If the filename has an extension ‘.mp4’, ‘.ogv’, ‘.webm’, the codec will be set accordingly, but you can still set it if you don’t like the default. For other extensions, the output filename must be set accordingly. <imagepercnt>: int, intensities most lower and upper percent will be clipped default: 1 <code>: video code, check, write_videofile method of moviepy Clip object to know all allowed video code <fps>: # of frames per second,default:15 Output: <clip>: a clip object from moviepy module To do: 1. input a frame function Example: images = np.random.rand(100,100,20) rz.imageprocess.imagesequencetovideo(images, 'video.mp4') History: 20180720 <videoname> can accept an Path object 20180430 RZ fixed the fps and videocodec input bug ''' from RZutilpy.imageprocess import imreadmulti, touint8 from RZutilpy.array import split from RZutilpy.math import normalizerange from RZutilpy.system import Path from numpy import ndarray, stack, percentile, all from moviepy.editor import ImageSequenceClip from os.path import splitext # check input if isinstance(images, ndarray): ndim = images.ndim if ndim == 3: images = stack((images, images, images), axis=2) return imagesequencetovideo(images, videoname, videocodec=videocodec, fps=fps) elif ndim == 4: assert images.shape[2] == 3, 'wrong RGB images input!' else: raise ValueError('image dimension is wrong !') # split to image list images = split(images) elif isinstance(images, str): # glob wildcard pattern print('load the images...') images = imreadmulti(images) print('done!') return imagesequencetovideo(images, videoname, videocodec=videocodec, fps=fps) elif isinstance(images, list): # ensure all element is rgb images, note that we cannot accept rgba images assert all([i.ndim == 3 and i.shape[2]==3 for i in images]), \ 'make sure all images are RGB images' else: raise ValueError('Input images format is wrong!') # videoname = Path(videoname) if ~isinstance(videoname, Path) else videoname # normalize convert all images to uint8 images = list(map(touint8, images)) # do it clip = ImageSequenceClip(images, fps=fps) _, ext = splitext(videoname) if ext in ['.mp4', '.ogv', '.webm']: clip.write_videofile(videoname.str) else: assert videocodec is not None, 'for this video format, please input videocodec' clip.write_videofile(videoname.str, codec=videocodec) return clip
def dcminfo2json(dcm, filename): ''' Save the scanning info from a dicom file into json format. THis is useful to keep all scanning parameters when converting dcm files to nifti. Also useful to prepare BIDS format Input: <dcm>: dicom file, can be (1), a string or a path-lib object towards ONE '*.dcm' file (2), a string or a path-lib object towards the directory contain '*.dcm' file In this case, we only read the head info from the 1st dcm (3), a pydicom object <filename>: a string or a path-lib object, with or without '.json' suffix are all OK Note: 1. Currently this program only support Siemens dicom files 2. Echo time, repetition time, slice timing are all in ms 3. we do some modification on raw dcm header (1) we remove '[Unknown]' parameters (2) we remove 'Patient*' parameters (3) we remove 'CSA' parameters (4) we rename '[MosaicRefAcqTimes]' to 'SliceTiming' (5) we remove 'Referenced Image Sequence' (6) we add 'FOV' and 'ismosaic' 4. If you want to access individual DataElement, you must call dcm[tag], otherwise you can only get a RawDataElment object, which is weird. To obtain a list of DataElement. You can do like ele = [dcm[i.tag] for i in dcm.values()] History: 20190413 RZ created ''' from RZutilpy.rzio import savejson, matchfiles from RZutilpy.system import Path from pydicom import dcmread from pydicom.dataset import FileDataset import pydicom if not isinstance(dcm, FileDataset): dcm = Path(dcm) # judge it is a directory or a dcm file dcm = Path(matchfiles((dcm/'*.dcm').str)[0]) if dcm.suffix=='' else dcm # assert dcm.suffix=='.dcm', 'dcm input seems incorrect!' dcm = dcmread(dcm.str) assert isinstance(dcm, FileDataset), 'dcm here should be an pydicom FileDataset object!' # read the tag, keywords and values tmp = [dcm[i.tag] for i in dcm.values()] values = [i.value for i in tmp] descriptions = [i.description() for i in tmp] # first remove some unknown parameters data = [(i,j) for (i,j) in zip(descriptions, values) if i != '[Unknown]'] # remove Patient information data = [(i,j) for (i,j) in data if 'Patient' not in i] # remove CSA header info data = [(i,j) for (i,j) in data if 'CSA' not in i] # remove pixel data data = dict(data) data.pop('Pixel Data', None) # manually add and adjust some information data['FOV'] = dcm[int('0051', 16), int('100c', 16)].value data['ismosaic'] = dcm[int('0051', 16), int('1016', 16)].value data.pop('Referenced Image Sequence', None) data['SliceTiming'] = data.pop('[MosaicRefAcqTimes]', None) try: data.pop('Icon Image Sequence') data.pop('Source Image Sequence') except: pass # deal with pydicom data type issue... This is stupid for i,j in data.items(): if isinstance(j, pydicom.multival.MultiValue): data[i] = list(j) elif isinstance(j, pydicom.valuerep.DSfloat) or isinstance(j, pydicom.valuerep.IS): data[i] = float(j) elif isinstance(j, pydicom.uid.UID) or isinstance(j, pydicom.valuerep.PersonName3): data[i] = str(j) #import ipdb;ipdb.set_trace();import matplotlib.pyplot as plt; # save json and return the dict savejson(filename, data) return data
def findminoutlier(funcfiles, outputDir=None): ''' def findminoutlier(func, outputDir=None): Find the minimal outlier from a set of runs of functional data. This function is derived direct from the script generate from afni_proc. We want to isolate this part so we can use the minimal outlier to perform epi2anat alignment separately Input: <funcfiles>: can be (1): a string, a wildercard to matchfile multiple functional files (2): a list of strings of fullfile paths <outputDir>: output directory, can be a string or a path object Output: ''' from RZutilpy.rzio import matchfiles from RZutilpy.system import unix_wrapper, Path, makedirs from os import getcwd from numpy import loadtxt funcfiles = matchfiles(funcfiles) if isinstance(funcfiles, str) else funcfiles assert len(funcfiles) >= 0, 'can not find data files!' nRuns = len(funcfiles) nVols_list = [ int(unix_wrapper(f'3dinfo -nt {i}', verbose=0, wantreturn=True)) for i in funcfiles ] # deal with Path outputDir = Path(getcwd()) if outputDir is None else outputDir outputDir = Path(outputDir) makedirs(outputDir) # create outputDir if not exist # remove existing files unix_wrapper(f'rm {outputDir}/out.pre_ss_warn.txt', verbose=0) unix_wrapper(f'rm {outputDir}/outcount.r**.1D', verbose=0) unix_wrapper(f'rm {outputDir}/out.min_outlier.txt', verbose=0) unix_wrapper(f'rm {outputDir}/vr_base_min_outlier*', verbose=0) # make outcount files pressfile = open(f'{outputDir}/out.pre_ss_warn.txt', 'w') for i in range(nRuns): unix_wrapper( f'3dToutcount -automask -fraction -polort 2 -legendre \ {funcfiles[i]} > {outputDir}/outcount.r{i+1:02d}.1D') outcount = loadtxt(f'{outputDir}/outcount.r{i+1:02d}.1D') if outcount[0] >= 0.4: print( f"** TR #0 outliers: possible pre-steady state TRs in run {i+1:02d}", file=pressfile) pressfile.close() # combine all files unix_wrapper( f'cat {outputDir}/outcount.r*.1D > {outputDir}/outcount_rall.1D') # find the which run and which volume outcountfile = loadtxt(f'{outputDir}/outcount_rall.1D') miniindex = outcountfile.argmin() i = 0 while i <= nRuns: if miniindex < sum(nVols_list[:i + 1]): minoutrun = i + 1 minouttr = miniindex - sum(nVols_list[:i]) break i = i + 1 min_outlier_vol_file = open(f'{outputDir}/out.min_outlier.txt', 'w') print(f'min outlier: run {minoutrun}, TR {minouttr}', file=min_outlier_vol_file) min_outlier_vol_file.close() # extract this vr_min_outlier unix_wrapper( f'3dbucket -prefix {outputDir}/vr_base_min_outlier "{funcfiles[minoutrun-1]}[{minouttr}]" ' ) print(f'min outlier: run {minoutrun}, TR {minouttr}')
def makedirs(name, mode=None, exist_ok=True, wantassert=True): ''' makedirs(name, mode=None, exist_ok=True, wantassert=True): Wrapper of os.makedirs, except that we default exist_ok=True, In other words, do nothing if the folder exists, and create it if it does not.We return True or False to indicate the success of creating the folder Note: 1. a name without suffixes will be treated as a folder 2. a name with suffixes will be treated as a file 3. The user homedir sign '~' is fine, we replace it with <name> can be either (1) a str folder name, like '/User/ruyuan/testpath' (2) a filename. like '/User/ruyuan/testpath/test.py'. in this case, we create the folder '/User/ruyuan/testpath/' '~/testpath' is OK only a filename 'test.py' is also ok, in this case we do not create any folder since we are already in this folder (3) a Path or path-like object <mode>: folder permission, default:None <exist_ok>: whether OK if it is exist, if the folder exists, we do nothing <wantassert>: raise error if fail which means we want to save something in the current folder. Then we choose to do nothing special. Example: # make a folder name 'here' if it does not exist makedirs('~/here') # make a folder name 'home' if it does not exist makedirs('/home') # make a folder name 'home' if it does not exist makedirs('/home/heihei.png') # This is useful when saving a file but not sure whether the folder exists # if not, we create the folder for this file, the usage is below makedirs('/home/heihei.png') History: 20190117 RZ fixed the bug, create a folder for a existing file 20180714 <name> now can be a path-like object or a string 20180622 switch to use pathlib module ''' from RZutilpy.system import Path # convert to rzpath object name = Path(name) if not isinstance(name, Path) else name # change name to its parent folder if it is a file or have suffixes name = name.parent if name.suffixes != [] or name.is_file() else name # we cannot set the default in the func, so using this... # note that we make all parent folder if they do not exist if mode is None: try: name.mkdir(parents=True, exist_ok=exist_ok) return True # this is helpful for assert except: print('Failed to make the dir %s!' % name.str) if wantassert: raise NameError('Failed to make the dir %s!' % name.str) return False else: try: name.mkdir(mode=mode, parents=True, exist_ok=exist_ok) return True except: print('Failed to make the dir %s!' % name.str) if wantassert: raise NameError('Failed to make the dir %s!' % name.str) return False