def glm_matlab_from_files(bold_file, tr, paradigm_csv_file, output_dir, mask_file, hf_cut=128, hack_mask=False): """ Only mono-session #TODO: compute mask if mask_file does not exist #TODO: handle contrasts """ # Functional mask # if not op.exists(mask_file): # pyhrf.verbose(1, 'Mask file does not exist. Computing mask from '\ # 'BOLD data') # compute_mask_files(bold_files, mask_file, False, 0.4, 0.9) #split BOLD into 3D vols: bold, hbold = read_volume(bold_file) bold_files = [] tmp_path = tempfile.mkdtemp(dir=pyhrf.cfg['global']['tmp_path']) for iscan, bscan in enumerate(bold): f = op.join(tmp_path, 'bold_%06d.nii' %iscan) write_volume(bscan, f, hbold) bold_files.append(f) bold_files = ';'.join(bold_files) script_path = op.join(op.dirname(pyhrf.__file__),'../../script/SPM') spm_path = pyhrf.cfg['global']['spm_path'] matlab_code = "cd %s;paradigm_file='%s';TR=%f;mask_file='%s';" \ "bold_files='%s';output_path='%s';" \ "HF_cut=%f;spm_path='%s';api=1;hack_mask=%d;glm_intra_subj;exit" \ %(script_path,paradigm_csv_file,tr,mask_file,bold_files,output_dir, hf_cut,spm_path,hack_mask) matlab_cmd = 'matlab -nosplash -nodesktop -r "%s"'%matlab_code if op.exists(op.join(output_dir,'SPM.mat')): #remove SPM.mat so that SPM won't ask over ask overwriting os.remove(op.join(output_dir,'SPM.mat')) #print 'matlab cmd:' #print matlab_cmd os.system(matlab_cmd) # Fix shape of outputs if necessary # eg if input data has shape (n,m,1) then SPM will write outputs of # shape (n,m) so that they are not consistent with their QForm input_shape = bscan.shape for foutput in glob.glob(op.join(output_dir, '*.img')): data, h = read_volume(foutput) if data.ndim < 3: sm = ','.join([ [':','np.newaxis'][d==1] \ for d in input_shape ] ) exec('data = data[%s]' %sm) assert data.shape == input_shape write_volume(data, foutput, h) shutil.rmtree(tmp_path) #TODO: maybe find a better way to grab beta file names beta_files = sorted(glob.glob(op.join(output_dir,'beta_*.img'))) return beta_files
def split_big_parcels(parcel_file, output_file, max_size=400): print 'split_big_parcels ...' roiMask, roiHeader = read_volume(parcel_file) roiIds = np.unique(roiMask) background = roiIds.min() labels = roiMask[np.where(roiMask>background)].astype(int) if (np.bincount(labels) <= max_size).all(): pyhrf.verbose(1, 'no parcel to split') return graphs = parcels_to_graphs(roiMask, kerMask3D_6n) for roiId in roiIds: if roiId != background: roi_size = (roiMask==roiId).sum() if roi_size > max_size: print 'roi %d, size = %d' %(roiId, roi_size) nparcels = int(np.ceil(roi_size*1./max_size)) print 'split into %d parcels ...' %(nparcels) split_parcel(labels, graphs, roiId, nparcels, inplace=True, verbosity=1) final_roi_mask = np.zeros_like(roiMask) final_roi_mask[np.where(roiMask>background)] = labels #print np.bincount(labels) assert (np.bincount(labels) <= max_size).all() write_volume(final_roi_mask, output_file, roiHeader)
def load_many_hrf_territories(nb_hrf_territories): from pyhrf.tools.io import read_volume fn = pyhrf.get_data_file_name('simu_hrf_%d_territories.nii' \ %nb_hrf_territories) assert op.exists(fn) territories = read_volume(fn)[0] return territories[np.where(np.ones_like(territories))]
def compute_consensus_clusters_parallel( K_clus, consensus_matrices, clustcount_matrices, totalcount_matrices, num_voxels, remote_mask_fn, clusters_consensi ): """ """ import nisl.io as ionisl import os import sys sys.path.append("/home/pc174679/pyhrf/pyhrf-tree_trunk/script/WIP/Scripts_IRMf_BB/Parcellations/") sys.path.append("/home/pc174679/pyhrf/pyhrf-tree_trunk/script/WIP/Scripts_IRMf_Adultes_Solv/") sys.path.append( "/home/pc174679/pyhrf/pyhrf-tree_trunk/script/WIP/Scripts_IRMf_Adultes_Solv/Scripts_divers_utiles/Scripts_utiles/" ) sys.path.append("/home/pc174679/local/installations/consensus-cluster-0.6") from Random_parcellations import random_parcellations, subsample_data_on_time from Divers_parcellations_test import * from pyhrf.tools.io import read_volume print " * Consensus with ", K_clus, " clusters" c_mat = np.array(clustcount_matrices[int(K_clus)]) t_mat = np.array(totalcount_matrices[int(K_clus)]) consensus_mat = gen_consensus_matrix(num_voxels, 0, c_mat, t_mat) parc_name = "Subsampled_data_with_" + str(K_clus) + "clusters" output_sub = os.sep.join((output_path, parc_name)) nifti_masker = ionisl.NiftiMasker(remote_mask_fn) mask_shape = read_volume(remote_mask_fn)[0].shape Nm = nifti_masker.fit(remote_mask_fn) # Nm = nifti_masker.fit(np.ones((mask_shape))) # clusterize the consensus matrix if clusterize_cons_mat: labels_cons_mat = hcluster_consensus(consensus_mat, num_voxels, K_clus, linkage="average") labels_cons_mat_inv = Nm.inverse_transform(labels_cons_mat).get_data() clusters_consensi[int(K_clus)] = np.zeros((K_clus)) # compute cluster consensus, for each clustering clusters_consensi[int(K_clus)] = compute_cc_mat(K_clus, consensus_mat, labels_cons_mat, num_voxels) return ( clusters_consensi.astype("int32"), consensus_mat.astype("int32"), labels_cons_mat.astype("int16"), labels_cons_mat_inv.astype("int16"), )
def test_split_vol_cc_3D(self): fn = 'subj0_parcellation.nii.gz' mask_file = pyhrf.get_data_file_name(fn) #print mask_file mask = read_volume(mask_file)[0].astype(int) #print np.unique(mask), mask.dtype mask[np.where(mask==2)] = 1 #print np.unique(mask), mask.dtype km = kerMask3D_6n for i,mcc in enumerate(split_mask_into_cc_iter(mask, kerMask=km)): #print mcc #print '' pass #print i assert i==1 #2 rois
def from_simulation_dict(self, simulation, mask=None): bold = simulation['bold'] if isinstance(bold, xndarray): bold = bold.reorient(['time','voxel']) nvox = bold.data.shape[1] ss = [range(bold.data.shape[0])] #one session print 'BOLD SHAPE=',bold.data.shape else: nvox = bold.shape[1] ss = [range(bold.shape[0])] #one session onsets = simulation['paradigm'].stimOnsets #onsets = dict(zip(ons.keys(),ons.values()])) durations = simulation['paradigm'].stimDurations #durations = dict(zip(dur.keys(),[o for o in dur.values()])) # print '' # print 'onsets:' # print onsets # print '' tr = simulation['tr'] s = simulation labelsVol = s.get('labels_vol',np.ones((1,nvox,))) if mask is None: if isinstance(labelsVol, xndarray): default_mshape = labelsVol.data.shape[1:] else: default_mshape = labelsVol.shape[1:] #print 'default_mask_shape:', default_mshape roiMask = simulation.get('mask', np.ones(default_mshape, dtype=np.int32)) else: roiMask = read_volume(mask)[0] #print 'roiMask:', roiMask.shape if len(roiMask.shape) == 3: data_type = 'volume' else: data_type = 'surface' #simulation ?? return FmriData(onsets, bold, tr, ss, roiMask, stimDurations=durations, simulation=[simulation], data_type=data_type)
def handle_mask(self, mask_file, bold_files, tr, onsets, durations, output_dir, mesh_file=None): if mesh_file is None: #Volumic if self.force_input_parcellation: if not op.exists(mask_file): raise IOError("Input parcellation is forced but " \ "mask file %s not found" %mask_file) else: #TODO: check if n-ary return mask_file FMRIAnalyser.handle_mask(self, mask_file, bold_files, onsets, durations, mesh_file) mask, mask_obj = read_volume(mask_file) roi_ids = np.unique(mask) if len(roi_ids) <= 2: glm_output_dir = op.join(output_dir, 'GLM') if not op.exists(glm_output_dir): os.makedirs(glm_output_dir) return glm_parcellation(bold_file, tr)
def make_parcellation_cubed_blobs_from_file(parcellation_file, output_path, roi_ids=None, bg_parcel=0, skip_existing=False): p,mp = read_volume(parcellation_file) p = p.astype(np.int32) if bg_parcel==0 and p.min() == -1: p += 1 #set background to 0 if roi_ids is None: roi_ids = np.unique(p) pyhrf.verbose(1,'%d rois to extract' %(len(roi_ids)-1)) tmp_dir = pyhrf.get_tmp_path('blob_parcellation') tmp_parcel_mask_file = op.join(tmp_dir, 'parcel_for_blob.nii') out_files = [] for roi_id in roi_ids: if roi_id != bg_parcel: #discard background output_blob_file = op.join(output_path, 'parcel_%d_cubed_blob.arg'\ %roi_id) out_files.append(output_blob_file) if skip_existing and os.path.exists(output_blob_file): continue parcel_mask = (p==roi_id).astype(np.int32) write_volume(parcel_mask, tmp_parcel_mask_file, mp) pyhrf.verbose(3,'Extract ROI %d -> %s' %(roi_id,output_blob_file)) cmd = 'AimsGraphConvert -i %s -o %s --bucket' \ %(tmp_parcel_mask_file, output_blob_file) pyhrf.verbose(3,'Cmd: %s' %(cmd)) os.system(cmd) if op.exists(tmp_parcel_mask_file): os.remove(tmp_parcel_mask_file) return out_files
def project_fmri_from_kernels(input_mesh, kernels_file, fmri_data_file, output_tex, bin_threshold=None, ): pyhrf.verbose(2,'Project data onto mesh using kernels ...') if 0: print 'Projecting ...' print 'func data:', fmri_data_file print 'Mesh file:', input_mesh print 'Save as:', output_tex pyhrf.verbose(2,'Call AimsFunctionProjection -op 1 ...') data_files = [] output_texs = [] p_ids = None if bin_threshold is not None: d,h = read_volume(fmri_data_file) if np.allclose(d.astype(int), d): tmp_dir = pyhrf.get_tmp_path() p_ids = np.unique(d) pyhrf.verbose(2, 'bin threshold: %f' %bin_threshold) pyhrf.verbose(2, 'pids(n=%d): %d...%d' \ %(len(p_ids),min(p_ids),max(p_ids))) for i,p_id in enumerate(p_ids): if p_id != 0: new_p = np.zeros_like(d) new_p[np.where(d==p_id)] = i + 1 #0 is background ifn = op.join(tmp_dir,'pmask_%d.nii'%p_id) write_volume(new_p, ifn, h) data_files.append(ifn) ofn = op.join(tmp_dir,'ptex_%d.gii'%p_id) output_texs.append(ofn) else: data_files.append(fmri_data_file) output_texs.append(output_tex) else: data_files.append(fmri_data_file) output_texs.append(output_tex) pyhrf.verbose(3, 'input data files: %s' %str(data_files)) pyhrf.verbose(3, 'output data files: %s' %str(output_texs)) for data_file, o_tex in zip(data_files, output_texs): projection = [ 'AimsFunctionProjection', '-op', '1', '-d', kernels_file, '-d1', data_file, '-m', input_mesh, '-o', o_tex ] cmd = ' '.join(map(str,projection)) pyhrf.verbose(3, 'cmd: %s' %cmd) os.system(cmd) if bin_threshold is not None: pyhrf.verbose(2, 'Binary threshold of texture at %f' %bin_threshold) o_tex = output_texs[0] data,data_gii = read_texture(o_tex) data = (data>bin_threshold).astype(np.int32) print 'data:', data.dtype if p_ids is not None: for pid, o_tex in zip(p_ids[1:], output_texs[1:]): pdata,pdata_gii = read_texture(o_tex) data += (pdata>bin_threshold).astype(np.int32) * pid #assert (np.unique(data) == p_ids).all() write_texture(data, output_tex, intent='NIFTI_INTENT_LABEL')
def load_vol_bold_and_mask(bold_files, mask_file): # Handle mask if not op.exists(mask_file): pyhrf.verbose(1,'Mask file %s does not exist. Mask is '\ ' computed from BOLD ...' %mask_file) bf = 'subj0_bold_session0.nii.gz' # HACK if bold_files[0] == pyhrf.get_data_file_name(bf): max_frac = .99999 #be sure to keep non zero voxels cc = 0 # default BOLD vol has 2 ROIs else: max_frac = .9 cc = 1 compute_mask_files(bold_files[0], mask_file, False, .4, max_frac, cc=cc) mask_loaded_from_file = False else: mask_loaded_from_file = True pyhrf.verbose(1,'Assuming orientation for mask file: ' + \ string.join(MRI3Daxes, ',')) pyhrf.verbose(2,'Read mask from: %s' %mask_file) mask, mask_meta_obj = read_volume(mask_file) if not np.allclose(np.round(mask),mask): raise Exception("Mask is not n-ary (%s)" %mask_file) mask = mask.astype(np.int32) pyhrf.verbose(1,'Mask has shape %s' %str(mask.shape)) pyhrf.verbose(1,'Mask min value: %d' %mask.min()) pyhrf.verbose(1,'Mask max value: %d' %mask.max()) mshape = mask.shape if mask.min() == -1: mask += 1 #Load BOLD: lastScan = 0 sessionScans = [] bolds = [] pyhrf.verbose(1,'Assuming orientation for BOLD files: ' + \ string.join(MRI4Daxes, ',')) #print 'type of bold_files[0]:', type(bold_files[0]) #print 'bold_files:', bold_files if type(bold_files[0]) is list: bold_files = bold_files[0] for bold_file in bold_files: if not op.exists(bold_file): raise Exception('File not found: ' + bold_file) b, bold_meta = read_volume(bold_file) bolds.append(b) sessionScans.append(np.arange(lastScan, lastScan+b.shape[TIME_AXIS], dtype=int)) lastScan += b.shape[TIME_AXIS] bold = np.concatenate(tuple(bolds), axis=TIME_AXIS) pyhrf.verbose(1,'BOLD has shape %s' %str(bold.shape)) discard_bad_data(bold, mask) # #HACK # if mask_file != DEFAULT_MASK_VOL_FILE: # write_volume(mask,'./treated_mask.nii') # sys.exit(0) return mask, mask_meta_obj, mask_loaded_from_file, bold, sessionScans
def parcellation_for_jde(fmri_data, avg_parcel_size=250, output_dir=None, method='gkm', glm_drift='Cosine', glm_hfcut=128): """ method: gkm, ward, ward_and_gkm """ if output_dir is None: output_dir = tempfile.mkdtemp(prefix='pyhrf_JDE_parcellation_GLM', dir=pyhrf.cfg['global']['tmp_path']) glm_output_dir = op.join(output_dir, 'GLM_for_parcellation') if not op.exists(glm_output_dir): os.makedirs(glm_output_dir) pyhrf.verbose(1, 'GLM for parcellation') # if fmri_data.data_type == 'volume': # paradigm_file, bold_file, mask_file = fmri_data.save(glm_output_dir) # beta_files = glm_nipy_from_files(bold_file, fmri_data.tr, paradigm_file, # glm_output_dir, mask_file, # drift_model=glm_drift, hfcut=glm_hfcut) # elif fmri_data.data_type == 'surface': # beta_files = glm_nipy(fmri_data, glm_output_dir, # drift_model=glm_drift, hfcut=glm_hfcut) g, dm, cons = glm_nipy(fmri_data, drift_model=glm_drift, hfcut=glm_hfcut) pval_files = [] if cons is not None: func_data = [('con_pval_%s' %cname, con.pvalue()) \ for cname, con in cons.iteritems()] else: reg_cst_drift = re.compile(".*constant.*|.*drift.*") func_data = [('beta_%s' %reg_name, g.beta[ir]) \ for ir,reg_name in enumerate(dm.names) \ if not reg_cst_drift.match(reg_name)] for name, data in func_data: val_vol = expand_array_in_mask(data, fmri_data.roiMask>0) val_fn = op.join(glm_output_dir, '%s.nii' %name) write_volume(val_vol, val_fn, fmri_data.meta_obj) pval_files.append(val_fn) mask_file = op.join(glm_output_dir,'mask.nii') write_volume(fmri_data.roiMask>0, mask_file, fmri_data.meta_obj) nvox = fmri_data.get_nb_vox_in_mask() nparcels = round_nb_parcels(nvox * 1. / avg_parcel_size) pyhrf.verbose(1, 'Parcellation from GLM outputs, method: %s, ' \ 'nb parcels: %d' %(method, nparcels)) if fmri_data.data_type == 'volume': parcellation_file = op.join(output_dir, 'parcellation_%s_np%d.nii' %(method, nparcels)) make_parcellation_from_files(pval_files, mask_file, parcellation_file, nparcels, method) parcellation,_ = read_volume(parcellation_file) else: mesh_file = fmri_data.data_files[-1] parcellation_file = op.join(output_dir, 'parcellation_%s_np%d.gii' %(method, nparcels)) make_parcellation_surf_from_files(pval_files, mesh_file, parcellation_file, nparcels, method, verbose=1) parcellation,_ = read_texture(parcellation_file) #print parcellation_file pyhrf.verbose(1, parcellation_report(parcellation)) return parcellation, parcellation_file
def setUp(self): pf = 'subj0_parcellation.nii.gz' fnm = pyhrf.get_data_file_name(pf) m,mh = read_volume(fnm) self.graph = parcels_to_graphs(m.astype(int), kerMask3D_6n, toDiscard=[0])[1]
def BMA_consensus_cluster_parallel( cfg, remote_path, remote_BOLD_fn, remote_mask_fn, Y, nifti_masker, num_vox, K_clus, K_clusters, parc, alpha, prop, nbItRFIR, onsets, durations, output_sub_parc, rescale=True, averg_bold=False, ): """ Performs all steps for one clustering case (Kclus given, number l of the parcellation given) remote_path: path on the cluster, where results will be stored """ import os import sys sys.path.append("/home/pc174679/pyhrf/pyhrf-tree_trunk/script/WIP/Scripts_IRMf_BB/Parcellations/") sys.path.append("/home/pc174679/pyhrf/pyhrf-tree_trunk/script/WIP/Scripts_IRMf_Adultes_Solv/") sys.path.append( "/home/pc174679/pyhrf/pyhrf-tree_trunk/script/WIP/Scripts_IRMf_Adultes_Solv/Scripts_divers_utiles/Scripts_utiles/" ) sys.path.append("/home/pc174679/local/installations/consensus-cluster-0.6") from Random_parcellations import random_parcellations, subsample_data_on_time from Divers_parcellations_test import * from RFIR_evaluation_parcellations import JDE_estim, RFIR_estim, clustering_from_RFIR from Random_parcellations import hrf_roi_to_vox from pyhrf.tools.io import remote_copy, remote_mkdir from nisl import io # nifti_masker.mask=remote_mask_fn # Creation of the necessary paths --> do not do here parc_name = "Subsampled_data_with_" + str(K_clus) + "clusters" parc_name_clus = parc_name + "rnd_number_" + str(parc + 1) remote_sub = os.sep.join((remote_path, parc_name)) # if not os.path.exists(remote_sub): # os.path.exists(remote_sub) # print 'remote_sub:', remote_sub # os.makedirs(remote_sub) remote_sub_parc = os.sep.join((remote_sub, parc_name_clus)) # if not os.path.exists(remote_sub_parc): # os.makedirs(remote_sub_parc) output_RFIR_parc = os.sep.join((output_sub_parc, "RFIR_estim")) ################################### ## 1st STEP: SUBSAMPLING print "--- Subsample data ---" Ysub = subsample_data_on_time(Y, remote_mask_fn, K_clus, alpha, prop, nifti_masker, rescale=rescale) print "Ysub:", Ysub print "remote_sub_prc:", remote_sub_parc Ysub_name = "Y_sub_" + str(K_clus) + "clusters_" + "rnd_number_" + str(parc + 1) + ".nii" Ysub_fn = os.sep.join((remote_sub_parc, Ysub_name)) Ysub_masked = nifti_masker.inverse_transform(Ysub).get_data() write_volume(Ysub_masked, Ysub_fn) ################################### ## 2D STEP: RFIR print "--- Performs RFIR estimation ---" remote_RFIR_parc_clus = os.sep.join((remote_sub_parc, "RFIR_estim")) # if not os.path.exists(remote_RFIR_parc):os.makedirs(remote_RFIR_parc) # remote_RFIR_parc_clus = os.sep.join((remote_RFIR_parc, parc_name_clus)) # if not os.path.exists(remote_RFIR_parc_clus):os.makedirs(remote_RFIR_parc_clus) print " * output path for RFIR ", remote_RFIR_parc_clus print " * RFIR for subsampling nb ", str(parc + 1), " with ", K_clus, " clusters" RFIR_estim(nbItRFIR, onsets, durations, Ysub_fn, remote_mask_fn, remote_RFIR_parc, avg_bold=averg_bold) hrf_fn = os.sep.join((remote_RFIR_parc_clus, "rfir_ehrf.nii")) # remote_copy([hrf_fn], remote_host, # remote_user, remote_path)[0] ################################### ## 3D STEP: CLUSTERING FROM RFIR RESULTS name_hrf = "rfir_ehrf.nii" from pyhrf.tools.io import write_volume, read_volume from pyhrf.tools.io import read_volume, write_volume import nisl.io as ionisl from sklearn.feature_extraction import image from sklearn.cluster import WardAgglomeration from scipy.spatial.distance import cdist, pdist hrf_fn = os.sep.join((remote_RFIR_parc_clus, name_hrf)) hrf = read_volume(hrf_fn)[0] hrf_t_fn = add_suffix(hrf_fn, "transpose") # taking only 1st condition to parcellate write_volume(hrf[:, :, :, :, 0], hrf_t_fn) nifti_masker = ionisl.NiftiMasker(remote_mask_fn) Nm = nifti_masker.fit(hrf_t_fn) # features: coeff of the HRF HRF = Nm.fit_transform(hrf_t_fn) mask, meta_data = read_volume(remote_mask_fn) shape = mask.shape connectivity = image.grid_to_graph(n_x=shape[0], n_y=shape[1], n_z=shape[2], mask=mask) # features used for clustering features = HRF.transpose() ward = WardAgglomeration(n_clusters=K_clus, connectivity=connectivity, memory="nisl_cache") ward.fit(HRF) labels_tot = ward.labels_ + 1 # Kelbow, Perc_WSS, all_parc_from_RFIR_fns, all_parc_RFIR = \ # clustering_from_RFIR(K_clusters, remote_RFIR_parc_clus, remote_mask_fn, name_hrf, plots=False) # labels_tot = all_parc_RFIR[str(Kelbow)] # to retrieve clustering with as many clusters as determined in K_clusters # labels_tot = all_parc_RFIR[str(K_clus)] # Parcellation retrieved: for K=Kelbow # clusters_RFIR_fn = all_parc_from_RFIR[str(Kelbow)] # clustering_rfir_fn = os.path.join(remote_RFIR_parc_clus, 'output_clustering_elbow.nii') # write_volume(read_volume(clusters_RFIR_fn)[0], clustering_rfir_fn, meta_bold) # labels_tot = nifti_masker.fit_transform([clusters_RFIR_fn])[0] # labels_tot = read_volume(clusters_RFIR_fn)[0] # labels_name='labels_' + str(int(K_clus)) + '_' + str(parc+1) + '.pck' # name_f = os.sep.join((remote_sub_parc, labels_name)) # pickle_labels=open(name_f, 'w') # cPickle.dump(labels_tot,f) # pickle_labels.close() # remote_copy(pickle_labels, remote_user, # remote_host, output_sub_parc) ################################# ## Prepare consensus clustering print "Prepare consensus clustering" clustcount, totalcount = upd_similarity_matrix(labels_tot) print "results:", clustcount return clustcount.astype(np.bool)