def test_mask(): # example from http://www.jarrodmillman.com/rcsds/lectures/glm_intro.html # it should be pointed out that hypothesis just looks at simple linear regression data = np.arange(1000000) data = data.reshape((100,100,100)) mask1 = np.ones((100,100,100)) mask2 = np.zeros((100,100,100)) mask3 = np.ones((200,200,100)) assert_equal(make_mask(data, mask1), data) assert_equal(make_mask(data,mask2), mask2) assert_equal(make_mask(data,mask3,fit=True).shape, data.shape)
def test_mask(): # example from http://www.jarrodmillman.com/rcsds/lectures/glm_intro.html # it should be pointed out that hypothesis just looks at simple linear regression data = np.arange(1000000) data = data.reshape((100,100,100)) mask1 = np.ones((100,100,100)) mask2 = np.zeros((100,100,100)) mask3 = np.ones((200,200,100)) assert_equal(make_mask(data, mask1), data) assert_equal(make_mask(data,mask2), mask2) assert_equal(make_mask(data,mask3,fit=True).shape, data.shape) x= False try: make_mask(data,mask3,fit=False) except ValueError: x=True assert(x==True)
neighbors = 1 q = .15 prop_t = .15 prop_beta = .15 # assign subjects a number for i, name in enumerate(sub_list): # the mask for each subject path_to_data = project_path + "data/ds009/" + name mask = nib.load(path_to_data + '/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data() rachels_ones = np.ones((64, 64, 34)) fitted_mask = make_mask(rachels_ones, mask_data, fit=True) fitted_mask[fitted_mask > 0] = 1 ##################################### # Run bh_procedure for each subject # ##################################### p_3d = np.load("../data/p-values/" + name + "_pvalue.npy") p_1d = np.ravel(p_3d) mask = fitted_mask mask_1d = np.ravel(mask) p_bh = p_1d[mask_1d == 1] bh_first = bh_procedure(p_bh, q) bh_3d = masking_reshape_end(bh_first, mask, off_value=.5) bh_3d[bh_3d < .5] = 0
#loop through each person's T-statistic for model in [ "_tstat.npy", "_tstat_rough_full.npy", "_tstat_smooth_simple.npy", "_tstat_rough_simple.npy" ]: t_mean = np.zeros((64, 64, 34, 24)) count = 0 for i in sub_list: t_stat = np.load(t_data + i + model) mask = nib.load(path_to_data + i + '/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data() t_mean[..., count] = make_mask(t_stat, mask_data, fit=True) count += 1 t_mean = np.mean(t_mean, axis=3) final = present_3d(t_mean) plt.imshow(final, interpolation='nearest', cmap='seismic') plt.title("Mean T-Statistic Value Across 25 Subjects") zero_out = max(abs(np.min(final)), np.max(final)) plt.clim(-zero_out, zero_out) plt.colorbar() plt.show() # # ##################################### # ########## Clustering########## # #####################################
img = nib.load(smooth_data + i + "_bold_smoothed.nii") data = img.get_data() behav = pd.read_table(path_to_data + i + behav_suffix, sep=" ") num_TR = float(behav["NumTRs"]) n_vols = num_TR hrf_matrix_all = np.loadtxt("../data/hrf/" + i + "_hrf_all.txt") hrf_matrix_1 = np.loadtxt("../data/hrf/" + i + "_hrf_1.txt") hrf_matrix_2 = np.loadtxt("../data/hrf/" + i + "_hrf_2.txt") hrf_matrix_3 = np.loadtxt("../data/hrf/" + i + "_hrf_3.txt") mask = nib.load(path_to_data + i + '/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data() mask_data = make_mask(np.ones(data.shape[:-1]), mask_data, fit=True) mask_data = mask_data != 0 mask_data = mask_data.astype(int) # PCA to_2d = masking_reshape_start(data, mask_data) X_pca = to_2d - np.mean(to_2d, 0) - np.mean(to_2d, 1)[:, None] cov = X_pca.T.dot(X_pca) U, S, V = npl.svd(cov) pca_addition = U[:, :6] # ~40% coverage #START DOING GLM
behav = pd.read_table(path_to_data + name + behav_suffix, sep = " ") num_TR = float(behav["NumTRs"]) img = nib.load(smooth_data + i + "_bold_smoothed.nii") data = img.get_data() n_vols = data.shape[-1] convolve = np.loadtxt(hrf_data + i + "_hrf_all.txt") residual_final = np.zeros((data.shape)) t_final = np.zeros((data.shape[:-1])) p_final = np.zeros((data.shape[:-1])) beta_final = np.zeros((data.shape[:-1])) mask = nib.load(path_to_data + i +'/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data() mask_data = make_mask(np.ones(data.shape[:-1]), mask_data, fit = True) mask_data = mask_data != 0 mask_data = mask_data.astype(int) #Run per slice in order to correct for time for j in range(data.shape[2]): data_slice = data[:, :, j, :] #Create design matrix X = np.ones((n_vols, 9)) X[:, 1] = convolve[:, j] X[:, 2] = np.linspace(-1, 1, num = X.shape[0]) #drift X[:, 3:] = fourier_creation(n_vols, 3)[:, 1:]
plt.colorbar() plt.title('Significant p-values (No mask)') plt.savefig(location_of_images + "NOMASK_significant_p_slice.png") plt.close() print("# ==== END No Mask, bh_procedure ==== #") print("# ==== BEGIN varying the Q value = .005 (FDR) ==== #") Q = .005 significant_pvals = bh_procedure(p_vals, Q) # Reshape significant_pvals reshaped_sig_p = np.reshape(significant_pvals, data.shape[:-1]) slice_reshaped_sig_p = reshaped_sig_p[..., 7] masked_data = make_mask(original_slice, reshaped_sig_p, fit=False) plt.imshow(present_3d(masked_data)) plt.clim(0, 1600) plt.colorbar() plt.title('Slice with Significant p-values (Q = .005)') plt.savefig(location_of_images + "significant_p_slice1.png") plt.close() print("# ==== END plot with Q = .005 done. ==== #") print("# ==== BEGIN varying the Q value = .05 (FDR) ==== #") Q = .05 significant_pvals = bh_procedure(p_vals, Q) # Reshape significant_pvals
pathtodata + i + "/model/model001/onsets/task001_run001/cond_all.txt", TR, n_vols) convolved = np.convolve(neural_prediction, hrf_at_trs) # hrf_at_trs sample N = len(neural_prediction) # N == n_vols == 173 M = len(hrf_at_trs) # M == 12 np_hrf = convolved[:N] B, t, df, p = t_stat(data, np_hrf, np.array([0, 1])) #Simple mask function mask = nib.load(pathtodata + i + '/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data() t_mean[..., int(i[-1])] = make_mask(np.reshape(t, (64, 64, 34)), mask_data, fit=True) final = present_3d(np.mean(t_mean, axis=3)) ####################### # Plot the results # ####################### plt.imshow(final, interpolation='nearest', cmap='seismic') plt.title("Mean T-Statistic Value Across 25 Subjects") zero_out = max(abs(np.min(final)), np.max(final)) plt.clim(-zero_out, zero_out) plt.colorbar() plt.savefig("../../../images/hypothesis_testing.png") plt.close()
########## Clustering ############## ##################################### #Mean across all subject t_mean = np.zeros((64, 64, 34,24)) #loop through each person's T-statistic count=0 for i in sub_list: t_stat = np.load(t_data+i+"_tstat.npy") mask = nib.load(path_to_data+i+'/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data() t_mean[...,count] = make_mask(t_stat, mask_data, fit=True) count+=1 t_mean = np.mean(t_mean,axis=3) final = present_3d(t_mean) plt.imshow(final,interpolation='nearest', cmap='seismic') plt.title("Mean T-Statistic Value Across 25 Subjects") zero_out=max(abs(np.min(final)),np.max(final)) plt.clim(-zero_out,zero_out) plt.colorbar() plt.show() #Cluster data_new = t_mean[...,10:15]
#plt.title('(OLD BH FUNCTION) Significant p-values (No mask)') #plt.savefig(location_of_images+"OLD_significant_p_slice_NOMASK.png") #plt.close() #print("Initial plot with NO MASK (using old bh function) done.") print("# ==== BEGIN varying the Q value = .005 (FDR) ==== #") Q = .005 significant_pvals = bh_procedure(p_vals, Q) # Reshape significant_pvals reshaped_sig_p = np.reshape(significant_pvals, data.shape[:-1]) slice_reshaped_sig_p = reshaped_sig_p[...,7] masked_data = make_mask(original_slice, reshaped_sig_p, fit=False) plt.imshow(present_3d(masked_data)) plt.clim(0, 1600) plt.colorbar() plt.title('Slice with Significant p-values (Q = .005)') plt.savefig(location_of_images+"significant_p_slice1.png") plt.close() print("# ==== END plot with Q = .005 done. ==== #") print("# ==== BEGIN varying the Q value = .05 (FDR) ==== #") Q = .05 significant_pvals = bh_procedure(p_vals, Q) # Reshape significant_pvals
all_stimuli = np.array( sorted(list(cond2[:, 0]) + list(cond3[:, 0]) + list(cond1[:, 0]))) # could also just x_s_array my_hrf = convolution_specialized(all_stimuli, np.ones(len(all_stimuli)), hrf_single, np.linspace(0, 239 * 2 - 2, 239)) B, t, df, p = t_stat(data, my_hrf, np.array([0, 1])) ############### # tgrouping # ############### mask = nib.load(pathtodata + '/anatomy/inplane001_brain_mask.nii.gz') mask = mask.get_data() inner_ones = np.ones(data.shape[:-1]) mask = make_mask(inner_ones, mask, True) mask[mask > 0] = 1 t_vals = t t_vals_3d = t_vals.reshape(data.shape[:-1]) pro = [.25, .1, .1, .05, .025] folks = [1, 1, 5, 5, 10] plt.close() for i in np.arange(5): start, cutoff = t_grouping_neighbor(t_vals_3d, mask, pro[i],
####################### # convolution # ####################### all_stimuli=np.array(sorted(list(cond2[:,0])+list(cond3[:,0])+list(cond1[:,0]))) # could also just x_s_array my_hrf = convolution_specialized(all_stimuli,np.ones(len(all_stimuli)),hrf_single,np.linspace(0,239*2-2,239)) B,t,df,p = t_stat(data, my_hrf, np.array([0,1])) ############### # tgrouping # ############### mask = nib.load(pathtodata + '/anatomy/inplane001_brain_mask.nii.gz') mask = mask.get_data() inner_ones=np.ones(data.shape[:-1]) mask= make_mask(inner_ones,mask,True) mask[mask>0]=1 t_vals=t t_vals_3d=t_vals.reshape(data.shape[:-1]) pro=[.25,.1,.1,.05,.025] folks=[1,1,5,5,10] plt.close() for i in np.arange(5): start,cutoff=t_grouping_neighbor(t_vals_3d,mask,pro[i],prop=True,neighbors= folks[i],abs_on=True)
sys.stdout.write("\b" * (toolbar_width + 1)) # return to start of line, after '[' # Set up lists to store proportion of p-values above 0.05. unmasked_prop = [] # Unmasked (all voxels) masked_prop = [] # Masked. for i in sub_list: residuals = np.load(residual_data + i + "_residual.npy") sw_pvals = check_sw(residuals) unmasked_prop.append(np.mean(sw_pvals > 0.05)) mask = nib.load(path_to_data + i + '/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data() masked_pvals = make_mask(sw_pvals, mask_data, fit=True) masked_pvals[masked_pvals > 1] = 1 pvals_in_brain = sw_pvals.ravel()[masked_pvals.ravel() != 0] masked_prop.append(np.mean(pvals_in_brain > 0.05)) if (i[-3:] == "010"): # Save image plots of unmasked p-values for subject 10. plt.imshow(present_3d(sw_pvals), cmap=plt.get_cmap('gray')) plt.colorbar() plt.xticks([]) plt.yticks([]) plt.title("p-values for " + i + " (Unmasked Data)") plt.savefig(location_of_images + i + 'sw.png') plt.close() # Save image plots of masked p-values for a single subject.
#plt.colorbar() #plt.title('(OLD BH FUNCTION) Significant p-values (No mask)') #plt.savefig(location_of_images+"OLD_significant_p_slice_NOMASK.png") #plt.close() #print("Initial plot with NO MASK (using old bh function) done.") print("# ==== BEGIN varying the Q value = .005 (FDR) ==== #") Q = .005 significant_pvals = bh_procedure(p_vals, Q) # Reshape significant_pvals reshaped_sig_p = np.reshape(significant_pvals, data.shape[:-1]) slice_reshaped_sig_p = reshaped_sig_p[..., 7] masked_data = make_mask(original_slice, reshaped_sig_p, fit=False) plt.imshow(present_3d(masked_data)) plt.clim(0, 1600) plt.colorbar() plt.title('Slice with Significant p-values (Q = .005)') plt.savefig(location_of_images + "significant_p_slice1.png") plt.close() print("# ==== END plot with Q = .005 done. ==== #") print("# ==== BEGIN varying the Q value = .05 (FDR) ==== #") Q = .05 significant_pvals = bh_procedure(p_vals, Q) # Reshape significant_pvals
img = nib.load(path_to_data + name + "/BOLD/task001_run001/bold.nii.gz") data = img.get_data().astype(float) # Load mask. mask = nib.load(path_to_data + name + "/anatomy/inplane001_brain_mask.nii.gz") mask_data = mask.get_data() # Drop the appropriate number of volumes from the beginning. first_n_vols = data.shape[-1] num_TR_cut = int(first_n_vols - num_TR) data = data[..., num_TR_cut:] # Now fit a mask to the 3-d image for each time point. my_mask = np.zeros(data.shape) for i in range(my_mask.shape[-1]): my_mask[..., i] = make_mask(data[..., i], mask_data, fit=True) # Reshape stuff to 2-d (voxel by time) and mask the data. # This should cut down the number of volumes by more than 50%. my_mask_2d = my_mask.reshape((-1, my_mask.shape[-1])) data_2d = data.reshape((-1, data.shape[-1])) masked_data_2d = data_2d[my_mask_2d.sum(1) != 0, :] # Subtract means over voxels (columns). masked_data_2d = masked_data_2d - np.mean(masked_data_2d, 0) # Subtract means over time (rows) masked_data_2d = masked_data_2d - np.mean(masked_data_2d, axis=1)[:, None] # PCA analysis on MASKED data: # Do SVD on the time by time matrix and transform into first k principal components.
np.savetxt(pathtodata+ i+ "/model/model001/onsets/task001_run001/cond_all.txt",cond_all) neural_prediction = events2neural(pathtodata+ i+ "/model/model001/onsets/task001_run001/cond_all.txt",TR,n_vols) convolved = np.convolve(neural_prediction, hrf_at_trs) # hrf_at_trs sample N = len(neural_prediction) # N == n_vols == 173 M = len(hrf_at_trs) # M == 12 np_hrf=convolved[:N] B,t,df,p = t_stat(data, np_hrf, np.array([0,1])) #Simple mask function mask = nib.load(pathtodata+i+'/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data() t_mean[...,int(i[-1])] = make_mask(np.reshape(t,(64,64,34)), mask_data, fit=True) final = present_3d(np.mean(t_mean,axis=3)) ####################### # Plot the results # ####################### plt.imshow(final,interpolation='nearest', cmap='seismic') plt.title("Mean T-Statistic Value Across 25 Subjects") zero_out=max(abs(np.min(final)),np.max(final)) plt.clim(-zero_out,zero_out) plt.colorbar()
data = img.get_data().astype(float) # Load mask. mask = nib.load(path_to_data + name + '/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data() # Drop the appropriate number of volumes from the beginning. first_n_vols = data.shape[-1] num_TR_cut = int(first_n_vols - num_TR) data = data[..., num_TR_cut:] # Now fit a mask to the 3-d image for each time point. my_mask = np.zeros(data.shape) for i in range(my_mask.shape[-1]): my_mask[..., i] = make_mask(data[..., i], mask_data, fit=True) # Reshape stuff to 2-d (voxel by time) and mask the data. # This should cut down the number of volumes by more than 50%. my_mask_2d = my_mask.reshape((-1, my_mask.shape[-1])) data_2d = data.reshape((-1, data.shape[-1])) masked_data_2d = data_2d[my_mask_2d.sum(1) != 0, :] # Subtract means over voxels (columns). masked_data_2d = masked_data_2d - np.mean(masked_data_2d, 0) # Subtract means over time (rows) masked_data_2d = masked_data_2d - np.mean(masked_data_2d, axis=1)[:, None] # PCA analysis on MASKED data: # Do SVD on the time by time matrix and transform into first k principal components.
# Progress bar toolbar_width=len(sub_list) sys.stdout.write("GLM, : ") sys.stdout.write("[%s]" % (" " * toolbar_width)) sys.stdout.flush() sys.stdout.write("\b" * (toolbar_width+1)) # return to start of line, after '[' for i in sub_list: residuals = np.load(residual_data+i+"_residual.npy") sw_pvals = check_sw(residuals) print(i+" proportion of voxels with p-value above 0.05 (unmasked): "+str(np.mean(sw_pvals > 0.05))) mask = nib.load(path_to_data+i+'/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data() masked_pvals = make_mask(sw_pvals, mask_data, fit=True) pvals_in_brain = sw_pvals.ravel()[masked_pvals.ravel() != 0] print(i+" proportion of voxels with p-value above 0.05 (masked): "+str(np.mean(pvals_in_brain > 0.05))) sys.stdout.write("-") sys.stdout.flush() # Save image plots of masked and unmasked p-values for a single subject. plt.imshow(present_3d(sw_pvals), cmap=plt.get_cmap('gray')) plt.savefig(location_of_images+i+'sw.png') plt.close() plt.imshow(present_3d(masked_pvals), cmap=plt.get_cmap('gray')) plt.savefig(location_of_images+i+'swmasked.png') plt.close() sys.stdout.write("\n")
from hypothesis import t_stat from Image_Visualizing import present_3d, make_mask from tgrouping import t_grouping_neighbor i = 'sub001' ##################################### ########## Clustering ########## ##################################### t_stat = np.load(t_data+i+"_tstat.npy") mask = nib.load(path_to_data+i+'/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data() ones = np.ones((64,64,34)) fitted_mask = make_mask(ones, mask_data, fit = True) fitted_mask[fitted_mask > 0] = 1 data_new= t_stat*fitted_mask data_new = data_new[...,19:22] X = np.reshape(data_new, (-1, 1)) connectivity = grid_to_graph(n_x= data_new.shape[0], n_y = data_new.shape[1], n_z = data_new.shape[2]) n_clusters = 5 # number of regions ward = AgglomerativeClustering(n_clusters=n_clusters, linkage='ward', connectivity=connectivity).fit(X) label = np.reshape(ward.labels_, data_new.shape)