def fourier_predict_underlying_noise(y_mean,p): """ predicts the underlying noise using fourier series and glm Parameters: ----------- y_mean: 1 dimensional np.array p: number of fourier series (pairs) Returns: -------- X: glm_matrix (first column is all 1s) fitted: the fitted values from glm residuals: the residuals betwen fitted and y_mean MRSS: MRSS from glm function (general output from glm_diagnostics) Note: ----- Does a backwards approach to fouriers (possibly sacrificing orthogonality), wants to look at maximum period first """ n= y_mean.shape[0] X=fourier_creation(n,p) beta, junk=glm_multiple(y_mean,X) MRSS, fitted, residuals = glm_diagnostics(beta, X, y_mean) return X,MRSS,fitted,residuals
def test_glm(): # Read in the image data. img = nib.load(pathtoclassdata + "ds114_sub009_t2r1.nii") data = img.get_data()[..., 4:] # Read in the convolutions. convolved = np.loadtxt(pathtoclassdata + "ds114_sub009_t2r1_conv.txt")[4:] # Create design matrix. actual_design = np.ones((len(convolved), 2)) actual_design[:, 1] = convolved # Calculate betas, copied from the exercise. data_2d = np.reshape(data, (-1, data.shape[-1])) actual_B = npl.pinv(actual_design).dot(data_2d.T) actual_B_4d = np.reshape(actual_B.T, img.shape[:-1] + (-1,)) # Run function. exp_B_4d, exp_design = glm(data, convolved) assert_almost_equal(actual_B_4d, exp_B_4d) assert_almost_equal(actual_design, exp_design) # Pick a single voxel to check diagnostics. # Calculate actual fitted values, residuals, and MRSS of voxel. actual_fitted = actual_design.dot(actual_B_4d[42, 32, 19]) actual_residuals = data[42, 32, 19] - actual_fitted actual_MRSS = np.sum(actual_residuals**2)/(actual_design.shape[0] - npl.matrix_rank(actual_design)) # Calculate using glm_diagnostics function. exp_MRSS, exp_fitted, exp_residuals = glm_diagnostics(exp_B_4d, exp_design, data) assert_almost_equal(actual_fitted, exp_fitted[42, 32, 19]) assert_almost_equal(actual_residuals, exp_residuals[42, 32, 19]) assert_almost_equal(actual_MRSS, exp_MRSS[42, 32, 19])
def compare_outliers(data, conv, plot=False): """ Return standard deviation across voxels for 4D array `data` Parameters ---------- data : 4D array 4D array from FMRI run with last axis indexing volumes. conv : 2D array of the convolved time course Returns ------- meanMRSS : mean MRSS of simple regression on the convolved time course. outmeanMRSS : mean MRSS of simple regression on the convolved time course after dropping the extended rms outliers. """ rms_values = vol_rms_diff(data) rms_outliers, rms_thresholds = iqr_outliers(rms_values) extended_indices = extend_diff_outliers(rms_outliers) X = np.ones((len(conv), 2)) X[:, 1] = conv B, junk = glm_multiple(data, X) MRSS, fitted, residuals = glm_diagnostics(B, X, data) meanMRSS = np.mean(MRSS) mask = np.ones(X.shape[0]) mask[extended_indices] = 0 outX = X[mask.nonzero()[0],:] outB, junk = glm_multiple(data[...,mask.nonzero()[0]], outX) outMRSS, outfitted, outresiduals = glm_diagnostics(outB, outX, data[...,mask.nonzero()[0]]) outmeanMRSS = np.mean(outMRSS) if plot==True: rms_values = np.resize(rms_values, len(rms_values)+1) rms_values[-1] = 0 plt.plot(rms_values, "k") plt.plot(extended_indices, rms_values[extended_indices], "ro") plt.axhline(rms_thresholds[0], ls="--") plt.axhline(rms_thresholds[1], ls="--") hand_out = mlines.Line2D([], [], color="r", marker="o", ls="None", label="Outliers") hand_thresh = mlines.Line2D([], [], color="b", ls="--", label="Thresholds") plt.legend(handles=[hand_out, hand_thresh], numpoints=1) return meanMRSS, outmeanMRSS
################################################# # Hemogrlobin response for each condition # ################################################# plt.plot(X_np[:,1]+X_np[:,2]+X_np[:,3],label="All Conditions",color="#000019") plt.plot([0,239],[0,0]) colors=["#000099","#1A1AFF","#9999FF"] for i in range(3): plt.plot(X_np[:,(i+1)]-2*(i+1),label="Condition " +str(i+1),color=colors[i]) plt.plot([0,239],[-2*(i+1),-2*(i+1)],color="#FF0000") plt.legend(loc='center right', shadow=True,fontsize="smaller") plt.title("Hemogoblin predicted response for different conditions") plt.xlabel("Time") plt.ylabel("Hemoglobin response") plt.savefig(location_of_images+'all_cond_time.png') plt.close() MRSS_my, fitted_my, residuals_my = glm_diagnostics(B_my, X_my, data) print("MRSS using multiple regression: "+str(np.mean(MRSS_my))) plt.plot(data[41, 47, 2],label="actual HR response") plt.plot(fitted_my[41, 47, 2],label="predicted HR response") plt.title("Subject 001, voxel (41,47,2) HR Fitted vs actual") plt.legend(loc='upper left', shadow=True,fontsize="smaller") plt.savefig(location_of_images+"fitted_vs_actual_mult_regression.png") plt.close()
data_slice = data[:, :, j, :] #Create design matrix X = np.ones((n_vols, 9)) X[:, 1] = convolve[:, j] X[:, 2] = np.linspace(-1, 1, num = X.shape[0]) #drift X[:, 3:] = fourier_creation(n_vols, 3)[:, 1:] beta, t, df, p = t_stat_mult_regression(data_slice, X) #take only first coefficient t = t[1, :] p = p[1, :] MRSS, fitted, residuals = glm_diagnostics(beta, X, data_slice) beta = beta[:, 1] #insert into the proper slice beta_final[:, :, j] = beta.reshape(data_slice.shape[:-1]) t_final[:, :, j] = t.reshape(data_slice.shape[:-1]) p_final[:, :, j] = p.reshape(data_slice.shape[:-1]) residual_final[:, :, j, :] = residuals.reshape(data_slice.shape) np.save("../data/betas/" + i + "_beta.npy", beta_final) np.save("../data/t_stat/" + i + "_tstat.npy", t_final) np.save("../data/residual/" + i + "_residual.npy", residual_final) np.save("../data/p-values/" + i + "_pvalue.npy", p_final) np.save("../data/X/" + i + "_covX.npy", np.cov(X.T))
X_cond[:, 3] = hrf_matrix_3[:, j] # 1 more X_cond[:, 4] = np.linspace(-1, 1, num = X.shape[0]) #drift # one X_cond[:, 5:11] = fourier_creation(X.shape[0], 3)[:, 1:] # six more X_cond[:, 11:] = pca_addition #START CREATING MODELS ################### # MODEL 1 # ################### # hrf (simple) beta1, t,df1, p = t_stat_mult_regression(data_slice, X[:, 0:2]) MRSS1, fitted, residuals = glm_diagnostics(beta1, X[:, 0:2], data_slice) model1_slice = np.zeros(len(MRSS1)) rank1 = npl.matrix_rank(X[:, 0:2]) count = 0 for value in MRSS1: model1_slice[count] = adjR2(value, np.array(data_slice[count, :]), df1, rank1) count += 1 adjr2_1 = adjr2_1 + model1_slice.tolist() aic_1 = aic_1 + AIC_2(MRSS1, data_slice, df1, rank1).tolist()
################# # First Attempt # ################# # First approach allowed for fourier strength to be fit to each voxel, # potentially overcorrecting and masking some response to neural stimulation # X matrix X = np.ones((n_vols, 9)) # changed since fourier needs more X[:, 1] = convolution_specialized(cond_all[:, 0], np.ones(len(cond_all)), hrf_single, all_tr_times) X[:, 2] = np.linspace(-1, 1, num=X.shape[0]) # drift X[:, 3:] = fourier_creation(X.shape[0], 3)[:, 1:] # modeling voxel hemodynamic response beta, junk = glm_multiple(data, X) MRSS, fitted, residuals = glm_diagnostics(beta, X, data) # individual voxel analysis plt.plot(all_tr_times, data[41, 47, 2], label="actual", color="b") plt.plot(all_tr_times, fitted[41, 47, 2], label="predicted", color="r") plt.title("Data for sub001, voxel [41, 47, 2],fourier 3 fit to voxel") plt.xlabel("Time") plt.ylabel("Hemodynamic response") plt.legend(loc="upper right", shadow=True, fontsize="smaller") plt.savefig(location_of_images + "noise_correction__fit_to_voxel_fitted.png") plt.close() plt.plot(all_tr_times, residuals[41, 47, 2], label="residuals", color="b") plt.plot([0, max(all_tr_times)], [0, 0], label="origin (residual=0)", color="k") plt.title("Residual for sub001, voxel [41, 47, 2],fourier 3 fit to voxel")
############################# ############################# # Analysis and diagonistics # ############################# ############################# ####################### # a. (my) convolution # ####################### # Now get the estimated coefficients and design matrix for doing # regression on the convolved time course. B_my, X_my = glm(data, my_hrf) # Some diagnostics. MRSS_my, fitted_my, residuals_my = glm_diagnostics(B_my, X_my, data) # Print out the mean MRSS. print("MRSS using 'my' convolution function: "+str(np.mean(MRSS_my))) # Plot the time course for a single voxel with the fitted values. # Looks pretty bad. plt.plot(data[41, 47, 2]) #change from cherry-picking plt.plot(fitted_my[41, 47, 2]) plt.savefig(location_of_images+"glm_plot_my.png") plt.close() ################## # b. np.convolve # ##################
# First Attempt # ################# # First approach allowed for fourier strength to be fit to each voxel, # potentially overcorrecting and masking some response to neural stimulation # X matrix X = np.ones((n_vols, 6)) X[:, 1] = convolution_specialized(cond_all[:, 0], np.ones(len(cond_all)), hrf_single, all_tr_times) X[:, 2] = np.linspace(-1, 1, num=X.shape[0]) #drift X[:, 3:] = fourier_creation(X.shape[0], 3)[:, 1:] # modeling voxel hemodynamic response beta, junk = glm_multiple(data, X) MRSS, fitted, residuals = glm_diagnostics(beta, X, data) # individual voxel analysis plt.plot(all_tr_times, data[41, 47, 2], label="actual", color="b") plt.plot(all_tr_times, fitted[41, 47, 2], label="predicted", color="r") plt.title("Data for sub001, voxel [41, 47, 2],fourier 3 fit to voxel") plt.xlabel("Time") plt.ylabel("Hemodynamic response") plt.legend(loc='upper right', shadow=True, fontsize="smaller") plt.savefig(location_of_images + 'noise_correction__fit_to_voxel_fitted.png') plt.close() plt.plot(all_tr_times, residuals[41, 47, 2], label="residuals", color="b") plt.plot([0, max(all_tr_times)], [0, 0], label="origin (residual=0)",
N = len(neural_prediction) # N == n_vols == 173 M = len(hrf_at_trs) # M == 12 np_hrf=convolved[:N] ################### # From GLM function ################### np_B, np_X = glm(data, np_hrf) #################################### # GLM Diagnostics (to get residuals) ################################### np_MRSS, np_fitted, np_residuals = glm_diagnostics(np_B, np_X, data) ########################### #Shapiro-Wilks on Residuals ########################### # Shapiro-Wilks: tests the null hypothesis that the data was # drawn from a normal distribution. # Using 4-d residuals. sw_pvals = check_sw(np_residuals) print("Proportion of voxels with p-value above 0.05 (unmasked): "+str(np.mean(sw_pvals > 0.05))) # Load mask. mask = nib.load(pathtodata+'/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data()
############################# ############################# # Analysis and diagonistics # ############################# ############################# ####################### # a. (my) convolution # ####################### # Now get the estimated coefficients and design matrix for doing # regression on the convolved time course. B_my, X_my = glm(data, my_hrf) # Some diagnostics. MRSS_my, fitted_my, residuals_my = glm_diagnostics(B_my, X_my, data) # Print out the mean MRSS. print("MRSS using 'my' convolution function: " + str(np.mean(MRSS_my))) # Plot the time course for a single voxel with the fitted values. # Looks pretty bad. plt.plot(data[41, 47, 2]) #change from cherry-picking plt.plot(fitted_my[41, 47, 2]) plt.savefig(location_of_images + "glm_plot_my.png") plt.close() ################## # b. np.convolve # ##################
hrf_at_trs) # hrf_at_trs sample data N = len(neural_prediction) # N == n_vols == 173 M = len(hrf_at_trs) # M == 12 np_hrf = convolved[:N] ################### # From GLM function ################### np_B, np_X = glm(data, np_hrf) #################################### # GLM Diagnostics (to get residuals) ################################### np_MRSS, np_fitted, np_residuals = glm_diagnostics(np_B, np_X, data) ########################### #Shapiro-Wilks on Residuals ########################### # Shapiro-Wilks: tests the null hypothesis that the data was # drawn from a normal distribution. # Using 4-d residuals. sw_pvals = check_sw(np_residuals) print("Proportion of voxels with p-value above 0.05 (unmasked): " + str(np.mean(sw_pvals > 0.05))) # Load mask. mask = nib.load(pathtodata + '/anatomy/inplane001_brain_mask.nii.gz') mask_data = mask.get_data()
X_cond[:, 2] = hrf_matrix_2[:, j] # 1 more X_cond[:, 3] = hrf_matrix_3[:, j] # 1 more X_cond[:, 4] = np.linspace(-1, 1, num=X.shape[0]) #drift # one X_cond[:, 5:11] = fourier_creation(X.shape[0], 3)[:, 1:] # six more X_cond[:, 11:] = pca_addition #START CREATING MODELS ################### # MODEL 1 # ################### # 1.1 hrf (simple) beta1, t, df1, p = t_stat_mult_regression(data_slice, X[:, 0:2]) MRSS1, fitted, residuals = glm_diagnostics(beta1, X[:, 0:2], data_slice) model1_slice = np.zeros(len(MRSS1)) rank1 = npl.matrix_rank(X[:, 0:2]) count = 0 for value in MRSS1: model1_slice[count] = adjR2(value, np.array(data_slice[count, :]), df1, rank1) count += 1 #model1=model1+model1_slice.tolist() adjr2_1 = adjr2_1 + model1_slice.tolist()
###PCA SHIT### #to_2d= masking_reshape_start(data,mask) #X_pca= to_2d - np.mean(to_2d,0) - np.mean(to_2d,1)[:,None] #cov = X_pca.T.dot(X_pca) #U, S, V = npl.svd(cov) #pca_addition= U[:,:6] # ~40% coverage #Create design matrix X = np.ones((n_vols, 9)) X[:, 1] = convolve[:, j] X[:, 2] = np.linspace(-1, 1, num=X.shape[0]) #drift X[:, 3:] = fourier_creation(n_vols, 3)[:, 1:] beta, t, df, p = t_stat_mult_regression(data_slice, X) MRSS, fitted, residuals = glm_diagnostics(beta, X, data_slice) ########################### # Fitted VS Residual Plot # ########################### plt.scatter(fitted[30, 40, :], residuals[30, 40, :]) min_max = (np.min(fitted[30, 40, :]), np.max(fitted[30, 40, :])) plt.plot([min_max[0], min_max[1]], [0, 0], color="k") plt.xlim(min_max[0], min_max[1]) plt.title("Subject 001, voxel: [30,40,15]") plt.xlabel("Fitted Values") plt.ylabel("Residuals") plt.savefig(location_of_images + 'Fitted_v_Residuals.png')
plt.plot(X_np[:,1]+X_np[:,2]+X_np[:,3],label="All Conditions",color="#000019") plt.plot([0,239],[0,0]) colors=["#000099","#1A1AFF","#9999FF"] for i in range(3): plt.plot(X_np[:,(i+1)]-2*(i+1),label="Condition " +str(i+1),color=colors[i]) plt.plot([0,239],[-2*(i+1),-2*(i+1)],color="#FF0000") plt.legend(loc='center right', shadow=True,fontsize="smaller") plt.title("Hemogoblin predicted response for different conditions") plt.xlabel("Time") plt.ylabel("Hemoglobin response") plt.savefig(location_of_images+'all_cond_time.png') plt.close() MRSS_my, fitted_my, residuals_my = glm_diagnostics(B_my, X_my, data) print("MRSS using multiple regression: "+str(np.mean(MRSS_my))) plt.plot(data[41, 47, 2],label="actual HR response") plt.plot(fitted_my[41, 47, 2],label="predicted HR response") plt.title("Subject 001, voxel (41,47,2) HR Fitted vs actual") plt.legend(loc='upper left', shadow=True,fontsize="smaller") plt.savefig(location_of_images+"fitted_vs_actual_mult_regression.png") plt.close()