def fourier_predict_underlying_noise(y_mean,p): """ predicts the underlying noise using fourier series and glm Parameters: ----------- y_mean: 1 dimensional np.array p: number of fourier series (pairs) Returns: -------- X: glm_matrix (first column is all 1s) fitted: the fitted values from glm residuals: the residuals betwen fitted and y_mean MRSS: MRSS from glm function (general output from glm_diagnostics) Note: ----- Does a backwards approach to fouriers (possibly sacrificing orthogonality), wants to look at maximum period first """ n= y_mean.shape[0] X=fourier_creation(n,p) beta, junk=glm_multiple(y_mean,X) MRSS, fitted, residuals = glm_diagnostics(beta, X, y_mean) return X,MRSS,fitted,residuals
def test_glm():
# Read in the image data.
img = nib.load(pathtoclassdata + "ds114_sub009_t2r1.nii")
data = img.get_data()[..., 4:]
# Read in the convolutions.
convolved = np.loadtxt(pathtoclassdata + "ds114_sub009_t2r1_conv.txt")[4:]
# Create design matrix.
actual_design = np.ones((len(convolved), 2))
actual_design[:, 1] = convolved
# Calculate betas, copied from the exercise.
data_2d = np.reshape(data, (-1, data.shape[-1]))
actual_B = npl.pinv(actual_design).dot(data_2d.T)
actual_B_4d = np.reshape(actual_B.T, img.shape[:-1] + (-1,))
# Run function.
exp_B_4d, exp_design = glm(data, convolved)
assert_almost_equal(actual_B_4d, exp_B_4d)
assert_almost_equal(actual_design, exp_design)
# Pick a single voxel to check diagnostics.
# Calculate actual fitted values, residuals, and MRSS of voxel.
actual_fitted = actual_design.dot(actual_B_4d[42, 32, 19])
actual_residuals = data[42, 32, 19] - actual_fitted
actual_MRSS = np.sum(actual_residuals**2)/(actual_design.shape[0] - npl.matrix_rank(actual_design))
# Calculate using glm_diagnostics function.
exp_MRSS, exp_fitted, exp_residuals = glm_diagnostics(exp_B_4d, exp_design, data)
assert_almost_equal(actual_fitted, exp_fitted[42, 32, 19])
assert_almost_equal(actual_residuals, exp_residuals[42, 32, 19])
assert_almost_equal(actual_MRSS, exp_MRSS[42, 32, 19])
def compare_outliers(data, conv, plot=False):
""" Return standard deviation across voxels for 4D array `data`
Parameters
----------
data : 4D array
4D array from FMRI run with last axis indexing volumes.
conv : 2D array of the convolved time course
Returns
-------
meanMRSS : mean MRSS of simple regression on the convolved time course.
outmeanMRSS : mean MRSS of simple regression on the convolved time course after dropping the extended rms outliers.
"""
rms_values = vol_rms_diff(data)
rms_outliers, rms_thresholds = iqr_outliers(rms_values)
extended_indices = extend_diff_outliers(rms_outliers)
X = np.ones((len(conv), 2))
X[:, 1] = conv
B, junk = glm_multiple(data, X)
MRSS, fitted, residuals = glm_diagnostics(B, X, data)
meanMRSS = np.mean(MRSS)
mask = np.ones(X.shape[0])
mask[extended_indices] = 0
outX = X[mask.nonzero()[0],:]
outB, junk = glm_multiple(data[...,mask.nonzero()[0]], outX)
outMRSS, outfitted, outresiduals = glm_diagnostics(outB, outX, data[...,mask.nonzero()[0]])
outmeanMRSS = np.mean(outMRSS)
if plot==True:
rms_values = np.resize(rms_values, len(rms_values)+1)
rms_values[-1] = 0
plt.plot(rms_values, "k")
plt.plot(extended_indices, rms_values[extended_indices], "ro")
plt.axhline(rms_thresholds[0], ls="--")
plt.axhline(rms_thresholds[1], ls="--")
hand_out = mlines.Line2D([], [], color="r", marker="o", ls="None", label="Outliers")
hand_thresh = mlines.Line2D([], [], color="b", ls="--", label="Thresholds")
plt.legend(handles=[hand_out, hand_thresh], numpoints=1)
return meanMRSS, outmeanMRSS
#################################################
# Hemogrlobin response for each condition #
#################################################
plt.plot(X_np[:,1]+X_np[:,2]+X_np[:,3],label="All Conditions",color="#000019")
plt.plot([0,239],[0,0])
colors=["#000099","#1A1AFF","#9999FF"]
for i in range(3):
plt.plot(X_np[:,(i+1)]-2*(i+1),label="Condition " +str(i+1),color=colors[i])
plt.plot([0,239],[-2*(i+1),-2*(i+1)],color="#FF0000")
plt.legend(loc='center right', shadow=True,fontsize="smaller")
plt.title("Hemogoblin predicted response for different conditions")
plt.xlabel("Time")
plt.ylabel("Hemoglobin response")
plt.savefig(location_of_images+'all_cond_time.png')
plt.close()
MRSS_my, fitted_my, residuals_my = glm_diagnostics(B_my, X_my, data)
print("MRSS using multiple regression: "+str(np.mean(MRSS_my)))
plt.plot(data[41, 47, 2],label="actual HR response")
plt.plot(fitted_my[41, 47, 2],label="predicted HR response")
plt.title("Subject 001, voxel (41,47,2) HR Fitted vs actual")
plt.legend(loc='upper left', shadow=True,fontsize="smaller")
plt.savefig(location_of_images+"fitted_vs_actual_mult_regression.png")
plt.close()
data_slice = data[:, :, j, :]
#Create design matrix
X = np.ones((n_vols, 9))
X[:, 1] = convolve[:, j]
X[:, 2] = np.linspace(-1, 1, num = X.shape[0]) #drift
X[:, 3:] = fourier_creation(n_vols, 3)[:, 1:]
beta, t, df, p = t_stat_mult_regression(data_slice, X)
#take only first coefficient
t = t[1, :]
p = p[1, :]
MRSS, fitted, residuals = glm_diagnostics(beta, X, data_slice)
beta = beta[:, 1]
#insert into the proper slice
beta_final[:, :, j] = beta.reshape(data_slice.shape[:-1])
t_final[:, :, j] = t.reshape(data_slice.shape[:-1])
p_final[:, :, j] = p.reshape(data_slice.shape[:-1])
residual_final[:, :, j, :] = residuals.reshape(data_slice.shape)
np.save("../data/betas/" + i + "_beta.npy", beta_final)
np.save("../data/t_stat/" + i + "_tstat.npy", t_final)
np.save("../data/residual/" + i + "_residual.npy", residual_final)
np.save("../data/p-values/" + i + "_pvalue.npy", p_final)
np.save("../data/X/" + i + "_covX.npy", np.cov(X.T))
X_cond[:, 3] = hrf_matrix_3[:, j] # 1 more
X_cond[:, 4] = np.linspace(-1, 1, num = X.shape[0]) #drift # one
X_cond[:, 5:11] = fourier_creation(X.shape[0], 3)[:, 1:] # six more
X_cond[:, 11:] = pca_addition
#START CREATING MODELS
###################
# MODEL 1 #
###################
# hrf (simple)
beta1, t,df1, p = t_stat_mult_regression(data_slice, X[:, 0:2])
MRSS1, fitted, residuals = glm_diagnostics(beta1, X[:, 0:2], data_slice)
model1_slice = np.zeros(len(MRSS1))
rank1 = npl.matrix_rank(X[:, 0:2])
count = 0
for value in MRSS1:
model1_slice[count] = adjR2(value, np.array(data_slice[count, :]), df1, rank1)
count += 1
adjr2_1 = adjr2_1 + model1_slice.tolist()
aic_1 = aic_1 + AIC_2(MRSS1, data_slice, df1, rank1).tolist()
#################
# First Attempt #
#################
# First approach allowed for fourier strength to be fit to each voxel,
# potentially overcorrecting and masking some response to neural stimulation
# X matrix
X = np.ones((n_vols, 9)) # changed since fourier needs more
X[:, 1] = convolution_specialized(cond_all[:, 0], np.ones(len(cond_all)), hrf_single, all_tr_times)
X[:, 2] = np.linspace(-1, 1, num=X.shape[0]) # drift
X[:, 3:] = fourier_creation(X.shape[0], 3)[:, 1:]
# modeling voxel hemodynamic response
beta, junk = glm_multiple(data, X)
MRSS, fitted, residuals = glm_diagnostics(beta, X, data)
# individual voxel analysis
plt.plot(all_tr_times, data[41, 47, 2], label="actual", color="b")
plt.plot(all_tr_times, fitted[41, 47, 2], label="predicted", color="r")
plt.title("Data for sub001, voxel [41, 47, 2],fourier 3 fit to voxel")
plt.xlabel("Time")
plt.ylabel("Hemodynamic response")
plt.legend(loc="upper right", shadow=True, fontsize="smaller")
plt.savefig(location_of_images + "noise_correction__fit_to_voxel_fitted.png")
plt.close()
plt.plot(all_tr_times, residuals[41, 47, 2], label="residuals", color="b")
plt.plot([0, max(all_tr_times)], [0, 0], label="origin (residual=0)", color="k")
plt.title("Residual for sub001, voxel [41, 47, 2],fourier 3 fit to voxel")
#############################
#############################
# Analysis and diagonistics #
#############################
#############################
#######################
# a. (my) convolution #
#######################
# Now get the estimated coefficients and design matrix for doing
# regression on the convolved time course.
B_my, X_my = glm(data, my_hrf)
# Some diagnostics.
MRSS_my, fitted_my, residuals_my = glm_diagnostics(B_my, X_my, data)
# Print out the mean MRSS.
print("MRSS using 'my' convolution function: "+str(np.mean(MRSS_my)))
# Plot the time course for a single voxel with the fitted values.
# Looks pretty bad.
plt.plot(data[41, 47, 2]) #change from cherry-picking
plt.plot(fitted_my[41, 47, 2])
plt.savefig(location_of_images+"glm_plot_my.png")
plt.close()
##################
# b. np.convolve #
##################
# First Attempt #
#################
# First approach allowed for fourier strength to be fit to each voxel,
# potentially overcorrecting and masking some response to neural stimulation
# X matrix
X = np.ones((n_vols, 6))
X[:, 1] = convolution_specialized(cond_all[:, 0], np.ones(len(cond_all)),
hrf_single, all_tr_times)
X[:, 2] = np.linspace(-1, 1, num=X.shape[0]) #drift
X[:, 3:] = fourier_creation(X.shape[0], 3)[:, 1:]
# modeling voxel hemodynamic response
beta, junk = glm_multiple(data, X)
MRSS, fitted, residuals = glm_diagnostics(beta, X, data)
# individual voxel analysis
plt.plot(all_tr_times, data[41, 47, 2], label="actual", color="b")
plt.plot(all_tr_times, fitted[41, 47, 2], label="predicted", color="r")
plt.title("Data for sub001, voxel [41, 47, 2],fourier 3 fit to voxel")
plt.xlabel("Time")
plt.ylabel("Hemodynamic response")
plt.legend(loc='upper right', shadow=True, fontsize="smaller")
plt.savefig(location_of_images + 'noise_correction__fit_to_voxel_fitted.png')
plt.close()
plt.plot(all_tr_times, residuals[41, 47, 2], label="residuals", color="b")
plt.plot([0, max(all_tr_times)], [0, 0],
label="origin (residual=0)",
N = len(neural_prediction) # N == n_vols == 173
M = len(hrf_at_trs) # M == 12
np_hrf=convolved[:N]
###################
# From GLM function
###################
np_B, np_X = glm(data, np_hrf)
####################################
# GLM Diagnostics (to get residuals)
###################################
np_MRSS, np_fitted, np_residuals = glm_diagnostics(np_B, np_X, data)
###########################
#Shapiro-Wilks on Residuals
###########################
# Shapiro-Wilks: tests the null hypothesis that the data was
# drawn from a normal distribution.
# Using 4-d residuals.
sw_pvals = check_sw(np_residuals)
print("Proportion of voxels with p-value above 0.05 (unmasked): "+str(np.mean(sw_pvals > 0.05)))
# Load mask.
mask = nib.load(pathtodata+'/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
#############################
#############################
# Analysis and diagonistics #
#############################
#############################
#######################
# a. (my) convolution #
#######################
# Now get the estimated coefficients and design matrix for doing
# regression on the convolved time course.
B_my, X_my = glm(data, my_hrf)
# Some diagnostics.
MRSS_my, fitted_my, residuals_my = glm_diagnostics(B_my, X_my, data)
# Print out the mean MRSS.
print("MRSS using 'my' convolution function: " + str(np.mean(MRSS_my)))
# Plot the time course for a single voxel with the fitted values.
# Looks pretty bad.
plt.plot(data[41, 47, 2]) #change from cherry-picking
plt.plot(fitted_my[41, 47, 2])
plt.savefig(location_of_images + "glm_plot_my.png")
plt.close()
##################
# b. np.convolve #
##################
hrf_at_trs) # hrf_at_trs sample data
N = len(neural_prediction) # N == n_vols == 173
M = len(hrf_at_trs) # M == 12
np_hrf = convolved[:N]
###################
# From GLM function
###################
np_B, np_X = glm(data, np_hrf)
####################################
# GLM Diagnostics (to get residuals)
###################################
np_MRSS, np_fitted, np_residuals = glm_diagnostics(np_B, np_X, data)
###########################
#Shapiro-Wilks on Residuals
###########################
# Shapiro-Wilks: tests the null hypothesis that the data was
# drawn from a normal distribution.
# Using 4-d residuals.
sw_pvals = check_sw(np_residuals)
print("Proportion of voxels with p-value above 0.05 (unmasked): " +
str(np.mean(sw_pvals > 0.05)))
# Load mask.
mask = nib.load(pathtodata + '/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
X_cond[:, 2] = hrf_matrix_2[:, j] # 1 more
X_cond[:, 3] = hrf_matrix_3[:, j] # 1 more
X_cond[:, 4] = np.linspace(-1, 1, num=X.shape[0]) #drift # one
X_cond[:, 5:11] = fourier_creation(X.shape[0], 3)[:, 1:] # six more
X_cond[:, 11:] = pca_addition
#START CREATING MODELS
###################
# MODEL 1 #
###################
# 1.1 hrf (simple)
beta1, t, df1, p = t_stat_mult_regression(data_slice, X[:, 0:2])
MRSS1, fitted, residuals = glm_diagnostics(beta1, X[:, 0:2],
data_slice)
model1_slice = np.zeros(len(MRSS1))
rank1 = npl.matrix_rank(X[:, 0:2])
count = 0
for value in MRSS1:
model1_slice[count] = adjR2(value, np.array(data_slice[count, :]),
df1, rank1)
count += 1
#model1=model1+model1_slice.tolist()
adjr2_1 = adjr2_1 + model1_slice.tolist()
###PCA SHIT###
#to_2d= masking_reshape_start(data,mask)
#X_pca= to_2d - np.mean(to_2d,0) - np.mean(to_2d,1)[:,None]
#cov = X_pca.T.dot(X_pca)
#U, S, V = npl.svd(cov)
#pca_addition= U[:,:6] # ~40% coverage
#Create design matrix
X = np.ones((n_vols, 9))
X[:, 1] = convolve[:, j]
X[:, 2] = np.linspace(-1, 1, num=X.shape[0]) #drift
X[:, 3:] = fourier_creation(n_vols, 3)[:, 1:]
beta, t, df, p = t_stat_mult_regression(data_slice, X)
MRSS, fitted, residuals = glm_diagnostics(beta, X, data_slice)
###########################
# Fitted VS Residual Plot #
###########################
plt.scatter(fitted[30, 40, :], residuals[30, 40, :])
min_max = (np.min(fitted[30, 40, :]), np.max(fitted[30, 40, :]))
plt.plot([min_max[0], min_max[1]], [0, 0], color="k")
plt.xlim(min_max[0], min_max[1])
plt.title("Subject 001, voxel: [30,40,15]")
plt.xlabel("Fitted Values")
plt.ylabel("Residuals")
plt.savefig(location_of_images + 'Fitted_v_Residuals.png')
plt.plot(X_np[:,1]+X_np[:,2]+X_np[:,3],label="All Conditions",color="#000019")
plt.plot([0,239],[0,0])
colors=["#000099","#1A1AFF","#9999FF"]
for i in range(3):
plt.plot(X_np[:,(i+1)]-2*(i+1),label="Condition " +str(i+1),color=colors[i])
plt.plot([0,239],[-2*(i+1),-2*(i+1)],color="#FF0000")
plt.legend(loc='center right', shadow=True,fontsize="smaller")
plt.title("Hemogoblin predicted response for different conditions")
plt.xlabel("Time")
plt.ylabel("Hemoglobin response")
plt.savefig(location_of_images+'all_cond_time.png')
plt.close()
MRSS_my, fitted_my, residuals_my = glm_diagnostics(B_my, X_my, data)
print("MRSS using multiple regression: "+str(np.mean(MRSS_my)))
plt.plot(data[41, 47, 2],label="actual HR response")
plt.plot(fitted_my[41, 47, 2],label="predicted HR response")
plt.title("Subject 001, voxel (41,47,2) HR Fitted vs actual")
plt.legend(loc='upper left', shadow=True,fontsize="smaller")
plt.savefig(location_of_images+"fitted_vs_actual_mult_regression.png")
plt.close()
