def test_mask():
# example from http://www.jarrodmillman.com/rcsds/lectures/glm_intro.html
# it should be pointed out that hypothesis just looks at simple linear regression
data = np.arange(1000000)
data = data.reshape((100,100,100))
mask1 = np.ones((100,100,100))
mask2 = np.zeros((100,100,100))
mask3 = np.ones((200,200,100))
assert_equal(make_mask(data, mask1), data)
assert_equal(make_mask(data,mask2), mask2)
assert_equal(make_mask(data,mask3,fit=True).shape, data.shape)
def test_mask():
# example from http://www.jarrodmillman.com/rcsds/lectures/glm_intro.html
# it should be pointed out that hypothesis just looks at simple linear regression
data = np.arange(1000000)
data = data.reshape((100,100,100))
mask1 = np.ones((100,100,100))
mask2 = np.zeros((100,100,100))
mask3 = np.ones((200,200,100))
assert_equal(make_mask(data, mask1), data)
assert_equal(make_mask(data,mask2), mask2)
assert_equal(make_mask(data,mask3,fit=True).shape, data.shape)
x= False
try:
make_mask(data,mask3,fit=False)
except ValueError:
x=True
assert(x==True)
neighbors = 1
q = .15
prop_t = .15
prop_beta = .15
# assign subjects a number
for i, name in enumerate(sub_list):
# the mask for each subject
path_to_data = project_path + "data/ds009/" + name
mask = nib.load(path_to_data + '/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
rachels_ones = np.ones((64, 64, 34))
fitted_mask = make_mask(rachels_ones, mask_data, fit=True)
fitted_mask[fitted_mask > 0] = 1
#####################################
# Run bh_procedure for each subject #
#####################################
p_3d = np.load("../data/p-values/" + name + "_pvalue.npy")
p_1d = np.ravel(p_3d)
mask = fitted_mask
mask_1d = np.ravel(mask)
p_bh = p_1d[mask_1d == 1]
bh_first = bh_procedure(p_bh, q)
bh_3d = masking_reshape_end(bh_first, mask, off_value=.5)
bh_3d[bh_3d < .5] = 0
#loop through each person's T-statistic
for model in [
"_tstat.npy", "_tstat_rough_full.npy", "_tstat_smooth_simple.npy",
"_tstat_rough_simple.npy"
]:
t_mean = np.zeros((64, 64, 34, 24))
count = 0
for i in sub_list:
t_stat = np.load(t_data + i + model)
mask = nib.load(path_to_data + i +
'/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
t_mean[..., count] = make_mask(t_stat, mask_data, fit=True)
count += 1
t_mean = np.mean(t_mean, axis=3)
final = present_3d(t_mean)
plt.imshow(final, interpolation='nearest', cmap='seismic')
plt.title("Mean T-Statistic Value Across 25 Subjects")
zero_out = max(abs(np.min(final)), np.max(final))
plt.clim(-zero_out, zero_out)
plt.colorbar()
plt.show()
#
# #####################################
# ########## Clustering##########
# #####################################
img = nib.load(smooth_data + i + "_bold_smoothed.nii")
data = img.get_data()
behav = pd.read_table(path_to_data + i + behav_suffix, sep=" ")
num_TR = float(behav["NumTRs"])
n_vols = num_TR
hrf_matrix_all = np.loadtxt("../data/hrf/" + i + "_hrf_all.txt")
hrf_matrix_1 = np.loadtxt("../data/hrf/" + i + "_hrf_1.txt")
hrf_matrix_2 = np.loadtxt("../data/hrf/" + i + "_hrf_2.txt")
hrf_matrix_3 = np.loadtxt("../data/hrf/" + i + "_hrf_3.txt")
mask = nib.load(path_to_data + i + '/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
mask_data = make_mask(np.ones(data.shape[:-1]), mask_data, fit=True)
mask_data = mask_data != 0
mask_data = mask_data.astype(int)
# PCA
to_2d = masking_reshape_start(data, mask_data)
X_pca = to_2d - np.mean(to_2d, 0) - np.mean(to_2d, 1)[:, None]
cov = X_pca.T.dot(X_pca)
U, S, V = npl.svd(cov)
pca_addition = U[:, :6] # ~40% coverage
#START DOING GLM
behav = pd.read_table(path_to_data + name + behav_suffix, sep = " ")
num_TR = float(behav["NumTRs"])
img = nib.load(smooth_data + i + "_bold_smoothed.nii")
data = img.get_data()
n_vols = data.shape[-1]
convolve = np.loadtxt(hrf_data + i + "_hrf_all.txt")
residual_final = np.zeros((data.shape))
t_final = np.zeros((data.shape[:-1]))
p_final = np.zeros((data.shape[:-1]))
beta_final = np.zeros((data.shape[:-1]))
mask = nib.load(path_to_data + i +'/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
mask_data = make_mask(np.ones(data.shape[:-1]), mask_data, fit = True)
mask_data = mask_data != 0
mask_data = mask_data.astype(int)
#Run per slice in order to correct for time
for j in range(data.shape[2]):
data_slice = data[:, :, j, :]
#Create design matrix
X = np.ones((n_vols, 9))
X[:, 1] = convolve[:, j]
X[:, 2] = np.linspace(-1, 1, num = X.shape[0]) #drift
X[:, 3:] = fourier_creation(n_vols, 3)[:, 1:]
plt.colorbar()
plt.title('Significant p-values (No mask)')
plt.savefig(location_of_images + "NOMASK_significant_p_slice.png")
plt.close()
print("# ==== END No Mask, bh_procedure ==== #")
print("# ==== BEGIN varying the Q value = .005 (FDR) ==== #")
Q = .005
significant_pvals = bh_procedure(p_vals, Q)
# Reshape significant_pvals
reshaped_sig_p = np.reshape(significant_pvals, data.shape[:-1])
slice_reshaped_sig_p = reshaped_sig_p[..., 7]
masked_data = make_mask(original_slice, reshaped_sig_p, fit=False)
plt.imshow(present_3d(masked_data))
plt.clim(0, 1600)
plt.colorbar()
plt.title('Slice with Significant p-values (Q = .005)')
plt.savefig(location_of_images + "significant_p_slice1.png")
plt.close()
print("# ==== END plot with Q = .005 done. ==== #")
print("# ==== BEGIN varying the Q value = .05 (FDR) ==== #")
Q = .05
significant_pvals = bh_procedure(p_vals, Q)
# Reshape significant_pvals
pathtodata + i + "/model/model001/onsets/task001_run001/cond_all.txt",
TR, n_vols)
convolved = np.convolve(neural_prediction, hrf_at_trs) # hrf_at_trs sample
N = len(neural_prediction) # N == n_vols == 173
M = len(hrf_at_trs) # M == 12
np_hrf = convolved[:N]
B, t, df, p = t_stat(data, np_hrf, np.array([0, 1]))
#Simple mask function
mask = nib.load(pathtodata + i + '/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
t_mean[..., int(i[-1])] = make_mask(np.reshape(t, (64, 64, 34)),
mask_data,
fit=True)
final = present_3d(np.mean(t_mean, axis=3))
#######################
# Plot the results #
#######################
plt.imshow(final, interpolation='nearest', cmap='seismic')
plt.title("Mean T-Statistic Value Across 25 Subjects")
zero_out = max(abs(np.min(final)), np.max(final))
plt.clim(-zero_out, zero_out)
plt.colorbar()
plt.savefig("../../../images/hypothesis_testing.png")
plt.close()
########## Clustering ##############
#####################################
#Mean across all subject
t_mean = np.zeros((64, 64, 34,24))
#loop through each person's T-statistic
count=0
for i in sub_list:
t_stat = np.load(t_data+i+"_tstat.npy")
mask = nib.load(path_to_data+i+'/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
t_mean[...,count] = make_mask(t_stat, mask_data, fit=True)
count+=1
t_mean = np.mean(t_mean,axis=3)
final = present_3d(t_mean)
plt.imshow(final,interpolation='nearest', cmap='seismic')
plt.title("Mean T-Statistic Value Across 25 Subjects")
zero_out=max(abs(np.min(final)),np.max(final))
plt.clim(-zero_out,zero_out)
plt.colorbar()
plt.show()
#Cluster
data_new = t_mean[...,10:15]
#plt.title('(OLD BH FUNCTION) Significant p-values (No mask)')
#plt.savefig(location_of_images+"OLD_significant_p_slice_NOMASK.png")
#plt.close()
#print("Initial plot with NO MASK (using old bh function) done.")
print("# ==== BEGIN varying the Q value = .005 (FDR) ==== #")
Q = .005
significant_pvals = bh_procedure(p_vals, Q)
# Reshape significant_pvals
reshaped_sig_p = np.reshape(significant_pvals, data.shape[:-1])
slice_reshaped_sig_p = reshaped_sig_p[...,7]
masked_data = make_mask(original_slice, reshaped_sig_p, fit=False)
plt.imshow(present_3d(masked_data))
plt.clim(0, 1600)
plt.colorbar()
plt.title('Slice with Significant p-values (Q = .005)')
plt.savefig(location_of_images+"significant_p_slice1.png")
plt.close()
print("# ==== END plot with Q = .005 done. ==== #")
print("# ==== BEGIN varying the Q value = .05 (FDR) ==== #")
Q = .05
significant_pvals = bh_procedure(p_vals, Q)
# Reshape significant_pvals
#plt.title('(OLD BH FUNCTION) Significant p-values (No mask)')
#plt.savefig(location_of_images+"OLD_significant_p_slice_NOMASK.png")
#plt.close()
#print("Initial plot with NO MASK (using old bh function) done.")
print("# ==== BEGIN varying the Q value = .005 (FDR) ==== #")
Q = .005
significant_pvals = bh_procedure(p_vals, Q)
# Reshape significant_pvals
reshaped_sig_p = np.reshape(significant_pvals, data.shape[:-1])
slice_reshaped_sig_p = reshaped_sig_p[...,7]
masked_data = make_mask(original_slice, reshaped_sig_p, fit=False)
plt.imshow(present_3d(masked_data))
plt.clim(0, 1600)
plt.colorbar()
plt.title('Slice with Significant p-values (Q = .005)')
plt.savefig(location_of_images+"significant_p_slice1.png")
plt.close()
print("# ==== END plot with Q = .005 done. ==== #")
print("# ==== BEGIN varying the Q value = .05 (FDR) ==== #")
Q = .05
significant_pvals = bh_procedure(p_vals, Q)
# Reshape significant_pvals
all_stimuli = np.array(
sorted(list(cond2[:, 0]) + list(cond3[:, 0]) +
list(cond1[:, 0]))) # could also just x_s_array
my_hrf = convolution_specialized(all_stimuli, np.ones(len(all_stimuli)),
hrf_single, np.linspace(0, 239 * 2 - 2, 239))
B, t, df, p = t_stat(data, my_hrf, np.array([0, 1]))
###############
# tgrouping #
###############
mask = nib.load(pathtodata + '/anatomy/inplane001_brain_mask.nii.gz')
mask = mask.get_data()
inner_ones = np.ones(data.shape[:-1])
mask = make_mask(inner_ones, mask, True)
mask[mask > 0] = 1
t_vals = t
t_vals_3d = t_vals.reshape(data.shape[:-1])
pro = [.25, .1, .1, .05, .025]
folks = [1, 1, 5, 5, 10]
plt.close()
for i in np.arange(5):
start, cutoff = t_grouping_neighbor(t_vals_3d,
mask,
pro[i],
####################### # convolution # ####################### all_stimuli=np.array(sorted(list(cond2[:,0])+list(cond3[:,0])+list(cond1[:,0]))) # could also just x_s_array my_hrf = convolution_specialized(all_stimuli,np.ones(len(all_stimuli)),hrf_single,np.linspace(0,239*2-2,239)) B,t,df,p = t_stat(data, my_hrf, np.array([0,1])) ############### # tgrouping # ############### mask = nib.load(pathtodata + '/anatomy/inplane001_brain_mask.nii.gz') mask = mask.get_data() inner_ones=np.ones(data.shape[:-1]) mask= make_mask(inner_ones,mask,True) mask[mask>0]=1 t_vals=t t_vals_3d=t_vals.reshape(data.shape[:-1]) pro=[.25,.1,.1,.05,.025] folks=[1,1,5,5,10] plt.close() for i in np.arange(5): start,cutoff=t_grouping_neighbor(t_vals_3d,mask,pro[i],prop=True,neighbors= folks[i],abs_on=True)
sys.stdout.write("\b" *
(toolbar_width + 1)) # return to start of line, after '['
# Set up lists to store proportion of p-values above 0.05.
unmasked_prop = [] # Unmasked (all voxels)
masked_prop = [] # Masked.
for i in sub_list:
residuals = np.load(residual_data + i + "_residual.npy")
sw_pvals = check_sw(residuals)
unmasked_prop.append(np.mean(sw_pvals > 0.05))
mask = nib.load(path_to_data + i + '/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
masked_pvals = make_mask(sw_pvals, mask_data, fit=True)
masked_pvals[masked_pvals > 1] = 1
pvals_in_brain = sw_pvals.ravel()[masked_pvals.ravel() != 0]
masked_prop.append(np.mean(pvals_in_brain > 0.05))
if (i[-3:] == "010"):
# Save image plots of unmasked p-values for subject 10.
plt.imshow(present_3d(sw_pvals), cmap=plt.get_cmap('gray'))
plt.colorbar()
plt.xticks([])
plt.yticks([])
plt.title("p-values for " + i + " (Unmasked Data)")
plt.savefig(location_of_images + i + 'sw.png')
plt.close()
# Save image plots of masked p-values for a single subject.
#plt.colorbar()
#plt.title('(OLD BH FUNCTION) Significant p-values (No mask)')
#plt.savefig(location_of_images+"OLD_significant_p_slice_NOMASK.png")
#plt.close()
#print("Initial plot with NO MASK (using old bh function) done.")
print("# ==== BEGIN varying the Q value = .005 (FDR) ==== #")
Q = .005
significant_pvals = bh_procedure(p_vals, Q)
# Reshape significant_pvals
reshaped_sig_p = np.reshape(significant_pvals, data.shape[:-1])
slice_reshaped_sig_p = reshaped_sig_p[..., 7]
masked_data = make_mask(original_slice, reshaped_sig_p, fit=False)
plt.imshow(present_3d(masked_data))
plt.clim(0, 1600)
plt.colorbar()
plt.title('Slice with Significant p-values (Q = .005)')
plt.savefig(location_of_images + "significant_p_slice1.png")
plt.close()
print("# ==== END plot with Q = .005 done. ==== #")
print("# ==== BEGIN varying the Q value = .05 (FDR) ==== #")
Q = .05
significant_pvals = bh_procedure(p_vals, Q)
# Reshape significant_pvals
img = nib.load(path_to_data + name + "/BOLD/task001_run001/bold.nii.gz")
data = img.get_data().astype(float)
# Load mask.
mask = nib.load(path_to_data + name + "/anatomy/inplane001_brain_mask.nii.gz")
mask_data = mask.get_data()
# Drop the appropriate number of volumes from the beginning.
first_n_vols = data.shape[-1]
num_TR_cut = int(first_n_vols - num_TR)
data = data[..., num_TR_cut:]
# Now fit a mask to the 3-d image for each time point.
my_mask = np.zeros(data.shape)
for i in range(my_mask.shape[-1]):
my_mask[..., i] = make_mask(data[..., i], mask_data, fit=True)
# Reshape stuff to 2-d (voxel by time) and mask the data.
# This should cut down the number of volumes by more than 50%.
my_mask_2d = my_mask.reshape((-1, my_mask.shape[-1]))
data_2d = data.reshape((-1, data.shape[-1]))
masked_data_2d = data_2d[my_mask_2d.sum(1) != 0, :]
# Subtract means over voxels (columns).
masked_data_2d = masked_data_2d - np.mean(masked_data_2d, 0)
# Subtract means over time (rows)
masked_data_2d = masked_data_2d - np.mean(masked_data_2d, axis=1)[:, None]
# PCA analysis on MASKED data:
# Do SVD on the time by time matrix and transform into first k principal components.
np.savetxt(pathtodata+ i+ "/model/model001/onsets/task001_run001/cond_all.txt",cond_all)
neural_prediction = events2neural(pathtodata+ i+ "/model/model001/onsets/task001_run001/cond_all.txt",TR,n_vols)
convolved = np.convolve(neural_prediction, hrf_at_trs) # hrf_at_trs sample
N = len(neural_prediction) # N == n_vols == 173
M = len(hrf_at_trs) # M == 12
np_hrf=convolved[:N]
B,t,df,p = t_stat(data, np_hrf, np.array([0,1]))
#Simple mask function
mask = nib.load(pathtodata+i+'/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
t_mean[...,int(i[-1])] = make_mask(np.reshape(t,(64,64,34)), mask_data, fit=True)
final = present_3d(np.mean(t_mean,axis=3))
#######################
# Plot the results #
#######################
plt.imshow(final,interpolation='nearest', cmap='seismic')
plt.title("Mean T-Statistic Value Across 25 Subjects")
zero_out=max(abs(np.min(final)),np.max(final))
plt.clim(-zero_out,zero_out)
plt.colorbar()
data = img.get_data().astype(float)
# Load mask.
mask = nib.load(path_to_data + name +
'/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
# Drop the appropriate number of volumes from the beginning.
first_n_vols = data.shape[-1]
num_TR_cut = int(first_n_vols - num_TR)
data = data[..., num_TR_cut:]
# Now fit a mask to the 3-d image for each time point.
my_mask = np.zeros(data.shape)
for i in range(my_mask.shape[-1]):
my_mask[..., i] = make_mask(data[..., i], mask_data, fit=True)
# Reshape stuff to 2-d (voxel by time) and mask the data.
# This should cut down the number of volumes by more than 50%.
my_mask_2d = my_mask.reshape((-1, my_mask.shape[-1]))
data_2d = data.reshape((-1, data.shape[-1]))
masked_data_2d = data_2d[my_mask_2d.sum(1) != 0, :]
# Subtract means over voxels (columns).
masked_data_2d = masked_data_2d - np.mean(masked_data_2d, 0)
# Subtract means over time (rows)
masked_data_2d = masked_data_2d - np.mean(masked_data_2d, axis=1)[:, None]
# PCA analysis on MASKED data:
# Do SVD on the time by time matrix and transform into first k principal components.
# Progress bar
toolbar_width=len(sub_list)
sys.stdout.write("GLM, : ")
sys.stdout.write("[%s]" % (" " * toolbar_width))
sys.stdout.flush()
sys.stdout.write("\b" * (toolbar_width+1)) # return to start of line, after '['
for i in sub_list:
residuals = np.load(residual_data+i+"_residual.npy")
sw_pvals = check_sw(residuals)
print(i+" proportion of voxels with p-value above 0.05 (unmasked): "+str(np.mean(sw_pvals > 0.05)))
mask = nib.load(path_to_data+i+'/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
masked_pvals = make_mask(sw_pvals, mask_data, fit=True)
pvals_in_brain = sw_pvals.ravel()[masked_pvals.ravel() != 0]
print(i+" proportion of voxels with p-value above 0.05 (masked): "+str(np.mean(pvals_in_brain > 0.05)))
sys.stdout.write("-")
sys.stdout.flush()
# Save image plots of masked and unmasked p-values for a single subject.
plt.imshow(present_3d(sw_pvals), cmap=plt.get_cmap('gray'))
plt.savefig(location_of_images+i+'sw.png')
plt.close()
plt.imshow(present_3d(masked_pvals), cmap=plt.get_cmap('gray'))
plt.savefig(location_of_images+i+'swmasked.png')
plt.close()
sys.stdout.write("\n")
from hypothesis import t_stat
from Image_Visualizing import present_3d, make_mask
from tgrouping import t_grouping_neighbor
i = 'sub001'
#####################################
########## Clustering ##########
#####################################
t_stat = np.load(t_data+i+"_tstat.npy")
mask = nib.load(path_to_data+i+'/anatomy/inplane001_brain_mask.nii.gz')
mask_data = mask.get_data()
ones = np.ones((64,64,34))
fitted_mask = make_mask(ones, mask_data, fit = True)
fitted_mask[fitted_mask > 0] = 1
data_new= t_stat*fitted_mask
data_new = data_new[...,19:22]
X = np.reshape(data_new, (-1, 1))
connectivity = grid_to_graph(n_x= data_new.shape[0], n_y = data_new.shape[1], n_z = data_new.shape[2])
n_clusters = 5 # number of regions
ward = AgglomerativeClustering(n_clusters=n_clusters,
linkage='ward', connectivity=connectivity).fit(X)
label = np.reshape(ward.labels_, data_new.shape)
