def t_grouping_neighbor(t_3d,
mask,
cutoff,
neighbors=None,
prop=False,
abs_on=False,
binary=True,
off_value=0,
masked_value=.5):
"""
Masks a 3d array, does t_binary_grouping, and does neighboring
Parameters
----------
t_3d: t-value of the betas 3d numpy array
mask: a 3d numpy array of 0s and 1s that has the same shape as t_3d
cutoff: the limit for the false discovery rate
neighbors: number of neighbors for neighbor smoothing (must have binary be true)
prop: logical~ if the cutoff is a proportion or a value
abs_on: logical~ if we want to take absolute value of the t input
binary: if binary, then off_value is ignored and 0 is used as the
off_value, 1 as the on value
off_value: the value of those not selected
Returns
-------
output_3d: a 3d numpy array same size as the t_3d with either:
(1) binary on_off values for inside the mask and "masked_value"
for values outside mask or (2) t values to the accepted values,
and "off_values" for lost values, and "masked_value" for values
outside mask. MOREOVER, it can have had neighbor smoothing applied
the binary case
cutoff: the limit for the false discovery rate
"""
if neighbors != None and binary == False:
return False
t_1d = masking_reshape_start(t_3d, mask)
t_1d = np.ravel(t_1d)
zero_one, cutoff = t_binary_grouping(t_1d, cutoff, prop, abs_on)
if not binary:
t_1d = t_1d * zero_one + off_value * (1 - zero_one)
else:
t_1d = zero_one
output_3d = masking_reshape_end(t_1d, mask, masked_value)
if neighbors != None:
output_3d = neighbor_smoothing(output_3d, neighbors)
return output_3d, cutoff
def t_grouping_neighbor(t_3d, mask, cutoff, neighbors = None,
prop = False, abs_on = False, binary = True, off_value = 0, masked_value = .5):
"""
Masks a 3d array, does t_binary_grouping, and does neighboring
Parameters
----------
t_3d: t-value of the betas 3d numpy array
mask: a 3d numpy array of 0s and 1s that has the same shape as t_3d
cutoff: the limit for the false discovery rate
neighbors: number of neighbors for neighbor smoothing (must have binary be true)
prop: logical~ if the cutoff is a proportion or a value
abs_on: logical~ if we want to take absolute value of the t input
binary: if binary, then off_value is ignored and 0 is used as the
off_value, 1 as the on value
off_value: the value of those not selected
Returns
-------
output_3d: a 3d numpy array same size as the t_3d with either:
(1) binary on_off values for inside the mask and "masked_value"
for values outside mask or (2) t values to the accepted values,
and "off_values" for lost values, and "masked_value" for values
outside mask. MOREOVER, it can have had neighbor smoothing applied
the binary case
cutoff: the limit for the false discovery rate
"""
if neighbors != None and binary == False:
return False
t_1d = masking_reshape_start(t_3d, mask)
t_1d = np.ravel(t_1d)
zero_one, cutoff = t_binary_grouping(t_1d, cutoff, prop, abs_on)
if not binary:
t_1d = t_1d*zero_one + off_value*(1 - zero_one)
else:
t_1d = zero_one
output_3d = masking_reshape_end(t_1d, mask, masked_value)
if neighbors != None:
output_3d = neighbor_smoothing(output_3d, neighbors)
return output_3d, cutoff
def test_2(): # checks BOTH masking_reshape_start and masking_reshape_end # 3d happy=np.arange(27) happy=happy.reshape((3,3,3)) mask=np.zeros((3,3,3)) mask[:2,1:,2:]=1 joy=masking_reshape_start(happy,mask) output=masking_reshape_end(joy,mask,off_value=0) test=np.zeros((3,3,3)) test[:2,1:,2:]=happy[:2,1:,2:] assert(np.all(test==output))
rachels_ones = np.ones((64, 64, 34))
fitted_mask = make_mask(rachels_ones, mask_data, fit=True)
fitted_mask[fitted_mask > 0] = 1
#####################################
# Run bh_procedure for each subject #
#####################################
p_3d = np.load("../data/p-values/" + name + "_pvalue.npy")
p_1d = np.ravel(p_3d)
mask = fitted_mask
mask_1d = np.ravel(mask)
p_bh = p_1d[mask_1d == 1]
bh_first = bh_procedure(p_bh, q)
bh_3d = masking_reshape_end(bh_first, mask, off_value=.5)
bh_3d[bh_3d < .5] = 0
bh_3d_1_good = 1 - bh_3d
bh_final = neighbor_smoothing_binary(bh_3d_1_good, neighbors)
bh_mean[..., i] = bh_3d_1_good
#####################################
# Run t_grouping for each subject #
#####################################
t_3d = np.load("../data/t_stat/" + name + "_tstat.npy")
#mask = fitted_mask
t_group = t_grouping_neighbor(t_3d,
mask,
fitted_mask = make_mask(rachels_ones, mask_data, fit=True)
fitted_mask[fitted_mask > 0] = 1
mask_new = mask_data[::2, ::2, :]
assert (mask_new.shape == fitted_mask.shape)
p_vals_3d = p_vals.reshape(data.shape[:-1])
# call masking_reshape_start function, reshaped to 1d output
to_1d = masking_reshape_start(p_vals_3d, mask_new)
# call bh_procedure where Q = .45
bh_1d = bh_procedure(to_1d, Q)
# call masking_reshape_end function, to reshape back into 3d shape
to_3d = 2 * masking_reshape_end(bh_1d, mask_new, .5) - 1
# plot this
plt.imshow(present_3d(to_3d), interpolation='nearest', cmap='seismic')
plt.title(str(Q) + " with masked p-values")
plt.savefig(location_of_images + "masked_pval_" + str(Q) + ".png")
plt.figure()
plt.close()
print("# ==== END mask_phase_2_dimension_change plotting ==== #")
# Smoothing/clustering on the masked p-values
print("# ==== BEGIN smoothed masked p-value plotting ==== #")
# Where the neighbor_smoothing function should start
print("# == NEIGHBOR SMOOTHING 5")
fitted_mask = make_mask(rachels_ones, mask_data, fit = True)
fitted_mask[fitted_mask > 0] = 1
mask_new = mask_data[::2, ::2, :]
assert(mask_new.shape == fitted_mask.shape)
p_vals_3d = p_vals.reshape(data.shape[:-1])
# call masking_reshape_start function, reshaped to 1d output
to_1d = masking_reshape_start(p_vals_3d, mask_new)
# call bh_procedure where Q = .45
bh_1d=bh_procedure(to_1d, Q)
# call masking_reshape_end function, to reshape back into 3d shape
to_3d = 2*masking_reshape_end(bh_1d, mask_new, .5) - 1
# plot this
plt.imshow(present_3d(to_3d),interpolation='nearest', cmap='seismic')
plt.title(str(Q) + " with masked p-values")
#plt.colorbar()
plt.savefig(location_of_images + "masked_pval_" + str(Q) + ".png")
plt.figure()
plt.close()
print("# ==== END mask_phase_2_dimension_change plotting ==== #")
# Smoothing/clustering on the masked p-values
print("# ==== BEGIN smoothed masked p-value plotting ==== #")
#test = to_3d
# Benjamini Hochberg Analysis #
###############################
toolbar_width=len(q1)
sys.stdout.write("Benjamini Hochberg: ")
sys.stdout.write("[%s]" % (" " * toolbar_width))
sys.stdout.flush()
sys.stdout.write("\b" * (toolbar_width+1)) # return to start of line, after '['
bh=[] # values a*6 + b - 1
count_a=0
for a,b in itertools.product(range(len(q1)),range(5)):
bh_first = bh_procedure(p_bh,q1[a])
bh_3d = masking_reshape_end(bh_first,mask,off_value=.5)
bh_3d[bh_3d<.5]=0
bh_3d_1_good = 1-bh_3d
first = neighbor_smoothing_binary(bh_3d_1_good,neighbors1[b])
bh.append(first)
if count_a==a and b==4:
sys.stdout.write("-")
sys.stdout.flush()
count_a+=1
###############################
# Benjamini Hochberg Analysis #
###############################
toolbar_width = len(q1)
sys.stdout.write("Benjamini Hochberg: ")
sys.stdout.write("[%s]" % (" " * toolbar_width))
sys.stdout.flush()
sys.stdout.write("\b" *
(toolbar_width + 1)) # return to start of line, after '['
bh = [] # values a*6 + b - 1
count_a = 0
for a, b in itertools.product(range(len(q1)), range(5)):
bh_first = bh_procedure(p_bh, q1[a])
bh_3d = masking_reshape_end(bh_first, mask, off_value=.5)
bh_3d[bh_3d < .5] = 0
bh_3d_1_good = 1 - bh_3d
first = neighbor_smoothing_binary(bh_3d_1_good, neighbors1[b])
bh.append(first)
if count_a == a and b == 4:
sys.stdout.write("-")
sys.stdout.flush()
count_a += 1
sys.stdout.write("\n")
#------------------#
rachels_ones = np.ones((64, 64, 34))
fitted_mask = make_mask(rachels_ones, mask_data, fit = True)
fitted_mask[fitted_mask > 0] = 1
#####################################
# Run bh_procedure for each subject #
#####################################
p_3d = np.load("../data/p-values/" + name + "_pvalue.npy")
p_1d = np.ravel(p_3d)
mask = fitted_mask
mask_1d = np.ravel(mask)
p_bh = p_1d[mask_1d == 1]
bh_first = bh_procedure(p_bh, q)
bh_3d = masking_reshape_end(bh_first, mask, off_value = .5)
bh_3d[bh_3d < .5] = 0
bh_3d_1_good = 1 - bh_3d
bh_final = neighbor_smoothing_binary(bh_3d_1_good, neighbors)
bh_mean[..., i] = bh_3d_1_good
#####################################
# Run t_grouping for each subject #
#####################################
t_3d = np.load("../data/t_stat/" + name + "_tstat.npy")
#mask = fitted_mask
t_group = t_grouping_neighbor(t_3d, mask, prop_t, neighbors = neighbors,
prop = True, abs_on = True, binary = True, off_value = 0, masked_value = .5)[0]
