def compile_corrs_by_freq(bo_path, corr_path, freq):
"""
Compiles correlation values - as well as other subject/electrode specific paramters - creates the compiled pandas dataframe used for figures
Parameters
----------
path_to_npz_data : string
Path to npz files - I know this isn't a great way to do this :/
corr_path : npz file
npz file containing correlation values (loop outside - for each electrode)
Returns
----------
results : dataframe
compiled dataframe with: Subject, electrode, correlation, samples, and sample rate
"""
def parse_path_name(path_name):
underscore_count = os.path.basename(path_name).count('_')
if os.path.splitext(os.path.basename(path_name))[0].split(
"_", underscore_count)[-1] == 'within':
f_name = os.path.basename(
path_name)[0:os.path.basename(path_name).find('_' + freq)]
electrode = os.path.splitext(os.path.basename(path_name))[0].split(
"_", underscore_count)[-2]
else:
f_name = os.path.basename(
path_name)[0:os.path.basename(path_name).find('_' + freq)]
electrode = os.path.splitext(os.path.basename(path_name))[0].split(
"_", underscore_count)[-1]
return f_name, electrode
f_name, electrode = parse_path_name(corr_path)
corr_data = np.load(corr_path, mmap_mode='r')
tempR = np.round(corr_data['coord'], 2)
tempmeancorr = z2r(np.mean(r2z(corr_data['corrs'])))
tempsamplerate = np.mean(
se.load(os.path.join(bo_path, f_name + '.bo'), field='sample_rate'))
tempsamples = se.load(os.path.join(bo_path, f_name + '.bo'),
field='sessions').shape[0]
kurt_vals = se.load(os.path.join(bo_path, f_name + '.bo'),
field='kurtosis')
thresh_bool = kurt_vals > 10
tempelecs = sum(~thresh_bool)
tempsessions = se.load(os.path.join(bo_path, f_name + '.bo'),
field='sessions').max()
tempthresholded = sum(thresh_bool)
return pd.DataFrame({
'R': [tempR],
'Correlation': [tempmeancorr],
'Subject': [f_name],
'Electrode': [electrode],
'Sample rate': [tempsamplerate],
'Samples': [tempsamples],
'Total Electrodes': [tempelecs],
'Sessions': [tempsessions],
'Number thresholded': [tempthresholded]
})
def electrode_search(fname, threshold=10):
kurt_vals = se.load(fname, field='kurtosis')
thresh_bool = kurt_vals > threshold
locs = se.load(fname, field='locs')
if sum(~thresh_bool) > 1:
locs = pd.DataFrame(locs, columns=['x', 'y', 'z'])
return locs[~thresh_bool]
def electrode_search(fname, threshold=10):
# searching original .bo
fname = os.path.basename(os.path.splitext(fname)[0])
fname = os.path.join(config['og_bodir'], fname.split('_' + freq)[0] + '.bo')
try:
kurt_vals = se.load(fname, field='kurtosis')
except:
kurt_vals = se.load(sys.argv[1], field='kurtosis')
thresh_bool = kurt_vals > threshold
return sum(~thresh_bool)
def electrode_search(fname, threshold=10):
# searching original .bo
fname = os.path.basename(os.path.splitext(fname)[0])
fname = os.path.join(config['og_bodir'],
fname.split('_' + freq)[0] + '.bo')
kurt_vals = se.load(fname, field='kurtosis')
thresh_bool = kurt_vals > threshold
locs = se.load(fname, field='locs')
if sum(~thresh_bool) > 1:
locs = pd.DataFrame(locs, columns=['x', 'y', 'z'])
return locs[~thresh_bool]
def density_by_voxel_plot(locs, r=20, vox_size=4, outfile=None, save_nii=None):
sub_nii = se.load('std', vox_size=4)
sub_locs = sub_nii.get_locs().values
point_tree = spatial.cKDTree(locs)
density_locs = np.array([])
for l in sub_locs:
density_locs = np.append(
density_locs,
np.divide(len(point_tree.query_ball_point(l, r)),
np.shape(locs)[0]))
bo_nii = se.Brain(data=np.atleast_2d(density_locs), locs=sub_locs)
nii_bo = se.helpers._brain_to_nifti(bo_nii, sub_nii)
ni_plt.plot_glass_brain(nii_bo,
colorbar=True,
threshold=None,
vmax=.1,
vmin=0,
display_mode='lyrz')
if save_nii:
nii_bo.save(save_nii)
if not outfile is None:
plt.savefig(outfile)
else:
plt.show()
def test_bo_save(tmpdir):
p = tmpdir.mkdir("sub").join("example")
print(p)
print(type(p))
nii.save(filepath=p.strpath)
test_bo = se.load(os.path.join(p.strpath + '.nii'))
assert isinstance(test_bo, se.Nifti)
def density_within_r_plot(locs, r, vox_size=4, outfile=None):
nii = se.load('std', vox_size=vox_size)
full_locs = nii.get_locs().values
point_tree = spatial.cKDTree(locs)
density_locs = np.array([])
for l in locs:
density_locs = np.append(
density_locs,
np.divide(len(point_tree.query_ball_point(l, r)),
np.shape(locs)[0]))
bo_nii = se.Brain(data=np.atleast_2d(density_locs), locs=locs)
nii_bo = se.helpers._brain_to_nifti(bo_nii, nii)
ni_plt.plot_glass_brain(nii_bo,
colorbar=True,
threshold=None,
vmax=.1,
vmin=0)
if not outfile is None:
plt.savefig(outfile)
else:
plt.show()
def interp_corr(locs,
corrs,
width=10,
vox_size=10,
outfile=None,
save_nii=None):
nii = se.load('std', vox_size=vox_size)
full_locs = nii.get_locs().values
W = np.exp(_log_rbf(full_locs, locs, width=width))
interp_corrs = np.dot(corrs, W.T)
bo_nii = se.Brain(data=interp_corrs, locs=full_locs)
nii_bo = _brain_to_nifti(bo_nii, nii)
ni_plt.plot_glass_brain(nii_bo,
colorbar=True,
threshold=None,
vmax=1,
vmin=0)
#ni_plt.plot_glass_brain(nii_bo, colorbar=True, threshold=None, vmax=1, vmin=0, display_mode='lyrz')
if save_nii:
nii_bo.save(save_nii)
if not outfile is None:
plt.savefig(outfile)
else:
plt.show()
def test_bo_save(tmpdir):
p = tmpdir.mkdir("sub").join("example")
print(p)
print(type(p))
bo.save(fname=p.strpath)
test_bo = se.load(os.path.join(p.strpath + '.bo'))
assert isinstance(test_bo, se.Brain)
def test_electrode_contingencies_3_locations_can_subset():
random_seed = np.random.seed(123)
noise = 0
# load mini model
gray = se.Brain(se.load('gray', vox_size=20))
# extract 20 locations
gray_locs = gray.locs.iloc[:5]
# create model from 10 locations
mo_locs = gray_locs.sample(4, random_state=random_seed).sort_values(
['x', 'y', 'z'])
# create covariance matrix from random seed
c = se.create_cov(cov='random', n_elecs=5)
# pull out model from covariance matrix
data = c[:, mo_locs.index][mo_locs.index, :]
# create model from subsetted covariance matrix and locations
model = se.Model(numerator=np.array(data),
denominator=np.ones(np.shape(data)),
locs=mo_locs,
n_subs=1)
# create brain object from the remaining locations - first find remaining locations
sub_locs = gray_locs[~gray_locs.index.isin(mo_locs.index)]
sub_locs = sub_locs.append(
gray_locs.sample(1,
random_state=random_seed).sort_values(['x', 'y',
'z']))
# create a brain object with all gray locations
bo = se.simulate_bo(n_samples=5,
sample_rate=1000,
locs=gray_locs,
noise=noise,
random_seed=random_seed)
# parse brain object to create synthetic patient data
data = bo.data.iloc[:, sub_locs.index]
# put data and locations together in new sample brain object
bo_sample = se.Brain(data=data.as_matrix(),
locs=sub_locs,
sample_rate=1000)
# predict activity at all unknown locations
recon = model.predict(bo_sample, nearest_neighbor=False)
# actual = bo.data.iloc[:, unknown_ind]
actual = bo.data.iloc[:, recon.locs.index]
corr_vals = _corr_column(actual.as_matrix(), recon.data.as_matrix())
assert 1 >= corr_vals.mean() >= -1
assert np.allclose(zscore(recon_3), recon.data, equal_nan=True)
def most_informative_locs_plot(df, vox_size=5, width=10, outfile=None):
locs = compile_df_locs(df['R'])
sub_nii = se.load('std', vox_size=vox_size)
sub_locs = sub_nii.get_locs().values
point_tree = spatial.cKDTree(locs)
most_info = np.array([])
z_df = df.copy(deep=True)
z_df['Correlation'] = r2z(z_df['Correlation'])
for l in sub_locs:
most_info = np.append(
most_info,
z_df['Correlation'][point_tree.query_ball_point(l, width)].mean())
bo_nii = se.Brain(data=np.atleast_2d(z2r(most_info)), locs=sub_locs)
nii_bo = se.helpers._brain_to_nifti(bo_nii, sub_nii)
ni_plt.plot_glass_brain(nii_bo,
colorbar=True,
threshold=None,
vmax=1,
vmin=0,
display_mode='lyrz')
if not outfile is None:
plt.savefig(outfile)
else:
plt.show()
def recon_elec(elec_ind):
recon_outfile_across = os.path.join(
results_dir,
os.path.basename(
os.path.splitext(bo_fname)[0] + '_' + str(elec_ind) + '.npz'))
recon_outfile_within = os.path.join(
results_dir,
os.path.basename(
os.path.splitext(bo_fname)[0] + '_' + str(elec_ind) +
'_within.npz'))
if os.path.exists(recon_outfile_across) and os.path.exists(
recon_outfile_within):
return
freq_bo = se.load(bo_fname)
bo = BandBrain(freq_bo, og_bo)
electrode = bo.get_locs().iloc[elec_ind]
R_K_subj = bo.get_locs().values
R_K_removed, other_inds = remove_electrode(R_K_subj, R_K_subj, elec_ind)
known_inds, unknown_inds, e_ind = known_unknown(R, R_K_removed, R_K_subj,
elec_ind)
electrode_ind = get_rows(R, electrode.values)
actual = bo[:, elec_ind]
bo = bo[:, other_inds]
print('bo indexed for ' + str(elec_ind))
if not os.path.exists(recon_outfile_across):
bo_r = mo_s.predict(bo, recon_loc_inds=e_ind)
print(electrode)
c = _corr_column(bo_r.data.as_matrix(), actual.get_zscore_data())
print(c)
np.savez(recon_outfile_across, coord=electrode, corrs=c)
if not os.path.exists(recon_outfile_within):
Model = se.Model(bo, locs=R_K_subj)
m_locs = Model.get_locs().as_matrix()
known_inds, unknown_inds, e_ind = known_unknown(
m_locs, R_K_removed, m_locs, elec_ind)
bo_r = Model.predict(bo)
bo_r = bo_r[:, unknown_inds]
print(electrode)
c = _corr_column(bo_r.data.as_matrix(), actual.get_zscore_data())
print(c)
np.savez(recon_outfile_within, coord=electrode, corrs=c)
def helper2(time_index):
# needs time_index, nifti, and path
nifti = None
path = None
slice_index = range(-4, 52, 7)
vmax = 3.5
symmetric_cbar = False
alpha = 0.5
fname = 'test.bo'
bo = se.load(fname)
nii = bo.to_nii(template='std', vox_size=6)
num_slice = len(slice_index)
nrow = int(np.floor(np.sqrt(num_slice)))
ncol = int(np.ceil(num_slice / nrow))
fig, ax = plt.subplots(nrows=nrow, ncols=ncol,
figsize=(10, 10)) # move out of for loop, set ax
ax = ax.reshape(-1)
displays = []
# nifti = _utils.check_niimg_4d(nifti)
for currax, loc in enumerate(slice_index):
nii_i = image.index_img(nifti, time_index[0])
if loc == slice_index[len(slice_index) - 1]:
display = ni_plt.plot_stat_map(nii_i,
cut_coords=[loc],
colorbar=True,
symmetric_cbar=symmetric_cbar,
figure=fig,
axes=ax[currax],
vmax=vmax,
alpha=alpha,
**kwargs)
else:
display = ni_plt.plot_stat_map(nii_i,
cut_coords=[loc],
colorbar=False,
symmetric_cbar=symmetric_cbar,
figure=fig,
axes=ax[currax],
vmax=vmax,
alpha=alpha,
**kwargs)
displays.append(display)
plt.savefig(os.path.join(path, str(time_index[0]) + '.png'), format='png')
for i in time_index[1:]:
nii_i = image.index_img(nifti, i)
for display in displays:
display.add_overlay(nii_i, colorbar=False, vmax=vmax, alpha=alpha)
plt.savefig(os.path.join(path, str(i)) + '.png', format='png')
plt.close()
return os.getpid()
def electrode_search(fname, threshold=10):
basefname = os.path.basename(fname)
freq = fname.split('_')[-1].split('.bo')[0]
og_fname = os.path.join(config['og_bodir'],
basefname.split('_' + freq)[0] + '.bo')
if not freq in set([
'delta', 'theta', 'alpha', 'beta', 'lgamma', 'hgamma', 'broadband',
'raw'
]):
og_fname = fname
kurt_vals = se.load(og_fname, field='kurtosis')
thresh_bool = kurt_vals > threshold
return sum(~thresh_bool)
def density_by_voxel(locs, r=20, vox_size=4):
sub_nii = se.load('std', vox_size=4)
sub_locs = sub_nii.get_locs().values
point_tree = spatial.cKDTree(locs)
density_locs = np.array([])
for l in sub_locs:
density_locs = np.append(
density_locs,
np.divide(len(point_tree.query_ball_point(l, r)),
np.shape(locs)[0]))
return density_locs
def test_electrode_contingencies_2_subset():
random_seed = np.random.seed(123)
noise = 0
gray = se.Brain(se.load('gray', vox_size=20))
# extract locations
gray_locs = gray.locs.iloc[:5]
mo_locs = gray_locs
c = se.create_cov(cov='random', n_elecs=5)
data = c[:, mo_locs.index][mo_locs.index, :]
model = se.Model(numerator=np.array(data),
denominator=np.ones(np.shape(data)),
locs=mo_locs,
n_subs=1)
# create brain object from the remaining locations - first find remaining locations
sub_locs = mo_locs.sample(2, random_state=random_seed).sort_values(
['x', 'y', 'z'])
# create a brain object with all gray locations
bo = se.simulate_bo(n_samples=5,
sample_rate=1000,
locs=gray_locs,
noise=noise,
random_seed=random_seed)
# parse brain object to create synthetic patient data
data = bo.data.iloc[:, sub_locs.index]
# put data and locations together in new sample brain object
bo_sample = se.Brain(data=data.values, locs=sub_locs, sample_rate=1000)
# predict activity at all unknown locations
recon = model.predict(bo_sample, nearest_neighbor=False)
actual = bo.data.iloc[:, recon.locs.index]
corr_vals = _corr_column(actual.values, recon.data.values)
#assert np.allclose(zscore(recon_2), recon.data, equal_nan=True)
assert 1 >= corr_vals.mean() >= -1
def most_informative_locs(df, vox_size=5, width=10):
locs = compile_df_locs(df['R'])
#locs = df['R']
sub_nii = se.load('std', vox_size=vox_size)
sub_locs = sub_nii.get_locs().values
point_tree = spatial.cKDTree(locs)
most_info = np.array([])
z_df = df.copy(deep=True)
z_df['Correlation'] = r2z(z_df['Correlation'])
for l in sub_locs:
most_info = np.append(
most_info,
z_df['Correlation'][point_tree.query_ball_point(l, width)].mean())
return z2r(most_info)
def electrode_search(fname, threshold=10):
kurt_vals = se.load(fname, field='kurtosis')
thresh_bool = kurt_vals > threshold
return sum(~thresh_bool)
import supereeg as se import os from visbrain.objects import BrainObj, SceneObj, SourceObj cmap = "viridis_spliced" nii_bo_dir = '../../../data/niis/supplemental_3/freq' fig_dir = '../../../paper/figs/source/supplemental_3/freq' b1 = se.load(os.path.join(nii_bo_dir, 'delta_best.bo')) b2 = se.load(os.path.join(nii_bo_dir, 'theta_best.bo')) b3 = se.load(os.path.join(nii_bo_dir, 'alpha_best.bo')) b4 = se.load(os.path.join(nii_bo_dir, 'beta_best.bo')) b5 = se.load(os.path.join(nii_bo_dir, 'lgamma_best.bo')) b6 = se.load(os.path.join(nii_bo_dir, 'hgamma_best.bo')) data1 = b1.get_data().values.ravel() xyz1 = b1.locs.values data2 = b2.get_data().values.ravel() xyz2 = b2.locs.values data3 = b3.get_data().values.ravel() xyz3 = b3.locs.values data4 = b4.get_data().values.ravel() xyz4 = b4.locs.values data5 = b5.get_data().values.ravel() xyz5 = b5.locs.values
=============================
Filtering electrodes
=============================
This example filters electrodes based on kurtosis thresholding (default=10).
"""
# Code source: Lucy Owen & Andrew Heusser
# License: MIT
# import
import supereeg as se
# load example data
bo = se.load('example_filter')
# plot filtered data as default
bo.plot_data()
# plot filtered locations as default
bo.plot_locs()
# 37 locations
bo.info()
# or you can set filter to None if you want to plot original data
bo.filter = None
# plot unfiltered data
bo.plot_data()
os.makedirs(results_dir)
except OSError as err:
print(err)
locs_data = np.load(os.path.join(locs_dir, 'locs.npz'))
loc_list = locs_data['loc_list']
locs = locs_data['locs']
subs = list(range(1, len(glob.glob(os.path.join(bo_dir, '*.bo'))) + 1))
for i, loc in enumerate(loc_list):
ind = np.random.choice(subs, 1)
bo = se.load(os.path.join(bo_dir, 'sub-%d.bo' % ind))
bo.filter = None
d = cdist(loc, bo.get_locs().values, metric='Euclidean')
for l in range(len(loc)):
min_ind = list(zip(*np.where(d == d.min())))[0]
loc[min_ind[0], :] = bo.get_locs().values[min_ind[1], :]
d[min_ind[0]] = np.inf
d[:, min_ind[1]] = np.inf
sub_inds = get_rows(bo.get_locs().values, loc)
subbo = bo[:, sub_inds]
subbo.save(
from builtins import str
import pytest
import os
import supereeg as se
import numpy as np
import pandas as pd
import nibabel as nib
bo = se.simulate_bo(n_samples=10, sample_rate=100)
nii = se.load('example_nifti')
bo_n = se.Brain(nii)
mo = se.load('example_model')
bo_m = se.Brain(mo)
def test_create_bo():
assert isinstance(bo, se.Brain)
def test_bo_data_nifti():
assert isinstance(bo_n, se.Brain)
def test_bo_data_model():
assert isinstance(bo_m, se.Brain)
def test_bo_data_df():
assert isinstance(bo.data, pd.DataFrame)
def test_bo_locs_df():
assert isinstance(bo.locs, pd.DataFrame)
import glob as glob
import supereeg as se
cmap = "hot_r"
nii_bo_dir = '../../../data/niis/best_locs'
fig_dir = '../../../paper/figs/source/best_locs'
# Colorbar default arguments
CBAR_STATE = dict(cbtxtsz=20, txtsz=20., txtcolor='white', width=.1, cbtxtsh=3.,
rect=(-.3, -2., 1., 4.))
KW = dict(title_size=14., zoom=2)
template_brain = 'B3'
bo = se.load(os.path.join(nii_bo_dir, 'best_90th.bo'))
data1 = bo.get_data().values.ravel()
xyz1 = bo.locs.values
s_obj_1 = SourceObj('iEEG', xyz1, data=data1, cmap=cmap)
s_obj_1.color_sources(data=data1)
sc = SceneObj(bgcolor='white', size=(1000, 500))
b_obj_proj_left = BrainObj(template_brain, hemisphere='left', translucent=False)
b_obj_proj_left.project_sources(s_obj_1, clim=(0, 2), cmap='binary')
sc.add_to_subplot(b_obj_proj_left, row=0, col=0, rotate='left', use_this_cam=True)
b_obj_proj_left = BrainObj(template_brain, hemisphere='left', translucent=False)
thresh_bool = kurt_vals > threshold
locs = se.load(fname, field='locs')
if sum(~thresh_bool) > 1:
locs = pd.DataFrame(locs, columns=['x', 'y', 'z'])
return locs[~thresh_bool]
bos =glob.glob(os.path.join(config['bo_datadir'], '*.bo'))
union_locs = []
bo_locs = []
for i in bos:
try:
locs = electrode_search(i)
if not locs.empty:
bo = se.load(i)
bo_locs.append(bo.get_locs().as_matrix())
if union_locs == []:
union_locs = locs.as_matrix()
else:
union_locs = np.vstack((union_locs, locs.as_matrix()))
except:
pass
locations, l_indices = _unique(union_locs)
results = os.path.join(results_dir, 'locs.npz')
np.savez(results, locs = locations, loc_list = bo_locs)
def test_return_type_mo_with_bo():
mo = se.load('example_data', return_type='mo')
assert isinstance(mo, se.Model)
def test_return_type_bo_with_nii():
bo = se.load('example_nifti', return_type='bo')
assert isinstance(bo, se.Brain)
def test_return_type_bo_with_mo():
bo = se.load('example_model', return_type='bo')
assert isinstance(bo, se.Brain)
HR.fit(xs, ys)
except:
errors += 1
try:
mean_height = HR.coef_[0] * midpoint + HR.intercept_
except:
mean_height = 0
mh_data[electrode][timepoint] = mean_height
mhq.put((mh_data, qpos))
if __name__ == "__main__":
fname = sys.argv[1]
bo = se.load(fname)
data = np.asarray(bo.data.T)
locs = bo.locs
sample_rate = bo.sample_rate
del bo
# filter out 60 Hz + harmonics
data = butter_filt(data, filterfreq=60, samplerate=sample_rate[0], axis=1)
# reshape data into 1D arrays for faster FFTs
og_shape = data.shape
data = data.ravel()
# get the log spaced frequencies
numfreqs = 50
nyq = np.floor_divide(sample_rate[0], 2.0)
freq_range = filterfreqs[freqnames.index(freq)]
bodir = config['datadir']
bo_files = glob.glob(os.path.join(bodir, '*.bo'))
for s, bo_file in enumerate(bo_files):
try:
fname = os.path.splitext(os.path.basename(bo_file))[0]
bo_f_file = os.path.join(config['resultsdir'],
fname + '_' + freq + '.bo')
if not os.path.exists(bo_f_file):
bo = se.load(bo_file)
bo.filter = None
# f_data = butter_filt(bo.data.values, 60, bo.sample_rate[0])
f_data = power_breakdown(bo.data.values, freq_range, SAMPLE_RATE)
bo_f = se.Brain(data=f_data,
locs=bo.locs,
sample_rate=bo.sample_rate,
sessions=bo.sessions.values,
kurtosis=bo.kurtosis)
bo_f.save(bo_f_file)
print('saving: ' + bo_f_file)
def test_nii_load(tmpdir):
p = tmpdir.mkdir("sub").join("example")
test_bo.to_nii(filepath=p.strpath)
nii = se.load(os.path.join(p.strpath + '.nii'))
assert isinstance(nii, nib.nifti1.Nifti1Image)
