def clear_ROIs(subs=cfg.subs, bands=cfg.bands):
"""
"""
print(
f"\n======== Clear ROIs =============================================\n"
)
# ITERATE SUBJECT LIST AND BAND LIST ===============================================
TimeStart = time.time()
iterator = list(product(enumerate(subs), enumerate(bands)))
for i, ((iSub, sub), (iBand, band)) in enumerate(iterator):
# load subject paths
SP = cf.sub_params(sub)
# load raw
fname = os.path.join(SP['DerivativesPath'],
f"{sub}_band-{band['name']}_raw.fif")
raw = mne.io.read_raw_fif(fname, preload=True)
# get list non ROI channels only
ChNames = [ChName for ChName in raw.ch_names if ChName[:3] == 'ROI']
raw.drop_channels(ChNames)
raw.save(fname, picks='all', overwrite=True)
cf.display_progress(
f"{sub} {band['name']}, {len(ChNames)} ROIs removed", i,
len(iterator), TimeStart)
def electrode_positions(subs=cfg.subs):
""" Make one figure per channel with the position of the channel
Concatenate for each subject
"""
print("\n======== electrode positions ======================================\n")
FiguresPath =cf.check_path(['..','Figures' , 'overview' + cfg.out ])
# iterate over subjects
for sub in subs:
cf.print_d(f'Electrode positions for {sub}')
# load subject params
SP = cf.sub_params(sub)
for iCh, ChName in enumerate(SP['ChNames']):
fig, ax = plt.subplots(1,1,figsize=(6,2))
pl.clear_axes(np.array([ax]))
fig.suptitle(f"{sub} {ChName}")
pl.channel_position_plot(ax,SP['coords'],iCh)
fig_name = os.path.join(FiguresPath,f"elec_positions_{sub}_{ChName}_temp.pdf")
fig.savefig(fig_name, format='pdf')
plt.close()
cf.concatenate_pdfs(FiguresPath,'temp',f"elec_positions_{sub}.pdf", remove=True)
def referencing_wrap(subs=cfg.subs, RefMethod=cfg.RefMethod):
print(
f"\n======== Applying reference ======================================\n"
)
# ITERATE SUBJECT LIST =============================================================================
TimeStart = time.time()
for iSub, sub in enumerate(subs):
cf.display_progress(f"{sub}", iSub, len(subs), TimeStart)
# add paths to parameters
SP = cf.sub_params(sub)
# load raw
raw = mne.io.read_raw_fif(SP['RawFile'], preload=True)
raw.drop_channels(SP['ChInfo']['bad'])
RefRaw, bads = referencing(raw, RefMethod)
# update bad channel list and save referenced epochs
SP['ChInfo']['bad'] += bads
json.dump(SP['ChInfo'], open(SP['ChInfoFile'], 'w'))
FileName = os.path.join(SP['DerivativesPath'], f"{sub}_raw.fif")
RefRaw.save(FileName, overwrite=True)
# move to next subject ===============================================
cf.print_d(f"{cfg.RefMethod} reference done!")
print(f"\n")
def rejection_wrap(subs=cfg.subs, figures=True):
""" Identify bad channels early in the processing and discard them
completely from raw files.
Rejection based on std of V and dVdt
"""
print(
"\n======== rejection ================================================\n"
)
print(f"SUB\tTOTAL\tKEPT\tREJECTED")
for iSub, sub in enumerate(subs):
SP = cf.sub_params(sub)
raw = mne.io.read_raw_fif(SP['RawFile'], preload=True)
kept, stats, thresholds = rejection(raw)
if figures: rejection_figure(SP, kept, stats, thresholds)
# mark bad channels in ChInfo and save
SP['ChInfo']['bad'] = [
ch for iCh, ch in enumerate(raw.ch_names) if ~kept[iCh]
]
json.dump(SP['ChInfo'], open(SP['ChInfoFile'], 'w'))
print(
f"{SP['sub'][4:]}\t{len(kept)}\t{np.sum(kept)}\t{np.sum(~kept)}\t{int(100*np.sum(~kept)/len(kept))}%"
)
def add_ROI_from_atlas(subs=cfg.subs, bands=cfg.bands):
""" Add a virtual channels with the average of channels
inside ROIs from atlas
Parameters
----------
ROI : list of str or int
list of channels to include by name or index
optionally ROI can be 'all'
ROIname : str
name of the virtual channel
"""
print(
f"\n======== Add ROIs from atlas =====================================\n"
)
# ITERATE SUBJECT LIST AND BAND LIST ===============================================
TimeStart = time.time()
iterator = list(product(enumerate(subs), enumerate(bands)))
for i, ((iSub, sub), (iBand, band)) in enumerate(iterator):
# load subject paths
SP = cf.sub_params(sub)
# load raw
fname = os.path.join(SP['DerivativesPath'],
f"{sub}_band-{band['name']}_raw.fif")
raw = mne.io.read_raw_fif(fname, preload=True)
# get list non ROI channels only
ChNames = [ChName for ChName in raw.ch_names if 'ROI' not in ChName]
coords = cf.get_coords(SP, picks=ChNames)
labels = cf.get_ROI_labels(coords)
for label in list(set(labels)):
ROI = [ChNames[j] for j, l in enumerate(labels) if l == label]
if len(ROI) > 0: # avoid adding empoty sets
# ROI center of mass
ROIcoords = cf.get_coords(SP, picks=ROI).mean(axis=0)
ROIname = f"{str(len(SP['ChInfo']['ChNames'])+1).zfill(3)}-ROI-{label}"
# add virtual channel to raw
raw = add_ROI(raw, ROI, ROIname, ROIcoords)
# update subject info
SP['ChInfo']['ChNames'] += [ROIname]
SP['ChInfo']['coords'] += [ROIcoords.tolist()]
cf.display_progress(
f"{raw.ch_names[-1]} ({len(ROI)} channels) added to {sub} {band['name']}",
i, len(iterator), TimeStart)
# save
json.dump(SP['ChInfo'], open(SP['ChInfoFile'], 'w'))
raw.save(fname, picks='all', overwrite=True)
def subject_overview(subs = cfg.subs, bands = cfg.bands, EpochsSetups = cfg.EpochsSetups,
avg = True, TrialWise = True, key = None, picks = 'good'):
""" Make one figure per channel with
channel position and one evoked plot per band
Parameters
----------
key : str
metadata key to plot separate traces by metadata value
"""
print("\n======== subject overview =========================================\n")
# iterate over subjects
for sub in subs:
# load subject params
SP = cf.sub_params(sub)
# preload raw data for all bands
epochs = []
iterator = product(enumerate(bands),enumerate(EpochsSetups))
for i, ((iBand, band),(iEp, EpochsSetup)) in enumerate(iterator):
cf.print_d(f"{sub}, loading {band['name']} epochs...")
# get list of channels
PicksList = cf.get_picks(SP, picks, band = band)
if PicksList == []: continue
# epochs (phase epochs are complex and cannot be resampled)
try: epochs += [cf.load_epochs(sub, band, EpochsSetup, picks = PicksList).resample(100)]
except: epochs += [cf.load_epochs(sub, band, EpochsSetup, picks = PicksList)]
# iterate over channels, one figure per channel
TimeStart = time.time()
for iCh, ChName in enumerate(PicksList):
cf.display_progress(f"{sub}, {ChName}", iCh, len(PicksList),TimeStart)
channel_overview(sub, ChName, bands, EpochsSetups, EpochsList=epochs, key=key, plot = False, save = True, temp=True)
# concatenate figures subject wise
FileName = f"{sub}_band-{band['name']}_{picks}_overview.pdf"
FiguresPath = cf.check_path(['..', 'Figures', 'overview' + cfg.out])
cf.concatenate_pdfs(FiguresPath, 'temp', FileName, remove = True)
# concatenate for all subjects
FileName = f"band-{band['name']}_{picks}_overview.pdf"
FiguresPath = cf.check_path(['..', 'Figures', 'overview' + cfg.out])
cf.concatenate_pdfs(FiguresPath, 'sub', FileName, remove = False)
def regression_wrap(model, subs=cfg.subs, bands=cfg.bands, alpha=cfg.alpha):
print(
"\n======== regressions ===================================================\n"
)
# ITERATE SUBJECT LIST AND BAND LIST ===============================================
time_start = time.time()
iterator = list(product(enumerate(subs), enumerate(bands)))
for i, ((i_sub, sub), (i_band, band)) in enumerate(iterator):
dp = cf.sub_params(sub)
cf.display_progress(f"{sub} {band['name']}", i, len(iterator),
time_start)
# load epochs
epochs = cf.load_epochs(sub, band, model['ep_setup'])[model['query']]
scores, ps = stepwise_regression(epochs, model['predictors'],
model['predictors_stepwise'])
# save data into dataframe
l = len(epochs.ch_names)
coords = cf.get_coords(epochs)
data = np.array([
l * [sub], epochs.ch_names, coords[:, 0], coords[:, 1],
coords[:, 2], l * [band['name']]
]).T
df = pd.DataFrame(data,
columns=['sub', 'ch_name', 'x', 'y', 'z',
'band']) #.astype('object')
# for each sw_predictor save time point with max significant score
for i_p, p in enumerate([''] + model['predictors_stepwise']):
idx = np.squeeze((scores * (ps < alpha))[i_p].argmax(axis=-1))
df['r2' + p] = [scores[i_p, i_ch, idx[i_ch]] for i_ch in range(l)]
df['p' + p] = [ps[i_p, i_ch, idx[i_ch]] for i_ch in range(l)]
df['t' + p] = [epochs.times[idx[i_ch]] for i_ch in range(l)]
file_name = os.path.join(
dp['derivatives_path'],
f"{sub}_band-{band['name']}_reg-{model['tag']}.csv")
df.to_csv(file_name)
def epochs2TFR_wrap(subs=cfg.subs,
bands=cfg.bands,
EpochsSetups=cfg.EpochsSetups):
""" Loop over subjects and bands and make and save
mne epochs objects with band power.
"""
print(
f"\n======== mne.epochs 2 mne.epochsTFR ==============================\n"
)
# ITERATE SUBJECT LIST AND BAND LIST ===============================================
TimeStart = time.time()
iterator = list(
product(enumerate(subs), enumerate(EpochsSetups), enumerate(bands)))
for i, ((iSub, sub), (iEp, EpochsSetup), (iBand,
band)) in enumerate(iterator):
cf.display_progress(f"{sub}, {band['name']} power", i, len(iterator),
TimeStart)
# load subject data and raw file only once per subject
if iBand == 0:
# add paths to parameters
SP = cf.sub_params(sub)
# load epochs
EpochsFile = os.path.join(
SP['DerivativesPath'],
f"{sub}_epochs-{EpochsSetup['name']}-epo.fif")
epochs = mne.read_epochs(EpochsFile)
EpochsBand = raw2TFR(epochs, band)
# save
fname = os.path.join(
SP['DerivativesPath'],
f"{sub}_band-{band['name']}_epochs-{EpochsSetup['name']}-epo.fif")
EpochsBand.save(fname, overwrite=True)
cf.print_d(f"done! elapsed time {int((time.time() - TimeStart)/60.)} min")
print(f"\n")
def ch_info(subs=cfg.subs):
print(
"\n======== Subject info =============================================\n"
)
for sub in subs:
SP = cf.sub_params(sub)
# load raw to extract ch info
raw = mne.io.read_raw_fif(SP['RawFile'])
ChInfo = {}
ChInfo['ChNames'] = raw.ch_names
PositionsDict = raw.get_montage().get_positions()['ch_pos']
ChInfo['coords'] = [
PositionsDict[ChName].tolist() for ChName in raw.ch_names
]
#'ROIalbels' : cf.get_ROI_labels(raw.get_montage().get_positions()['ch_pos'])
json.dump(ChInfo, open(SP['ChInfoFile'], 'w'))
def rejection_epochs(subs=cfg.subs, EpochsSetups=cfg.EpochsSetups):
""" Identify bad channels early in the processing and discard them
completely from epochs files.
Rejection based on std of V and dVdt
"""
print(
"\n======== rejection ==============================================\n"
)
for iSub, sub in enumerate(subs):
for iEp, EpochsSetup in enumerate(EpochsSetups):
SP = cf.sub_params(sub)
# load epochs
EpochsFile = os.path.join(
SP['DerivativesPath'],
f"{sub}_epochs-{EpochsSetup['name']}-epo.fif")
epochs = mne.read_epochs(EpochsFile)
data = np.swapaxes(epochs.get_data(), 0,
1).reshape(len(epochs.ch_names), -1)
# std of voltage and voltage increments
s_v = np.std(data, axis=-1)
s_dv = np.std(np.diff(data, axis=-1), axis=-1)
# use robust scaler to get relative values
x = np.squeeze(RobustScaler().fit_transform(s_v.reshape(-1, 1)))
y = np.squeeze(RobustScaler().fit_transform(s_dv.reshape(-1, 1)))
# identify good channels (those within the thresholds)
kept = (x > cfg.V_low) * (x < cfg.V_high) * (y > cfg.dV_low) * (
y < cfg.dV_high)
print(f"CH\tKEPT\tstd(V)\tstd(dVdt)")
for iCh, k in enumerate(kept):
print(
f"{iCh}\t{k}\t{np.around(s_v[iCh],decimals=2)}\t{np.around(s_dv[iCh],decimals=2)}"
)
def raw2TFR_wrap(subs=cfg.subs, bands=cfg.bands, skip=cfg.skip):
""" Loop over subjects and bands and make and save
mne raw objects with band power.
"""
print(
f"\n======== mne.raw 2 mne.rawTFR ====================================\n"
)
# ITERATE SUBJECT LIST AND BAND LIST ===============================================
TimeStart = time.time()
iterator = list(product(enumerate(subs), enumerate(bands)))
for i, ((iSub, sub), (iBand, band)) in enumerate(iterator):
cf.display_progress(f"{sub}, {band['name']} power", i, len(iterator),
TimeStart)
# load subject data and raw file only once per subject
if iBand == 0:
# add paths to parameters
SP = cf.sub_params(sub)
# load raw
fname = os.path.join(SP['DerivativesPath'], f"{sub}_raw.fif")
raw = mne.io.read_raw_fif(fname, preload=True)
# skip if file already exists
fname = os.path.join(SP['DerivativesPath'],
f"{sub}_band-{band['name']}_raw.fif")
if os.path.isfile(fname) and skip: continue
RawBand = raw2TFR(raw, band)
# save
RawBand.save(fname, picks='all', overwrite=True)
cf.print_d(f"done! elapsed time {int((time.time() - TimeStart)/60.)} min")
print(f"\n")
def reg_statistics(model,
predictors=[''],
subs=cfg.subs,
bands=cfg.bands,
alpha=cfg.alpha,
filter_tag='',
plot=True,
save=True):
# load data
df = cf.load_df(f"reg_{model['tag']}", bands=bands, subs=subs)
L = len(df)
# filter out regressions with negative score
df = df.query(f'r2>0 and p<{alpha}')
if predictors == 'all': predictors = [''] + model['predictors']
keys = ['p' + p for p in predictors]
conditions = ['r2' + p for p in predictors]
fig, ax = plt.subplots(len(keys),
3,
figsize=(12, 3 * len(keys)),
gridspec_kw={'width_ratios': [1, 2, 0.5]})
if len(keys) == 1: ax = np.array([ax])
fig.suptitle(f"Regression stats summary", fontsize=15)
colors = plt.get_cmap('viridis')(np.linspace(0, 1, len(bands)))
for i, (k, c) in enumerate(zip(keys, conditions)):
# scatter r2 vs p -------------------------------------------------------------
for i_band, band in enumerate(bands):
ax[i, 0].scatter(df[df['band'] == band['name']][k],
df[df['band'] == band['name']][c],
c=[colors[i_band]],
alpha=0.3)
ax[i, 0].set_xlabel(k)
ax[i, 0].set_ylabel(c)
ax[i, 0].axvline(alpha, color='k', linewidth=0.5)
ax[i, 0].axhline(0, color='k', linewidth=0.5)
ax[i, 0].spines['right'].set_visible(False)
ax[i, 0].spines['top'].set_visible(False)
ax[i, 0].tick_params(axis='both', which='both', size=0, labelsize=9)
# number of significant channels ----------------------------------------------
n = []
for sub in subs:
# load subject params
dp = cf.sub_params(sub)
# significant channels for subjec-band pair
n_sub = []
# iterate over bands -----------------------------------------------
for i_band, band in enumerate(bands):
n_sub += [
np.sum(
np.array(df[(df['sub'] == sub) &
(df['band'] == band['name'])][k]) < alpha)
]
# go to next band --------------------------------------------------
n += [n_sub]
# go to next subject ======================================================
n = np.array(n).T
# width and positions of bars
width = 0.8 / len(bands)
x = np.arange(len(subs))
colors = plt.get_cmap('viridis')(np.linspace(0, 1, len(bands)))
for i_band, band in enumerate(bands):
ax[i, 1].bar(x + i_band * width,
n[i_band],
width,
color=colors[i_band])
ax[i, 2].bar(i_band * width,
np.sum(n[i_band]),
width,
label=band['name'],
color=colors[i_band])
ax[i, 1].set_ylabel("# significant channels")
ax[i, 1].set_xticks(np.arange(len(subs)) + 0.4)
ax[i, 1].set_xticklabels(subs, rotation=45)
ax[i, 1].spines['right'].set_visible(False)
ax[i, 1].spines['top'].set_visible(False)
ax[i, 1].tick_params(axis='both', which='both', size=0, labelsize=9)
ax[i, 2].set_xticks([0.4])
ax[i, 2].set_xlim([-0.2, 1])
ax[i, 2].set_xticklabels(['Total'])
ax[i, 2].spines['right'].set_visible(False)
ax[i, 2].spines['top'].set_visible(False)
ax[i, 2].tick_params(axis='both', which='both', size=0, labelsize=9)
if i > 0: L = len(df)
ax[i, 2].set_title(f"{np.sum(n)}/{L} {int(100*np.sum(n)/L)}%")
ax[0, 2].legend(frameon=False, loc=(1, 0.2), fontsize=9)
fig.subplots_adjust(left=0.1, right=0.9, wspace=0.3, hspace=0.5)
# save figure
fig_name = os.path.join(
cf.check_path(['..', 'Figures' + cfg.out, f"reg_{model['tag']}"]),
f"stats_reg_{model['tag']}.pdf")
if save: fig.savefig(fig_name, format='pdf', dpi=100)
if plot: plt.show()
else: plt.close()
def metadata_overview(features, query=None, subs=cfg.subs, key=None, plot_subs = True, plot = True, save = True):
print("\n======== metadata overview ========================================\n")
# colect metadata
metadata_list = []
for sub in cfg.subs:
SP = cf.sub_params(sub)
sub_metadata = pd.read_csv(SP['metadata_file'])
if query!=None: sub_metadata=sub_metadata.query(query)
metadata_list += [sub_metadata]
# make figure
fig, ax = plt.subplots(1+len(subs)*plot_subs,len(features),figsize=(2*len(features),2*(1+len(subs)*plot_subs)))
# add extra dimension if missing
if not plot_subs: ax = np.array([ax])
fig.suptitle(f"Metadata")
metadata_list = [pd.concat(metadata_list)] + metadata_list*plot_subs
labels = ['All subjects'] + subs*plot_subs
iterator = product(enumerate(metadata_list),enumerate(features))
for (i_m, metadata),(i_f, feature) in iterator:
# OPTION 1: plot all epochs together
if key == None:
# plot all subs together
data = metadata[feature].values
x = np.arange(data.min()-1.5,data.max()+2)
h, _ = np.histogram(data,x,density=True)
pl.trace_plot(ax[i_m,i_f], x[:-1]+0.5, h, ylabel = '' + labels[i_m]*(i_f==0), title = '' + feature*(i_m==len(metadata_list)-1))
ax[i_m,i_f].set_xticks(np.arange(data.min(),data.max()+2))
# OPTION 2: separate epochs by condition
else:
conditions=list(set(metadata[key]))
conditions.sort()
queries = [f"{key} =='{c}'" for c in conditions]
colors = plt.get_cmap('viridis')(np.linspace(0, 1, len(conditions)))
for i_q, query in enumerate(queries):
data = metadata[feature].values
x = np.arange(data.min()-1.5,data.max()+2)
data_query = metadata.query(query)[feature].values
h, _ = np.histogram(data_query,x,density=True)
pl.trace_plot(ax[i_m,i_f], x[:-1]+0.5, h, ylabel = '' + labels[i_m]*(i_f==0), xlabel = '' + feature*(i_m==len(metadata_list)-1) ,color=colors[i_q],label = conditions[i_q])
ax[i_m,i_f].set_xticks(np.arange(data.min(),data.max()+2))
ax[0,-1].legend(frameon=False,loc=(0.5,0.5))
fig.subplots_adjust(top=0.8,bottom=0.2,wspace=0.3, hspace=0.3)
if save:
FiguresPath =cf.check_path(['..','Figures' , 'overview'+ cfg.out])
fig_name = os.path.join(FiguresPath,f"metadata.pdf")
fig.savefig(fig_name, format='pdf')
if plot: plt.show()
else: plt.close()
def spike_detection(sub, ch_name, tw):
dp = cf.sub_params(sub)
# load epochs
filename = os.path.join(dp['derivatives_path'], f'{sub}_raw.fif')
raw = mne.io.read_raw_fif(filename, preload=True).crop(tmin=tw[0],
tmax=tw[1])
amp = np.array([raw.get_data(picks=ch_name)])
t = np.linspace(tw[0], tw[1], amp.shape[-1])
# compute wavelets
f = np.arange(5, 150, 2)
power = np.squeeze(
mne.time_frequency.tfr_array_morlet(amp,
sp.srate,
f,
n_cycles=f / 2.,
output='power')[0])
power = power / power.std(axis=-1)[:, np.newaxis] #*f[:,np.newaxis]
amp = np.squeeze(amp)
timewindowsize = 0.1 # s
d = int(timewindowsize * sp.srate)
C = []
for i in range(len(t) - d):
cf.print_d(f"{i/(len(t) - d)}")
corr = np.corrcoef(power[:, i:i + d])
corr = corr[np.triu_indices_from(corr, k=1)]
C += [corr]
C = np.array(C).T
fig, ax = plt.subplots(4, 1, figsize=(6, 6))
fig.suptitle(f'{sub} {ch_name}')
coords = dp['coords'][dp['ch_names'].index(ch_name)]
pl.channel_position_plot(ax[0], [coords], 0)
pl.trace_plot(ax[1],
t,
amp,
xlims=[tw[0] + 0.5, tw[1] - 0.5],
ylabel='V (uV)')
pl.imshow_plot(ax[2],
power,
title='',
ylabel='f (Hz)',
xlims=[tw[0] + 0.5, tw[1] - 0.5],
ylims=f,
colorbar='power (std)')
pl.trace_plot(ax[3],
t[:-d],
C,
xlims=[tw[0] + 0.5, tw[1] - 0.5],
ylabel='correlation',
xlabel='t (s)',
plot_std=True,
plot_p595=True,
mode='avg')
fig_name = os.path.join(cf.check_path(['..', 'Figures', 'spikes']),
f'{sub}_{ch_name}_{int(tw[0])}.pdf')
fig.savefig(fig_name, format='pdf')
plt.close()
def num_significant_channels(key,
test,
tag,
split=None,
alpha=sp.alpha,
subs=sp.subject_list,
bands=sp.bands):
""" Make a bar plot with number of significant channels
per subject per band for some test or regression
Parameters
----------
key : str
name of the column to extract
test : str
name of the test/regression that works as a label for files
queries : dict
generated with cf.compose_queries, contains mne queries, file tag, queries list
split : int or str
if int divide the time window into 'split' chunks and run one test in each
if string separate epochs by values of metadata column 'split',
f.ex. split = 'w_position' run the test for each word position
alpha : float
level of significance (leave out p>alpha)
subs : list of str
list of subjects to include f.e. [sub-01, sub-02]
bands : list of dict
list of bands to include
"""
print(f"\n{test} {tag} summary figure")
df = cf.load_df(test + '_' + tag)
if split == None: labels = [''] # no split
elif isinstance(split, int):
labels = list(np.unique(df['tw'].to_numpy())) # split by time windows
elif isinstance(split, str):
labels = list(np.unique(df[split].to_numpy())) # split by conditions
num_plots = len(labels)
fig, ax = plt.subplots(num_plots,
1,
figsize=(len(subs), 3 * num_plots),
sharex=True)
fig.suptitle(fr"{test} {tag}, {key} $\leq {alpha}$")
if split == None: ax = np.array([ax])
# loop over plots
for i_plot, label in enumerate(labels):
# number of significant channels
n = []
for sub in subs:
# load subject params
dp = cf.sub_params(sub)
# significant channels for subjec-band pair
n_sub = []
# iterate over bands -----------------------------------------------
for i_band, band in enumerate(bands):
if split == None:
n_sub += [
np.sum(
np.array(df[(df['sub'] == sub) & (
df['band'] == band['name'])][key]) < alpha)
]
title = ''
elif isinstance(split, int):
n_sub += [
np.sum(
np.array(df[
(df['sub'] == sub)
& (df['band'] == band['name'])
& ([x == label for x in df['tw'].to_numpy()])]
[key]) < alpha)
]
title = str(label) + ' s'
elif isinstance(split, str):
n_sub += [
np.sum(
np.array(df[(df['sub'] == sub)
& (df['band'] == band['name'])
& (df[split] == label)][key]) < alpha)
]
title = f"{split} = {label}"
# go to next band --------------------------------------------------
n += [n_sub]
# go to next subject ======================================================
n = np.array(n).T
# width and positions of bars
width = 0.8 / len(bands)
x = np.arange(len(subs))
colors = plt.get_cmap('viridis')(np.linspace(0, 1, len(bands)))
for i_band, band in enumerate(bands):
ax[i_plot].bar(x + i_band * width,
n[i_band],
width,
label=band['name'],
color=colors[i_band])
ax[i_plot].set_ylabel("# significant channels")
ax[i_plot].set_title(title)
ax[i_plot].set_xticks(np.arange(len(subs)) + 0.4)
ax[i_plot].set_xticklabels(subs)
ax[i_plot].spines['right'].set_visible(False)
ax[i_plot].spines['top'].set_visible(False)
ax[i_plot].tick_params(axis='both', which='both', size=0, labelsize=9)
ax[i_plot].legend(frameon=False, loc=(1, 0.2), fontsize=9)
fig.subplots_adjust(right=0.8)
# save figure
fig_name = os.path.join(cf.check_path(['..', 'Figures', test + sp.out]),
f"summary_{test}_{tag}_{key}.pdf")
fig.savefig(fig_name, format='pdf', dpi=100)
if sp.plot: plt.show()
plt.close()
def source2raw(subs=cfg.subs):
from neo.io import BlackrockIO
TimeStart = time.time()
# ITERATE SUBJECT LIST =============================================================================
for iSub, sub in enumerate(subs):
print(
f"\n--- {sub} ----------------------------------------------------\n"
)
# add paths to parameters
SP = cf.sub_params(sub)
cf.display_progress(f"Loading source data, {sub}", iSub, len(subs),
TimeStart)
ieeg = []
ttl = []
# 2 files per session
for i, source_file in enumerate(SP['source_files']):
reader = BlackrockIO(filename=source_file)
blks = reader.read(lazy=False)
ttl_ = np.squeeze(1. * (np.diff(1. * (np.array(
blks[0].segments[-1].analogsignals[0]).T[0, :] > 4000.)) >
0)).astype(np.int8)
ieeg_ = np.array(blks[0].segments[-1].analogsignals[1],
dtype=np.int16).T
# start five seconds before first fixation
t0 = np.argmax(ttl_ > 0.5) - int(5 * cfg.source_srate)
ttl += [ttl_[t0:]]
ieeg += [ieeg_[:, t0:]]
# loop over sessions
ieeg_full = []
ttl_full = []
for i in range(0, len(ttl), 2):
m = min(len(ttl[i]), len(ttl[i + 1]))
ieeg_session = np.vstack((ieeg[i][:, :m], ieeg[i + 1][:, :m]))
ieeg_full += [ieeg_session]
ttl_full += [ttl[i][:m]]
# concatenate sessions
ieeg = np.concatenate(ieeg_full, axis=-1)
ttl = np.concatenate(ttl_full)
print(
f"\n{ieeg.shape[0]} channels, {int(len(ttl)/cfg.source_srate)} seconds, {np.sum(ttl)} events, {round(ieeg.nbytes/1e9,2)} GB of data loaded"
)
cf.print_d("saving...")
# save one ieeg, one ttl file per subject
fname = os.path.join(SP['RawPath'], f"{sub}_ttl.csv")
ttl.tofile(fname, sep=',')
if len(SP['ChNames']) != ieeg.shape[0]:
dp = fix_channels(dp, len(ieeg))
cf.display_progress(f"Making MNE Raw, {sub} ", iSub, len(subs),
TimeStart)
# Create MNE info
info = mne.create_info(SP['ChNames'],
sfreq=cfg.source_srate,
ch_types='eeg')
# Finally, create the Raw object
raw = mne.io.RawArray(ieeg, info)
montage = mne.channels.make_dig_montage(
dict(zip(SP['ChNames'], SP['coords'])))
raw.set_montage(montage)
#print(raw.get_montage().get_positions()['ch_pos']['001-AH1'])
# downsample for memory
if cfg.srate >= raw.info['sfreq']:
print(
f"Error: Original sampling freq smaller than or equal to target sampling freq ({raw.info['sfreq']} Hz <={cfg.srate} Hz)"
)
else:
# resample raw
cf.display_progress(
f"Resampling from {cfg.source_srate} to {cfg.srate}, {sub}",
iSub, len(subs), TimeStart)
raw.resample(cfg.srate)
cf.display_progress(
f"Applying notch filtering at {cfg.landline_noise} Hz, and 4 harmonics, {sub}",
iSub, len(subs), TimeStart)
raw.notch_filter(cfg.landline_noise * np.arange(1, 5),
filter_length='auto',
phase='zero',
picks='all')
cf.print_d(f"Saving mne raw files for {sub}")
fname = os.path.join(SP['RawPath'], f"{sub}_raw.fif")
raw.save(fname, picks='all', overwrite=True)
print(' ')
def channel_overview(sub, ChName, bands, EpochsSetups, EpochsList=None,
key=None, plot = True, save = False, temp=False):
""" Same as subject overview but for one single channel
Parameters
----------
key : str
metadata key to plot separate traces by metadata value
"""
# load subject params
SP = cf.sub_params(sub)
#if ChName not in SP['ChNames']: sys.exit(f"{sub} {ChName} not found (o_0) ")
# make figure
fig, ax = plt.subplots(3,len(EpochsSetups),figsize=(5*len(EpochsSetups),4.5))
# add extra dimension if missing
if len(EpochsSetups) == 1: ax = np.array([ax]).T
title = f"{sub} {ChName}"
if 'sample' in EpochsSetups[0].keys(): title += f"\nsample: {EpochsSetups[0]['sample']}"
fig.suptitle(title)
# plot channel cosition
pl.channel_position_plot(ax[0,0],cf.get_coords(SP,picks=[ChName]))
pl.clear_axes(ax[0,:])
# LOOP OVER BANDS AND EPOCH SETUPS -----------------------------------------
iterator = product(enumerate(bands),enumerate(EpochsSetups))
for i, ((iBand, band),(iEp, EpochsSetup)) in enumerate(iterator):
if EpochsList == None: epochs = cf.load_epochs(sub,band,EpochsSetup)
else: epochs = EpochsList[iEp].copy()
epochs = epochs.pick_channels([ChName])
xticks = [x*SP['soa'] for x in EpochsSetup['xticks']]
# avg PLOT -----------------------------------------------------------
if band['method'] in [None,'filter']: ylabel = f"{band['name']} band\n"*(iEp==0) + f"\nV (z-scored)"
elif band['method'] in ['complex']: ylabel = f"{band['name']} band\n"*(iEp==0) + f"\nITC"
else: ylabel = f"{band['name']} power\n"*(iEp==0) + f"\ndB (z-scored)"
# OPTION 1: plot all epochs together ----------------------------------
if key == None:
x = np.squeeze(epochs.get_data(picks=ChName))
# plot avg trace
pl.trace_plot(ax[2,iEp], epochs.times, x, ylabel = ylabel,
vlines = [SP['soa']*x for x in EpochsSetup['xticks']], plot_sem = True)
# OPTION 2: separate epochs by condition -------------------------------
else:
conditions=list(set(epochs.metadata[key]))
conditions.sort()
if isinstance(conditions[0],str):
queries = [f"{key} =='{c}'" for c in conditions]
elif len(conditions)>10:
conditions = np.linspace(min(conditions),max(conditions),5)
queries = [f"{key} >={conditions[i]} and {key} <{conditions[i+1]}" for i in range(len(conditions)-1)]
else:
queries = [f"{key} =={c}" for c in conditions]
colors = plt.get_cmap('viridis')(np.linspace(0, 1, len(conditions)))
for i_q, query in enumerate(queries):
x = np.squeeze(epochs.copy()[query].get_data(picks=ChName))
# plot avg trace per condition
pl.trace_plot(ax[2,iEp], epochs.times, x, ylabel = '' + band['name']*(iEp==0),
vlines = xticks, plot_sem = True, color=colors[i_q],label = conditions[i_q])
# make legend
if iEp==len(EpochsSetups)-1 and key!=None:
ax[-1,-1].legend(frameon=False,title=key,loc=(1,0.2),fontsize = 7) #
# TrialWise PLOT -----------------------------------------------------------
if band['method'] in [None,'filter']: ylabel, cb = f"{band['name']} band", f"V (z-scored)"
elif band['method'] in ['complex']: ylabel, cb =f"{band['name']} band", r"$\theta$"
else: ylabel, cb = f"{band['name']} power", f"dB (z-scored)"
try: order = EpochsSetup['order']
except: order = key
if order!=None:
values = epochs.metadata[order].values
idx =np.argsort(values)
yticks = [j for j, x in enumerate(np.diff(values[idx])!=0) if x]
yticklabels = [str(values[idx][y]) for y in yticks]
ylabel=(iEp==0)*ylabel + f"\n\n{order}"
else:
idx = []
yticks = []
yticklabels = []
ylabel=(iEp==0)*ylabel + f"\n\nepoch"
m = np.squeeze(epochs.copy().get_data(picks=ChName))
if isinstance(m[0,0],complex):
m = np.angle(m)
vmin, vmax = -np.pi, np.pi
else:
m = (m-m.mean())/m.std()
vmin = np.percentile(m,15)
vmax = np.percentile(m,85)
pl.imshow_plot(ax[1,iEp], m,vmin=vmin,vmax=vmax, ylabel = ylabel, xlims = epochs.times,
title = EpochsSetup['name'], yticks = yticks, yticklabels = yticklabels, vlines = xticks, colorbar = cb,cmap='jet',order=idx)
ax[-1,0].set_xlabel('t (s)')
fig.subplots_adjust(wspace=0.3, hspace=0.3)
if save:
FiguresPath = cf.check_path(['..','Figures' , 'overview' + cfg.out ])
fig_name = os.path.join(FiguresPath,f"{sub}_{ChName}{'_temp'*temp}.pdf")
fig.savefig(fig_name, format='pdf')
if plot: plt.show()
else: plt.close()
def reg_single_channel(sub,
ch_name,
band,
model,
alpha=cfg.alpha,
plot=True,
save=True,
temp=True,
n=4):
# load subject params
dp = cf.sub_params(sub)
# load epochs
all_epochs = cf.load_epochs(sub, band, model['ep_setup'],
picks=[ch_name]).decimate(5)
epochs = all_epochs[model['query']]
# regression
scores, ps = stepwise_regression(epochs, model['predictors'],
model['predictors_stepwise'])
predictors = model['predictors_stepwise']
# FIGURE --------------------------------------------------------
fig, ax = plt.subplots(len(predictors) + 1,
3,
figsize=(15, 3 * (len(predictors) + 1)))
fig.suptitle(f"{sub} {ch_name} {band['name']}\n{model['query']}",
fontsize=15)
# ch_position
pl.channel_position_plot(ax[0, 2], cf.get_coords(epochs, picks=[ch_name]))
# channel summary
pl.channel_summary_plot(ax[0, 1], sub, ch_name, model['tag'],
model['predictors'])
t_sig = pl.score_plot(
ax[0, 0],
scores,
ps,
epochs.times,
model['predictors_stepwise'],
vlines=[dp['soa'] * x for x in model['ep_setup']['xticks']],
xlims=[model['ep_setup']['tmin'], model['ep_setup']['tmax']])
ax[1, 0].set_title(model['query'])
if np.sum(np.array([len(t_sig[pr]) == 2 for pr in predictors])) == 0:
plt.close()
return
for i_p, predictor in enumerate(predictors):
predictor_values = epochs.metadata[predictor].values
N = min(n, len(set(predictor_values)))
predictor_thresholds = np.linspace(np.nanmin(predictor_values),
np.nanmax(predictor_values), N + 1)
#predictor_thresholds = np.percentile(predictor_values, np.linspace(0,100,N+1))
colors = plt.get_cmap('autumn')(np.linspace(0., 1., N))
for i in range(N):
y = np.squeeze(epochs[
f'{predictor}>={predictor_thresholds[i]} and {predictor}<={predictor_thresholds[i+1]}']
.get_data(picks=ch_name))
label = f"({round(predictor_thresholds[i])},{round(predictor_thresholds[i+1])})"
pl.trace_plot(
ax[i_p + 1, 0],
epochs.times,
y,
ylabel='zscored power',
title=predictor,
xlims=[model['ep_setup']['tmin'], model['ep_setup']['tmax']],
vlines=[dp['soa'] * x for x in model['ep_setup']['xticks']],
color=colors[i],
label=label,
legend=True,
plot_sem=True,
xlabel='t (s)')
if len(t_sig[predictor]) == 2:
ax[i_p + 1, 0].axvspan(t_sig[predictor][0] - cfg.smooth * 0.5,
t_sig[predictor][1] + cfg.smooth * 0.5,
color='gray',
alpha=0.3,
lw=0)
pl.response_plot(ax[i_p + 1, 1],
ch_name,
all_epochs.copy(),
predictor,
tmin=t_sig[predictor][0] - cfg.smooth * 0.5,
tmax=t_sig[predictor][1] + cfg.smooth * 0.5)
else:
pl.clear_axes(np.array([ax[i_p + 1, 1]]))
values = all_epochs.metadata[predictor].values
order = np.argsort(values)
yticks = [j for j, x in enumerate(np.diff(values[order]) != 0) if x]
yticklabels = [str(values[order][idx]) for idx in yticks]
m = np.squeeze(all_epochs.copy().get_data(picks=ch_name))
m = (m - m.mean()) / m.std()
vmin = np.percentile(m, 15)
vmax = np.percentile(m, 85)
pl.imshow_plot(ax[i_p + 1, 2],
m,
vmin=vmin,
vmax=vmax,
ylabel=predictor,
xlims=all_epochs.times,
yticks=yticks,
yticklabels=yticklabels,
colorbar='z-score',
cmap='RdBu_r',
order=order)
fig.subplots_adjust(left=0.1,
bottom=0.1,
right=0.9,
top=0.9,
wspace=0.3,
hspace=0.4)
if save:
fig_name = os.path.join(
cf.check_path(['..', 'Figures' + cfg.out, f"reg_{model['tag']}"]),
f"{sub}_{ch_name}_{band['name']}_temp.pdf")
fig.savefig(fig_name, format='pdf', dpi=100)
if plot: plt.show()
else: plt.close()
def reg_single_channel2(sub,
ch_name,
band_name,
model,
extra_band,
predictor2,
predictors='all',
alpha=cfg.alpha,
plot=True,
save=False,
temp=False,
n=4,
twsize=cfg.tws):
# load subject params
dp = cf.sub_params(sub)
if predictors == 'all': predictors = model['predictors']
ep_setup = model['ep_setup']
# raw file
mneraw_file = os.path.join(dp['derivatives_path'],
f"{sub}_band-{band_name}_raw.fif")
raw = mne.io.read_raw_fif(mneraw_file, preload=True)
all_epochs = cf.load_epochs(dp,
raw.copy().pick_channels([ch_name]),
model['ep_setup'])
epochs = all_epochs[model['query']]
# get predictor values
X = np.array(epochs.metadata[model['predictors']])
X[np.isinf(X)] = -15.5
# get responses averaged over time windows
tmin, tmax = model['ep_setup']['tmin'], model['ep_setup']['tmax'] - twsize
tws = [[t, t + cfg.tws]
for t in np.arange(tmin, tmax, (tmax - tmin) / 100.)]
y = []
t = []
for t1, t2 in tws:
# epoch average
e = epochs.copy()
y += [np.squeeze(e.crop(t1, t2).get_data(picks=ch_name).mean(axis=-1))]
t += [0.5 * (t1 + t2)]
y = np.swapaxes(np.array([y]).T, -1, -2)
# compute regressions
R = regression(X, y, t)
# FIGURE --------------------------------------------------------
fig, ax = plt.subplots(len(predictors) + 2,
3,
figsize=(15, 3 * (len(predictors) + 2)))
fig.suptitle(
f"{sub} {ch_name} {band_name}\n{model['predictors']}\n{model['query']}",
fontsize=15)
# ch_position
#pl.clear_axes(ax[0,:])
pl.channel_position_plot(ax[0, 0],
[dp['coords'][dp['ch_names'].index(ch_name)]], 0)
# channel summary
pl.channel_summary_plot(ax[0, 1], sub, ch_name, model['tag'],
model['predictors'])
# raster plot
y = cf.load_epochs(dp,
raw.copy().pick_channels([ch_name]),
model['ep_setup']).get_data(picks=ch_name)
#y = (y - np.median(y))/np.subtract(*np.percentile(y, [75, 25]))
y = (y - np.median(y)) / y.std()
pl.imshow_plot(
ax[1, 1],
y[:, 0, :],
title='All trials',
xlims=[model['ep_setup']['tmin'], model['ep_setup']['tmax']],
vlines=[dp['soa'] * x for x in ep_setup['xticks']],
ylabel='epoch',
colorbar='zscore',
vmin=-1,
vmax=1,
cmap='RdBu_r')
# score plot
t_sig = pl.score_plot(
ax[1, 0],
R,
model['predictors'],
vlines=[dp['soa'] * x for x in ep_setup['xticks']],
xlims=[model['ep_setup']['tmin'], model['ep_setup']['tmax']])
ax[1, 0].set_title(model['query'])
if np.sum(np.array([len(t_sig[pr]) == 2 for pr in predictors])) == 0:
plt.close()
return
for i_p, predictor in enumerate(predictors):
predictor_values = epochs.metadata[predictor].values
predictor_values[np.isinf(predictor_values)] = -15.5
N = min(n, len(set(predictor_values)))
predictor_thresholds = np.linspace(np.nanmin(predictor_values),
np.nanmax(predictor_values), N + 1)
colors = plt.get_cmap('autumn')(np.linspace(0., 1., N))
for i in range(N):
y = np.squeeze(epochs[
f'{predictor}>={predictor_thresholds[i]} and {predictor}<={predictor_thresholds[i+1]}']
.get_data(picks=ch_name))
label = f"({round(predictor_thresholds[i])},{round(predictor_thresholds[i+1])})"
pl.trace_plot(
ax[i_p + 2, 0],
epochs.times,
y,
ylabel='zscored power',
title=predictor,
xlims=[model['ep_setup']['tmin'], model['ep_setup']['tmax']],
vlines=[dp['soa'] * x for x in ep_setup['xticks']],
color=colors[i],
label=label,
legend=True,
plot_sem=True,
xlabel='t (s)')
if len(t_sig[predictor]) == 2:
ax[i_p + 2, 0].axvspan(t_sig[predictor][0],
t_sig[predictor][1],
color='gray',
alpha=0.3,
lw=0)
pl.response_plot(ax[i_p + 2, 1],
ch_name,
all_epochs.copy(),
predictor,
tmin=t_sig[predictor][0],
tmax=t_sig[predictor][1])
else:
pl.clear_axes(np.array([ax[i_p + 2, 1]]))
# Third column
order = np.argsort(
np.squeeze(epochs.copy().crop(
t_sig[predictor2][0],
t_sig[predictor2][1]).get_data(picks=ch_name).mean(axis=-1)))
# raster plot
y = epochs.get_data(picks=ch_name)
y = (y - np.median(y)) / y.std()
pl.imshow_plot(
ax[1, 2],
y[order, 0, :],
title=f"{band_name} {predictor2} ordered",
xlims=[model['ep_setup']['tmin'], model['ep_setup']['tmax']],
vlines=[dp['soa'] * x for x in ep_setup['xticks']],
ylabel='epoch',
colorbar='zscore',
vmin=-1,
vmax=1,
cmap='RdBu_r')
for i in range(len(extra_band)):
# raw file
mneraw_file = os.path.join(dp['derivatives_path'],
f"{sub}_band-{extra_band[i]}_raw.fif")
raw = mne.io.read_raw_fif(mneraw_file, preload=True)
all_epochs = cf.load_epochs(dp,
raw.copy().pick_channels([ch_name]),
model['ep_setup'])
epochs = all_epochs[model['query']]
# raster plot
y = epochs.get_data(picks=ch_name)
y = (y - np.median(y)) / y.std()
pl.imshow_plot(
ax[2 + i, 2],
y[order, 0, :],
title=f"{extra_band[i]}",
xlims=[model['ep_setup']['tmin'], model['ep_setup']['tmax']],
vlines=[dp['soa'] * x for x in ep_setup['xticks']],
ylabel='epoch',
colorbar='zscore',
vmin=-1,
vmax=1,
cmap='RdBu_r')
pl.clear_axes(np.array(ax[0, 2]))
fig.subplots_adjust(left=0.1,
bottom=0.1,
right=0.9,
top=0.9,
wspace=0.3,
hspace=0.4)
if save:
#i_band = [band['name'] for band in cfg.bands].index(band_name)
fig_name = os.path.join(
cf.check_path(['..', 'Figures' + cfg.out, f"reg_{model['tag']}"]),
f"{sub}_{ch_name}_{band_name}_temp.pdf")
fig.savefig(fig_name, format='pdf', dpi=100)
if plot: plt.show()
else: plt.close()
def ttest(EpochsSetups,
EOIlabel,
split=None,
subs=cfg.subs,
bands=cfg.bands,
picks='good',
alpha=cfg.alpha,
paired=False,
PopMean=0,
alternative='greater',
Bonferroni=False,
fdr=True,
figure=True):
""" Run ttest on epochs looping over subjects and bands
Use split to run tests on different subsets of the data
Parameters
----------
EpochsSetups : list of dict
split : str or None
separate epochs by values of metadata column 'split',
f.ex. split = 'w_position' run the test independently for each word position
subs : list of str
subject list
bands : list of dict
Outputs
-------
"""
print(
f"\n======== t-test ==================================================\n {EOIlabel}\n"
)
from scipy.stats import ttest_ind, ttest_1samp
# ITERATE SUBJECT LIST AND BAND LIST =======================================
TimeStart = time.time()
iterator = list(product(enumerate(bands), enumerate(subs)))
for i, ((iBand, band), (iSub, sub)) in enumerate(iterator):
cf.display_progress(f"{sub} {band['name']}", i, len(iterator),
TimeStart)
# load subject params
SP = cf.sub_params(sub)
if 'EOI' in picks and SP['ChInfo'][f"{picks}-{band['name']}"] == []:
continue
# colect data
samples, values = get_samples(sub,
band,
EpochsSetups,
split=split,
picks=picks,
paired=paired,
avg=True)
# perform tests
pValues = []
# ---- 2 sample test --------
if len(samples[0]) == 2:
for s1, s2 in samples:
# s1 = [n_observations (i.e. n_epochs), n_channels]
pValues += [
ttest_ind(s1 - PopMean,
s2,
equal_var=False,
alternative=alternative)[1]
]
# ---- 1 sample test --------
if len(samples[0]) == 1:
for s1 in samples:
pValues += [
ttest_1samp(s1[0],
PopMean,
axis=0,
alternative=alternative)[1]
]
pValues = np.array(pValues)
# Bonferroni correction
if split != None and Bonferroni: pValues *= len(samples)
# fdr correction
if fdr:
pValues = mne.stats.fdr_correction(
pValues[:], cfg.alpha)[1].reshape(pValues.shape)
RejectedNull = np.sum(pValues < alpha, axis=0)
# boolean with True for significant chs
SignificantMask = (RejectedNull > 0)
# List of significant ch names
SignificantList = [
ch for j, ch in enumerate(cf.get_picks(SP, picks=picks, band=band))
if SignificantMask[j]
]
# save to ChInfoFile
SP['ChInfo'][f"{EOIlabel}-{band['name']}"] = SignificantList
json.dump(SP['ChInfo'], open(SP['ChInfoFile'], 'w'))
print(
f"\r{sub} {band['name']} {len(SignificantList)}/{len(SignificantMask)} significant channels"
)
if figure:
ttest_figure_signle_sub(SP, band, EpochsSetups, split, samples,
values, SignificantMask, SignificantList,
PopMean, pValues, picks, alpha)
# concatenate for each band
if figure and iSub == len(subs) - 1:
#ttest_figure_all_subs()
# concatenate figures
FiguresPath = cf.check_path(['..', 'Figures', 'ttest' + cfg.out])
FigureName = f"ttest-{EOIlabel}_band-{band['name']}.pdf"
cf.concatenate_pdfs(FiguresPath,
f'sub*temp',
FigureName,
remove=True)