def split_TFR(
BIDS_PATH,
subj,
bloc,
by="VTC",
lobound=None,
hibound=None,
stage="1600TFR",
filt_order=3,
filt_cutoff=0.05,
):
if by == "VTC":
event_id = {"IN": 1, "OUT": 0}
INidx, OUTidx, VTC_epochs, idx_trimmed = get_VTC_epochs(
LOGS_DIR,
subj,
bloc,
lobound=lobound,
hibound=hibound,
stage=stage[:-3] + "epo",
save_epochs=False,
filt_order=filt_order,
filt_cutoff=filt_cutoff,
)
epo_path, epo_filename = get_SAflow_bids(
BIDS_PATH, subj, bloc, stage=stage[:-3] + "epo", cond=None
)
epo_events = mne.read_events(
epo_filename, verbose=False
) # get events from the epochs file (so no resp event)
TFR_path, TFR_filename = get_SAflow_bids(
BIDS_PATH, subj, bloc, stage=stage, cond=None
)
TFR = mne.time_frequency.read_tfrs(TFR_filename)
for i, event in enumerate(epo_events):
if i in INidx:
TFR[0].events[i, 2] = 1
if i in OUTidx:
TFR[0].events[i, 2] = 0
TFR[0].event_id = event_id
return TFR
def load_VTC_data(BIDS_PATH, LOGS_DIR, SUBJ_LIST, BLOCS_LIST):
VTC_alldata = []
for subj in SUBJ_LIST:
all_subj = [] ## all the data of one subject
for run in BLOCS_LIST:
# get events from epochs file
epo_path, epo_filename = get_SAflow_bids(
BIDS_PATH, subj, run, "epo", cond=None
)
events_epoched = mne.read_events(
epo_filename, verbose=False
) # get events from the epochs file (so no resp event)
# get events from original file (only 599 events)
events_fname, events_fpath = get_SAflow_bids(
BIDS_PATH, subj, run, stage="preproc_raw", cond=None
)
raw = read_raw_fif(
events_fpath, preload=False, verbose=False
) # , min_duration=2/epochs.info['sfreq'])
all_events = mne.find_events(
raw, min_duration=2 / raw.info["sfreq"], verbose=False
)
stim_idx = []
for i in range(len(all_events)):
if all_events[i, 2] in [21, 31]:
stim_idx.append(i)
all_events = all_events[stim_idx]
# compute VTC for all trials
log_file = find_logfile(subj, run, os.listdir(LOGS_DIR))
VTC, INbounds, OUTbounds, INzone, OUTzone = get_VTC_from_file(
LOGS_DIR + log_file, lobound=None, hibound=None
)
epochs_VTC = []
for event_time in events_epoched[:, 0]:
idx = list(all_events[:, 0]).index(event_time)
epochs_VTC.append(VTC[idx])
all_subj.append(np.array(epochs_VTC))
VTC_alldata.append(all_subj)
return VTC_alldata
def load_PSD_data(BIDS_PATH, SUBJ_LIST, BLOCS_LIST, time_avg=True, stage="PSD"):
"""
Returns a list containing n_subj lists of n_blocs matrices of shape n_freqs X n_channels X n_trials
"""
PSD_alldata = []
for subj in SUBJ_LIST:
all_subj = [] ## all the data of one subject
for run in BLOCS_LIST:
SAflow_bidsname, SAflow_bidspath = get_SAflow_bids(
BIDS_PATH, subj, run, stage=stage, cond=None
)
mat = loadmat(SAflow_bidspath)["PSD"]
if time_avg == True:
mat = np.mean(mat, axis=2) # average PSDs in time across epochs
all_subj.append(mat)
PSD_alldata.append(all_subj)
return PSD_alldata
def new_split_trials(subj, run, by="VTC", inout_bounds=None):
condA = []
condB = []
if by == "odd":
cond = "5050"
elif by == "VTC" or by == "VTCprec":
cond = f"{inout_bounds[0]}{inout_bounds[1]}"
for idx_freq, freq_bounds in enumerate(FREQS):
_, PSDpath = get_SAflow_bids(
BIDS_PATH,
subj,
run,
stage=f"-epoenv4001200_{FREQS_NAMES[idx_freq]}",
cond=cond,
)
epochs = mne.read_epochs(PSDpath)
if by == "VTC":
condA_epochs = epochs["FreqIN"]
condB_epochs = epochs["FreqOUT"]
elif by == "VTCprec":
epochs_prec = annotate_precursor_events(BIDS_PATH, subj, run)
epochs.events = epochs_prec.events
epochs.event_id = epochs_prec.event_id
condA_epochs = epochs["FreqIN"]
condB_epochs = epochs["FreqOUT"]
elif by == "odd":
condA_epochs = epochs[["FreqIN", "FreqOUT", "FreqHit"]]
condB_epochs = epochs["RareHit"]
condA.append(condA_epochs.get_data())
condB.append(condB_epochs.get_data())
try:
print(np.array(condA).shape)
print(np.array(condB).shape)
condA = np.mean(np.array(condA), axis=3).transpose(1, 2, 0)
condB = np.mean(np.array(condB), axis=3).transpose(1, 2, 0)
except:
breakpoint()
return condA, condB
def annotate_precursor_events(BIDS_PATH, subj, bloc):
_, epopath = get_SAflow_bids(BIDS_PATH,
subj,
bloc,
stage="-epo4001200",
cond=None)
epochs = mne.read_epochs(epopath)
# find events
events = epochs.events
files_list = os.listdir(LOGS_DIR)
logfile = LOGS_DIR + find_logfile(subj, bloc, files_list)
(
IN_idx,
OUT_idx,
VTC_raw,
VTC_filtered,
IN_mask,
OUT_mask,
performance_dict,
df_response,
RT_to_VTC,
) = get_VTC_from_file(subj, bloc, files_list, inout_bounds=[50, 50])
events = annotate_events(logfile, events, inout_idx=[IN_idx, OUT_idx])
event_id = get_present_events(events)
events_precursor = []
for idx, ev in enumerate(events):
if ev[2] == 310:
if events[idx - 1][2] == 2111 or events[idx - 1][2] == 2110:
events_precursor.append(events[idx - 1])
events_precursor = np.array(events_precursor)
epochs.events = events_precursor
epochs.event_id = event_id
return epochs
for run in BLOCS_LIST:
CONDS_LIST = args.split
by = args.by
stage = args.input_stage
if by == "VTC":
INepochs, OUTepochs = new_split_trials(subj=subj,
run=run,
by="VTC",
inout_bounds=CONDS_LIST)
precINepochs, precOUTepochs = new_split_trials(
subj=subj, run=run, by="VTCprec", inout_bounds=CONDS_LIST)
VTCepochs_path, VTCepochs_filename = get_SAflow_bids(
BIDS_PATH,
subj=subj,
run=run,
stage=stage,
cond=f"{CONDS_LIST[0]}{CONDS_LIST[1]}",
)
INepochs_path, INepochs_filename = get_SAflow_bids(
BIDS_PATH,
subj=subj,
run=run,
stage=stage,
cond="IN{}".format(CONDS_LIST[0]),
)
OUTepochs_path, OUTepochs_filename = get_SAflow_bids(
BIDS_PATH,
subj=subj,
run=run,
stage=stage,
from saflow.neuro import saflow_preproc, find_rawfile
from saflow import BIDS_PATH, SUBJ_LIST, BLOCS_LIST
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
"-s",
"--subject",
default='04',
type=str,
help="Subject to process",
)
parser.add_argument(
"-i",
"--ica",
default = True,
type = bool,
help="Preprocessing with or without ica"
)
args = parser.parse_args()
if __name__ == "__main__":
#for subj in SUBJ_LIST:
subj = args.subject
ica = args.ica
for bloc in BLOCS_LIST:
file_path = get_SAflow_bids(BIDS_PATH, subj=subj, run=bloc, stage='raw_ds')[1]
save_path = get_SAflow_bids(BIDS_PATH, subj=subj, run=bloc, stage='preproc_raw')[1]
report_path = get_SAflow_bids(BIDS_PATH, subj=subj, run=bloc, stage='preproc_report')[1]
saflow_preproc(file_path, save_path, report_path, ica=ica)
parser = argparse.ArgumentParser()
parser.add_argument(
"-s",
"--subject",
default="04",
type=str,
help="Subject to process",
)
args = parser.parse_args()
if __name__ == "__main__":
subj = args.subject
for bloc in BLOCS_LIST:
preproc_path, preproc_filename = get_SAflow_bids(BIDS_PATH,
subj,
bloc,
stage="preproc_raw",
cond=None)
epoched_path, epoched_filename = get_SAflow_bids(BIDS_PATH,
subj,
bloc,
stage="-epo4001200",
cond=None)
ARlog_path, ARlog_filename = get_SAflow_bids(BIDS_PATH,
subj,
bloc,
stage="ARlog4001200",
cond=None)
if not os.path.isfile(epoched_filename):
epochs_clean, AR_log = segment_files(preproc_filename,
tmin=0.4,
type=str,
help="Subject to process",
)
args = parser.parse_args()
### OPEN SEGMENTED FILES AND COMPUTE PSDS
if __name__ == "__main__":
subj = args.subject
tmin = 0.4
tmax = 1.2
for bloc in BLOCS_LIST:
# Generate filenames
_, epopath = get_SAflow_bids(BIDS_PATH,
subj,
bloc,
stage="-epo4001200",
cond=None)
_, rawpath = get_SAflow_bids(BIDS_PATH,
subj,
bloc,
stage="preproc_raw",
cond=None)
_, ARpath = get_SAflow_bids(BIDS_PATH,
subj,
bloc,
stage="ARlog4001200",
cond=None)
_, PSDpath = get_SAflow_bids(BIDS_PATH,
subj,
bloc,
# Correction for multiple comparisons
# freq_perms = list(itertools.chain.from_iterable(allchans_accperms))
toplot_featimp = features_importances_list
figpath_featimp = IMG_DIR + classif_name + "_features_importances" + ".png"
freq_titles = []
for idx_pval, pval in enumerate(pvals_list):
if pval < alpha:
freq_titles.append(FREQS_NAMES[idx_pval] + " *")
else:
freq_titles.append(FREQS_NAMES[idx_pval])
titles = freq_titles
_, data_fname = get_SAflow_bids(BIDS_PATH,
subj="04",
run="2",
stage="-epo")
epochs = mne.read_epochs(data_fname)
ch_xy = epochs.pick_types(
meg=True, ref_meg=False).info # Find the channel's position
if "singlefeat" in classif_name:
# Setup the min and max value of the color scale
vmax_pval = np.max(np.max(np.asarray(toplot_pval)))
vmin_pval = np.min(np.min(np.asarray(toplot_pval)))
vmax_acc = np.max(np.max(np.asarray(toplot_acc)))
vmin_acc = np.min(np.min(np.asarray(toplot_acc)))
array_topoplot(
toplot_pval,
ch_xy,
def prepare_data(
BIDS_PATH,
SUBJ_LIST,
BLOCS_LIST,
conds_list,
stage="PSD",
CHAN=0,
FREQ=None,
balance=False,
avg=False,
level="group",
):
# Prepare data
print("Prepping data :")
print("=====SUBJ=======")
print(f"{SUBJ_LIST}")
print("=====BLOCS=======")
print(f"{BLOCS_LIST}")
print("=====FREQ=======")
print(f"{FREQ}")
print("=====CHAN=======")
print(f"{CHAN}")
print("============")
X = []
y = []
groups = []
for i_subj, subj in enumerate(SUBJ_LIST):
for i_cond, cond in enumerate(conds_list):
X_subj = []
for i_run, run in enumerate(BLOCS_LIST):
_, fpath_cond = get_SAflow_bids(BIDS_PATH,
subj,
run,
stage=stage,
cond=cond)
with open(fpath_cond, "rb") as f:
data = pickle.load(f)
if avg:
X_subj.append(np.mean(data[:, CHAN, FREQ], axis=0))
else:
for x in data[:, CHAN, FREQ]:
X.append(x)
y.append(i_cond)
if level == "group":
groups.append(i_subj)
elif level == "subject":
groups.append(i_run)
if avg:
X.append(np.mean(np.array(X_subj), axis=0))
y.append(i_cond)
groups.append(i_subj)
if balance:
X, y, groups = balance_data(X, y, groups)
if type(CHAN) is int and type(FREQ) is int:
X = np.array(X).reshape(-1, 1)
elif len(CHAN) != 1: #
X = np.array(X).reshape(-1, len(X[0])) # ??
y = np.asarray(y)
groups = np.asarray(groups)
return X, y, groups
def split_trials(
BIDS_PATH,
LOGS_DIR,
subj,
run,
stage="PSD",
by="VTC",
keep_errors=False,
equalize=False,
lobound=None,
hibound=None,
filt_order=3,
filt_cutoff=0.05,
freq_names=None,
oddball="hits",
):
"""
If stage is 'env', freq_names must be specified.
"""
# Split epochs indices
if by == "VTC":
INidx, OUTidx, VTC_epochs, idx_trimmed = get_VTC_epochs(
BIDS_PATH,
LOGS_DIR,
subj,
run,
lobound=lobound,
hibound=hibound,
save_epochs=False,
filt_order=filt_order,
filt_cutoff=filt_cutoff,
)
condA_idx = INidx
condB_idx = OUTidx
print("{} IN epochs".format(len(INidx)))
print("{} OUT epochs".format(len(OUTidx)))
elif by == "odd":
freqs_hits_idx, freqs_miss_idx, rares_hits_idx, rares_miss_idx = get_odd_epochs(
BIDS_PATH, LOGS_DIR, subj, run, stage="-epo"
)
if oddball == "all":
condA_idx = np.sort(np.concatenate((freqs_hits_idx, freqs_miss_idx)))
condB_idx = np.sort(np.concatenate((rares_hits_idx, rares_miss_idx)))
elif oddball == "hits":
condA_idx = freqs_hits_idx
condB_idx = rares_hits_idx
elif oddball == "miss":
condA_idx = freqs_miss_idx
condB_idx = rares_miss_idx
elif oddball == "rares":
condA_idx = rares_hits_idx
condB_idx = rares_miss_idx
print("{} condA epochs".format(len(condA_idx)))
print("{} condB epochs".format(len(condB_idx)))
elif by == "resp":
freqs_hits_idx, freqs_miss_idx, rares_hits_idx, rares_miss_idx = get_odd_epochs(
BIDS_PATH, LOGS_DIR, subj, run, stage="-epo"
)
condA_idx = np.sort(np.concatenate((freqs_hits_idx, rares_miss_idx)))
condB_idx = np.sort(np.concatenate((freqs_miss_idx, rares_hits_idx)))
print("{} Resp epochs".format(len(condA_idx)))
print("{} NoResp epochs".format(len(condB_idx)))
condA_idx = [int(x) for x in condA_idx]
condB_idx = [int(x) for x in condB_idx]
# Load epochs
if stage == "PSD":
fname, fpath = get_SAflow_bids(BIDS_PATH, subj, run, stage, cond=None)
with open(fpath, "rb") as f:
data = pickle.load(f)
try:
condA = data[condA_idx]
condB = data[condB_idx]
except IndexError:
print("condB empty")
condB = []
elif "env" in stage:
condA = []
condB = []
for freq in freq_names:
fname, fpath = get_SAflow_bids(BIDS_PATH, subj, run, stage, cond=freq)
data = mne.read_epochs(fpath)
condA.append(data[condA_idx])
condB.append(data[condB_idx])
return condA, condB
def split_PSD_data(
BIDS_PATH,
SUBJ_LIST,
BLOCS_LIST,
by="VTC",
lobound=None,
hibound=None,
stage="PSD",
filt_order=3,
filt_cutoff=0.1,
):
"""
This func splits the PSD data into two conditions. It returns a list of 2 (cond1 and cond2), each containing a list of n_subject matrices of shape n_freqs X n_channels X n_trials
"""
PSD_alldata = load_PSD_data(
BIDS_PATH, SUBJ_LIST, BLOCS_LIST, time_avg=True, stage=stage
)
PSD_cond1 = []
PSD_cond2 = []
for subj_idx, subj in enumerate(SUBJ_LIST):
subj_cond1 = []
subj_cond2 = []
for bloc_idx, bloc in enumerate(BLOCS_LIST):
print("Splitting sub-{}_run-{}".format(subj, bloc))
# Obtain indices of the two conditions
if by == "VTC":
INidx, OUTidx, VTC_epochs, idx_trimmed = get_VTC_epochs(
LOGS_DIR,
subj,
bloc,
lobound=lobound,
hibound=hibound,
save_epochs=False,
filt_order=filt_order,
filt_cutoff=filt_cutoff,
)
cond1_idx = INidx
cond2_idx = OUTidx
if by == "odd":
# Get indices of freq and rare events
ev_fname, ev_fpath = get_SAflow_bids(BIDS_PATH, subj, bloc, stage="epo")
events_artrej = mne.read_events(ev_fpath)
log_file = LOGS_DIR + find_logfile(subj, bloc, os.listdir(LOGS_DIR))
events_fname, events_fpath = get_SAflow_bids(
BIDS_PATH, subj, bloc, stage="preproc_raw", cond=None
)
raw = read_raw_fif(
events_fpath, preload=False, verbose=False
) # , min_duration=2/epochs.info['sfreq'])
try:
events = mne.find_events(
raw, min_duration=1 / raw.info["sfreq"], verbose=False
)
except ValueError:
events = mne.find_events(
raw, min_duration=2 / raw.info["sfreq"], verbose=False
)
events_noerr, events_comerr, events_omerr = remove_errors(
log_file, events
)
events_trimmed, idx_trimmed = trim_events(events_noerr, events_artrej)
cond1_idx = []
cond2_idx = []
for idx, ev in enumerate(events_trimmed):
if ev[2] == 21: # Frequent events
cond1_idx.append(idx)
if ev[2] == 31:
cond2_idx.append(idx)
cond1_idx = np.array(cond1_idx)
cond2_idx = np.array(cond2_idx)
# Add this to keep the same number of trials in both conditions
random.seed(0)
cond1_idx = random.choices(cond1_idx, k=len(cond2_idx))
print(
"N trials retained for each condition : {}".format(len(cond2_idx))
)
# Pick the data of each condition
if bloc_idx == 0: # if first bloc, init ndarray size using the first matrix
subj_cond1 = PSD_alldata[subj_idx][bloc_idx][:, :, cond1_idx]
subj_cond2 = PSD_alldata[subj_idx][bloc_idx][:, :, cond2_idx]
else: # if not first bloc, just concatenate along the trials dimension
subj_cond1 = np.concatenate(
(subj_cond1, PSD_alldata[subj_idx][bloc_idx][:, :, cond1_idx]),
axis=2,
)
subj_cond2 = np.concatenate(
(subj_cond2, PSD_alldata[subj_idx][bloc_idx][:, :, cond2_idx]),
axis=2,
)
PSD_cond1.append(subj_cond1)
PSD_cond2.append(subj_cond2)
splitted_PSD = [PSD_cond1, PSD_cond2]
return splitted_PSD
def get_VTC_epochs(
BIDS_PATH,
LOGS_DIR,
subj,
run,
stage="-epo",
lobound=None,
hibound=None,
save_epochs=False,
filt_order=3,
filt_cutoff=0.1,
):
"""
This functions allows to use the logfile to split the epochs obtained in the epo.fif file.
It works by comparing the timestamps of IN and OUT events to the timestamps in the epo file events
It returns IN and OUT indices that are to be used in the split_PSD_data function
"""
### Get events after artifact rejection have been performed
epo_path, epo_filename = get_SAflow_bids(
BIDS_PATH, subj, run, stage=stage, cond=None
)
events_artrej = mne.read_events(
epo_filename, verbose=False
) # get events from the epochs file (so no resp event)
### Find logfile to extract VTC
behav_list = os.listdir(LOGS_DIR)
log_file = LOGS_DIR + find_logfile(subj, run, behav_list)
(
INidx,
OUTidx,
VTC_raw,
VTC_filtered,
IN_mask,
OUT_mask,
performance_dict,
df_response_out,
) = get_VTC_from_file(
subj, run, behav_list, inout_bounds=[lobound, hibound], filt_cutoff=filt_cutoff
)
### Get original events and split them using the VTC
events_fname, events_fpath = get_SAflow_bids(
BIDS_PATH, subj, run, stage="preproc_raw", cond=None
)
raw = read_raw_fif(
events_fpath, preload=False, verbose=False
) # , min_duration=2/epochs.info['sfreq'])
try:
events = mne.find_events(raw, min_duration=1 / raw.info["sfreq"], verbose=False)
except ValueError:
events = mne.find_events(raw, min_duration=2 / raw.info["sfreq"], verbose=False)
(
events_noerr,
events_comerr,
events_omerr,
events_comcorr,
events_omcorr,
) = remove_errors(log_file, events)
# Keep only events that are correct and clean
events_trimmed, idx_trimmed = trim_events(events_comcorr, events_artrej)
# Write INidx and OUTidx as indices of clean events
INidx, OUTidx = trim_INOUT_idx(INidx, OUTidx, events_trimmed, events)
VTC_epo = np.array([VTC_raw[idx] for idx in idx_trimmed])
return INidx, OUTidx, VTC_epo, idx_trimmed
def get_odd_epochs(BIDS_PATH, LOGS_DIR, subj, bloc, stage="-epo"):
"""
Returns an array of indices of Freqs and Rares epochs. Retains only clean epochs.
"""
### Get events after artifact rejection have been performed
epo_path, epo_filename = get_SAflow_bids(
BIDS_PATH, subj, bloc, stage=stage, cond=None
)
events_artrej = mne.read_events(
epo_filename, verbose=False
) # get events from the epochs file (so no resp event)
### Get original events from the raw file, to compare them to the events left in the epochs file
events_fname, events_fpath = get_SAflow_bids(
BIDS_PATH, subj, bloc, stage="preproc_raw", cond=None
)
raw = read_raw_fif(
events_fpath, preload=False, verbose=False
) # , min_duration=2/epochs.info['sfreq'])
try:
events = mne.find_events(raw, min_duration=1 / raw.info["sfreq"], verbose=False)
except ValueError:
events = mne.find_events(raw, min_duration=2 / raw.info["sfreq"], verbose=False)
# Get the list of hits/miss events
log_file = LOGS_DIR + find_logfile(subj, bloc, os.listdir(LOGS_DIR))
(
events_noerr,
events_comerr,
events_omerr,
events_comcorr,
events_omcorr,
) = remove_errors(log_file, events)
# Keep only events that are clean, and split them by condition
# Start with correct events
events_noerr_trimmed, idx_noerr_trimmed = trim_events(events_noerr, events_artrej)
freqs_hits_idx = np.array(
[idx_noerr_trimmed[i] for i, x in enumerate(events_noerr_trimmed) if x[2] == 21]
)
rares_hits_idx = np.array(
[idx_noerr_trimmed[i] for i, x in enumerate(events_noerr_trimmed) if x[2] == 31]
)
# Then commission errors
if events_comerr.size > 0:
events_comerr_trimmed, idx_comerr_trimmed = trim_events(
events_comerr, events_artrej
)
rares_miss_idx = np.array(idx_comerr_trimmed)
else:
rares_miss_idx = np.array([])
# And finally ommission errors
if events_omerr.size > 0:
events_omerr_trimmed, idx_omerr_trimmed = trim_events(
events_omerr, events_artrej
)
freqs_miss_idx = np.array(idx_omerr_trimmed)
else:
freqs_miss_idx = np.array([])
return freqs_hits_idx, freqs_miss_idx, rares_hits_idx, rares_miss_idx