def run_decoder(
subject,
decoder,
epoch,
hemis=["Pair", "Lateralized", "Averaged"],
ogl=False,
ntasks=n_jobs,
):
"""
Parallelize across areas and hemis.
"""
set_n_threads(1)
from multiprocessing import Pool
from glob import glob
from pymeg import aggregate_sr as asr
from conf_analysis.meg import srtfr
if ogl:
clusters = srtfr.get_ogl_clusters()
filenames = glob(
join(oglinpath, "missing_ogl_S%i_*_%s_agg.hdf" % (subject, epoch)))
else:
clusters = srtfr.get_clusters()
filenames = glob(join(inpath, "S%i_*_%s_agg.hdf" % (subject, epoch)))
print(filenames)
areas = clusters.keys()
areas = [x for x in areas if not x.startswith("NSWFRONT")]
print("Areas:", areas)
meta = augment_meta(preprocessing.get_meta_for_subject(
subject, "stimulus"))
# meta = meta.dropna(subset=['contrast_probe'])
args = []
for area, hemi in product(areas, hemis):
if hemi == "Pair":
area = [area + "_RH", area + "_LH"]
a = (meta, asr.delayed_agg(filenames, hemi=hemi,
cluster=area), decoder, hemi)
args.append(a)
print("Processing %i tasks" % (len(args)))
from contextlib import closing
scores = []
# for arglist in range(0, len(args), ntasks):
with closing(Pool(ntasks, maxtasksperchild=1)) as p:
sc = p.starmap(
_par_apply_decode,
args # ,[arglist : arglist + ntasks]
) # , chunksize=ntasks)
sc = [s for s in sc if s is not None]
scores.extend(sc)
print("Concat ", len(scores))
scores = pd.concat(scores)
filename = get_save_path(subject, decoder, epoch, ogl=ogl)
# outpath + "/concatogl_S%i-%s-%s-decoding.hdf" % (subject, decoder, epoch)
scores.loc[:, "subject"] = subject
print("Saving as ", filename)
scores.to_hdf(filename, "decoding")
p.terminate()
return scores
def get_power(subject,
session=None,
decim=3,
F=None,
folder='/home/nwilming/conf_meg/sr_freq_labeled_three_layer/'):
from os.path import join
sstring = join(folder, 'S%i-SESS*-lcmv.hdf' % (subject))
files = glob(sstring)
if session is not None:
files = [file for file in files if 'SESS%i' % session in file]
if F is not None:
files = [file for file in files if str(F) in file]
print(files)
if len(files) == 0:
raise RuntimeError(
'No files found for sub %i, session %s; \n search path was: %s' %
(subject, str(session), sstring))
df = []
for f in preprocessing.ensure_iter(files):
subject, session, F = find_numbers(f)
d = pd.read_hdf(f)
if d.time.min() < -0.75:
d.time += 0.75
if decim is not None:
def decim_func(df, x):
df = df.reset_index().set_index('time')
df = df.sort_index()
return df.iloc[slice(0, len(df), x)]
d = d.groupby('trial').apply(lambda x: decim_func(x, decim))
del d['trial']
d.loc[:, 'subject'] = subject
d.loc[:, 'session'] = session
d.loc[:, 'F'] = F
# d.loc[:, 'tune'] = tuning
d = d.reset_index().set_index(
['time', 'trial', 'subject', 'session', 'F'])
d = combine_areas(d, hemi=True)
df.append(d)
df = pd.concat(df)
meta = preprocessing.get_meta_for_subject(subject, 'stimulus')
return df, meta
def build_auc_design_matrix(subject,
cluster,
latency,
hemi="Lateralized",
freq_bands=None,
ogl=False):
from glob import glob
from pymeg import aggregate_sr as asr
if not ogl:
filenames = glob(
join(inpath, "S%i_*_%s_agg.hdf" % (subject, "stimulus")))
data = asr.delayed_agg(filenames, hemi=hemi, cluster=cluster)()
else:
filenames = glob(
join(inpath,
"ogl/slimdown/ogl_S%i_*_%s_agg.hdf" % (subject, "stimulus")))
data = asr.delayed_agg(filenames,
hemi="Pair",
cluster=["%s_LH" % cluster,
"%s_RH" % cluster])().copy()
data = data.groupby(["hemi", "trial", "freq"]).mean()
data.head()
hemi = np.asarray(data.index.get_level_values("hemi")).astype("str")
trial = np.asarray(data.index.get_level_values("trial")).astype(int)
freq = np.asarray(data.index.get_level_values("freq")).astype(int)
# cl = hemi.copy()
cl = [cluster] * len(hemi)
index = pd.MultiIndex.from_arrays(
[hemi, cl, trial, freq],
names=["hemi", "cluster", "trial", "freq"])
data.index = index
data.head()
data = pd.pivot_table(data, values=latency, index='trial', columns='freq')
if freq_bands is not None:
data = data.loc[:, freq_bands[0]:freq_bands[1]].mean(1)
trial_index = data.index.get_level_values("trial").unique()
meta = preprocessing.get_meta_for_subject(subject, "stimulus")
meta.set_index("hash", inplace=True)
meta = meta.loc[trial_index]
return data, meta
def get_ortho_midc(subject):
"""
Compute orthogonalized decoding values for IPS/PostCeS.
"""
from pymeg import aggregate_sr as asr
from glob import glob
inpath = "/home/nwilming/conf_meg/sr_labeled/aggs"
cluster = "JWG_M1"
# First load low level averaged stimulus data
filenames = glob(join(inpath, "S%i_*_%s_agg.hdf" % (subject, "stimulus")))
data = asr.delayed_agg(filenames,
hemi="Lateralized",
cluster=[cluster] + ["JWG_IPS_PCeS"])()
meta = augment_meta(preprocessing.get_meta_for_subject(
subject, "stimulus"))
scores = ortho_midc_decoder(meta,
data,
"JWG_IPS_PCeS",
"JWG_M1",
latency=1.1)
scores.loc[:, "subject"] = subject
return scores
def get_contrasts(contrasts, subject, baseline_per_condition=False, scratch=False):
if subject <= 8:
hemi = "rh_is_ipsi"
else:
hemi = "lh_is_ipsi"
hemis = [hemi, "avg"]
new_contrasts = {}
for key, value in contrasts.items():
new_contrasts[key+'lat'] = [value[0], value[1], hemi]
new_contrasts[key+'avg'] = [value[0], value[1], 'avg']
contrasts = new_contrasts
print(contrasts)
from os.path import join
stim, resp = [], []
for session in range(0, 4):
stim.append(
join(data_path, "sr_labeled/S%i-SESS%i-stimulus*.hdf" % (subject, session))
)
resp.append(
join(data_path, "sr_labeled/S%i-SESS%i-response*.hdf" % (subject, session))
)
if scratch:
from subprocess import run
import os
tmpdir = os.environ["TMPDIR"]
command = "cp {stim} {tmpdir} & cp {resp} {tmpdir}".format(
stim=stim, resp=resp, tmpdir=tmpdir
)
logging.info("Copying data with following command: %s" % command)
p = run(command, shell=True, check=True)
stim = join(data_path, tmpdir, "S%i-SESS%i-stimulus*.hdf" % (subject, session))
resp = join(data_path, tmpdir, "S%i-SESS%i-response*.hdf" % (subject, session))
logging.info("Copied data")
meta = preprocessing.get_meta_for_subject(subject, "stimulus")
response_left = meta.response == 1
stimulus = meta.side == 1
meta = augment_data(meta, response_left, stimulus)
meta["high_conf_high_contrast"] = (meta.confidence == 2) & (meta.mc > 0.5)
meta["high_conf_low_contrast"] = (meta.confidence == 2) & (meta.mc <= 0.5)
meta["low_conf_high_contrast"] = (meta.confidence == 1) & (meta.mc > 0.5)
meta["low_conf_low_contrast"] = (meta.confidence == 1) & (meta.mc <= 0.5)
cps = []
with Cache() as cache:
try:
contrast = compute_contrast(
contrasts,
#hemis,
stim,
stim,
meta,
(-0.25, 0),
baseline_per_condition=baseline_per_condition,
n_jobs=1,
cache=cache,
all_clusters=get_clusters(),
)
contrast.loc[:, "epoch"] = "stimulus"
cps.append(contrast)
except ValueError as e:
# No objects to concatenate
print(e)
pass
try:
contrast = compute_contrast(
contrasts,
#hemis,
resp,
stim,
meta,
(-0.25, 0),
baseline_per_condition=baseline_per_condition,
n_jobs=1,
cache=cache,
)
contrast.loc[:, "epoch"] = "response"
cps.append(contrast)
except ValueError as e:
# No objects to concatenate
print(e)
pass
contrast = pd.concat(cps)
del cps
contrast.loc[:, "subject"] = subject
contrast.set_index(
["subject", "contrast", "hemi", "epoch", "cluster"], append=True, inplace=True
)
return contrast
def decode(
subject,
area,
epoch_type="stimulus",
only_glasser=False,
BEM="three_layer",
debug=False,
target="response",
):
mne.set_log_level("WARNING")
pymeglcmv.logging.getLogger().setLevel(logging.INFO)
set_n_threads(1)
labels = get_labels(subject, only_glasser)
labels = [x for x in labels if any([cl for cl in areas_to_labels[area] if cl in x.name])]
print(labels)
if len(labels) < 1:
raise RuntimeError('Expecting at least two labels')
label = labels.pop()
for l in labels:
label += l
#label = labels[0] + labels[1]
print('Selecting this label for area %s:'%area, label)
#return
logging.info("Reading stimulus data")
if epoch_type == "stimulus":
# data_cov, epochs = get_stim_epoch(subject, session)
data = [get_stim_epoch(subject, i) for i in range(4)]
elif epoch_type == "response":
# data_cov, epochs = get_response_epoch(subject, session)
data = [get_response_epoch(subject, i) for i in range(4)]
else:
raise RuntimeError("Did not recognize epoch")
logging.info("Setting up source space and forward model")
fwds = [get_leadfield(subject, session, BEM)[0] for session in range(4)]
fois = np.arange(10, 150, 5)
lfois = np.arange(1, 10, 1)
tfr_params = {
"F": {
"foi": fois,
"cycles": fois * 0.1,
"time_bandwidth": 2,
"n_jobs": 1,
"est_val": fois,
"est_key": "F",
},
"LF": {
"foi": lfois,
"cycles": lfois * 0.25,
"time_bandwidth": 2,
"n_jobs": 1,
"est_val": lfois,
"est_key": "LF",
},
}
events = [d[1].events[:, 2] for d in data]
filters = []
for (data_cov, epochs), forward in zip(data, fwds):
filters.append(
pymeglcmv.setup_filters(epochs.info, forward, data_cov, None, [label])
)
set_n_threads(1)
F_tfrdata, events, F_freq, times = decoding.get_lcmv(
tfr_params["F"], [d[1] for d in data], filters, njobs=6
)
LF_tfrdata, events, LF_freq, times = decoding.get_lcmv(
tfr_params["LF"], [d[1] for d in data], filters, njobs=6
)
tfrdata = np.hstack([F_tfrdata, LF_tfrdata])
del LF_tfrdata, F_tfrdata
freq = np.concatenate([F_freq, LF_freq])
meta = augment_meta(
preprocessing.get_meta_for_subject(
subject, epoch_type, sessions=range(4)
).set_index("hash")
)
# Kick out trials with RT < 0.225
choice_rt = meta.choice_rt
valid_trials = choice_rt[choice_rt>=0.225].index.values
valid_trials = np.isin(events, valid_trials)
tfrdata = tfrdata[valid_trials]
events = events[valid_trials]
# How many kicked out?
n_out = (~valid_trials).sum()
n_all = len(events)
print('Kicking out %i/%i (%0.2f percent) trials due to RT'%(n_out, n_all, n_out/n_all))
all_s = []
for target in ["response", "unsigned_confidence", "signed_confidence"]:
fname = "/home/nwilming/S%i_trg%s_ROI_%s.hdf" % (subject, target, area)
try:
k = pd.read_hdf(fname)
except FileNotFoundError:
target_vals = meta.loc[:, target]
dcd = decoding.Decoder(target_vals)
k = dcd.classify(
tfrdata, times, freq, events, area,
average_vertices=False, use_phase=True
)
k.loc[:, "target"] = target
k.to_hdf(fname, "df")
all_s.append(k)
all_s = pd.concat(all_s)
all_s.loc[:, 'ROI'] = area
all_s.to_hdf("/home/nwilming/S%i_ROI_%s.hdf" % (subject, area), "df")
return k
def motor_decoder(
subject,
low_level_data,
low_level_area,
motor_data,
motor_area,
low_level_peak,
target_col="response",
save_filename=None,
save_prefix=None,
add_meta={},
odd_times=None,
):
"""
This signal predicts motor activity from a set of time points in early visual cortex.
The idea is to train to decoders simultaneously:
1) Train a motor decoder and predict log(probability) of a choice. Do this
on the training set.
2) Train a decoder from early visual cortex data that predicts the
log(probability) of a choice. Train on training set, evaluate on test.
"""
meta = augment_meta(preprocessing.get_meta_for_subject(
subject, "stimulus"))
meta = meta.set_index("hash")
low_level_data = low_level_data()
motor_data = motor_data()
# Iterate over all high level time points:
times = motor_data.columns.get_level_values("time").values
dt = np.diff(times)[0]
t_idx = (-0.4 < times) & (times < 0.1)
cnt = 0
time_points = times[t_idx]
if odd_times is not None:
if odd_times:
time_points = time_points[:-1:2]
else:
time_points = time_points[1::2]
for high_level_latency in time_points:
print("High level latency:", high_level_latency)
target_time_point = times[np.argmin(abs(times - high_level_latency))]
md = prep_motor_data(motor_area, motor_data, target_time_point)
# Buld target vector
motor_target = (meta.loc[md.index, target_col]).astype(int)
all_scores = []
for low_level_latency in np.arange(-0.1, 0.2 + dt, dt):
lld_data = prep_low_level_data(low_level_area, low_level_data,
low_level_peak, low_level_latency)
# low_level_meta = meta.loc[lld_data.index, :]
assert all(lld_data.index == motor_target.index)
scores = chained_categorize(motor_target, md, lld_data)
scores.loc[:, "low_level_latency"] = low_level_latency
all_scores.append(scores)
save_all_scores(
all_scores,
add_meta,
save_filename,
save_prefix,
cnt,
high_level_latency,
low_level_peak,
low_level_area,
motor_area,
)
cnt += 1
del all_scores
del md
del motor_target
del motor_data
del low_level_data