def _stats(analysis):
"""2nd order stats across subjects"""
# if already computed lets just load it
ana_name = 'stats_' + analysis['name'] + '_vhp'
if op.exists(paths('score', analysis=ana_name)):
return load('score', analysis=ana_name)
# gather scores across subjects
scores = list()
for subject in range(1, 21):
kwargs = dict(subject=subject, analysis=analysis['name'] + '_vhp')
fname = paths('score', **kwargs)
if op.exists(fname):
score, times = load(**kwargs)
else:
score, times = _decod(subject, analysis)
scores.append(score)
scores = np.array(scores)
# compute stats across subjects
p_values = stats(scores - analysis['chance'])
diag_offdiag = scores - np.tile([np.diag(sc) for sc in scores],
[len(times), 1, 1]).transpose(1, 0, 2)
p_values_off = stats(diag_offdiag)
# Save stats results
out = dict(scores=scores,
p_values=p_values,
p_values_off=p_values_off,
times=times,
analysis=analysis)
save(out, 'score', analysis=ana_name)
return out
def _stats(analysis):
"""2nd order stats across subjects"""
# if already computed lets just load it
ana_name = 'stats_' + analysis['name'] + '_vhp'
if op.exists(paths('score', analysis=ana_name)):
return load('score', analysis=ana_name)
# gather scores across subjects
scores = list()
for subject in range(1, 21):
kwargs = dict(subject=subject, analysis=analysis['name'] + '_vhp')
fname = paths('score', **kwargs)
if op.exists(fname):
score, times = load(**kwargs)
else:
score, times = _decod(subject, analysis)
scores.append(score)
scores = np.array(scores)
# compute stats across subjects
p_values = stats(scores - analysis['chance'])
diag_offdiag = scores - np.tile([np.diag(sc) for sc in scores],
[len(times), 1, 1]).transpose(1, 0, 2)
p_values_off = stats(diag_offdiag)
# Save stats results
out = dict(scores=scores, p_values=p_values, p_values_off=p_values_off,
times=times, analysis=analysis)
save(out, 'score', analysis=ana_name)
return out
def _analyze_continuous(analysis):
"""Regress prediction error as a function of visibility and contrast for
each time point"""
ana_name = analysis['name'] + '-continuous'
# don't recompute if not necessary
fname = paths('score', analysis=ana_name)
if os.path.exists(fname):
return load('score', analysis=ana_name)
# gather data
n_subject = 20
n_time = 151
scores = dict(visibility=np.zeros((n_subject, n_time, 4)),
contrast=np.zeros((n_subject, n_time, 3)))
R = dict(visibility=np.zeros((n_subject, n_time)),
contrast=np.zeros((n_subject, n_time)),)
for s, subject in enumerate(subjects):
gat, _, events_sel, events = load('decod', subject=subject,
analysis=analysis['name'])
events = events.iloc[events_sel].reset_index()
y_pred = np.transpose(get_diagonal_ypred(gat), [1, 0, 2])[..., 0]
for factor in ['visibility', 'contrast']:
# subscore per condition (e.g. each visibility rating)
scores[factor][s, :, :] = _subscore(y_pred, events,
analysis, factor)
# correlate residuals with factor
R[factor][s, :] = _subregress(y_pred, events,
analysis, factor, True)
times = gat.train_times_['times']
save([scores, R, times], 'score', analysis=ana_name,
overwrite=True, upload=True)
return [scores, R, times]
def _analyze_toi(analysis):
"""Subscore each analysis as a function of the reported visibility"""
ana_name = analysis['name'] + '-toi'
# don't recompute if not necessary
fname = paths('score', analysis=ana_name)
if os.path.exists(fname):
return load('score', analysis=ana_name)
# gather data
n_subject = 20
scores = dict(visibility=np.zeros((n_subject, len(tois), 4)),
contrast=np.zeros((n_subject, len(tois), 3)))
R = dict(visibility=np.zeros((n_subject, len(tois))),
contrast=np.zeros((n_subject, len(tois))),)
for s, subject in enumerate(subjects):
gat, _, events_sel, events = load('decod', subject=subject,
analysis=analysis['name'])
events = events.iloc[events_sel].reset_index()
for t, toi in enumerate(tois):
# Average predictions on single trials across time points
y_pred = _average_ypred_toi(gat, toi, analysis)
# visibility
for factor in ['visibility', 'contrast']:
# subscore per condition (e.g. each visibility rating)
scores[factor][s, t, :] = _subscore(y_pred, events,
analysis, factor)
# correlate residuals with factor
R[factor][s, t] = _subregress(y_pred, events,
analysis, factor, True)
save([scores, R], 'score', analysis=ana_name, overwrite=True, upload=True)
return [scores, R]
def _check_epochs(subject):
"""Some triggers values are misread, but it should not concern more than
5 trials"""
epochs = load('epochs', subject=subject, preload=False)
bhv_fname = paths('behavior', subject=subject)
mat = loadmat(bhv_fname, squeeze_me=True, struct_as_record=False)
trials = mat["trials"]
diff = epochs.events[:, 2] - [trial.ttl_value for trial in trials]
assert_true(len(np.unique(diff)) <= 5)
def _check_epochs(subject):
"""Some triggers values are misread, but it should not concern more than
5 trials"""
epochs = load('epochs', subject=subject, preload=False)
bhv_fname = paths('behavior', subject=subject)
mat = loadmat(bhv_fname, squeeze_me=True, struct_as_record=False)
trials = mat["trials"]
diff = epochs.events[:, 2] - [trial.ttl_value for trial in trials]
assert_true(len(np.unique(diff)) <= 5)
def _epoch_raw(subject, block, overwrite=False):
"""high pass filter raw data, make consistent channels and epoch."""
# Checks if preprocessing has already been done
epo_fname = paths('epo_block', subject=subject, block=block)
if op.exists(epo_fname) and not overwrite:
return
print(subject, block)
# Load raw data
raw = load('sss', subject=subject, block=block, preload=True)
# Explicit picking of channel to ensure same channels across subjects
picks = ['STI101', 'EEG060', 'EOG061', 'EOG062', 'ECG063', 'EEG064',
'MISC004']
# Potentially add forgotten channels
ch_type = dict(STI='stim', EEG='eeg', EOG='eog', ECG='ecg',
MIS='misc')
missing_chans = list()
for channel in picks:
if channel not in raw.ch_names:
missing_chans.append(channel)
if missing_chans:
info = create_info(missing_chans, raw.info['sfreq'],
[ch_type[ch[:3]] for ch in missing_chans])
raw.add_channels([RawArray(
np.zeros((len(missing_chans), raw.n_times)), info,
raw.first_samp)], force_update_info=True)
# Select same channels order across subjects
picks = [np.where(np.array(raw.ch_names) == ch)[0][0] for ch in picks]
picks = np.r_[np.arange(306), picks]
# high pass filtering
raw.filter(.1, None, l_trans_bandwidth=.05, filter_length='30s',
n_jobs=1)
# Ensure same sampling rate
if raw.info['sfreq'] != 1000.0:
raw.resample(1000.0)
# Select events
events = find_events(raw, stim_channel='STI101', shortest_event=1)
sel = np.where(events[:, 2] <= 255)[0]
events = events[sel, :]
# Compensate for delay (as measured manually with photodiod
events[1, :] += .050 * raw.info['sfreq']
# Epoch continuous data
epochs = Epochs(raw, events, reject=None, tmin=-.600, tmax=1.8,
picks=picks, baseline=None)
save(epochs, 'epo_block', subject=subject, block=block, overwrite=True,
upload=False)
def _decimate_epochs(subject, overwrite=True):
# check overwrite
epo_fname = paths('epochs_decim', subject=subject)
if op.exists(epo_fname) and not overwrite:
return
# load non decimated data
epochs = load('epochs', subject=subject, preload=True)
epochs._data = low_pass_filter(epochs._data, epochs.info['sfreq'], 30.,
n_jobs=-1)
epochs.crop(-.200, 1.600)
epochs.decimate(10)
save(epochs, 'epochs_decim', subject=subject, overwrite=True, upload=True)
def _concatenate_epochs(subject, overwrite=False):
"""Concatenate epoched blocks and check that matches with behavior file."""
epo_fname = paths('epochs', subject=subject)
if op.exists(epo_fname) and not overwrite:
return
print(subject)
epochs = list()
for block in range(1, 6):
this_epochs = load('epo_block', subject=subject, block=block,
preload=False)
epochs.append(this_epochs)
epochs = concatenate_epochs(epochs)
save(epochs, 'epochs', subject=subject, overwrite=True, upload=False)
def _decod(subject, analysis):
from mne.decoding import GeneralizationAcrossTime
# if already computed let's just load it from disk
fname_kwargs = dict(subject=subject, analysis=analysis['name'] + '_vhp')
score_fname = paths('score', **fname_kwargs)
if op.exists(score_fname):
return load('score', **fname_kwargs)
epochs = _get_epochs(subject)
events = load('behavior', subject=subject)
# Let's not recompute everything, this is just a control analysis
print(subject, analysis['name'])
epochs._data = epochs.get_data()
epochs.preload = True
epochs.crop(0., .900)
epochs.decimate(2)
query, condition = analysis['query'], analysis['condition']
sel = range(len(events)) if query is None else events.query(query).index
sel = [ii for ii in sel if ~np.isnan(events[condition][sel][ii])]
y = np.array(events[condition], dtype=np.float32)
print analysis['name'], np.unique(y[sel]), len(sel)
if len(sel) == 0:
return
# Apply analysis
gat = GeneralizationAcrossTime(clf=analysis['clf'],
cv=analysis['cv'],
scorer=analysis['scorer'],
n_jobs=-1)
print(subject, analysis['name'], 'fit')
gat.fit(epochs[sel], y=y[sel])
print(subject, analysis['name'], 'score')
score = gat.score(epochs[sel], y=y[sel])
print(subject, analysis['name'], 'save')
# save space
gat.estimators_ = None
gat.y_pred_ = None
# Save analysis
save([score, epochs.times],
'score',
overwrite=True,
upload=True,
**fname_kwargs)
return score, epochs.times
def _decimate_epochs(subject, overwrite=True):
# check overwrite
epo_fname = paths('epochs_decim', subject=subject)
if op.exists(epo_fname) and not overwrite:
return
# load non decimated data
epochs = load('epochs', subject=subject, preload=True)
epochs._data = low_pass_filter(epochs._data,
epochs.info['sfreq'],
30.,
n_jobs=-1)
epochs.crop(-.200, 1.600)
epochs.decimate(10)
save(epochs, 'epochs_decim', subject=subject, overwrite=True, upload=True)
def _concatenate_epochs(subject, overwrite=False):
"""Concatenate epoched blocks and check that matches with behavior file."""
epo_fname = paths('epochs', subject=subject)
if op.exists(epo_fname) and not overwrite:
return
print(subject)
epochs = list()
for block in range(1, 6):
this_epochs = load('epo_block',
subject=subject,
block=block,
preload=False)
epochs.append(this_epochs)
epochs = concatenate_epochs(epochs)
save(epochs, 'epochs', subject=subject, overwrite=True, upload=False)
def _decod(subject, analysis):
from mne.decoding import GeneralizationAcrossTime
# if already computed let's just load it from disk
fname_kwargs = dict(subject=subject, analysis=analysis['name'] + '_vhp')
score_fname = paths('score', **fname_kwargs)
if op.exists(score_fname):
return load('score', **fname_kwargs)
epochs = _get_epochs(subject)
events = load('behavior', subject=subject)
# Let's not recompute everything, this is just a control analysis
print(subject, analysis['name'])
epochs._data = epochs.get_data()
epochs.preload = True
epochs.crop(0., .900)
epochs.decimate(2)
query, condition = analysis['query'], analysis['condition']
sel = range(len(events)) if query is None else events.query(query).index
sel = [ii for ii in sel if ~np.isnan(events[condition][sel][ii])]
y = np.array(events[condition], dtype=np.float32)
print analysis['name'], np.unique(y[sel]), len(sel)
if len(sel) == 0:
return
# Apply analysis
gat = GeneralizationAcrossTime(clf=analysis['clf'],
cv=analysis['cv'],
scorer=analysis['scorer'],
n_jobs=-1)
print(subject, analysis['name'], 'fit')
gat.fit(epochs[sel], y=y[sel])
print(subject, analysis['name'], 'score')
score = gat.score(epochs[sel], y=y[sel])
print(subject, analysis['name'], 'save')
# save space
gat.estimators_ = None
gat.y_pred_ = None
# Save analysis
save([score, epochs.times], 'score', overwrite=True, upload=True,
**fname_kwargs)
return score, epochs.times
def _subscore_pipeline(analysis): # FIXME merge with subscore
"""Subscore each analysis as a function of the reported visibility"""
ana_name = analysis['name'] + '-vis'
# don't recompute if not necessary
fname = paths('score', analysis=ana_name)
if os.path.exists(fname):
return load('score', analysis=ana_name)
# gather data
all_scores = list()
for subject in subjects:
gat, _, events_sel, events = load('decod',
subject=subject,
analysis=analysis['name'])
times = gat.train_times_['times']
# remove irrelevant trials
events = events.iloc[events_sel].reset_index()
scores = list()
gat.score_mode = 'mean-sample-wise'
for vis in range(4):
sel = np.where(events['detect_button'] == vis)[0]
# If target present, we use the AUC against all absent trials
if len(sel) < 5:
scores.append(np.nan * np.empty(gat.y_pred_.shape[:2]))
continue
if analysis['name'] == 'target_present':
sel = np.r_[sel,
np.where(
events['target_present'] == False)[0]] # noqa
score = subscore(gat, sel)
scores.append(score)
all_scores.append(scores)
all_scores = np.array(all_scores)
# stats
pval = list()
for vis in range(4):
pval.append(stats(all_scores[:, vis, :, :] - analysis['chance']))
save([all_scores, pval, times],
'score',
analysis=ana_name,
overwrite=True,
upload=True)
return all_scores, pval, times
def _correlate(analysis):
"""Correlate estimator prediction with a visibility reports"""
ana_name = analysis['name'] + '-Rvis'
# don't recompute if not necessary
fname = paths('score', analysis=ana_name)
if os.path.exists(fname):
return load('score', analysis=ana_name)
# gather data
all_R = list()
for subject in subjects:
gat, _, events_sel, events = load('decod',
subject=subject,
analysis=analysis['name'])
times = gat.train_times_['times']
# remove irrelevant trials
events = events.iloc[events_sel].reset_index()
y_vis = np.array(events['detect_button'])
# only analyse present trials
sel = np.where(events['target_present'])[0]
y_vis = y_vis[sel]
gat.y_pred_ = gat.y_pred_[:, :, sel, :]
# make 2D y_pred
y_pred = gat.y_pred_.transpose(2, 0, 1, 3)[..., 0]
y_pred = y_pred.reshape(len(y_pred), -1)
# regress
R = repeated_spearman(y_pred, y_vis)
# reshape and store
R = R.reshape(*gat.y_pred_.shape[:2])
all_R.append(R)
all_R = np.array(all_R)
# stats
pval = stats(all_R)
save([all_R, pval, times],
'score',
analysis=ana_name,
overwrite=True,
upload=True)
return all_R, pval, times
def _subscore_pipeline(analysis): # FIXME merge with subscore
"""Subscore each analysis as a function of the reported visibility"""
ana_name = analysis['name'] + '-vis'
# don't recompute if not necessary
fname = paths('score', analysis=ana_name)
if os.path.exists(fname):
return load('score', analysis=ana_name)
# gather data
all_scores = list()
for subject in subjects:
gat, _, events_sel, events = load('decod', subject=subject,
analysis=analysis['name'])
times = gat.train_times_['times']
# remove irrelevant trials
events = events.iloc[events_sel].reset_index()
scores = list()
gat.score_mode = 'mean-sample-wise'
for vis in range(4):
sel = np.where(events['detect_button'] == vis)[0]
# If target present, we use the AUC against all absent trials
if len(sel) < 5:
scores.append(np.nan * np.empty(gat.y_pred_.shape[:2]))
continue
if analysis['name'] == 'target_present':
sel = np.r_[sel,
np.where(events['target_present'] == False)[0]] # noqa
score = subscore(gat, sel)
scores.append(score)
all_scores.append(scores)
all_scores = np.array(all_scores)
# stats
pval = list()
for vis in range(4):
pval.append(stats(all_scores[:, vis, :, :] - analysis['chance']))
save([all_scores, pval, times],
'score', analysis=ana_name, overwrite=True, upload=True)
return all_scores, pval, times
def _analyze_continuous(analysis):
"""Regress prediction error as a function of visibility and contrast for
each time point"""
ana_name = analysis['name'] + '-continuous'
# don't recompute if not necessary
fname = paths('score', analysis=ana_name)
if os.path.exists(fname):
return load('score', analysis=ana_name)
# gather data
n_subject = 20
n_time = 151
scores = dict(visibility=np.zeros((n_subject, n_time, 4)),
contrast=np.zeros((n_subject, n_time, 3)))
R = dict(
visibility=np.zeros((n_subject, n_time)),
contrast=np.zeros((n_subject, n_time)),
)
for s, subject in enumerate(subjects):
gat, _, events_sel, events = load('decod',
subject=subject,
analysis=analysis['name'])
events = events.iloc[events_sel].reset_index()
y_pred = np.transpose(get_diagonal_ypred(gat), [1, 0, 2])[..., 0]
for factor in ['visibility', 'contrast']:
# subscore per condition (e.g. each visibility rating)
scores[factor][s, :, :] = _subscore(y_pred, events, analysis,
factor)
# correlate residuals with factor
R[factor][s, :] = _subregress(y_pred, events, analysis, factor,
True)
times = gat.train_times_['times']
save([scores, R, times],
'score',
analysis=ana_name,
overwrite=True,
upload=True)
return [scores, R, times]
def _correlate(analysis):
"""Correlate estimator prediction with a visibility reports"""
ana_name = analysis['name'] + '-Rvis'
# don't recompute if not necessary
fname = paths('score', analysis=ana_name)
if os.path.exists(fname):
return load('score', analysis=ana_name)
# gather data
all_R = list()
for subject in subjects:
gat, _, events_sel, events = load('decod', subject=subject,
analysis=analysis['name'])
times = gat.train_times_['times']
# remove irrelevant trials
events = events.iloc[events_sel].reset_index()
y_vis = np.array(events['detect_button'])
# only analyse present trials
sel = np.where(events['target_present'])[0]
y_vis = y_vis[sel]
gat.y_pred_ = gat.y_pred_[:, :, sel, :]
# make 2D y_pred
y_pred = gat.y_pred_.transpose(2, 0, 1, 3)[..., 0]
y_pred = y_pred.reshape(len(y_pred), -1)
# regress
R = repeated_spearman(y_pred, y_vis)
# reshape and store
R = R.reshape(*gat.y_pred_.shape[:2])
all_R.append(R)
all_R = np.array(all_R)
# stats
pval = stats(all_R)
save([all_R, pval, times], 'score', analysis=ana_name,
overwrite=True, upload=True)
return all_R, pval, times
def _analyze_toi(analysis):
"""Subscore each analysis as a function of the reported visibility"""
ana_name = analysis['name'] + '-toi'
# don't recompute if not necessary
fname = paths('score', analysis=ana_name)
if os.path.exists(fname):
return load('score', analysis=ana_name)
# gather data
n_subject = 20
scores = dict(visibility=np.zeros((n_subject, len(tois), 4)),
contrast=np.zeros((n_subject, len(tois), 3)))
R = dict(
visibility=np.zeros((n_subject, len(tois))),
contrast=np.zeros((n_subject, len(tois))),
)
for s, subject in enumerate(subjects):
gat, _, events_sel, events = load('decod',
subject=subject,
analysis=analysis['name'])
events = events.iloc[events_sel].reset_index()
for t, toi in enumerate(tois):
# Average predictions on single trials across time points
y_pred = _average_ypred_toi(gat, toi, analysis)
# visibility
for factor in ['visibility', 'contrast']:
# subscore per condition (e.g. each visibility rating)
scores[factor][s, t, :] = _subscore(y_pred, events, analysis,
factor)
# correlate residuals with factor
R[factor][s, t] = _subregress(y_pred, events, analysis, factor,
True)
save([scores, R], 'score', analysis=ana_name, overwrite=True, upload=True)
return [scores, R]
def _duration_toi(analysis):
"""Estimate temporal generalization
Re-align on diagonal, average per toi and compute stats."""
ana_name = analysis['name'] + '-duration-toi'
if os.path.exists(paths('score', analysis=ana_name)):
return load('score', analysis=ana_name)
all_scores, _, times = load('score', analysis=analysis['name'] + '-vis')
# Add average duration
n_subject = len(all_scores)
all_score_tois = np.zeros((n_subject, 4, len(tois), len(times)))
all_pval_tois = np.zeros((4, len(tois), len(times)))
for vis in range(4):
scores = all_scores[:, vis, ...]
# align score on training time
scores = [align_on_diag(score) for score in scores]
# center effect
scores = np.roll(scores, len(times) // 2, axis=2)
for t, toi in enumerate(tois):
toi = np.where((times >= toi[0]) & (times <= toi[1]))[0]
score_toi = np.mean(scores[:, toi, :], axis=1)
all_score_tois[:, vis, t, :] = score_toi
all_pval_tois[vis, t, :] = stats(score_toi - analysis['chance'])
save([all_score_tois, all_pval_tois, times], 'score', analysis=ana_name)
return [all_score_tois, all_pval_tois, times]
def _duration_toi(analysis):
"""Estimate temporal generalization
Re-align on diagonal, average per toi and compute stats."""
ana_name = analysis['name'] + '-duration-toi'
if os.path.exists(paths('score', analysis=ana_name)):
return load('score', analysis=ana_name)
all_scores, _, times = load('score', analysis=analysis['name'] + '-vis')
# Add average duration
n_subject = len(all_scores)
all_score_tois = np.zeros((n_subject, 4, len(tois), len(times)))
all_pval_tois = np.zeros((4, len(tois), len(times)))
for vis in range(4):
scores = all_scores[:, vis, ...]
# align score on training time
scores = [align_on_diag(score) for score in scores]
# center effect
scores = np.roll(scores, len(times) // 2, axis=2)
for t, toi in enumerate(tois):
toi = np.where((times >= toi[0]) & (times <= toi[1]))[0]
score_toi = np.mean(scores[:, toi, :], axis=1)
all_score_tois[:, vis, t, :] = score_toi
all_pval_tois[vis, t, :] = stats(score_toi - analysis['chance'])
save([all_score_tois, all_pval_tois, times], 'score', analysis=ana_name)
return [all_score_tois, all_pval_tois, times]
from toolbox.utils import find_in_df, build_contrast, save_to_dict
from utils import get_data
from config import (
data_path,
pyoutput_path,
subjects,
paths('report'),
contrasts,
open_browser,
chan_types,
)
report, run_id, _, logger = setup_provenance(script=__file__,
results_dir=paths('report'))
mne.set_log_level('INFO')
# force separation of magnetometers and gradiometers
if 'meg' in [i['name'] for i in chan_types]:
chan_types = [dict(name='mag'), dict(name='grad')] + \
[dict(name=i['name']) for i in chan_types
if i['name'] != 'meg']
for subject in subjects:
# Extract events from mat file
meg_fname = op.join(data_path, subject, 'preprocessed', subject + '_preprocessed')
bhv_fname = op.join(data_path, subject, 'behavior', subject + '_fixed.mat')
epochs, events = get_data(meg_fname, bhv_fname)
'''
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': config.DATABASE_FILE
}
}
if DEBUG:
EMAIL_HOST = 'localhost'
EMAIL_PORT = '1025'
else:
EMAIL_HOST = 'smtp.webfaction.com'
EMAIL_HOST_USER = '******'
if os.path.exists(config.paths(config.CURR_DIR, 'password.txt')):
EMAIL_HOST_PASSWORD = open(
config.paths(config.CURR_DIR, 'password.txt'),
'rt').read().strip()
else:
EMAIL_HOST_PASSWORD = ''
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'
# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
def _get_epochs(subject):
# if already computed, lets load it from disk
epo_fname = paths('epochs_vhp', subject=subject)
if op.exists(epo_fname):
return load('epochs_vhp', subject=subject, preload=True)
# high pass filter and epoch
for block in range(1, 6):
raw = load('sss', subject=subject, block=block, preload=True)
# Explicit picking of channel to ensure same channels across subjects
picks = ['STI101', 'EEG060', 'EOG061', 'EOG062', 'ECG063', 'EEG064',
'MISC004']
# Potentially add forgotten channels
ch_type = dict(STI='stim', EEG='eeg', EOG='eog', ECG='ecg',
MIS='misc')
missing_chans = list()
for channel in picks:
if channel not in raw.ch_names:
missing_chans.append(channel)
if missing_chans:
info = create_info(missing_chans, raw.info['sfreq'],
[ch_type[ch[:3]] for ch in missing_chans])
raw.add_channels([RawArray(
np.zeros((len(missing_chans), raw.n_times)), info,
raw.first_samp)], force_update_info=True)
# Select same channels order across subjects
picks = [np.where(np.array(raw.ch_names) == ch)[0][0] for ch in picks]
picks = np.r_[np.arange(306), picks]
# Filtered
raw.filter(2, 30, l_trans_bandwidth=.5, filter_length='30s',
n_jobs=1)
# Ensure same sampling rate
if raw.info['sfreq'] != 1000.0:
raw.resample(1000.0)
# Select events
events = find_events(raw, stim_channel='STI101', shortest_event=1)
sel = np.where(events[:, 2] <= 255)[0]
events = events[sel, :]
# Compensate for delay (as measured manually with photodiod
events[1, :] += int(.050 * raw.info['sfreq'])
# Epoch continuous data
this_epochs = Epochs(raw, events, reject=None, tmin=-.200, tmax=1.6,
picks=picks, baseline=None, decim=10)
save(this_epochs, 'epo_block', subject=subject, block=block)
this_epochs._data = None
raw.data = None
del this_epochs, raw
epochs = list()
for block in range(1, 6):
this_epochs = load('epo_block', subject=subject, block=block)
epochs.append(this_epochs)
epochs = concatenate_epochs(epochs)
# save for faster retest
save(epochs, 'epochs_vhp', subject=subject, overwrite=True, upload=False)
return epochs
y_pred = list()
for train in range(len(gat.y_pred_)):
y_pred_ = list()
for test in range(len(gat.y_pred_[train])):
y_pred__ = list()
for c in np.unique(y):
m = np.mean(gat.y_pred_[train][test][y == c, :], axis=0)
y_pred__.append(m)
y_pred_.append(y_pred__)
y_pred.append(y_pred_)
return y_pred
# Setup logs:
# mne.set_log_level('INFO')
report, run_id, _, logger = setup_provenance(
script=__file__, results_dir=paths('report'))
for data_type in data_types:
# logger.info(data_type)
for subscore in subscores:
# logger.info(subscore['name'])
# Gather subjects data
scores_list = list()
y_pred_list = list()
for subject in subjects:
# Load CV data
file = pkl_fname(typ, subject, subscore['contrast'])
with open(file) as f:
gat, _, sel, events = pickle.load(f)
# Put back trials predictions in order according to original
def _epoch_raw(subject, block, overwrite=False):
"""high pass filter raw data, make consistent channels and epoch."""
# Checks if preprocessing has already been done
epo_fname = paths('epo_block', subject=subject, block=block)
if op.exists(epo_fname) and not overwrite:
return
print(subject, block)
# Load raw data
raw = load('sss', subject=subject, block=block, preload=True)
# Explicit picking of channel to ensure same channels across subjects
picks = [
'STI101', 'EEG060', 'EOG061', 'EOG062', 'ECG063', 'EEG064', 'MISC004'
]
# Potentially add forgotten channels
ch_type = dict(STI='stim', EEG='eeg', EOG='eog', ECG='ecg', MIS='misc')
missing_chans = list()
for channel in picks:
if channel not in raw.ch_names:
missing_chans.append(channel)
if missing_chans:
info = create_info(missing_chans, raw.info['sfreq'],
[ch_type[ch[:3]] for ch in missing_chans])
raw.add_channels([
RawArray(np.zeros(
(len(missing_chans), raw.n_times)), info, raw.first_samp)
],
force_update_info=True)
# Select same channels order across subjects
picks = [np.where(np.array(raw.ch_names) == ch)[0][0] for ch in picks]
picks = np.r_[np.arange(306), picks]
# high pass filtering
raw.filter(.1, None, l_trans_bandwidth=.05, filter_length='30s', n_jobs=1)
# Ensure same sampling rate
if raw.info['sfreq'] != 1000.0:
raw.resample(1000.0)
# Select events
events = find_events(raw, stim_channel='STI101', shortest_event=1)
sel = np.where(events[:, 2] <= 255)[0]
events = events[sel, :]
# Compensate for delay (as measured manually with photodiod
events[1, :] += .050 * raw.info['sfreq']
# Epoch continuous data
epochs = Epochs(raw,
events,
reject=None,
tmin=-.600,
tmax=1.8,
picks=picks,
baseline=None)
save(epochs,
'epo_block',
subject=subject,
block=block,
overwrite=True,
upload=False)
# Author: Jean-Remi King <*****@*****.**>
#
# Licence: BSD 3-clause
"""Prepare source analysis for each subject"""
import os
import numpy as np
from mne.io import read_info
from config import (paths, load, save, subjects_id,
missing_mri, bad_watershed, bad_mri)
subjects_dir = paths('freesurfer')
os.environ['SUBJECTS_DIR'] = subjects_dir
# Anatomy pipeline ------------------------------------------------------------
from jr.meg import anatomy_pipeline
if True: # Runs Freesurfer anatomy pipeline
for meg_subject, subject in zip(range(1, 21), subjects_id):
if subject not in missing_mri + bad_watershed + bad_mri:
continue
# Check or compute bem segmentation
anatomy_pipeline(subject=subject, subjects_dir=subjects_dir,
overwrite=False)
if True: # Runs MNE/Freesurfer BEM models
from mne.bem import (make_bem_model, write_bem_surfaces,
make_bem_solution, write_bem_solution)
for subject in bad_watershed:
bem_dir = os.path.join(subjects_dir, subject, 'bem')
bem_fname = os.path.join(bem_dir, subject + '-5120-bem.fif')
bem_sol_fname = os.path.join(bem_dir, subject + '-5120-bem-sol.fif')
print(subject, analysis['name'], 'score')
score = tfd.score()
# Save analysis
print(subject, analysis['name'], 'save')
if analysis['name'] not in ['target_present', 'target_circAngle']:
save([tfd.td.y_pred_, sel, events, epochs.times[decim], frequencies],
'decod_tfr',
subject=subject, analysis=analysis['name'], overwrite=True)
save([score, epochs.times[decim], frequencies], 'score_tfr',
subject=subject, analysis=analysis['name'], overwrite=True)
for s, subject in enumerate(subjects): # Loop across each subject
print(subject)
epochs = load('epochs', subject=subject, preload=False)
events = load('behavior', subject=subject)
# Apply to each analysis
for analysis in analyses:
fname = paths('score_tfr', subject=subject, analysis=analysis['name'])
if client.metadata(fname)['exist']:
continue
if not epochs.preload:
epochs._data = epochs.get_data()
epochs.preload = True
epochs.decimate(2) # 500 Hz is enough
epochs.pick_types(meg=True, eeg=False, stim=False,
eog=False, ecg=False)
_run(epochs, events, analysis)
'''
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': config.DATABASE_FILE
}
}
if DEBUG:
EMAIL_HOST = 'localhost'
EMAIL_PORT = '1025'
else:
EMAIL_HOST = 'smtp.webfaction.com'
EMAIL_HOST_USER = '******'
if os.path.exists(config.paths(config.CURR_DIR,'password.txt')):
EMAIL_HOST_PASSWORD = open(config.paths(config.CURR_DIR,'password.txt'),'rt').read().strip()
else:
EMAIL_HOST_PASSWORD = ''
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# If running in a Windows environment this must be set to the same as your
# system time zone.
TIME_ZONE = 'America/Chicago'
# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'
analysis=analysis['name'],
overwrite=True)
save([score, epochs.times[decim], frequencies],
'score_tfr',
subject=subject,
analysis=analysis['name'],
overwrite=True)
for s, subject in enumerate(subjects): # Loop across each subject
print(subject)
epochs = load('epochs', subject=subject, preload=False)
events = load('behavior', subject=subject)
# Apply to each analysis
for analysis in analyses:
fname = paths('score_tfr', subject=subject, analysis=analysis['name'])
if client.metadata(fname)['exist']:
continue
if not epochs.preload:
epochs._data = epochs.get_data()
epochs.preload = True
epochs.decimate(2) # 500 Hz is enough
epochs.pick_types(meg=True,
eeg=False,
stim=False,
eog=False,
ecg=False)
_run(epochs, events, analysis)
# Author: Jean-Remi King <*****@*****.**>
#
# Licence: BSD 3-clause
"""Prepare source analysis for each subject"""
import os
import numpy as np
from mne.io import read_info
from config import (paths, load, save, subjects_id, missing_mri, bad_watershed,
bad_mri)
subjects_dir = paths('freesurfer')
os.environ['SUBJECTS_DIR'] = subjects_dir
# Anatomy pipeline ------------------------------------------------------------
from jr.meg import anatomy_pipeline
if True: # Runs Freesurfer anatomy pipeline
for meg_subject, subject in zip(range(1, 21), subjects_id):
if subject not in missing_mri + bad_watershed + bad_mri:
continue
# Check or compute bem segmentation
anatomy_pipeline(subject=subject,
subjects_dir=subjects_dir,
overwrite=False)
if True: # Runs MNE/Freesurfer BEM models
from mne.bem import (make_bem_model, write_bem_surfaces, make_bem_solution,
write_bem_solution)
for subject in bad_watershed:
bem_dir = os.path.join(subjects_dir, subject, 'bem')
bem_fname = os.path.join(bem_dir, subject + '-5120-bem.fif')
bem_sol_fname = os.path.join(bem_dir, subject + '-5120-bem-sol.fif')
def _get_epochs(subject):
# if already computed, lets load it from disk
epo_fname = paths('epochs_vhp', subject=subject)
if op.exists(epo_fname):
return load('epochs_vhp', subject=subject, preload=True)
# high pass filter and epoch
for block in range(1, 6):
raw = load('sss', subject=subject, block=block, preload=True)
# Explicit picking of channel to ensure same channels across subjects
picks = [
'STI101', 'EEG060', 'EOG061', 'EOG062', 'ECG063', 'EEG064',
'MISC004'
]
# Potentially add forgotten channels
ch_type = dict(STI='stim', EEG='eeg', EOG='eog', ECG='ecg', MIS='misc')
missing_chans = list()
for channel in picks:
if channel not in raw.ch_names:
missing_chans.append(channel)
if missing_chans:
info = create_info(missing_chans, raw.info['sfreq'],
[ch_type[ch[:3]] for ch in missing_chans])
raw.add_channels([
RawArray(np.zeros(
(len(missing_chans), raw.n_times)), info, raw.first_samp)
],
force_update_info=True)
# Select same channels order across subjects
picks = [np.where(np.array(raw.ch_names) == ch)[0][0] for ch in picks]
picks = np.r_[np.arange(306), picks]
# Filtered
raw.filter(2, 30, l_trans_bandwidth=.5, filter_length='30s', n_jobs=1)
# Ensure same sampling rate
if raw.info['sfreq'] != 1000.0:
raw.resample(1000.0)
# Select events
events = find_events(raw, stim_channel='STI101', shortest_event=1)
sel = np.where(events[:, 2] <= 255)[0]
events = events[sel, :]
# Compensate for delay (as measured manually with photodiod
events[1, :] += int(.050 * raw.info['sfreq'])
# Epoch continuous data
this_epochs = Epochs(raw,
events,
reject=None,
tmin=-.200,
tmax=1.6,
picks=picks,
baseline=None,
decim=10)
save(this_epochs, 'epo_block', subject=subject, block=block)
this_epochs._data = None
raw.data = None
del this_epochs, raw
epochs = list()
for block in range(1, 6):
this_epochs = load('epo_block', subject=subject, block=block)
epochs.append(this_epochs)
epochs = concatenate_epochs(epochs)
# save for faster retest
save(epochs, 'epochs_vhp', subject=subject, overwrite=True, upload=False)
return epochs
import numpy as np
from scipy.io import wavfile
import SignalUtils as su
from os import listdir
import random
import matplotlib.pyplot as plt
from keras.callbacks import EarlyStopping
from keras.models import Sequential
from keras.layers import Dense
from keras import optimizers
from config import get_mapping_paths as paths
paths_mapping = paths(
) # [("D:/dataset/combine/", 1), ("D:/dataset/other/", 0), ("D:/dataset/voice/", 1)]
files_mapping = []
for path_mapping in paths_mapping:
files_mapping.extend([(path_mapping[0] + file, path_mapping[1])
for file in listdir(path_mapping[0])])
random.shuffle(files_mapping)
test_index = int(0.6 * len(files_mapping))
train_samples = files_mapping[0:test_index]
test_samples = files_mapping[test_index:len(files_mapping)]
batch_size = 32
look_back = 5
epochs = 200
model_file = "d:/dataset/simple_model.h5"
callback = [EarlyStopping(monitor='val_loss', patience=5, mode='auto')]
model = Sequential()
import mne
from jr.plot import pretty_decod
from base import stats
from conditions import analyses
from config import load, subjects, subjects_id, report
# A priori defined ROI
rois = ['lingual', 'inferiortemporal', 'superiorparietal',
'rostralmiddlefrontal', 'precentral']
# Copy fsaverage labels for subjects without an mri
for subject in missing_mri:
to = op.join(paths('freesurfer'), subject, 'label')
if not op.exists(to):
shutil.copytree(op.join(paths('freesurfer'), 'fsaverage', 'label'), to)
sel_analyses = ['target_present', 'target_circAngle', 'detect_button_pst']
analyses = [ana for ana in analyses if ana['name'] in sel_analyses]
for analysis in analyses:
section = ['sources_%s' % analysis['name']]
chance = analysis['chance']
# Read data
evokeds = dict()
for subject, subject_id in zip(subjects, subjects_id):
if subject_id in bad_mri:
continue
print(subjects)
labels = mne.read_labels_from_annot(subject_id, parc='aparc',
