def _stats(analysis): """2nd order stats across subjects""" # if already computed lets just load it ana_name = 'stats_' + analysis['name'] + '_vhp' if op.exists(paths('score', analysis=ana_name)): return load('score', analysis=ana_name) # gather scores across subjects scores = list() for subject in range(1, 21): kwargs = dict(subject=subject, analysis=analysis['name'] + '_vhp') fname = paths('score', **kwargs) if op.exists(fname): score, times = load(**kwargs) else: score, times = _decod(subject, analysis) scores.append(score) scores = np.array(scores) # compute stats across subjects p_values = stats(scores - analysis['chance']) diag_offdiag = scores - np.tile([np.diag(sc) for sc in scores], [len(times), 1, 1]).transpose(1, 0, 2) p_values_off = stats(diag_offdiag) # Save stats results out = dict(scores=scores, p_values=p_values, p_values_off=p_values_off, times=times, analysis=analysis) save(out, 'score', analysis=ana_name) return out
def _stats(analysis): """2nd order stats across subjects""" # if already computed lets just load it ana_name = 'stats_' + analysis['name'] + '_vhp' if op.exists(paths('score', analysis=ana_name)): return load('score', analysis=ana_name) # gather scores across subjects scores = list() for subject in range(1, 21): kwargs = dict(subject=subject, analysis=analysis['name'] + '_vhp') fname = paths('score', **kwargs) if op.exists(fname): score, times = load(**kwargs) else: score, times = _decod(subject, analysis) scores.append(score) scores = np.array(scores) # compute stats across subjects p_values = stats(scores - analysis['chance']) diag_offdiag = scores - np.tile([np.diag(sc) for sc in scores], [len(times), 1, 1]).transpose(1, 0, 2) p_values_off = stats(diag_offdiag) # Save stats results out = dict(scores=scores, p_values=p_values, p_values_off=p_values_off, times=times, analysis=analysis) save(out, 'score', analysis=ana_name) return out
def _analyze_continuous(analysis): """Regress prediction error as a function of visibility and contrast for each time point""" ana_name = analysis['name'] + '-continuous' # don't recompute if not necessary fname = paths('score', analysis=ana_name) if os.path.exists(fname): return load('score', analysis=ana_name) # gather data n_subject = 20 n_time = 151 scores = dict(visibility=np.zeros((n_subject, n_time, 4)), contrast=np.zeros((n_subject, n_time, 3))) R = dict(visibility=np.zeros((n_subject, n_time)), contrast=np.zeros((n_subject, n_time)),) for s, subject in enumerate(subjects): gat, _, events_sel, events = load('decod', subject=subject, analysis=analysis['name']) events = events.iloc[events_sel].reset_index() y_pred = np.transpose(get_diagonal_ypred(gat), [1, 0, 2])[..., 0] for factor in ['visibility', 'contrast']: # subscore per condition (e.g. each visibility rating) scores[factor][s, :, :] = _subscore(y_pred, events, analysis, factor) # correlate residuals with factor R[factor][s, :] = _subregress(y_pred, events, analysis, factor, True) times = gat.train_times_['times'] save([scores, R, times], 'score', analysis=ana_name, overwrite=True, upload=True) return [scores, R, times]
def _analyze_toi(analysis): """Subscore each analysis as a function of the reported visibility""" ana_name = analysis['name'] + '-toi' # don't recompute if not necessary fname = paths('score', analysis=ana_name) if os.path.exists(fname): return load('score', analysis=ana_name) # gather data n_subject = 20 scores = dict(visibility=np.zeros((n_subject, len(tois), 4)), contrast=np.zeros((n_subject, len(tois), 3))) R = dict(visibility=np.zeros((n_subject, len(tois))), contrast=np.zeros((n_subject, len(tois))),) for s, subject in enumerate(subjects): gat, _, events_sel, events = load('decod', subject=subject, analysis=analysis['name']) events = events.iloc[events_sel].reset_index() for t, toi in enumerate(tois): # Average predictions on single trials across time points y_pred = _average_ypred_toi(gat, toi, analysis) # visibility for factor in ['visibility', 'contrast']: # subscore per condition (e.g. each visibility rating) scores[factor][s, t, :] = _subscore(y_pred, events, analysis, factor) # correlate residuals with factor R[factor][s, t] = _subregress(y_pred, events, analysis, factor, True) save([scores, R], 'score', analysis=ana_name, overwrite=True, upload=True) return [scores, R]
def _check_epochs(subject): """Some triggers values are misread, but it should not concern more than 5 trials""" epochs = load('epochs', subject=subject, preload=False) bhv_fname = paths('behavior', subject=subject) mat = loadmat(bhv_fname, squeeze_me=True, struct_as_record=False) trials = mat["trials"] diff = epochs.events[:, 2] - [trial.ttl_value for trial in trials] assert_true(len(np.unique(diff)) <= 5)
def _check_epochs(subject): """Some triggers values are misread, but it should not concern more than 5 trials""" epochs = load('epochs', subject=subject, preload=False) bhv_fname = paths('behavior', subject=subject) mat = loadmat(bhv_fname, squeeze_me=True, struct_as_record=False) trials = mat["trials"] diff = epochs.events[:, 2] - [trial.ttl_value for trial in trials] assert_true(len(np.unique(diff)) <= 5)
def _epoch_raw(subject, block, overwrite=False): """high pass filter raw data, make consistent channels and epoch.""" # Checks if preprocessing has already been done epo_fname = paths('epo_block', subject=subject, block=block) if op.exists(epo_fname) and not overwrite: return print(subject, block) # Load raw data raw = load('sss', subject=subject, block=block, preload=True) # Explicit picking of channel to ensure same channels across subjects picks = ['STI101', 'EEG060', 'EOG061', 'EOG062', 'ECG063', 'EEG064', 'MISC004'] # Potentially add forgotten channels ch_type = dict(STI='stim', EEG='eeg', EOG='eog', ECG='ecg', MIS='misc') missing_chans = list() for channel in picks: if channel not in raw.ch_names: missing_chans.append(channel) if missing_chans: info = create_info(missing_chans, raw.info['sfreq'], [ch_type[ch[:3]] for ch in missing_chans]) raw.add_channels([RawArray( np.zeros((len(missing_chans), raw.n_times)), info, raw.first_samp)], force_update_info=True) # Select same channels order across subjects picks = [np.where(np.array(raw.ch_names) == ch)[0][0] for ch in picks] picks = np.r_[np.arange(306), picks] # high pass filtering raw.filter(.1, None, l_trans_bandwidth=.05, filter_length='30s', n_jobs=1) # Ensure same sampling rate if raw.info['sfreq'] != 1000.0: raw.resample(1000.0) # Select events events = find_events(raw, stim_channel='STI101', shortest_event=1) sel = np.where(events[:, 2] <= 255)[0] events = events[sel, :] # Compensate for delay (as measured manually with photodiod events[1, :] += .050 * raw.info['sfreq'] # Epoch continuous data epochs = Epochs(raw, events, reject=None, tmin=-.600, tmax=1.8, picks=picks, baseline=None) save(epochs, 'epo_block', subject=subject, block=block, overwrite=True, upload=False)
def _decimate_epochs(subject, overwrite=True): # check overwrite epo_fname = paths('epochs_decim', subject=subject) if op.exists(epo_fname) and not overwrite: return # load non decimated data epochs = load('epochs', subject=subject, preload=True) epochs._data = low_pass_filter(epochs._data, epochs.info['sfreq'], 30., n_jobs=-1) epochs.crop(-.200, 1.600) epochs.decimate(10) save(epochs, 'epochs_decim', subject=subject, overwrite=True, upload=True)
def _concatenate_epochs(subject, overwrite=False): """Concatenate epoched blocks and check that matches with behavior file.""" epo_fname = paths('epochs', subject=subject) if op.exists(epo_fname) and not overwrite: return print(subject) epochs = list() for block in range(1, 6): this_epochs = load('epo_block', subject=subject, block=block, preload=False) epochs.append(this_epochs) epochs = concatenate_epochs(epochs) save(epochs, 'epochs', subject=subject, overwrite=True, upload=False)
def _decod(subject, analysis): from mne.decoding import GeneralizationAcrossTime # if already computed let's just load it from disk fname_kwargs = dict(subject=subject, analysis=analysis['name'] + '_vhp') score_fname = paths('score', **fname_kwargs) if op.exists(score_fname): return load('score', **fname_kwargs) epochs = _get_epochs(subject) events = load('behavior', subject=subject) # Let's not recompute everything, this is just a control analysis print(subject, analysis['name']) epochs._data = epochs.get_data() epochs.preload = True epochs.crop(0., .900) epochs.decimate(2) query, condition = analysis['query'], analysis['condition'] sel = range(len(events)) if query is None else events.query(query).index sel = [ii for ii in sel if ~np.isnan(events[condition][sel][ii])] y = np.array(events[condition], dtype=np.float32) print analysis['name'], np.unique(y[sel]), len(sel) if len(sel) == 0: return # Apply analysis gat = GeneralizationAcrossTime(clf=analysis['clf'], cv=analysis['cv'], scorer=analysis['scorer'], n_jobs=-1) print(subject, analysis['name'], 'fit') gat.fit(epochs[sel], y=y[sel]) print(subject, analysis['name'], 'score') score = gat.score(epochs[sel], y=y[sel]) print(subject, analysis['name'], 'save') # save space gat.estimators_ = None gat.y_pred_ = None # Save analysis save([score, epochs.times], 'score', overwrite=True, upload=True, **fname_kwargs) return score, epochs.times
def _decimate_epochs(subject, overwrite=True): # check overwrite epo_fname = paths('epochs_decim', subject=subject) if op.exists(epo_fname) and not overwrite: return # load non decimated data epochs = load('epochs', subject=subject, preload=True) epochs._data = low_pass_filter(epochs._data, epochs.info['sfreq'], 30., n_jobs=-1) epochs.crop(-.200, 1.600) epochs.decimate(10) save(epochs, 'epochs_decim', subject=subject, overwrite=True, upload=True)
def _concatenate_epochs(subject, overwrite=False): """Concatenate epoched blocks and check that matches with behavior file.""" epo_fname = paths('epochs', subject=subject) if op.exists(epo_fname) and not overwrite: return print(subject) epochs = list() for block in range(1, 6): this_epochs = load('epo_block', subject=subject, block=block, preload=False) epochs.append(this_epochs) epochs = concatenate_epochs(epochs) save(epochs, 'epochs', subject=subject, overwrite=True, upload=False)
def _decod(subject, analysis): from mne.decoding import GeneralizationAcrossTime # if already computed let's just load it from disk fname_kwargs = dict(subject=subject, analysis=analysis['name'] + '_vhp') score_fname = paths('score', **fname_kwargs) if op.exists(score_fname): return load('score', **fname_kwargs) epochs = _get_epochs(subject) events = load('behavior', subject=subject) # Let's not recompute everything, this is just a control analysis print(subject, analysis['name']) epochs._data = epochs.get_data() epochs.preload = True epochs.crop(0., .900) epochs.decimate(2) query, condition = analysis['query'], analysis['condition'] sel = range(len(events)) if query is None else events.query(query).index sel = [ii for ii in sel if ~np.isnan(events[condition][sel][ii])] y = np.array(events[condition], dtype=np.float32) print analysis['name'], np.unique(y[sel]), len(sel) if len(sel) == 0: return # Apply analysis gat = GeneralizationAcrossTime(clf=analysis['clf'], cv=analysis['cv'], scorer=analysis['scorer'], n_jobs=-1) print(subject, analysis['name'], 'fit') gat.fit(epochs[sel], y=y[sel]) print(subject, analysis['name'], 'score') score = gat.score(epochs[sel], y=y[sel]) print(subject, analysis['name'], 'save') # save space gat.estimators_ = None gat.y_pred_ = None # Save analysis save([score, epochs.times], 'score', overwrite=True, upload=True, **fname_kwargs) return score, epochs.times
def _subscore_pipeline(analysis): # FIXME merge with subscore """Subscore each analysis as a function of the reported visibility""" ana_name = analysis['name'] + '-vis' # don't recompute if not necessary fname = paths('score', analysis=ana_name) if os.path.exists(fname): return load('score', analysis=ana_name) # gather data all_scores = list() for subject in subjects: gat, _, events_sel, events = load('decod', subject=subject, analysis=analysis['name']) times = gat.train_times_['times'] # remove irrelevant trials events = events.iloc[events_sel].reset_index() scores = list() gat.score_mode = 'mean-sample-wise' for vis in range(4): sel = np.where(events['detect_button'] == vis)[0] # If target present, we use the AUC against all absent trials if len(sel) < 5: scores.append(np.nan * np.empty(gat.y_pred_.shape[:2])) continue if analysis['name'] == 'target_present': sel = np.r_[sel, np.where( events['target_present'] == False)[0]] # noqa score = subscore(gat, sel) scores.append(score) all_scores.append(scores) all_scores = np.array(all_scores) # stats pval = list() for vis in range(4): pval.append(stats(all_scores[:, vis, :, :] - analysis['chance'])) save([all_scores, pval, times], 'score', analysis=ana_name, overwrite=True, upload=True) return all_scores, pval, times
def _correlate(analysis): """Correlate estimator prediction with a visibility reports""" ana_name = analysis['name'] + '-Rvis' # don't recompute if not necessary fname = paths('score', analysis=ana_name) if os.path.exists(fname): return load('score', analysis=ana_name) # gather data all_R = list() for subject in subjects: gat, _, events_sel, events = load('decod', subject=subject, analysis=analysis['name']) times = gat.train_times_['times'] # remove irrelevant trials events = events.iloc[events_sel].reset_index() y_vis = np.array(events['detect_button']) # only analyse present trials sel = np.where(events['target_present'])[0] y_vis = y_vis[sel] gat.y_pred_ = gat.y_pred_[:, :, sel, :] # make 2D y_pred y_pred = gat.y_pred_.transpose(2, 0, 1, 3)[..., 0] y_pred = y_pred.reshape(len(y_pred), -1) # regress R = repeated_spearman(y_pred, y_vis) # reshape and store R = R.reshape(*gat.y_pred_.shape[:2]) all_R.append(R) all_R = np.array(all_R) # stats pval = stats(all_R) save([all_R, pval, times], 'score', analysis=ana_name, overwrite=True, upload=True) return all_R, pval, times
def _subscore_pipeline(analysis): # FIXME merge with subscore """Subscore each analysis as a function of the reported visibility""" ana_name = analysis['name'] + '-vis' # don't recompute if not necessary fname = paths('score', analysis=ana_name) if os.path.exists(fname): return load('score', analysis=ana_name) # gather data all_scores = list() for subject in subjects: gat, _, events_sel, events = load('decod', subject=subject, analysis=analysis['name']) times = gat.train_times_['times'] # remove irrelevant trials events = events.iloc[events_sel].reset_index() scores = list() gat.score_mode = 'mean-sample-wise' for vis in range(4): sel = np.where(events['detect_button'] == vis)[0] # If target present, we use the AUC against all absent trials if len(sel) < 5: scores.append(np.nan * np.empty(gat.y_pred_.shape[:2])) continue if analysis['name'] == 'target_present': sel = np.r_[sel, np.where(events['target_present'] == False)[0]] # noqa score = subscore(gat, sel) scores.append(score) all_scores.append(scores) all_scores = np.array(all_scores) # stats pval = list() for vis in range(4): pval.append(stats(all_scores[:, vis, :, :] - analysis['chance'])) save([all_scores, pval, times], 'score', analysis=ana_name, overwrite=True, upload=True) return all_scores, pval, times
def _analyze_continuous(analysis): """Regress prediction error as a function of visibility and contrast for each time point""" ana_name = analysis['name'] + '-continuous' # don't recompute if not necessary fname = paths('score', analysis=ana_name) if os.path.exists(fname): return load('score', analysis=ana_name) # gather data n_subject = 20 n_time = 151 scores = dict(visibility=np.zeros((n_subject, n_time, 4)), contrast=np.zeros((n_subject, n_time, 3))) R = dict( visibility=np.zeros((n_subject, n_time)), contrast=np.zeros((n_subject, n_time)), ) for s, subject in enumerate(subjects): gat, _, events_sel, events = load('decod', subject=subject, analysis=analysis['name']) events = events.iloc[events_sel].reset_index() y_pred = np.transpose(get_diagonal_ypred(gat), [1, 0, 2])[..., 0] for factor in ['visibility', 'contrast']: # subscore per condition (e.g. each visibility rating) scores[factor][s, :, :] = _subscore(y_pred, events, analysis, factor) # correlate residuals with factor R[factor][s, :] = _subregress(y_pred, events, analysis, factor, True) times = gat.train_times_['times'] save([scores, R, times], 'score', analysis=ana_name, overwrite=True, upload=True) return [scores, R, times]
def _correlate(analysis): """Correlate estimator prediction with a visibility reports""" ana_name = analysis['name'] + '-Rvis' # don't recompute if not necessary fname = paths('score', analysis=ana_name) if os.path.exists(fname): return load('score', analysis=ana_name) # gather data all_R = list() for subject in subjects: gat, _, events_sel, events = load('decod', subject=subject, analysis=analysis['name']) times = gat.train_times_['times'] # remove irrelevant trials events = events.iloc[events_sel].reset_index() y_vis = np.array(events['detect_button']) # only analyse present trials sel = np.where(events['target_present'])[0] y_vis = y_vis[sel] gat.y_pred_ = gat.y_pred_[:, :, sel, :] # make 2D y_pred y_pred = gat.y_pred_.transpose(2, 0, 1, 3)[..., 0] y_pred = y_pred.reshape(len(y_pred), -1) # regress R = repeated_spearman(y_pred, y_vis) # reshape and store R = R.reshape(*gat.y_pred_.shape[:2]) all_R.append(R) all_R = np.array(all_R) # stats pval = stats(all_R) save([all_R, pval, times], 'score', analysis=ana_name, overwrite=True, upload=True) return all_R, pval, times
def _analyze_toi(analysis): """Subscore each analysis as a function of the reported visibility""" ana_name = analysis['name'] + '-toi' # don't recompute if not necessary fname = paths('score', analysis=ana_name) if os.path.exists(fname): return load('score', analysis=ana_name) # gather data n_subject = 20 scores = dict(visibility=np.zeros((n_subject, len(tois), 4)), contrast=np.zeros((n_subject, len(tois), 3))) R = dict( visibility=np.zeros((n_subject, len(tois))), contrast=np.zeros((n_subject, len(tois))), ) for s, subject in enumerate(subjects): gat, _, events_sel, events = load('decod', subject=subject, analysis=analysis['name']) events = events.iloc[events_sel].reset_index() for t, toi in enumerate(tois): # Average predictions on single trials across time points y_pred = _average_ypred_toi(gat, toi, analysis) # visibility for factor in ['visibility', 'contrast']: # subscore per condition (e.g. each visibility rating) scores[factor][s, t, :] = _subscore(y_pred, events, analysis, factor) # correlate residuals with factor R[factor][s, t] = _subregress(y_pred, events, analysis, factor, True) save([scores, R], 'score', analysis=ana_name, overwrite=True, upload=True) return [scores, R]
def _duration_toi(analysis): """Estimate temporal generalization Re-align on diagonal, average per toi and compute stats.""" ana_name = analysis['name'] + '-duration-toi' if os.path.exists(paths('score', analysis=ana_name)): return load('score', analysis=ana_name) all_scores, _, times = load('score', analysis=analysis['name'] + '-vis') # Add average duration n_subject = len(all_scores) all_score_tois = np.zeros((n_subject, 4, len(tois), len(times))) all_pval_tois = np.zeros((4, len(tois), len(times))) for vis in range(4): scores = all_scores[:, vis, ...] # align score on training time scores = [align_on_diag(score) for score in scores] # center effect scores = np.roll(scores, len(times) // 2, axis=2) for t, toi in enumerate(tois): toi = np.where((times >= toi[0]) & (times <= toi[1]))[0] score_toi = np.mean(scores[:, toi, :], axis=1) all_score_tois[:, vis, t, :] = score_toi all_pval_tois[vis, t, :] = stats(score_toi - analysis['chance']) save([all_score_tois, all_pval_tois, times], 'score', analysis=ana_name) return [all_score_tois, all_pval_tois, times]
def _duration_toi(analysis): """Estimate temporal generalization Re-align on diagonal, average per toi and compute stats.""" ana_name = analysis['name'] + '-duration-toi' if os.path.exists(paths('score', analysis=ana_name)): return load('score', analysis=ana_name) all_scores, _, times = load('score', analysis=analysis['name'] + '-vis') # Add average duration n_subject = len(all_scores) all_score_tois = np.zeros((n_subject, 4, len(tois), len(times))) all_pval_tois = np.zeros((4, len(tois), len(times))) for vis in range(4): scores = all_scores[:, vis, ...] # align score on training time scores = [align_on_diag(score) for score in scores] # center effect scores = np.roll(scores, len(times) // 2, axis=2) for t, toi in enumerate(tois): toi = np.where((times >= toi[0]) & (times <= toi[1]))[0] score_toi = np.mean(scores[:, toi, :], axis=1) all_score_tois[:, vis, t, :] = score_toi all_pval_tois[vis, t, :] = stats(score_toi - analysis['chance']) save([all_score_tois, all_pval_tois, times], 'score', analysis=ana_name) return [all_score_tois, all_pval_tois, times]
from toolbox.utils import find_in_df, build_contrast, save_to_dict from utils import get_data from config import ( data_path, pyoutput_path, subjects, paths('report'), contrasts, open_browser, chan_types, ) report, run_id, _, logger = setup_provenance(script=__file__, results_dir=paths('report')) mne.set_log_level('INFO') # force separation of magnetometers and gradiometers if 'meg' in [i['name'] for i in chan_types]: chan_types = [dict(name='mag'), dict(name='grad')] + \ [dict(name=i['name']) for i in chan_types if i['name'] != 'meg'] for subject in subjects: # Extract events from mat file meg_fname = op.join(data_path, subject, 'preprocessed', subject + '_preprocessed') bhv_fname = op.join(data_path, subject, 'behavior', subject + '_fixed.mat') epochs, events = get_data(meg_fname, bhv_fname)
''' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': config.DATABASE_FILE } } if DEBUG: EMAIL_HOST = 'localhost' EMAIL_PORT = '1025' else: EMAIL_HOST = 'smtp.webfaction.com' EMAIL_HOST_USER = '******' if os.path.exists(config.paths(config.CURR_DIR, 'password.txt')): EMAIL_HOST_PASSWORD = open( config.paths(config.CURR_DIR, 'password.txt'), 'rt').read().strip() else: EMAIL_HOST_PASSWORD = '' # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name # although not all choices may be available on all operating systems. # If running in a Windows environment this must be set to the same as your # system time zone. TIME_ZONE = 'America/Chicago' # Language code for this installation. All choices can be found here: # http://www.i18nguy.com/unicode/language-identifiers.html
def _get_epochs(subject): # if already computed, lets load it from disk epo_fname = paths('epochs_vhp', subject=subject) if op.exists(epo_fname): return load('epochs_vhp', subject=subject, preload=True) # high pass filter and epoch for block in range(1, 6): raw = load('sss', subject=subject, block=block, preload=True) # Explicit picking of channel to ensure same channels across subjects picks = ['STI101', 'EEG060', 'EOG061', 'EOG062', 'ECG063', 'EEG064', 'MISC004'] # Potentially add forgotten channels ch_type = dict(STI='stim', EEG='eeg', EOG='eog', ECG='ecg', MIS='misc') missing_chans = list() for channel in picks: if channel not in raw.ch_names: missing_chans.append(channel) if missing_chans: info = create_info(missing_chans, raw.info['sfreq'], [ch_type[ch[:3]] for ch in missing_chans]) raw.add_channels([RawArray( np.zeros((len(missing_chans), raw.n_times)), info, raw.first_samp)], force_update_info=True) # Select same channels order across subjects picks = [np.where(np.array(raw.ch_names) == ch)[0][0] for ch in picks] picks = np.r_[np.arange(306), picks] # Filtered raw.filter(2, 30, l_trans_bandwidth=.5, filter_length='30s', n_jobs=1) # Ensure same sampling rate if raw.info['sfreq'] != 1000.0: raw.resample(1000.0) # Select events events = find_events(raw, stim_channel='STI101', shortest_event=1) sel = np.where(events[:, 2] <= 255)[0] events = events[sel, :] # Compensate for delay (as measured manually with photodiod events[1, :] += int(.050 * raw.info['sfreq']) # Epoch continuous data this_epochs = Epochs(raw, events, reject=None, tmin=-.200, tmax=1.6, picks=picks, baseline=None, decim=10) save(this_epochs, 'epo_block', subject=subject, block=block) this_epochs._data = None raw.data = None del this_epochs, raw epochs = list() for block in range(1, 6): this_epochs = load('epo_block', subject=subject, block=block) epochs.append(this_epochs) epochs = concatenate_epochs(epochs) # save for faster retest save(epochs, 'epochs_vhp', subject=subject, overwrite=True, upload=False) return epochs
y_pred = list() for train in range(len(gat.y_pred_)): y_pred_ = list() for test in range(len(gat.y_pred_[train])): y_pred__ = list() for c in np.unique(y): m = np.mean(gat.y_pred_[train][test][y == c, :], axis=0) y_pred__.append(m) y_pred_.append(y_pred__) y_pred.append(y_pred_) return y_pred # Setup logs: # mne.set_log_level('INFO') report, run_id, _, logger = setup_provenance( script=__file__, results_dir=paths('report')) for data_type in data_types: # logger.info(data_type) for subscore in subscores: # logger.info(subscore['name']) # Gather subjects data scores_list = list() y_pred_list = list() for subject in subjects: # Load CV data file = pkl_fname(typ, subject, subscore['contrast']) with open(file) as f: gat, _, sel, events = pickle.load(f) # Put back trials predictions in order according to original
def _epoch_raw(subject, block, overwrite=False): """high pass filter raw data, make consistent channels and epoch.""" # Checks if preprocessing has already been done epo_fname = paths('epo_block', subject=subject, block=block) if op.exists(epo_fname) and not overwrite: return print(subject, block) # Load raw data raw = load('sss', subject=subject, block=block, preload=True) # Explicit picking of channel to ensure same channels across subjects picks = [ 'STI101', 'EEG060', 'EOG061', 'EOG062', 'ECG063', 'EEG064', 'MISC004' ] # Potentially add forgotten channels ch_type = dict(STI='stim', EEG='eeg', EOG='eog', ECG='ecg', MIS='misc') missing_chans = list() for channel in picks: if channel not in raw.ch_names: missing_chans.append(channel) if missing_chans: info = create_info(missing_chans, raw.info['sfreq'], [ch_type[ch[:3]] for ch in missing_chans]) raw.add_channels([ RawArray(np.zeros( (len(missing_chans), raw.n_times)), info, raw.first_samp) ], force_update_info=True) # Select same channels order across subjects picks = [np.where(np.array(raw.ch_names) == ch)[0][0] for ch in picks] picks = np.r_[np.arange(306), picks] # high pass filtering raw.filter(.1, None, l_trans_bandwidth=.05, filter_length='30s', n_jobs=1) # Ensure same sampling rate if raw.info['sfreq'] != 1000.0: raw.resample(1000.0) # Select events events = find_events(raw, stim_channel='STI101', shortest_event=1) sel = np.where(events[:, 2] <= 255)[0] events = events[sel, :] # Compensate for delay (as measured manually with photodiod events[1, :] += .050 * raw.info['sfreq'] # Epoch continuous data epochs = Epochs(raw, events, reject=None, tmin=-.600, tmax=1.8, picks=picks, baseline=None) save(epochs, 'epo_block', subject=subject, block=block, overwrite=True, upload=False)
# Author: Jean-Remi King <*****@*****.**> # # Licence: BSD 3-clause """Prepare source analysis for each subject""" import os import numpy as np from mne.io import read_info from config import (paths, load, save, subjects_id, missing_mri, bad_watershed, bad_mri) subjects_dir = paths('freesurfer') os.environ['SUBJECTS_DIR'] = subjects_dir # Anatomy pipeline ------------------------------------------------------------ from jr.meg import anatomy_pipeline if True: # Runs Freesurfer anatomy pipeline for meg_subject, subject in zip(range(1, 21), subjects_id): if subject not in missing_mri + bad_watershed + bad_mri: continue # Check or compute bem segmentation anatomy_pipeline(subject=subject, subjects_dir=subjects_dir, overwrite=False) if True: # Runs MNE/Freesurfer BEM models from mne.bem import (make_bem_model, write_bem_surfaces, make_bem_solution, write_bem_solution) for subject in bad_watershed: bem_dir = os.path.join(subjects_dir, subject, 'bem') bem_fname = os.path.join(bem_dir, subject + '-5120-bem.fif') bem_sol_fname = os.path.join(bem_dir, subject + '-5120-bem-sol.fif')
print(subject, analysis['name'], 'score') score = tfd.score() # Save analysis print(subject, analysis['name'], 'save') if analysis['name'] not in ['target_present', 'target_circAngle']: save([tfd.td.y_pred_, sel, events, epochs.times[decim], frequencies], 'decod_tfr', subject=subject, analysis=analysis['name'], overwrite=True) save([score, epochs.times[decim], frequencies], 'score_tfr', subject=subject, analysis=analysis['name'], overwrite=True) for s, subject in enumerate(subjects): # Loop across each subject print(subject) epochs = load('epochs', subject=subject, preload=False) events = load('behavior', subject=subject) # Apply to each analysis for analysis in analyses: fname = paths('score_tfr', subject=subject, analysis=analysis['name']) if client.metadata(fname)['exist']: continue if not epochs.preload: epochs._data = epochs.get_data() epochs.preload = True epochs.decimate(2) # 500 Hz is enough epochs.pick_types(meg=True, eeg=False, stim=False, eog=False, ecg=False) _run(epochs, events, analysis)
''' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': config.DATABASE_FILE } } if DEBUG: EMAIL_HOST = 'localhost' EMAIL_PORT = '1025' else: EMAIL_HOST = 'smtp.webfaction.com' EMAIL_HOST_USER = '******' if os.path.exists(config.paths(config.CURR_DIR,'password.txt')): EMAIL_HOST_PASSWORD = open(config.paths(config.CURR_DIR,'password.txt'),'rt').read().strip() else: EMAIL_HOST_PASSWORD = '' # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name # although not all choices may be available on all operating systems. # If running in a Windows environment this must be set to the same as your # system time zone. TIME_ZONE = 'America/Chicago' # Language code for this installation. All choices can be found here: # http://www.i18nguy.com/unicode/language-identifiers.html LANGUAGE_CODE = 'en-us'
analysis=analysis['name'], overwrite=True) save([score, epochs.times[decim], frequencies], 'score_tfr', subject=subject, analysis=analysis['name'], overwrite=True) for s, subject in enumerate(subjects): # Loop across each subject print(subject) epochs = load('epochs', subject=subject, preload=False) events = load('behavior', subject=subject) # Apply to each analysis for analysis in analyses: fname = paths('score_tfr', subject=subject, analysis=analysis['name']) if client.metadata(fname)['exist']: continue if not epochs.preload: epochs._data = epochs.get_data() epochs.preload = True epochs.decimate(2) # 500 Hz is enough epochs.pick_types(meg=True, eeg=False, stim=False, eog=False, ecg=False) _run(epochs, events, analysis)
# Author: Jean-Remi King <*****@*****.**> # # Licence: BSD 3-clause """Prepare source analysis for each subject""" import os import numpy as np from mne.io import read_info from config import (paths, load, save, subjects_id, missing_mri, bad_watershed, bad_mri) subjects_dir = paths('freesurfer') os.environ['SUBJECTS_DIR'] = subjects_dir # Anatomy pipeline ------------------------------------------------------------ from jr.meg import anatomy_pipeline if True: # Runs Freesurfer anatomy pipeline for meg_subject, subject in zip(range(1, 21), subjects_id): if subject not in missing_mri + bad_watershed + bad_mri: continue # Check or compute bem segmentation anatomy_pipeline(subject=subject, subjects_dir=subjects_dir, overwrite=False) if True: # Runs MNE/Freesurfer BEM models from mne.bem import (make_bem_model, write_bem_surfaces, make_bem_solution, write_bem_solution) for subject in bad_watershed: bem_dir = os.path.join(subjects_dir, subject, 'bem') bem_fname = os.path.join(bem_dir, subject + '-5120-bem.fif') bem_sol_fname = os.path.join(bem_dir, subject + '-5120-bem-sol.fif')
def _get_epochs(subject): # if already computed, lets load it from disk epo_fname = paths('epochs_vhp', subject=subject) if op.exists(epo_fname): return load('epochs_vhp', subject=subject, preload=True) # high pass filter and epoch for block in range(1, 6): raw = load('sss', subject=subject, block=block, preload=True) # Explicit picking of channel to ensure same channels across subjects picks = [ 'STI101', 'EEG060', 'EOG061', 'EOG062', 'ECG063', 'EEG064', 'MISC004' ] # Potentially add forgotten channels ch_type = dict(STI='stim', EEG='eeg', EOG='eog', ECG='ecg', MIS='misc') missing_chans = list() for channel in picks: if channel not in raw.ch_names: missing_chans.append(channel) if missing_chans: info = create_info(missing_chans, raw.info['sfreq'], [ch_type[ch[:3]] for ch in missing_chans]) raw.add_channels([ RawArray(np.zeros( (len(missing_chans), raw.n_times)), info, raw.first_samp) ], force_update_info=True) # Select same channels order across subjects picks = [np.where(np.array(raw.ch_names) == ch)[0][0] for ch in picks] picks = np.r_[np.arange(306), picks] # Filtered raw.filter(2, 30, l_trans_bandwidth=.5, filter_length='30s', n_jobs=1) # Ensure same sampling rate if raw.info['sfreq'] != 1000.0: raw.resample(1000.0) # Select events events = find_events(raw, stim_channel='STI101', shortest_event=1) sel = np.where(events[:, 2] <= 255)[0] events = events[sel, :] # Compensate for delay (as measured manually with photodiod events[1, :] += int(.050 * raw.info['sfreq']) # Epoch continuous data this_epochs = Epochs(raw, events, reject=None, tmin=-.200, tmax=1.6, picks=picks, baseline=None, decim=10) save(this_epochs, 'epo_block', subject=subject, block=block) this_epochs._data = None raw.data = None del this_epochs, raw epochs = list() for block in range(1, 6): this_epochs = load('epo_block', subject=subject, block=block) epochs.append(this_epochs) epochs = concatenate_epochs(epochs) # save for faster retest save(epochs, 'epochs_vhp', subject=subject, overwrite=True, upload=False) return epochs
import numpy as np from scipy.io import wavfile import SignalUtils as su from os import listdir import random import matplotlib.pyplot as plt from keras.callbacks import EarlyStopping from keras.models import Sequential from keras.layers import Dense from keras import optimizers from config import get_mapping_paths as paths paths_mapping = paths( ) # [("D:/dataset/combine/", 1), ("D:/dataset/other/", 0), ("D:/dataset/voice/", 1)] files_mapping = [] for path_mapping in paths_mapping: files_mapping.extend([(path_mapping[0] + file, path_mapping[1]) for file in listdir(path_mapping[0])]) random.shuffle(files_mapping) test_index = int(0.6 * len(files_mapping)) train_samples = files_mapping[0:test_index] test_samples = files_mapping[test_index:len(files_mapping)] batch_size = 32 look_back = 5 epochs = 200 model_file = "d:/dataset/simple_model.h5" callback = [EarlyStopping(monitor='val_loss', patience=5, mode='auto')] model = Sequential()
import mne from jr.plot import pretty_decod from base import stats from conditions import analyses from config import load, subjects, subjects_id, report # A priori defined ROI rois = ['lingual', 'inferiortemporal', 'superiorparietal', 'rostralmiddlefrontal', 'precentral'] # Copy fsaverage labels for subjects without an mri for subject in missing_mri: to = op.join(paths('freesurfer'), subject, 'label') if not op.exists(to): shutil.copytree(op.join(paths('freesurfer'), 'fsaverage', 'label'), to) sel_analyses = ['target_present', 'target_circAngle', 'detect_button_pst'] analyses = [ana for ana in analyses if ana['name'] in sel_analyses] for analysis in analyses: section = ['sources_%s' % analysis['name']] chance = analysis['chance'] # Read data evokeds = dict() for subject, subject_id in zip(subjects, subjects_id): if subject_id in bad_mri: continue print(subjects) labels = mne.read_labels_from_annot(subject_id, parc='aparc',