def test_n_components_and_max_pca_components_none():
"""Test n_components and max_pca_components=None."""
raw = read_raw_fif(raw_fname).crop(1.5, stop).load_data()
events = read_events(event_name)
picks = pick_types(raw.info, eeg=True, meg=False)
epochs = Epochs(raw,
events,
event_id,
tmin,
tmax,
picks=picks,
baseline=(None, 0),
preload=True)
max_pca_components = None
n_components = None
random_state = 12345
tempdir = _TempDir()
output_fname = op.join(tempdir, 'test_ica-ica.fif')
ica = ICA(max_pca_components=max_pca_components,
n_components=n_components,
random_state=random_state)
with warnings.catch_warnings(record=True): # convergence
ica.fit(epochs)
ica.save(output_fname)
ica = read_ica(output_fname)
# ICA.fit() replaced max_pca_components, which was previously None,
# with the appropriate integer value.
assert_equal(ica.max_pca_components, epochs.info['nchan'])
assert_is_none(ica.n_components)
def clean_fif(fif_path, ica_sol_path, ica_bads_path, cleaned_fif_path):
raw = read_raw_fif(fif_path, preload=True)
ica = read_ica(ica_sol_path)
ica.exclude = read_ica_bads(ica_bads_path, logger)
logger.info(f"Excluding ICs {ica.exclude}")
ica.apply(raw)
raw.save(cleaned_fif_path, overwrite=True)
def apply_ICA(subject, runlist, badEEG, badICA):
from mne.io.pick import _picks_by_type as picks_by_type
from mne.preprocessing import read_ica
from mne.io import Raw
data_path = '/neurospin/meg/meg_tmp/MTT_MEG_Baptiste/MEG/'
for run in runlist:
raw_fname = (data_path + subject + '/' + run + '_trans_sss.fif')
# read raw data
raw = []
raw = Raw(raw_fname, preload=True)
raw.info['bads'] = badEEG
# Load ICA file for each data type (MEG and EEG)
for ch_type, picks in picks_by_type(raw.info, meg_combined=True):
# Read ICA file
ica = []
ica = read_ica(data_path + subject + '/mne_python/ICA/ICA_' +
ch_type + '_allRuns')
# Apply ICA projection
ica.exclude += badICA[ch_type]
raw = ica.apply(raw, copy=True)
# Interpolate bad EEG channels
raw = raw.copy().interpolate_bads()
# Save artifacts-corrected raw data
raw.save((data_path + subject + '/' + run + 'ICAcorr_trans_sss.fif'),
overwrite='True')
def test_ica_n_iter_(method):
"""Test that ICA.n_iter_ is set after fitting."""
_skip_check_picard(method)
raw = read_raw_fif(raw_fname).crop(0.5, stop).load_data()
n_components = 3
max_iter = 1
ica = ICA(n_components=n_components, max_iter=max_iter, method=method)
if method == 'infomax':
ica.fit(raw)
else:
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(raw)
assert_equal(ica.n_iter_, max_iter)
# Test I/O roundtrip.
tempdir = _TempDir()
output_fname = op.join(tempdir, 'test_ica-ica.fif')
_assert_ica_attributes(ica)
ica.save(output_fname)
ica = read_ica(output_fname)
_assert_ica_attributes(ica)
assert_equal(ica.n_iter_, max_iter)
def test_n_components_none():
"""Test n_components=None."""
raw = read_raw_fif(raw_fname).crop(1.5, stop).load_data()
events = read_events(event_name)
picks = pick_types(raw.info, eeg=True, meg=False)
epochs = Epochs(raw, events, event_id, tmin, tmax, picks=picks,
baseline=(None, 0), preload=True)
max_pca_components = 10
n_components = None
random_state = 12345
tempdir = _TempDir()
output_fname = op.join(tempdir, 'test_ica-ica.fif')
ica = ICA(max_pca_components=max_pca_components,
n_components=n_components, random_state=random_state)
with warnings.catch_warnings(record=True): # convergence
ica.fit(epochs)
ica.save(output_fname)
ica = read_ica(output_fname)
# ICA.fit() replaced max_pca_components, which was previously None,
# with the appropriate integer value.
assert_equal(ica.max_pca_components, 10)
assert_is_none(ica.n_components)
def test_fit_params(method, tmpdir):
"""Test fit_params for ICA."""
_skip_check_picard(method)
fit_params = {}
ICA(fit_params=fit_params, method=method) # test no side effects
assert fit_params == {}
# Test I/O roundtrip.
# Only picard and infomax support the "extended" keyword, so limit the
# tests to those.
if method in ['picard', 'infomax']:
tmpdir = str(tmpdir)
output_fname = op.join(tmpdir, 'test_ica-ica.fif')
raw = read_raw_fif(raw_fname).crop(0.5, stop).load_data()
n_components = 3
max_iter = 1
fit_params = dict(extended=True)
ica = ICA(fit_params=fit_params, n_components=n_components,
max_iter=max_iter, method=method)
fit_params_after_instantiation = ica.fit_params
if method == 'infomax':
ica.fit(raw)
else:
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(raw)
ica.save(output_fname)
ica = read_ica(output_fname)
assert ica.fit_params == fit_params_after_instantiation
def load_saved_icas(dir_icas):
icas = []
for dir, _, file in os.walk(dir_icas):
for fi in file:
ica = read_ica(os.path.join(dir, fi))
icas.append(ica)
return icas
def test_n_components_and_max_pca_components_none(method):
"""Test n_components and max_pca_components=None."""
_skip_check_picard(method)
raw = read_raw_fif(raw_fname).crop(1.5, stop).load_data()
events = read_events(event_name)
picks = pick_types(raw.info, eeg=True, meg=False)
epochs = Epochs(raw, events, event_id, tmin, tmax, picks=picks,
baseline=(None, 0), preload=True)
max_pca_components = None
n_components = None
random_state = 12345
tempdir = _TempDir()
output_fname = op.join(tempdir, 'test_ica-ica.fif')
ica = ICA(max_pca_components=max_pca_components, method=method,
n_components=n_components, random_state=random_state)
with pytest.warns(None): # convergence
ica.fit(epochs)
ica.save(output_fname)
ica = read_ica(output_fname)
# ICA.fit() replaced max_pca_components, which was previously None,
# with the appropriate integer value.
assert_equal(ica.max_pca_components, epochs.info['nchan'])
assert ica.n_components is None
def test_max_pca_components_none(method):
"""Test max_pca_components=None."""
_skip_check_picard(method)
raw = read_raw_fif(raw_fname).crop(1.5, stop).load_data()
events = read_events(event_name)
picks = pick_types(raw.info, eeg=True, meg=False)
epochs = Epochs(raw,
events,
event_id,
tmin,
tmax,
picks=picks,
baseline=(None, 0),
preload=True)
max_pca_components = None
n_components = 10
random_state = 12345
tempdir = _TempDir()
output_fname = op.join(tempdir, 'test_ica-ica.fif')
ica = ICA(max_pca_components=max_pca_components,
method=method,
n_components=n_components,
random_state=random_state)
with pytest.warns(None):
ica.fit(epochs)
ica.save(output_fname)
ica = read_ica(output_fname)
# ICA.fit() replaced max_pca_components, which was previously None,
# with the appropriate integer value.
assert_equal(ica.max_pca_components, epochs.info['nchan'])
assert_equal(ica.n_components, 10)
def inspect_ica(filt_path, ica_sol_path, ica_bads_path):
raw = read_raw_fif(filt_path, preload=True)
ica = read_ica(ica_sol_path)
if ica_bads_path.exists():
ica.exclude = read_ica_bads(ica_bads_path, logger)
ica.plot_sources(raw, block=True)
logger.info(f"Excluding ICs {ica.exclude}")
write_ica_bads(ica_bads_path, ica, logger)
def browse_components(subject, runlist, badEEG):
import numpy as np
import mne
import os.path as op
import os
from mne.report import Report
from mne.io.pick import _picks_by_type as picks_by_type
from mne.preprocessing import read_ica
from mne.io import Raw
data_path = '/neurospin/meg/meg_tmp/MTT_MEG_Baptiste/MEG/'
report = Report(subject)
raw_fnames = []
for run in runlist:
raw_fnames.append(data_path + subject + '/' + run + '_trans_sss.fif')
# read raw data
raw = Raw(raw_fnames, preload=True)
raw.info['bads'] = badEEG
# Highpass filter 1Hz on EOG/ECG channels
#picks = mne.pick_types(raw.info, meg=False,eeg=False, eog=True, ecg=True)
#raw.filter(l_freq = 1, h_freq=30, picks = picks)
#picks = mne.pick_types(raw.info, meg=True,eeg=True, eog=True, ecg=True)
#raw.filter(l_freq = None, h_freq=30, picks = picks,n_jobs = 4)
picks = mne.pick_types(raw.info, meg=False, eeg=False, eog=False, ecg=True)
raw.filter(l_freq=1, h_freq=30, picks=picks)
picks = mne.pick_types(raw.info, meg=False, eeg=False, eog=True, ecg=False)
raw.filter(l_freq=1, h_freq=5, picks=picks)
picks = mne.pick_types(raw.info, meg=True, eeg=True, eog=True, ecg=True)
raw.filter(l_freq=None, h_freq=30, picks=picks, n_jobs=4)
results_dir = op.join(data_path, subject, 'artefactICA')
if not op.exists(results_dir):
os.makedirs(results_dir)
# Plot all ICA component
for ch_type, picks in picks_by_type(
raw.info, meg_combined=True): # bad EEG channels are excluded
# Read ICA file for each data type (MEG and EEG) - created in preprocessing_ica
ica = read_ica(data_path + subject + '/mne_python/ICA/ICA_' + ch_type +
'_allRuns')
print subject + ' ' + ch_type
#fig = ica.plot_sources(raw, block = True) # the viewer will open and ICA component can be screened
#report.add_figs_to_section(fig, 'All ICA sources (%s)' % ch_type,'ICA decomposition')
fig = ica.plot_components(colorbar=True)
for i in range(np.shape(fig)[0]):
report.add_figs_to_section(fig[i],
'All ICA components (%s)' % ch_type,
'ICA decomposition')
report.save((results_dir + '/AllRuns_checkcomponents.html'),
open_browser=False,
overwrite=True)
del raw
def load_ica(subject, description, ica_data_root=None):
if ica_data_root is None:
# use default data root
import deepthought
data_root = os.path.join(deepthought.DATA_PATH, 'OpenMIIR')
ica_data_root = os.path.join(data_root, 'eeg', 'preprocessing', 'ica')
ica_filepath = os.path.join(ica_data_root,
'{}-{}-ica.fif'.format(subject, description))
return read_ica(ica_filepath)
def load_ica(subject, description, ica_data_root=None):
if ica_data_root is None:
# use default data root
import deepthought
data_root = os.path.join(deepthought.DATA_PATH, 'OpenMIIR')
ica_data_root = os.path.join(data_root, 'eeg', 'preprocessing', 'ica')
ica_filepath = os.path.join(ica_data_root,
'{}-{}-ica.fif'.format(subject, description))
return read_ica(ica_filepath)
def loading_icas ():
icas = []
for subj in range (len(subjects)):
icas_names = os.path.join(dir_icas,'S'+ str(subjects[subj]) + '_ica.fif')
loaded_ica = read_ica(icas_names[subj])
icas.append(loaded_ica)
return icas
def get_ica(subject):
"""
a function to get the ica corresponding to a subject ID
parameters:
-----------
- subject : str
correspond to the ID of the subject
returns: mne ICA object
-------
"""
path_ica = os.path.join(ICA_DIR, f'P{subject}-100p_64c-ica.fif')
return read_ica(path_ica)
def reconstructICA(subjectID):
subject = 'dh{:#02d}a'.format(subjectID)
logFile = open(outPath + subject + '/removedICAcomponents.log', 'w')
for block in ['1','2']:
logFile.write('Processing subject ' + subject + ', block ' + block + '\n')
subjectStub = subject + '/block' + block
raw_fname = rawPath + subjectStub + '.fif'
icaFile = rawPath + subjectStub + '_ica.fif'
outFile = outPath + subjectStub + '.fif'
if not os.path.isdir(outPath + subject):
os.makedirs(outPath + subject)
if os.path.exists(raw_fname):
raw = Raw(raw_fname, preload=True)
raw.info['bads'] = badChanLibrary[subjectID][int(block)]
ica = read_ica(icaFile)
#raw.ch_names = ica.ch_names
# Select and filter interesting channels
picks_eog = mne.pick_types(raw.info, meg=False, eeg=False, stim=False, eog=True, ecg=False)
picks_ecg = mne.pick_types(raw.info, meg=False, eeg=False, stim=False, eog=False, ecg=True)
logFile.write('Rejecting components that correlate with EOG and ECG channels...\n')
excluded_components = []
eogChannel = picks_eog[1]
eogNames = raw.ch_names[eogChannel]
eog_idx, eog_scores = ica.find_bads_eog(raw, eogNames)
for i in eog_idx:
if i not in excluded_components:
excluded_components.append(i)
logFile.write("Excluding component {}, Correlation with EOG: {} \n".format(i, eog_scores[i]))
ecgChannel = picks_ecg
ecgNames = raw.ch_names[ecgChannel]
ecg_idx, ecg_scores = ica.find_bads_ecg(raw, ecgNames)
for i in ecg_idx:
if i not in excluded_components:
excluded_components.append(i)
logFile.write("Excluding component {}, Correlation with ECG: {} \n".format(i, ecg_scores[i]))
# Save corrected fif file
if len(excluded_components) > 0:
logFile.write('Saving corrected fif file\n')
ica.apply(raw, exclude=excluded_components)
raw.save(outFile, overwrite=True)
else:
logFile.write('Nothing to do; creating symlink\n')
force_symlink(raw_fname, outFile)
logFile.close()
def cli(subjdirs):
"""
Show the variance explained for mne ica solution
EXAMPLES:
Show explained variances for ica solutions for each subject in FIF_DATASET and write them to file vars.txt:
$ ica_var FIF_DATASET/*/*-ica.fif >> vars.txt
"""
for fname in subjdirs:
with nostdout():
ica = read_ica(fname)
n_comp = ica.n_components_
tot_var = ica.pca_explained_variance_.sum()
n_comp_var = ica.pca_explained_variance_[:n_comp].sum()
PVE = n_comp_var / tot_var
click.echo(PVE)
def cli(subjdirs):
"""
Show the variance explained for mne ica solution
EXAMPLES:
Show explained variances for ica solutions for each subject in FIF_DATASET and write them to file vars.txt:
$ ica_var FIF_DATASET/*/*-ica.fif >> vars.txt
"""
for fname in subjdirs:
with nostdout():
ica = read_ica(fname)
n_comp = ica.n_components_
tot_var = ica.pca_explained_variance_.sum()
n_comp_var = ica.pca_explained_variance_[:n_comp].sum()
PVE = n_comp_var / tot_var
click.echo(PVE)
def preprocess_set_ICA_comp_fif_to_ts(fif_file, subject_id, n_comp_exclude,
is_sensor_space):
import os
import sys
import mne
from mne.preprocessing import read_ica
from nipype.utils.filemanip import split_filename as split_f
from ephypype.preproc import create_ts
subj_path, basename, ext = split_f(fif_file)
(data_path, sbj_name) = os.path.split(subj_path)
print(('*** SBJ %s' % subject_id + '***'))
# Read raw
current_dir = os.getcwd()
if os.path.exists(os.path.join(current_dir, '../ica', basename + '_ica' + ext)):
raw_ica_file = os.path.join(
current_dir, '../ica', basename + '_ica' + ext)
elif os.path.exists(os.path.join(current_dir, '../ica', basename + '_filt_ica' + ext)):
raw_ica_file = os.path.join(
current_dir, '../ica', basename + '_filt_ica' + ext)
elif os.path.exists(os.path.join(current_dir, '../ica', basename + '_filt_dsamp_ica' + ext)):
raw_ica_file = os.path.join(
current_dir, '../ica', basename + '_filt_dsamp_ica' + ext)
print(('*** raw_ica_file %s' % raw_ica_file + '***'))
raw = mne.io.read_raw_fif(raw_ica_file, preload=True)
# load ICA
if os.path.exists(os.path.join(current_dir, '../ica', basename + '_ica_solution.fif')):
ica_sol_file = os.path.join(
current_dir, '../ica', basename + '_ica_solution.fif')
elif os.path.exists(os.path.join(current_dir, '../ica', basename + '_filt_ica_solution.fif')):
ica_sol_file = os.path.join(
current_dir, '../ica', basename + '_filt_ica_solution.fif')
elif os.path.exists(os.path.join(current_dir, '../ica', basename + '_filt_dsamp_ica_solution.fif')):
ica_sol_file = os.path.join(
current_dir, '../ica', basename + '_filt_dsamp_ica_solution.fif')
if os.path.exists(ica_sol_file) is False:
print(('$$$ Warning, no %s found' % ica_sol_file))
sys.exit()
else:
ica = read_ica(ica_sol_file)
print(('\n *** ica.exclude before set components= ', ica.exclude))
if subject_id in n_comp_exclude:
print(('*** ICA to be excluded for sbj %s ' % subject_id))
print((' ' + str(n_comp_exclude[subject_id]) + '***'))
session_dict = n_comp_exclude[subject_id]
session_names = list(session_dict.keys())
componentes = []
for s in session_names:
componentes = session_dict[s]
if len(componentes) == 0:
print('\n no ICA to be excluded \n')
else:
print(('\n *** ICA to be excluded for session %s ' % s +
' ' + str(componentes) + ' *** \n'))
ica.exclude = componentes
print(('\n *** ica.exclude after set components = ', ica.exclude))
# apply ICA to raw data
new_raw_ica_file = os.path.join(subj_path, basename + '_ica-raw.fif')
raw_ica = ica.apply(raw)
raw_ica.save(new_raw_ica_file, overwrite=True)
# save ICA solution
print(ica_sol_file)
ica.save(ica_sol_file)
ts_file, channel_coords_file, channel_names_file, raw.info['sfreq'] = create_ts(
new_raw_ica_file)
if is_sensor_space:
return ts_file, channel_coords_file, channel_names_file, raw.info['sfreq']
else:
return raw_ica, channel_coords_file, channel_names_file, raw.info['sfreq']
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Apr 10 01:58:13 2020
@author: wexu
"""
import mne
import numpy as np
from config_SYN_Human import study_path, exp
import os
from mne.preprocessing import read_ica
for ID in exp.index:
fname = os.path.join(study_path, ID + '.edf')
ica = read_ica(fname.replace(".edf", "-ica.fif"))
epochs = mne.read_epochs(fname.replace(".edf", "_cleaned-epo.fif"))
ica.plot_sources(epochs, block=True)
print(ica.exclude)
ica.plot_components(inst=epochs)
print(ica.exclude)
ica.plot_sources(epochs, block=True)
np.save(fname.replace(".edf", 'ICA_excludes.npy'), ica.exclude)
def apply_ica(*, cfg, subject, session):
bids_basename = BIDSPath(subject=subject,
session=session,
task=cfg.task,
acquisition=cfg.acq,
run=None,
recording=cfg.rec,
space=cfg.space,
datatype=cfg.datatype,
root=cfg.deriv_root,
check=False)
fname_epo_in = bids_basename.copy().update(suffix='epo', extension='.fif')
fname_epo_out = bids_basename.copy().update(processing='ica',
suffix='epo',
extension='.fif')
fname_ica = bids_basename.copy().update(suffix='ica', extension='.fif')
fname_ica_components = bids_basename.copy().update(processing='ica',
suffix='components',
extension='.tsv')
report_fname = (bids_basename.copy().update(processing='ica',
suffix='report',
extension='.html'))
title = f'ICA artifact removal – sub-{subject}'
if session is not None:
title += f', ses-{session}'
if cfg.task is not None:
title += f', task-{cfg.task}'
# Load ICA.
msg = f'Reading ICA: {fname_ica}'
logger.debug(
**gen_log_kwargs(message=msg, subject=subject, session=session))
ica = read_ica(fname=fname_ica)
# Select ICs to remove.
tsv_data = pd.read_csv(fname_ica_components, sep='\t')
ica.exclude = (tsv_data.loc[tsv_data['status'] == 'bad',
'component'].to_list())
# Load epochs to reject ICA components.
msg = f'Input: {fname_epo_in}, Output: {fname_epo_out}'
logger.info(
**gen_log_kwargs(message=msg, subject=subject, session=session))
epochs = mne.read_epochs(fname_epo_in, preload=True)
epochs.drop_bad(cfg.ica_reject)
# Now actually reject the components.
msg = f'Rejecting ICs: {", ".join([str(ic) for ic in ica.exclude])}'
logger.info(
**gen_log_kwargs(message=msg, subject=subject, session=session))
epochs_cleaned = ica.apply(epochs.copy()) # Copy b/c works in-place!
msg = 'Saving reconstructed epochs after ICA.'
logger.info(
**gen_log_kwargs(message=msg, subject=subject, session=session))
epochs_cleaned.save(fname_epo_out, overwrite=True, split_naming='bids')
# Compare ERP/ERF before and after ICA artifact rejection. The evoked
# response is calculated across ALL epochs, just like ICA was run on
# all epochs, regardless of their respective experimental condition.
#
# We apply baseline correction here to (hopefully!) make the effects of
# ICA easier to see. Otherwise, individual channels might just have
# arbitrary DC shifts, and we wouldn't be able to easily decipher what's
# going on!
report = Report(report_fname, title=title, verbose=False)
picks = ica.exclude if ica.exclude else None
report.add_ica(ica=ica,
title='Effects of ICA cleaning',
inst=epochs.copy().apply_baseline(cfg.baseline),
picks=picks)
report.save(report_fname, overwrite=True, open_browser=cfg.interactive)
raw = mne.io.read_raw_fif(run_fname, preload=True, add_eeg_ref=False)
###############################################################################
# We change the channel type for ECG and EOG.
raw.set_channel_types({'EEG061': 'eog', 'EEG062': 'eog', 'EEG063': 'ecg',
'EEG064': 'misc'}) # EEG064 free floating el.
raw.rename_channels({'EEG061': 'EOG061', 'EEG062': 'EOG062',
'EEG063': 'ECG063'})
###############################################################################
# Bad sensors are repaired.
raw.info['bads'] = bads
raw.interpolate_bads()
raw.set_eeg_reference()
###############################################################################
# Now let's get to some serious ICA preprocessing
ica_name = op.join(meg_dir, subject, 'run_%02d-ica.fif' % run)
ica = read_ica(ica_name)
n_max_ecg = 3 # use max 3 components
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5)
ecg_inds, scores_ecg = ica.find_bads_ecg(ecg_epochs, method='ctps',
threshold=0.8)
ica.plot_sources(raw, exclude=ecg_inds)
ica.plot_scores(scores_ecg, exclude=ecg_inds)
ica.plot_properties(raw, ecg_inds)
ica.exclude += ecg_inds[:n_max_ecg]
for sub_string in sub_list:
# Define the Subject ID and paths
deriv_path = deriv_dir / sub_string
print(f'Preprocessing task-{task} data for {sub_string}')
print(f' Derivatives Folder: {deriv_path}')
# Load ICA Epochs
epochs_fif_file = deriv_path / \
f'{sub_string}_task-{task}_ref-FCz_desc-ica_epo.fif.gz'
epochs = read_epochs(epochs_fif_file)
# Load ICA
ica_file = deriv_path / \
f'{sub_string}_task-{task}_ref-FCz_desc-ica_ica.fif.gz'
ica = read_ica(ica_file)
# Plot ICA
ica.plot_components(inst=epochs, reject=None, psd_args=dict(fmax=70))
ica.exclude.sort()
ica.save(ica_file)
print(f'ICs Flagged for Removal: {ica.exclude}')
# Load json file for updating
json_file = deriv_path / \
f'{sub_string}_task-{task}_ref-FCz_desc-ica_ica.json'
with open(json_file, 'r') as f:
json_info = json.load(f)
# Only update description, proportion of flagged components,
# and flagged components
cond_names = ad.analysis_dict[subj][input_files].keys()
# sort names so that VS comes before FFA!
cond_names.sort(reverse=True)
for cond in cond_names:
if 'empty' not in cond:
raw_path = ad._scratch_folder + '/' + input_files + '/' + subj
in_fnames = ad.analysis_dict[subj][input_files][cond]['files']
for fname in in_fnames:
img_prefix = img_folder + '/' + cond
print 'In: ', fname
raw = Raw(fname, preload=True) # for finding events from raw, must be preloaded
ica = read_ica(ica_folder + '/' + cond + '-ica.fif')
# 2) identify bad components by analyzing latent sources.
title = 'Sources related to %s artifacts (red)'
# generate ECG epochs use detection via phase statistics
picks = mne.pick_types(raw.info, meg=True, eeg=False, eog=True, ecg=True, stim=False, exclude='bads')
# create_ecg_epochs is strange: it strips the channels of anything non M/EEG
# UNLESS picks=None
#picks=None
# This will work with the above, but uses MASSIVE RAM
# Not sure the ECG quality is good enough for the QRS-detector
ecg_inds, scores = ica.find_bads_ecg(raw, method='ctps', ch_name='ECG002', threshold=0.25)
if len(ecg_inds) < 1:
# destroy the ECG channel by changing it to an EMG!
fname = op.join(MEG_data_path, subject,
task + '_%d' % (day + subject_id) + '_tsss_mc.fif')
raw_tsss_mc = mne.io.read_raw_fif(fname, preload=True)
if op.isfile(fname.replace("tsss_mc", "annot")):
raw_tsss_mc.set_annotations(
mne.read_annotations(fname.replace("tsss_mc", "annot")))
print("Annotion loaded!")
raw_tsss_mc.filter(l_freq=1,
h_freq=40.0,
fir_design='firwin',
n_jobs=-1) # band-pass filter data
ica = read_ica(
op.join(MEG_data_path, subject,
task + '_%d' % (day + subject_id) + '-ica.fif'))
ICA_reject_threshold = np.load(
fname.replace("tsss_mc.fif", 'ICA_reject_threshold.npy'))
if op.isfile(fname.replace("tsss_mc.fif", 'ICA_excludes.npy')):
ica.exclude.extend(
np.load(fname.replace("tsss_mc.fif",
'ICA_excludes.npy')).tolist())
events = pd.read_csv(fname.replace('tsss_mc.fif', 'events.csv'))[[
"stim_onset", "button_press", "trigger_code"
]].astype(int).values
epochs = mne.Epochs(raw_tsss_mc,
events=events,
tmin=-0.2,
tmax=1,
def main():
#################################################
## SETUP
## Get list of subject files
subj_files = listdir(DAT_PATH)
subj_files = [file for file in subj_files if EXT.lower() in file.lower()]
## Set up FOOOF Objects
# Initialize FOOOF settings & objects objects
fooof_settings = FOOOFSettings(peak_width_limits=PEAK_WIDTH_LIMITS, max_n_peaks=MAX_N_PEAKS,
min_peak_amplitude=MIN_PEAK_AMP, peak_threshold=PEAK_THRESHOLD,
aperiodic_mode=APERIODIC_MODE)
fm = FOOOF(*fooof_settings, verbose=False)
fg = FOOOFGroup(*fooof_settings, verbose=False)
# Save out a settings file
fg.save('0-FOOOF_Settings', pjoin(RES_PATH, 'FOOOF'), save_settings=True)
# Set up the dictionary to store all the FOOOF results
fg_dict = dict()
for load_label in LOAD_LABELS:
fg_dict[load_label] = dict()
for side_label in SIDE_LABELS:
fg_dict[load_label][side_label] = dict()
for seg_label in SEG_LABELS:
fg_dict[load_label][side_label][seg_label] = []
## Initialize group level data stores
n_subjs, n_conds, n_times = len(subj_files), 3, N_TIMES
group_fooofed_alpha_freqs = np.zeros(shape=[n_subjs])
dropped_components = np.ones(shape=[n_subjs, 50]) * 999
dropped_trials = np.ones(shape=[n_subjs, 1500]) * 999
canonical_group_avg_dat = np.zeros(shape=[n_subjs, n_conds, n_times])
fooofed_group_avg_dat = np.zeros(shape=[n_subjs, n_conds, n_times])
# Set channel types
ch_types = {'LHor' : 'eog', 'RHor' : 'eog', 'IVer' : 'eog', 'SVer' : 'eog',
'LMas' : 'misc', 'RMas' : 'misc', 'Nose' : 'misc', 'EXG8' : 'misc'}
#################################################
## RUN ACROSS ALL SUBJECTS
# Run analysis across each subject
for s_ind, subj_file in enumerate(subj_files):
# Get subject label and print status
subj_label = subj_file.split('.')[0]
print('\nCURRENTLY RUNNING SUBJECT: ', subj_label, '\n')
#################################################
## LOAD / ORGANIZE / SET-UP DATA
# Load subject of data, apply apply fixes for channels, etc
eeg_dat = mne.io.read_raw_edf(pjoin(DAT_PATH, subj_file),
preload=True, verbose=False)
# Fix channel name labels
eeg_dat.info['ch_names'] = [chl[2:] for chl in \
eeg_dat.ch_names[:-1]] + [eeg_dat.ch_names[-1]]
for ind, chi in enumerate(eeg_dat.info['chs']):
eeg_dat.info['chs'][ind]['ch_name'] = eeg_dat.info['ch_names'][ind]
# Update channel types
eeg_dat.set_channel_types(ch_types)
# Set reference - average reference
eeg_dat = eeg_dat.set_eeg_reference(ref_channels='average',
projection=False, verbose=False)
# Set channel montage
chs = mne.channels.read_montage('standard_1020', eeg_dat.ch_names)
eeg_dat.set_montage(chs)
# Get event information & check all used event codes
evs = mne.find_events(eeg_dat, shortest_event=1, verbose=False)
# Pull out sampling rate
srate = eeg_dat.info['sfreq']
#################################################
## Pre-Processing: ICA
# High-pass filter data for running ICA
eeg_dat.filter(l_freq=1., h_freq=None, fir_design='firwin')
if RUN_ICA:
print("\nICA: CALCULATING SOLUTION\n")
# ICA settings
method = 'fastica'
n_components = 0.99
random_state = 47
reject = {'eeg': 20e-4}
# Initialize ICA object
ica = ICA(n_components=n_components, method=method,
random_state=random_state)
# Fit ICA
ica.fit(eeg_dat, reject=reject)
# Save out ICA solution
ica.save(pjoin(RES_PATH, 'ICA', subj_label + '-ica.fif'))
# Otherwise: load previously saved ICA to apply
else:
print("\nICA: USING PRECOMPUTED\n")
ica = read_ica(pjoin(RES_PATH, 'ICA', subj_label + '-ica.fif'))
# Find components to drop, based on correlation with EOG channels
drop_inds = []
for chi in EOG_CHS:
inds, _ = ica.find_bads_eog(eeg_dat, ch_name=chi, threshold=2.5,
l_freq=1, h_freq=10, verbose=False)
drop_inds.extend(inds)
drop_inds = list(set(drop_inds))
# Set which components to drop, and collect record of this
ica.exclude = drop_inds
dropped_components[s_ind, 0:len(drop_inds)] = drop_inds
# Apply ICA to data
eeg_dat = ica.apply(eeg_dat)
#################################################
## SORT OUT EVENT CODES
# Extract a list of all the event labels
all_trials = [it for it2 in EV_DICT.values() for it in it2]
# Create list of new event codes to be used to label correct trials (300s)
all_trials_new = [it + 100 for it in all_trials]
# This is an annoying way to collapse across the doubled event markers from above
all_trials_new = [it - 1 if not ind%2 == 0 else it for ind, it in enumerate(all_trials_new)]
# Get labelled dictionary of new event names
ev_dict2 = {k:v for k, v in zip(EV_DICT.keys(), set(all_trials_new))}
# Initialize variables to store new event definitions
evs2 = np.empty(shape=[0, 3], dtype='int64')
lags = np.array([])
# Loop through, creating new events for all correct trials
t_min, t_max = -0.4, 3.0
for ref_id, targ_id, new_id in zip(all_trials, CORR_CODES * 6, all_trials_new):
t_evs, t_lags = mne.event.define_target_events(evs, ref_id, targ_id, srate,
t_min, t_max, new_id)
if len(t_evs) > 0:
evs2 = np.vstack([evs2, t_evs])
lags = np.concatenate([lags, t_lags])
#################################################
## FOOOF
# Set channel of interest
ch_ind = eeg_dat.ch_names.index(CHL)
# Calculate PSDs over ~ first 2 minutes of data, for specified channel
fmin, fmax = 1, 50
tmin, tmax = 5, 125
psds, freqs = mne.time_frequency.psd_welch(eeg_dat, fmin=fmin, fmax=fmax,
tmin=tmin, tmax=tmax,
n_fft=int(2*srate), n_overlap=int(srate),
n_per_seg=int(2*srate),
verbose=False)
# Fit FOOOF across all channels
fg.fit(freqs, psds, FREQ_RANGE, n_jobs=-1)
# Save out FOOOF results
fg.save(subj_label + '_fooof', pjoin(RES_PATH, 'FOOOF'), save_results=True)
# Extract individualized CF from specified channel, add to group collection
fm = fg.get_fooof(ch_ind, False)
fooof_freq, _, _ = get_band_peak(fm.peak_params_, [7, 14])
group_fooofed_alpha_freqs[s_ind] = fooof_freq
# If not FOOOF alpha extracted, reset to 10
if np.isnan(fooof_freq):
fooof_freq = 10
#################################################
## ALPHA FILTERING
# CANONICAL: Filter data to canonical alpha band: 8-12 Hz
alpha_dat = eeg_dat.copy()
alpha_dat.filter(8, 12, fir_design='firwin', verbose=False)
alpha_dat.apply_hilbert(envelope=True, verbose=False)
# FOOOF: Filter data to FOOOF derived alpha band
fooof_dat = eeg_dat.copy()
fooof_dat.filter(fooof_freq-2, fooof_freq+2, fir_design='firwin')
fooof_dat.apply_hilbert(envelope=True)
#################################################
## EPOCH TRIALS
# Set epoch timings
tmin, tmax = -0.85, 1.1
# Epoch trials - raw data for trial rejection
epochs = mne.Epochs(eeg_dat, evs2, ev_dict2, tmin=tmin, tmax=tmax,
baseline=None, preload=True, verbose=False)
# Epoch trials - filtered version
epochs_alpha = mne.Epochs(alpha_dat, evs2, ev_dict2, tmin=tmin, tmax=tmax,
baseline=(-0.5, -0.35), preload=True, verbose=False)
epochs_fooof = mne.Epochs(fooof_dat, evs2, ev_dict2, tmin=tmin, tmax=tmax,
baseline=(-0.5, -0.35), preload=True, verbose=False)
#################################################
## PRE-PROCESSING: AUTO-REJECT
if RUN_AUTOREJECT:
print('\nAUTOREJECT: CALCULATING SOLUTION\n')
# Initialize and run autoreject across epochs
ar = AutoReject(n_jobs=4, verbose=False)
ar.fit(epochs)
# Save out AR solution
ar.save(pjoin(RES_PATH, 'AR', subj_label + '-ar.hdf5'), overwrite=True)
# Otherwise: load & apply previously saved AR solution
else:
print('\nAUTOREJECT: USING PRECOMPUTED\n')
ar = read_auto_reject(pjoin(RES_PATH, 'AR', subj_label + '-ar.hdf5'))
ar.verbose = 'tqdm'
# Apply autoreject to the original epochs object it was learnt on
epochs, rej_log = ar.transform(epochs, return_log=True)
# Apply autoreject to the copies of the data - apply interpolation, then drop same epochs
_apply_interp(rej_log, epochs_alpha, ar.threshes_, ar.picks_, ar.verbose)
epochs_alpha.drop(rej_log.bad_epochs)
_apply_interp(rej_log, epochs_fooof, ar.threshes_, ar.picks_, ar.verbose)
epochs_fooof.drop(rej_log.bad_epochs)
# Collect which epochs were dropped
dropped_trials[s_ind, 0:sum(rej_log.bad_epochs)] = np.where(rej_log.bad_epochs)[0]
#################################################
## SET UP CHANNEL CLUSTERS
# Set channel clusters - take channels contralateral to stimulus presentation
# Note: channels will be used to extract data contralateral to stimulus presentation
le_chs = ['P3', 'P5', 'P7', 'P9', 'O1', 'PO3', 'PO7'] # Left Side Channels
le_inds = [epochs.ch_names.index(chn) for chn in le_chs]
ri_chs = ['P4', 'P6', 'P8', 'P10', 'O2', 'PO4', 'PO8'] # Right Side Channels
ri_inds = [epochs.ch_names.index(chn) for chn in ri_chs]
#################################################
## TRIAL-RELATED ANALYSIS: CANONICAL vs. FOOOF
## Pull out channels of interest for each load level
# Channels extracted are those contralateral to stimulus presentation
# Canonical Data
lo1_a = np.concatenate([epochs_alpha['LeLo1']._data[:, ri_inds, :],
epochs_alpha['RiLo1']._data[:, le_inds, :]], 0)
lo2_a = np.concatenate([epochs_alpha['LeLo2']._data[:, ri_inds, :],
epochs_alpha['RiLo2']._data[:, le_inds, :]], 0)
lo3_a = np.concatenate([epochs_alpha['LeLo3']._data[:, ri_inds, :],
epochs_alpha['RiLo3']._data[:, le_inds, :]], 0)
# FOOOFed data
lo1_f = np.concatenate([epochs_fooof['LeLo1']._data[:, ri_inds, :],
epochs_fooof['RiLo1']._data[:, le_inds, :]], 0)
lo2_f = np.concatenate([epochs_fooof['LeLo2']._data[:, ri_inds, :],
epochs_fooof['RiLo2']._data[:, le_inds, :]], 0)
lo3_f = np.concatenate([epochs_fooof['LeLo3']._data[:, ri_inds, :],
epochs_fooof['RiLo3']._data[:, le_inds, :]], 0)
## Calculate average across trials and channels - add to group data collection
# Canonical data
canonical_group_avg_dat[s_ind, 0, :] = np.mean(lo1_a, 1).mean(0)
canonical_group_avg_dat[s_ind, 1, :] = np.mean(lo2_a, 1).mean(0)
canonical_group_avg_dat[s_ind, 2, :] = np.mean(lo3_a, 1).mean(0)
# FOOOFed data
fooofed_group_avg_dat[s_ind, 0, :] = np.mean(lo1_f, 1).mean(0)
fooofed_group_avg_dat[s_ind, 1, :] = np.mean(lo2_f, 1).mean(0)
fooofed_group_avg_dat[s_ind, 2, :] = np.mean(lo3_f, 1).mean(0)
#################################################
## FOOOFING TRIAL AVERAGED DATA
# Loop loop loads & trials segments
for seg_label, seg_time in zip(SEG_LABELS, SEG_TIMES):
tmin, tmax = seg_time[0], seg_time[1]
# Calculate PSDs across trials, fit FOOOF models to averages
for le_label, ri_label, load_label in zip(['LeLo1', 'LeLo2', 'LeLo3'],
['RiLo1', 'RiLo2', 'RiLo3'],
LOAD_LABELS):
## Calculate trial wise PSDs for left & right side trials
trial_freqs, le_trial_psds = periodogram(
epochs[le_label]._data[:, :, _time_mask(epochs.times, tmin, tmax, srate)],
srate, window='hann', nfft=4*srate)
trial_freqs, ri_trial_psds = periodogram(
epochs[ri_label]._data[:, :, _time_mask(epochs.times, tmin, tmax, srate)],
srate, window='hann', nfft=4*srate)
## FIT ALL CHANNELS VERSION
if FIT_ALL_CHANNELS:
## Average spectra across trials within a given load & side
le_avg_psd_contra = avg_func(le_trial_psds[:, ri_inds, :], 0)
le_avg_psd_ipsi = avg_func(le_trial_psds[:, le_inds, :], 0)
ri_avg_psd_contra = avg_func(ri_trial_psds[:, le_inds, :], 0)
ri_avg_psd_ipsi = avg_func(ri_trial_psds[:, ri_inds, :], 0)
## Combine spectra across left & right trials for given load
ch_psd_contra = np.vstack([le_avg_psd_contra, ri_avg_psd_contra])
ch_psd_ipsi = np.vstack([le_avg_psd_ipsi, ri_avg_psd_ipsi])
## Fit FOOOFGroup to all channels, average & and collect results
fg.fit(trial_freqs, ch_psd_contra, FREQ_RANGE)
fm = avg_fg(fg)
fg_dict[load_label]['Contra'][seg_label].append(fm.copy())
fg.fit(trial_freqs, ch_psd_ipsi, FREQ_RANGE)
fm = avg_fg(fg)
fg_dict[load_label]['Ipsi'][seg_label].append(fm.copy())
## COLLAPSE ACROSS CHANNELS VERSION
else:
## Average spectra across trials and channels within a given load & side
le_avg_psd_contra = avg_func(avg_func(le_trial_psds[:, ri_inds, :], 0), 0)
le_avg_psd_ipsi = avg_func(avg_func(le_trial_psds[:, le_inds, :], 0), 0)
ri_avg_psd_contra = avg_func(avg_func(ri_trial_psds[:, le_inds, :], 0), 0)
ri_avg_psd_ipsi = avg_func(avg_func(ri_trial_psds[:, ri_inds, :], 0), 0)
## Collapse spectra across left & right trials for given load
avg_psd_contra = avg_func(np.vstack([le_avg_psd_contra, ri_avg_psd_contra]), 0)
avg_psd_ipsi = avg_func(np.vstack([le_avg_psd_ipsi, ri_avg_psd_ipsi]), 0)
## Fit FOOOF, and collect results
fm.fit(trial_freqs, avg_psd_contra, FREQ_RANGE)
fg_dict[load_label]['Contra'][seg_label].append(fm.copy())
fm.fit(trial_freqs, avg_psd_ipsi, FREQ_RANGE)
fg_dict[load_label]['Ipsi'][seg_label].append(fm.copy())
#################################################
## SAVE OUT RESULTS
# Save out group data
np.save(pjoin(RES_PATH, 'Group', 'alpha_freqs_group'), group_fooofed_alpha_freqs)
np.save(pjoin(RES_PATH, 'Group', 'canonical_group'), canonical_group_avg_dat)
np.save(pjoin(RES_PATH, 'Group', 'fooofed_group'), fooofed_group_avg_dat)
np.save(pjoin(RES_PATH, 'Group', 'dropped_trials'), dropped_trials)
np.save(pjoin(RES_PATH, 'Group', 'dropped_components'), dropped_components)
# Save out second round of FOOOFing
for load_label in LOAD_LABELS:
for side_label in SIDE_LABELS:
for seg_label in SEG_LABELS:
fg = combine_fooofs(fg_dict[load_label][side_label][seg_label])
fg.save('Group_' + load_label + '_' + side_label + '_' + seg_label,
pjoin(RES_PATH, 'FOOOF'), save_results=True)
def test_ica_additional():
"""Test additional ICA functionality
"""
stop2 = 500
raw = io.Raw(raw_fname, preload=True).crop(0, stop, False).crop(1.5)
picks = pick_types(raw.info, meg=True, stim=False, ecg=False,
eog=False, exclude='bads')
test_cov = read_cov(test_cov_name)
events = read_events(event_name)
picks = pick_types(raw.info, meg=True, stim=False, ecg=False,
eog=False, exclude='bads')
epochs = Epochs(raw, events[:4], event_id, tmin, tmax, picks=picks,
baseline=(None, 0), preload=True)
# for testing eog functionality
picks2 = pick_types(raw.info, meg=True, stim=False, ecg=False,
eog=True, exclude='bads')
epochs_eog = Epochs(raw, events[:4], event_id, tmin, tmax, picks=picks2,
baseline=(None, 0), preload=True)
test_cov2 = deepcopy(test_cov)
ica = ICA(noise_cov=test_cov2, n_components=3, max_pca_components=4,
n_pca_components=4)
assert_true(ica.info is None)
ica.decompose_raw(raw, picks[:5])
assert_true(isinstance(ica.info, Info))
assert_true(ica.n_components_ < 5)
ica = ICA(n_components=3, max_pca_components=4,
n_pca_components=4)
assert_raises(RuntimeError, ica.save, '')
ica.decompose_raw(raw, picks=None, start=start, stop=stop2)
# test warnings on bad filenames
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
ica_badname = op.join(op.dirname(tempdir), 'test-bad-name.fif.gz')
ica.save(ica_badname)
read_ica(ica_badname)
assert_true(len(w) == 2)
# test decim
ica = ICA(n_components=3, max_pca_components=4,
n_pca_components=4)
raw_ = raw.copy()
for _ in range(3):
raw_.append(raw_)
n_samples = raw_._data.shape[1]
ica.decompose_raw(raw, picks=None, decim=3)
assert_true(raw_._data.shape[1], n_samples)
# test expl var
ica = ICA(n_components=1.0, max_pca_components=4,
n_pca_components=4)
ica.decompose_raw(raw, picks=None, decim=3)
assert_true(ica.n_components_ == 4)
# epochs extraction from raw fit
assert_raises(RuntimeError, ica.get_sources_epochs, epochs)
# test reading and writing
test_ica_fname = op.join(op.dirname(tempdir), 'test-ica.fif')
for cov in (None, test_cov):
ica = ICA(noise_cov=cov, n_components=2, max_pca_components=4,
n_pca_components=4)
with warnings.catch_warnings(record=True): # ICA does not converge
ica.decompose_raw(raw, picks=picks, start=start, stop=stop2)
sources = ica.get_sources_epochs(epochs)
assert_true(ica.mixing_matrix_.shape == (2, 2))
assert_true(ica.unmixing_matrix_.shape == (2, 2))
assert_true(ica.pca_components_.shape == (4, len(picks)))
assert_true(sources.shape[1] == ica.n_components_)
for exclude in [[], [0]]:
ica.exclude = [0]
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.exclude == ica_read.exclude)
# test pick merge -- add components
ica.pick_sources_raw(raw, exclude=[1])
assert_true(ica.exclude == [0, 1])
# -- only as arg
ica.exclude = []
ica.pick_sources_raw(raw, exclude=[0, 1])
assert_true(ica.exclude == [0, 1])
# -- remove duplicates
ica.exclude += [1]
ica.pick_sources_raw(raw, exclude=[0, 1])
assert_true(ica.exclude == [0, 1])
# test basic include
ica.exclude = []
ica.pick_sources_raw(raw, include=[1])
ica_raw = ica.sources_as_raw(raw)
assert_true(ica.exclude == [ica_raw.ch_names.index(e) for e in
ica_raw.info['bads']])
# test filtering
d1 = ica_raw._data[0].copy()
with warnings.catch_warnings(record=True): # dB warning
ica_raw.filter(4, 20)
assert_true((d1 != ica_raw._data[0]).any())
d1 = ica_raw._data[0].copy()
with warnings.catch_warnings(record=True): # dB warning
ica_raw.notch_filter([10])
assert_true((d1 != ica_raw._data[0]).any())
ica.n_pca_components = 2
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.n_pca_components == ica_read.n_pca_components)
# check type consistency
attrs = ('mixing_matrix_ unmixing_matrix_ pca_components_ '
'pca_explained_variance_ _pre_whitener')
f = lambda x, y: getattr(x, y).dtype
for attr in attrs.split():
assert_equal(f(ica_read, attr), f(ica, attr))
ica.n_pca_components = 4
ica_read.n_pca_components = 4
ica.exclude = []
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
for attr in ['mixing_matrix_', 'unmixing_matrix_', 'pca_components_',
'pca_mean_', 'pca_explained_variance_',
'_pre_whitener']:
assert_array_almost_equal(getattr(ica, attr),
getattr(ica_read, attr))
assert_true(ica.ch_names == ica_read.ch_names)
assert_true(isinstance(ica_read.info, Info))
assert_raises(RuntimeError, ica_read.decompose_raw, raw)
sources = ica.get_sources_raw(raw)
sources2 = ica_read.get_sources_raw(raw)
assert_array_almost_equal(sources, sources2)
_raw1 = ica.pick_sources_raw(raw, exclude=[1])
_raw2 = ica_read.pick_sources_raw(raw, exclude=[1])
assert_array_almost_equal(_raw1[:, :][0], _raw2[:, :][0])
os.remove(test_ica_fname)
# check scrore funcs
for name, func in score_funcs.items():
if name in score_funcs_unsuited:
continue
scores = ica.find_sources_raw(raw, target='EOG 061', score_func=func,
start=0, stop=10)
assert_true(ica.n_components_ == len(scores))
# check univariate stats
scores = ica.find_sources_raw(raw, score_func=stats.skew)
# check exception handling
assert_raises(ValueError, ica.find_sources_raw, raw,
target=np.arange(1))
params = []
params += [(None, -1, slice(2), [0, 1])] # varicance, kurtosis idx params
params += [(None, 'MEG 1531')] # ECG / EOG channel params
for idx, ch_name in product(*params):
ica.detect_artifacts(raw, start_find=0, stop_find=50, ecg_ch=ch_name,
eog_ch=ch_name, skew_criterion=idx,
var_criterion=idx, kurt_criterion=idx)
## score funcs epochs ##
# check score funcs
for name, func in score_funcs.items():
if name in score_funcs_unsuited:
continue
scores = ica.find_sources_epochs(epochs_eog, target='EOG 061',
score_func=func)
assert_true(ica.n_components_ == len(scores))
# check univariate stats
scores = ica.find_sources_epochs(epochs, score_func=stats.skew)
# check exception handling
assert_raises(ValueError, ica.find_sources_epochs, epochs,
target=np.arange(1))
# ecg functionality
ecg_scores = ica.find_sources_raw(raw, target='MEG 1531',
score_func='pearsonr')
with warnings.catch_warnings(record=True): # filter attenuation warning
ecg_events = ica_find_ecg_events(raw,
sources[np.abs(ecg_scores).argmax()])
assert_true(ecg_events.ndim == 2)
# eog functionality
eog_scores = ica.find_sources_raw(raw, target='EOG 061',
score_func='pearsonr')
with warnings.catch_warnings(record=True): # filter attenuation warning
eog_events = ica_find_eog_events(raw,
sources[np.abs(eog_scores).argmax()])
assert_true(eog_events.ndim == 2)
# Test ica fiff export
ica_raw = ica.sources_as_raw(raw, start=0, stop=100)
assert_true(ica_raw.last_samp - ica_raw.first_samp == 100)
assert_true(len(ica_raw._filenames) == 0) # API consistency
ica_chans = [ch for ch in ica_raw.ch_names if 'ICA' in ch]
assert_true(ica.n_components_ == len(ica_chans))
test_ica_fname = op.join(op.abspath(op.curdir), 'test-ica_raw.fif')
ica.n_components = np.int32(ica.n_components)
ica_raw.save(test_ica_fname, overwrite=True)
ica_raw2 = io.Raw(test_ica_fname, preload=True)
assert_allclose(ica_raw._data, ica_raw2._data, rtol=1e-5, atol=1e-4)
ica_raw2.close()
os.remove(test_ica_fname)
# Test ica epochs export
ica_epochs = ica.sources_as_epochs(epochs)
assert_true(ica_epochs.events.shape == epochs.events.shape)
sources_epochs = ica.get_sources_epochs(epochs)
assert_array_equal(ica_epochs.get_data(), sources_epochs)
ica_chans = [ch for ch in ica_epochs.ch_names if 'ICA' in ch]
assert_true(ica.n_components_ == len(ica_chans))
assert_true(ica.n_components_ == ica_epochs.get_data().shape[1])
assert_true(ica_epochs.raw is None)
assert_true(ica_epochs.preload is True)
# test float n pca components
ica.pca_explained_variance_ = np.array([0.2] * 5)
ica.n_components_ = 0
for ncomps, expected in [[0.3, 1], [0.9, 4], [1, 1]]:
ncomps_ = _check_n_pca_components(ica, ncomps)
assert_true(ncomps_ == expected)
def apply_ica(subject, run, session):
print("Processing subject: %s" % subject)
# Construct the search path for the data file. `sub` is mandatory
subject_path = op.join('sub-{}'.format(subject))
# `session` is optional
if session is not None:
subject_path = op.join(subject_path, 'ses-{}'.format(session))
subject_path = op.join(subject_path, config.kind)
bids_basename = make_bids_basename(subject=subject,
session=session,
task=config.task,
acquisition=config.acq,
run=None,
processing=config.proc,
recording=config.rec,
space=config.space
)
fpath_deriv = op.join(config.bids_root, 'derivatives',
config.PIPELINE_NAME, subject_path)
fname_in = \
op.join(fpath_deriv, bids_basename + '-epo.fif')
fname_out = \
op.join(fpath_deriv, bids_basename + '_cleaned-epo.fif')
# load epochs to reject ICA components
epochs = mne.read_epochs(fname_in, preload=True)
print("Input: ", fname_in)
print("Output: ", fname_out)
# load first run of raw data for ecg /eog epochs
print(" Loading one run from raw data")
bids_basename = make_bids_basename(subject=subject,
session=session,
task=config.task,
acquisition=config.acq,
run=config.runs[0],
processing=config.proc,
recording=config.rec,
space=config.space
)
if config.use_maxwell_filter:
raw_fname_in = \
op.join(fpath_deriv, bids_basename + '_sss_raw.fif')
else:
raw_fname_in = \
op.join(fpath_deriv, bids_basename + '_filt_raw.fif')
raw = mne.io.read_raw_fif(raw_fname_in, preload=True)
# run ICA on MEG and EEG
picks_meg = mne.pick_types(raw.info, meg=True, eeg=False,
eog=False, stim=False, exclude='bads')
picks_eeg = mne.pick_types(raw.info, meg=False, eeg=True,
eog=False, stim=False, exclude='bads')
all_picks = {'meg': picks_meg, 'eeg': picks_eeg}
for ch_type in config.ch_types:
report = None
print(ch_type)
picks = all_picks[ch_type]
# Load ICA
fname_ica = \
op.join(fpath_deriv, bids_basename + '_%s-ica.fif' % ch_type)
print('Reading ICA: ' + fname_ica)
ica = read_ica(fname=fname_ica)
pick_ecg = mne.pick_types(raw.info, meg=False, eeg=False,
ecg=True, eog=False)
# ECG
# either needs an ecg channel, or avg of the mags (i.e. MEG data)
ecg_inds = list()
if pick_ecg or ch_type == 'meg':
picks_ecg = np.concatenate([picks, pick_ecg])
# Create ecg epochs
if ch_type == 'meg':
reject = {'mag': config.reject['mag'],
'grad': config.reject['grad']}
elif ch_type == 'eeg':
reject = {'eeg': config.reject['eeg']}
ecg_epochs = create_ecg_epochs(raw, picks=picks_ecg, reject=reject,
baseline=(None, 0), tmin=-0.5,
tmax=0.5)
ecg_average = ecg_epochs.average()
ecg_inds, scores = \
ica.find_bads_ecg(ecg_epochs, method='ctps',
threshold=config.ica_ctps_ecg_threshold)
del ecg_epochs
report_fname = \
op.join(fpath_deriv,
bids_basename + '_%s-reject_ica.html' % ch_type)
report = Report(report_fname, verbose=False)
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores,
exclude=ecg_inds,
show=config.plot),
captions=ch_type.upper() + ' - ECG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(ecg_average,
exclude=ecg_inds,
show=config.plot),
captions=ch_type.upper() + ' - ECG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(ecg_average,
exclude=ecg_inds,
show=config.plot),
captions=ch_type.upper() + ' - ECG - ' +
'Corrections')
else:
# XXX : to check when EEG only is processed
print('no ECG channel is present. Cannot automate ICAs component '
'detection for ECG!')
# EOG
pick_eog = mne.pick_types(raw.info, meg=False, eeg=False,
ecg=False, eog=True)
eog_inds = list()
if pick_eog.any():
print('using EOG channel')
picks_eog = np.concatenate([picks, pick_eog])
# Create eog epochs
eog_epochs = create_eog_epochs(raw, picks=picks_eog, reject=None,
baseline=(None, 0), tmin=-0.5,
tmax=0.5)
eog_average = eog_epochs.average()
eog_inds, scores = ica.find_bads_eog(eog_epochs, threshold=3.0)
del eog_epochs
params = dict(exclude=eog_inds, show=config.plot)
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores, **params),
captions=ch_type.upper() + ' - EOG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(eog_average, **params),
captions=ch_type.upper() + ' - EOG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(eog_average, **params),
captions=ch_type.upper() + ' - EOG - ' +
'Corrections')
report.save(report_fname, overwrite=True, open_browser=False)
else:
print('no EOG channel is present. Cannot automate ICAs component '
'detection for EOG!')
ica_reject = (list(ecg_inds) + list(eog_inds) +
list(config.rejcomps_man[subject][ch_type]))
# now reject the components
print('Rejecting from %s: %s' % (ch_type, ica_reject))
epochs = ica.apply(epochs, exclude=ica_reject)
print('Saving cleaned epochs')
epochs.save(fname_out)
if report is not None:
fig = ica.plot_overlay(raw, exclude=ica_reject, show=config.plot)
report.add_figs_to_section(fig, captions=ch_type.upper() +
' - ALL(epochs) - Corrections')
if config.plot:
epochs.plot_image(combine='gfp', group_by='type', sigma=2.,
cmap="YlGnBu_r", show=config.plot)
# Read events from the stim channel
mask = 4096 + 256 # mask for excluding high order bits
events = mne.find_events(raw,
stim_channel='STI101',
consecutive='increasing',
mask=mask,
mask_type='not_and',
min_duration=0.003)
# Compensate for projector delay
events[:, 0] += int(round(0.0345 * raw.info['sfreq']))
# Load the ICA object
print(' Using ICA')
ica = read_ica(fname.ica(subject=subject))
# Make epochs. Because the original 1000Hz sampling rate is a bit excessive
# for what we're going for, we only read every 5th sample. This gives us a
# sampling rate of ~200Hz.
epochs = mne.Epochs(raw,
events,
events_id,
epoch_tmin,
epoch_tmax,
baseline=baseline,
decim=5,
preload=True)
# Save evoked plot to the report
with mne.open_report(fname.report(subject=subject)) as report:
subject = group_name + "%d" % subject_id
print("processing subject: %s" % subject)
tasks = ['AVLearn', 'AVLearn']
days = [100, 200]
for task, day in zip(tasks, days):
fname = op.join(MEG_data_path, subject,
task + '_%d' % (day + subject_id) + '_tsss_mc.fif')
raw_tsss_mc = mne.io.read_raw_fif(fname, preload=True)
raw_tsss_mc.filter(l_freq=None, h_freq=40.0,
fir_design='firwin') # low-pass filter data
ica = read_ica(fname.replace("_tsss_mc", "-ica"))
ica.exclude = np.load(fname.replace("tsss_mc.fif",
'ICA_excludes.npy')).tolist()
raw_tsss_mc = ica.apply(raw_tsss_mc, exclude=ica.exclude)
events = pd.read_csv(fname.replace('tsss_mc.fif', 'events.csv'))
artifacts = pd.read_csv(fname.replace('tsss_mc.fif', 'artifacts.csv'))
df = pd.merge(events, artifacts, on='stim_onset')
df['trigger_code2'] = df['trigger_code'] + df['block'] * 1000
df = df[df['artifacts'] == False]
events = df[["stim_onset", "button_press",
"trigger_code2"]].astype(int).values
picks = mne.pick_types(raw_tsss_mc.info,
meg=True,
def run_epochs(subject_id, tsss=False):
subject = "sub%03d" % subject_id
print("Processing subject: %s%s" % (subject,
(' (tSSS=%d)' % tsss) if tsss else ''))
data_path = op.join(meg_dir, subject)
# map to correct subject for bad channels
mapping = map_subjects[subject_id]
raw_list = list()
events_list = list()
print(" Loading raw data")
for run in range(1, 7):
bads = list()
bad_name = op.join('bads', mapping, 'run_%02d_raw_tr.fif_bad' % run)
if os.path.exists(bad_name):
with open(bad_name) as f:
for line in f:
bads.append(line.strip())
if tsss:
run_fname = op.join(data_path,
'run_%02d_filt_tsss_%d_raw.fif' % (run, tsss))
else:
run_fname = op.join(
data_path, 'run_%02d_filt_sss_'
'highpass-%sHz_raw.fif' % (run, l_freq))
raw = mne.io.read_raw_fif(run_fname, preload=True)
delay = int(round(0.0345 * raw.info['sfreq']))
events = mne.read_events(op.join(data_path, 'run_%02d-eve.fif' % run))
events[:, 0] = events[:, 0] + delay
events_list.append(events)
raw.info['bads'] = bads
raw.interpolate_bads()
raw_list.append(raw)
raw, events = mne.concatenate_raws(raw_list, events_list=events_list)
raw.set_eeg_reference(projection=True)
del raw_list
picks = mne.pick_types(raw.info,
meg=True,
eeg=True,
stim=True,
eog=True,
exclude=())
# Epoch the data
print(' Epoching')
epochs = mne.Epochs(raw,
events,
events_id,
tmin,
tmax,
proj=True,
picks=picks,
baseline=baseline,
preload=False,
decim=5,
reject=None,
reject_tmax=reject_tmax)
print(' Interpolating bad channels')
# ICA
if tsss:
ica_name = op.join(meg_dir, subject,
'run_concat-tsss_%d-ica.fif' % (tsss, ))
ica_out_name = ica_name
else:
ica_name = op.join(meg_dir, subject, 'run_concat-ica.fif')
ica_out_name = op.join(meg_dir, subject,
'run_concat_highpass-%sHz-ica.fif' % (l_freq, ))
print(' Using ICA')
ica = read_ica(ica_name)
ica.exclude = []
filter_label = '-tsss_%d' % tsss if tsss else '_highpass-%sHz' % l_freq
ecg_epochs = create_ecg_epochs(raw, tmin=-.3, tmax=.3, preload=False)
eog_epochs = create_eog_epochs(raw, tmin=-.5, tmax=.5, preload=False)
del raw
n_max_ecg = 3 # use max 3 components
ecg_epochs.decimate(5)
ecg_epochs.load_data()
ecg_epochs.apply_baseline((None, None))
ecg_inds, scores_ecg = ica.find_bads_ecg(ecg_epochs,
method='ctps',
threshold=0.8)
print(' Found %d ECG indices' % (len(ecg_inds), ))
ica.exclude.extend(ecg_inds[:n_max_ecg])
ecg_epochs.average().save(
op.join(data_path, '%s%s-ecg-ave.fif' % (subject, filter_label)))
np.save(
op.join(data_path, '%s%s-ecg-scores.npy' % (subject, filter_label)),
scores_ecg)
del ecg_epochs
n_max_eog = 3 # use max 2 components
eog_epochs.decimate(5)
eog_epochs.load_data()
eog_epochs.apply_baseline((None, None))
eog_inds, scores_eog = ica.find_bads_eog(eog_epochs)
print(' Found %d EOG indices' % (len(eog_inds), ))
ica.exclude.extend(eog_inds[:n_max_eog])
eog_epochs.average().save(
op.join(data_path, '%s%s-eog-ave.fif' % (subject, filter_label)))
np.save(
op.join(data_path, '%s%s-eog-scores.npy' % (subject, filter_label)),
scores_eog)
del eog_epochs
ica.save(ica_out_name)
epochs.load_data()
ica.apply(epochs)
print(' Getting rejection thresholds')
reject = get_rejection_threshold(epochs.copy().crop(None, reject_tmax))
epochs.drop_bad(reject=reject)
print(' Dropped %0.1f%% of epochs' % (epochs.drop_log_stats(), ))
print(' Writing to disk')
if tsss:
epochs.save(op.join(data_path, '%s-tsss_%d-epo.fif' % (subject, tsss)))
else:
epochs.save(
op.join(data_path, '%s_highpass-%sHz-epo.fif' % (subject, l_freq)))
def test_ica_additional():
"""Test additional functionality
"""
stop2 = 500
test_cov2 = deepcopy(test_cov)
ica = ICA(noise_cov=test_cov2, n_components=3, max_pca_components=4,
n_pca_components=4)
assert_true(ica.info is None)
ica.decompose_raw(raw, picks[:5])
assert_true(isinstance(ica.info, Info))
assert_true(ica.n_components_ < 5)
ica = ICA(n_components=3, max_pca_components=4,
n_pca_components=4)
assert_raises(RuntimeError, ica.save, '')
ica.decompose_raw(raw, picks=None, start=start, stop=stop2)
# epochs extraction from raw fit
assert_raises(RuntimeError, ica.get_sources_epochs, epochs)
# test reading and writing
test_ica_fname = op.join(op.dirname(tempdir), 'ica_test.fif')
for cov in (None, test_cov):
ica = ICA(noise_cov=cov, n_components=3, max_pca_components=4,
n_pca_components=4)
ica.decompose_raw(raw, picks=picks, start=start, stop=stop2)
sources = ica.get_sources_epochs(epochs)
assert_true(sources.shape[1] == ica.n_components_)
for exclude in [[], [0]]:
ica.exclude = [0]
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.exclude == ica_read.exclude)
# test pick merge -- add components
ica.pick_sources_raw(raw, exclude=[1])
assert_true(ica.exclude == [0, 1])
# -- only as arg
ica.exclude = []
ica.pick_sources_raw(raw, exclude=[0, 1])
assert_true(ica.exclude == [0, 1])
# -- remove duplicates
ica.exclude += [1]
ica.pick_sources_raw(raw, exclude=[0, 1])
assert_true(ica.exclude == [0, 1])
ica_raw = ica.sources_as_raw(raw)
assert_true(ica.exclude == [ica_raw.ch_names.index(e) for e in
ica_raw.info['bads']])
ica.n_pca_components = 2
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.n_pca_components ==
ica_read.n_pca_components)
ica.n_pca_components = 4
ica_read.n_pca_components = 4
ica.exclude = []
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.ch_names == ica_read.ch_names)
assert_true(isinstance(ica_read.info, Info)) # XXX improve later
assert_true(np.allclose(ica.mixing_matrix_, ica_read.mixing_matrix_,
rtol=1e-16, atol=1e-32))
assert_array_equal(ica.pca_components_,
ica_read.pca_components_)
assert_array_equal(ica.pca_mean_, ica_read.pca_mean_)
assert_array_equal(ica.pca_explained_variance_,
ica_read.pca_explained_variance_)
assert_array_equal(ica._pre_whitener, ica_read._pre_whitener)
# assert_raises(RuntimeError, ica_read.decompose_raw, raw)
sources = ica.get_sources_raw(raw)
sources2 = ica_read.get_sources_raw(raw)
assert_array_almost_equal(sources, sources2)
_raw1 = ica.pick_sources_raw(raw, exclude=[1])
_raw2 = ica_read.pick_sources_raw(raw, exclude=[1])
assert_array_almost_equal(_raw1[:, :][0], _raw2[:, :][0])
os.remove(test_ica_fname)
# check scrore funcs
for name, func in score_funcs.items():
if name in score_funcs_unsuited:
continue
scores = ica.find_sources_raw(raw, target='EOG 061', score_func=func,
start=0, stop=10)
assert_true(ica.n_components_ == len(scores))
# check univariate stats
scores = ica.find_sources_raw(raw, score_func=stats.skew)
# check exception handling
assert_raises(ValueError, ica.find_sources_raw, raw,
target=np.arange(1))
params = []
params += [(None, -1, slice(2), [0, 1])] # varicance, kurtosis idx params
params += [(None, 'MEG 1531')] # ECG / EOG channel params
for idx, ch_name in product(*params):
ica.detect_artifacts(raw, start_find=0, stop_find=50, ecg_ch=ch_name,
eog_ch=ch_name, skew_criterion=idx,
var_criterion=idx, kurt_criterion=idx)
## score funcs epochs ##
# check score funcs
for name, func in score_funcs.items():
if name in score_funcs_unsuited:
continue
scores = ica.find_sources_epochs(epochs_eog, target='EOG 061',
score_func=func)
assert_true(ica.n_components_ == len(scores))
# check univariate stats
scores = ica.find_sources_epochs(epochs, score_func=stats.skew)
# check exception handling
assert_raises(ValueError, ica.find_sources_epochs, epochs,
target=np.arange(1))
# ecg functionality
ecg_scores = ica.find_sources_raw(raw, target='MEG 1531',
score_func='pearsonr')
ecg_events = ica_find_ecg_events(raw, sources[np.abs(ecg_scores).argmax()])
assert_true(ecg_events.ndim == 2)
# eog functionality
eog_scores = ica.find_sources_raw(raw, target='EOG 061',
score_func='pearsonr')
eog_events = ica_find_eog_events(raw, sources[np.abs(eog_scores).argmax()])
assert_true(eog_events.ndim == 2)
# Test ica fiff export
ica_raw = ica.sources_as_raw(raw, start=0, stop=100)
assert_true(ica_raw.last_samp - ica_raw.first_samp == 100)
ica_chans = [ch for ch in ica_raw.ch_names if 'ICA' in ch]
assert_true(ica.n_components_ == len(ica_chans))
test_ica_fname = op.join(op.abspath(op.curdir), 'test_ica.fif')
ica_raw.save(test_ica_fname)
ica_raw2 = fiff.Raw(test_ica_fname, preload=True)
assert_array_almost_equal(ica_raw._data, ica_raw2._data)
ica_raw2.close()
os.remove(test_ica_fname)
# Test ica epochs export
ica_epochs = ica.sources_as_epochs(epochs)
assert_true(ica_epochs.events.shape == epochs.events.shape)
sources_epochs = ica.get_sources_epochs(epochs)
assert_array_equal(ica_epochs.get_data(), sources_epochs)
ica_chans = [ch for ch in ica_epochs.ch_names if 'ICA' in ch]
assert_true(ica.n_components_ == len(ica_chans))
assert_true(ica.n_components_ == ica_epochs.get_data().shape[1])
assert_true(ica_epochs.raw is None)
assert_true(ica_epochs.preload == True)
# regression test for plot method
assert_raises(ValueError, ica.plot_sources_raw, raw,
order=np.arange(50))
assert_raises(ValueError, ica.plot_sources_epochs, epochs,
order=np.arange(50))
session_no = ''
ica_check_eves = dict(FFA=['A','B'])
ica_cond = 'FFA'
ica_excludes = load_excludes(ica_excludes_path, subj, ica_cond)
print 30*'*'
print 'ICA excludes:', ica_excludes
print 30*'*'
raw_path = opj(ad._scratch_folder, input_files, subj)
in_fnames = ad.analysis_dict[subj][input_files][cond]['files']
events = mne.read_events(opj(eve_folder, cond + '-eve.fif'))
eve_dict, id_dict = \
split_events_by_trialtype(events, condition=cond)
for fname in in_fnames:
ica = read_ica(opj(ica_folder, cond + '-ica.fif'))
print 'In: ', fname
raw = Raw(fname, preload=performBandpassFilter)
rep_section_name = ''
if performBandpassFilter:
raw.filter(filter_params['highpass'],
filter_params['lowpass'],
method='iir', n_jobs=1
)
picks = mne.pick_types(raw.info, meg=True, eog=True)
for trial_type in trial_types:
epochs = mne.Epochs(raw, eve_dict[trial_type],
id_dict[trial_type],
def run_epochs(subject_id):
subject = "sub%03d" % subject_id
print("processing subject: %s" % subject)
data_path = op.join(meg_dir, subject)
all_epochs = list()
# Get all bad channels
mapping = map_subjects[subject_id] # map to correct subject
all_bads = list()
for run in range(1, 7):
bads = list()
bad_name = op.join('bads', mapping, 'run_%02d_raw_tr.fif_bad' % run)
if os.path.exists(bad_name):
with open(bad_name) as f:
for line in f:
bads.append(line.strip())
all_bads += [bad for bad in bads if bad not in all_bads]
for run in range(1, 7):
print " - Run %s" % run
run_fname = op.join(data_path, 'run_%02d_filt_sss_raw.fif' % run)
if not os.path.exists(run_fname):
continue
raw = mne.io.Raw(run_fname, preload=True, add_eeg_ref=False)
raw.set_channel_types({'EEG061': 'eog',
'EEG062': 'eog',
'EEG063': 'ecg',
'EEG064': 'misc'}) # EEG064 free floating el.
raw.rename_channels({'EEG061': 'EOG061',
'EEG062': 'EOG062',
'EEG063': 'ECG063'})
eog_events = mne.preprocessing.find_eog_events(raw)
eog_events[:, 0] -= int(0.25 * raw.info['sfreq'])
annotations = mne.Annotations(eog_events[:, 0] / raw.info['sfreq'],
np.repeat(0.5, len(eog_events)),
'BAD_blink', raw.info['meas_date'])
raw.annotations = annotations # Remove epochs with blinks
delay = int(0.0345 * raw.info['sfreq'])
events = mne.read_events(op.join(data_path,
'run_%02d_filt_sss-eve.fif' % run))
events[:, 0] = events[:, 0] + delay
raw.info['bads'] = all_bads
raw.interpolate_bads()
raw.set_eeg_reference()
picks = mne.pick_types(raw.info, meg=True, eeg=True, stim=True,
eog=True)
# Read epochs
epochs = mne.Epochs(raw, events, events_id, tmin, tmax, proj=True,
picks=picks, baseline=baseline, preload=True,
decim=2, reject=reject, add_eeg_ref=False)
# ICA
ica_name = op.join(meg_dir, subject, 'run_%02d-ica.fif' % run)
ica = read_ica(ica_name)
n_max_ecg = 3 # use max 3 components
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5)
ecg_inds, scores_ecg = ica.find_bads_ecg(ecg_epochs, method='ctps',
threshold=0.8)
ica.exclude += ecg_inds[:n_max_ecg]
ica.apply(epochs)
all_epochs.append(epochs)
epochs = mne.epochs.concatenate_epochs(all_epochs)
epochs.save(op.join(data_path, '%s-epo.fif' % subject))
return eve_dict, id_dict
# Set epoch parameters
tmin, tmax = -0.4, 0.6 # no need to take more than this, wide enough to see eyemov though
rej_tmin, rej_tmax = -0.2, 0.2 # reject trial only if blinks in the 400 ms middle portion!
baseline = (-0.2, 0.)
reject = dict(eog=150e-6, mag=4e-12, grad=4000e-13) # compare to standard rejection
raw_path = ad._scratch_folder + '/tsss_initial/007_SGF'
eve_path = ad._scratch_folder + '/events.fif/007_SGF/raw'
fname = raw_path + '/VS_1a_1_tsss_mc.fif'
raw = Raw(fname, preload=True)
ica = read_ica(raw_path + '/ica_pre.fif')
print 'Excluding', ica.exclude
raw_ica = ica.apply(raw, copy=True)
events = mne.read_events(eve_path + '/VS_1a_1-eve.fif')
picks = pick_types(raw.info, meg=True, eeg=False, stim=True, eog=True, misc=True)
eve_dict, id_dict = split_events_by_trialtype(events)
for trial_type in ['VS']:
epochs = mne.Epochs(raw, eve_dict[trial_type], id_dict,
tmin, tmax, picks=picks, verbose=False,
baseline=baseline, reject=reject, preload=True,
reject_tmin=rej_tmin, reject_tmax=rej_tmax) # Check rejection settings
epochs_ica = mne.Epochs(raw_ica, eve_dict[trial_type], id_dict,
tmin, tmax, picks=picks, verbose=False,
baseline=baseline, reject=None, preload=True)
def preprocess_set_ICA_comp_fif_to_ts(fif_file, n_comp_exclude, l_freq, h_freq,
down_sfreq, is_sensor_space):
import os
import numpy as np
import sys
import mne
from mne.io import Raw
from mne.preprocessing import read_ica
from mne.report import Report
from nipype.utils.filemanip import split_filename as split_f
report = Report()
subj_path, basename, ext = split_f(fif_file)
(data_path, sbj_name) = os.path.split(subj_path)
print '*** SBJ %s' % sbj_name + '***'
# n_session = int(filter(str.isdigit, basename))
# print '*** n session = %d' % n_session + '***'
# Read raw
raw = Raw(fif_file, preload=True)
# select sensors
select_sensors = mne.pick_types(raw.info, meg=True, ref_meg=False,
exclude='bads')
picks_meeg = mne.pick_types(raw.info, meg=True, eeg=True,
exclude='bads')
# save electrode locations
sens_loc = [raw.info['chs'][i]['loc'][:3] for i in select_sensors]
sens_loc = np.array(sens_loc)
channel_coords_file = os.path.abspath("correct_channel_coords.txt")
np.savetxt(channel_coords_file, sens_loc, fmt='%s')
# save electrode names
sens_names = np.array([raw.ch_names[pos] for pos in select_sensors],
dtype="str")
channel_names_file = os.path.abspath("correct_channel_names.txt")
np.savetxt(channel_names_file, sens_names, fmt='%s')
# filtering + downsampling
# TODO n_jobs=8
raw.filter(l_freq=l_freq, h_freq=h_freq, picks=picks_meeg,
method='iir',n_jobs=8)
# raw.resample(sfreq=down_sfreq, npad=0)
# load ICA
is_show = False # visualization
ica_filename = os.path.join(subj_path, basename + '-ica.fif')
if os.path.exists(ica_filename) is False:
print "$$$ Warning, no %s found" % ica_filename
sys.exit()
else:
ica = read_ica(ica_filename)
# AP 210316
'''
print '*** ica.exclude before set components= ', ica.exclude
if n_comp_exclude.has_key(sbj_name):
print '*** ICA to be excluded for sbj %s ' % sbj_name + ' ' + str(n_comp_exclude[sbj_name]) + '***'
matrix_c_ICA = n_comp_exclude[sbj_name]
if not matrix_c_ICA[n_session-1]:
print 'no ICA'
else:
print '*** ICA to be excluded for session %d ' %n_session + ' ' + str(matrix_c_ICA[n_session-1]) + '***'
ica.exclude = matrix_c_ICA[n_session-1]
'''
# AP new dict
print '*** ica.exclude before set components= ', ica.exclude
if n_comp_exclude.has_key(sbj_name):
print '*** ICA to be excluded for sbj %s ' % sbj_name + ' ' + str(n_comp_exclude[sbj_name]) + '***'
session_dict = n_comp_exclude[sbj_name]
session_names = session_dict.keys()
componentes = []
for s in session_names:
if basename.find(s) > -1:
componentes = session_dict[s]
break
if len(componentes) == 0:
print '\n no ICA to be excluded \n'
else:
print '\n *** ICA to be excluded for session %s ' % s + \
' ' + str(componentes) + ' *** \n'
ica.exclude = componentes
print '\n *** ica.exclude after set components = ', ica.exclude
fig1 = ica.plot_overlay(raw, show=is_show)
report.add_figs_to_section(fig1, captions=['Signal'],
section='Signal quality')
report_filename = os.path.join(subj_path, basename + "-report_NEW.html")
print report_filename
report.save(report_filename, open_browser=False, overwrite=True)
# 3) apply ICA to raw data and save solution and report
# check the amplitudes do not change
# raw_ica_file = os.path.abspath(basename[:i_raw] + 'ica-raw.fif')
raw_ica_file = os.path.join(subj_path, basename + '-preproc-raw.fif')
raw_ica = ica.apply(raw)
raw_ica.resample(sfreq=down_sfreq, npad=0)
raw_ica.save(raw_ica_file, overwrite=True)
# save ICA solution
print ica_filename
ica.save(ica_filename)
# 4) save data
data_noIca, times = raw[select_sensors, :]
data, times = raw_ica[select_sensors, :]
print data.shape
print raw.info['sfreq']
ts_file = os.path.abspath(basename + '_ica.npy')
np.save(ts_file, data)
print '*** TS FILE ' + ts_file + '***'
print '*** raw.info[sfreq] = ' + str(raw.info['sfreq'])
if is_sensor_space:
return ts_file, channel_coords_file, channel_names_file, raw.info['sfreq']
else:
return raw_ica_file, channel_coords_file, channel_names_file, raw.info['sfreq']
def test_ica_additional(method):
"""Test additional ICA functionality."""
_skip_check_picard(method)
tempdir = _TempDir()
stop2 = 500
raw = read_raw_fif(raw_fname).crop(1.5, stop).load_data()
raw.del_proj() # avoid warnings
raw.set_annotations(Annotations([0.5], [0.5], ['BAD']))
# XXX This breaks the tests :(
# raw.info['bads'] = [raw.ch_names[1]]
test_cov = read_cov(test_cov_name)
events = read_events(event_name)
picks = pick_types(raw.info, meg=True, stim=False, ecg=False,
eog=False, exclude='bads')[1::2]
epochs = Epochs(raw, events, None, tmin, tmax, picks=picks,
baseline=(None, 0), preload=True, proj=False)
epochs.decimate(3, verbose='error')
assert len(epochs) == 4
# test if n_components=None works
ica = ICA(n_components=None, max_pca_components=None,
n_pca_components=None, random_state=0, method=method, max_iter=1)
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(epochs)
# for testing eog functionality
picks2 = np.concatenate([picks, pick_types(raw.info, False, eog=True)])
epochs_eog = Epochs(raw, events[:4], event_id, tmin, tmax, picks=picks2,
baseline=(None, 0), preload=True)
del picks2
test_cov2 = test_cov.copy()
ica = ICA(noise_cov=test_cov2, n_components=3, max_pca_components=4,
n_pca_components=4, method=method)
assert (ica.info is None)
with pytest.warns(RuntimeWarning, match='normalize_proj'):
ica.fit(raw, picks[:5])
assert (isinstance(ica.info, Info))
assert (ica.n_components_ < 5)
ica = ICA(n_components=3, max_pca_components=4, method=method,
n_pca_components=4, random_state=0)
pytest.raises(RuntimeError, ica.save, '')
ica.fit(raw, picks=[1, 2, 3, 4, 5], start=start, stop=stop2)
# check passing a ch_name to find_bads_ecg
with pytest.warns(RuntimeWarning, match='longer'):
_, scores_1 = ica.find_bads_ecg(raw)
_, scores_2 = ica.find_bads_ecg(raw, raw.ch_names[1])
assert scores_1[0] != scores_2[0]
# test corrmap
ica2 = ica.copy()
ica3 = ica.copy()
corrmap([ica, ica2], (0, 0), threshold='auto', label='blinks', plot=True,
ch_type="mag")
corrmap([ica, ica2], (0, 0), threshold=2, plot=False, show=False)
assert (ica.labels_["blinks"] == ica2.labels_["blinks"])
assert (0 in ica.labels_["blinks"])
# test retrieval of component maps as arrays
components = ica.get_components()
template = components[:, 0]
EvokedArray(components, ica.info, tmin=0.).plot_topomap([0], time_unit='s')
corrmap([ica, ica3], template, threshold='auto', label='blinks', plot=True,
ch_type="mag")
assert (ica2.labels_["blinks"] == ica3.labels_["blinks"])
plt.close('all')
ica_different_channels = ICA(n_components=2, random_state=0).fit(
raw, picks=[2, 3, 4, 5])
pytest.raises(ValueError, corrmap, [ica_different_channels, ica], (0, 0))
# test warnings on bad filenames
ica_badname = op.join(op.dirname(tempdir), 'test-bad-name.fif.gz')
with pytest.warns(RuntimeWarning, match='-ica.fif'):
ica.save(ica_badname)
with pytest.warns(RuntimeWarning, match='-ica.fif'):
read_ica(ica_badname)
# test decim
ica = ICA(n_components=3, max_pca_components=4,
n_pca_components=4, method=method, max_iter=1)
raw_ = raw.copy()
for _ in range(3):
raw_.append(raw_)
n_samples = raw_._data.shape[1]
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(raw, picks=picks[:5], decim=3)
assert raw_._data.shape[1] == n_samples
# test expl var
ica = ICA(n_components=1.0, max_pca_components=4,
n_pca_components=4, method=method, max_iter=1)
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(raw, picks=None, decim=3)
assert (ica.n_components_ == 4)
ica_var = _ica_explained_variance(ica, raw, normalize=True)
assert (np.all(ica_var[:-1] >= ica_var[1:]))
# test ica sorting
ica.exclude = [0]
ica.labels_ = dict(blink=[0], think=[1])
ica_sorted = _sort_components(ica, [3, 2, 1, 0], copy=True)
assert_equal(ica_sorted.exclude, [3])
assert_equal(ica_sorted.labels_, dict(blink=[3], think=[2]))
# epochs extraction from raw fit
pytest.raises(RuntimeError, ica.get_sources, epochs)
# test reading and writing
test_ica_fname = op.join(op.dirname(tempdir), 'test-ica.fif')
for cov in (None, test_cov):
ica = ICA(noise_cov=cov, n_components=2, max_pca_components=4,
n_pca_components=4, method=method, max_iter=1)
with pytest.warns(None): # ICA does not converge
ica.fit(raw, picks=picks[:10], start=start, stop=stop2)
sources = ica.get_sources(epochs).get_data()
assert (ica.mixing_matrix_.shape == (2, 2))
assert (ica.unmixing_matrix_.shape == (2, 2))
assert (ica.pca_components_.shape == (4, 10))
assert (sources.shape[1] == ica.n_components_)
for exclude in [[], [0], np.array([1, 2, 3])]:
ica.exclude = exclude
ica.labels_ = {'foo': [0]}
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert (list(ica.exclude) == ica_read.exclude)
assert_equal(ica.labels_, ica_read.labels_)
ica.apply(raw)
ica.exclude = []
ica.apply(raw, exclude=[1])
assert (ica.exclude == [])
ica.exclude = [0, 1]
ica.apply(raw, exclude=[1])
assert (ica.exclude == [0, 1])
ica_raw = ica.get_sources(raw)
assert (ica.exclude == [ica_raw.ch_names.index(e) for e in
ica_raw.info['bads']])
# test filtering
d1 = ica_raw._data[0].copy()
ica_raw.filter(4, 20, fir_design='firwin2')
assert_equal(ica_raw.info['lowpass'], 20.)
assert_equal(ica_raw.info['highpass'], 4.)
assert ((d1 != ica_raw._data[0]).any())
d1 = ica_raw._data[0].copy()
ica_raw.notch_filter([10], trans_bandwidth=10, fir_design='firwin')
assert ((d1 != ica_raw._data[0]).any())
ica.n_pca_components = 2
ica.method = 'fake'
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert (ica.n_pca_components == ica_read.n_pca_components)
assert_equal(ica.method, ica_read.method)
assert_equal(ica.labels_, ica_read.labels_)
# check type consistency
attrs = ('mixing_matrix_ unmixing_matrix_ pca_components_ '
'pca_explained_variance_ pre_whitener_')
def f(x, y):
return getattr(x, y).dtype
for attr in attrs.split():
assert_equal(f(ica_read, attr), f(ica, attr))
ica.n_pca_components = 4
ica_read.n_pca_components = 4
ica.exclude = []
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
for attr in ['mixing_matrix_', 'unmixing_matrix_', 'pca_components_',
'pca_mean_', 'pca_explained_variance_',
'pre_whitener_']:
assert_array_almost_equal(getattr(ica, attr),
getattr(ica_read, attr))
assert (ica.ch_names == ica_read.ch_names)
assert (isinstance(ica_read.info, Info))
sources = ica.get_sources(raw)[:, :][0]
sources2 = ica_read.get_sources(raw)[:, :][0]
assert_array_almost_equal(sources, sources2)
_raw1 = ica.apply(raw, exclude=[1])
_raw2 = ica_read.apply(raw, exclude=[1])
assert_array_almost_equal(_raw1[:, :][0], _raw2[:, :][0])
os.remove(test_ica_fname)
# check score funcs
for name, func in get_score_funcs().items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(raw, target='EOG 061', score_func=func,
start=0, stop=10)
assert (ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(raw, start=0, stop=50, score_func=stats.skew)
# check exception handling
pytest.raises(ValueError, ica.score_sources, raw,
target=np.arange(1))
params = []
params += [(None, -1, slice(2), [0, 1])] # variance, kurtosis params
params += [(None, 'MEG 1531')] # ECG / EOG channel params
for idx, ch_name in product(*params):
ica.detect_artifacts(raw, start_find=0, stop_find=50, ecg_ch=ch_name,
eog_ch=ch_name, skew_criterion=idx,
var_criterion=idx, kurt_criterion=idx)
# Make sure detect_artifacts marks the right components.
# For int criterion, the doc says "E.g. range(2) would return the two
# sources with the highest score". Assert that's what it does.
# Only test for skew, since it's always the same code.
ica.exclude = []
ica.detect_artifacts(raw, start_find=0, stop_find=50, ecg_ch=None,
eog_ch=None, skew_criterion=0,
var_criterion=None, kurt_criterion=None)
assert np.abs(scores[ica.exclude]) == np.max(np.abs(scores))
evoked = epochs.average()
evoked_data = evoked.data.copy()
raw_data = raw[:][0].copy()
epochs_data = epochs.get_data().copy()
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_ecg(raw, method='ctps')
assert_equal(len(scores), ica.n_components_)
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_ecg(raw, method='correlation')
assert_equal(len(scores), ica.n_components_)
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_eog(raw)
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(epochs, method='ctps')
assert_equal(len(scores), ica.n_components_)
pytest.raises(ValueError, ica.find_bads_ecg, epochs.average(),
method='ctps')
pytest.raises(ValueError, ica.find_bads_ecg, raw,
method='crazy-coupling')
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_eog(raw)
assert_equal(len(scores), ica.n_components_)
raw.info['chs'][raw.ch_names.index('EOG 061') - 1]['kind'] = 202
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_eog(raw)
assert (isinstance(scores, list))
assert_equal(len(scores[0]), ica.n_components_)
idx, scores = ica.find_bads_eog(evoked, ch_name='MEG 1441')
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(evoked, method='correlation')
assert_equal(len(scores), ica.n_components_)
assert_array_equal(raw_data, raw[:][0])
assert_array_equal(epochs_data, epochs.get_data())
assert_array_equal(evoked_data, evoked.data)
# check score funcs
for name, func in get_score_funcs().items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(epochs_eog, target='EOG 061',
score_func=func)
assert (ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(epochs, score_func=stats.skew)
# check exception handling
pytest.raises(ValueError, ica.score_sources, epochs,
target=np.arange(1))
# ecg functionality
ecg_scores = ica.score_sources(raw, target='MEG 1531',
score_func='pearsonr')
with pytest.warns(RuntimeWarning, match='longer'):
ecg_events = ica_find_ecg_events(
raw, sources[np.abs(ecg_scores).argmax()])
assert (ecg_events.ndim == 2)
# eog functionality
eog_scores = ica.score_sources(raw, target='EOG 061',
score_func='pearsonr')
with pytest.warns(RuntimeWarning, match='longer'):
eog_events = ica_find_eog_events(
raw, sources[np.abs(eog_scores).argmax()])
assert (eog_events.ndim == 2)
# Test ica fiff export
ica_raw = ica.get_sources(raw, start=0, stop=100)
assert (ica_raw.last_samp - ica_raw.first_samp == 100)
assert_equal(len(ica_raw._filenames), 1) # API consistency
ica_chans = [ch for ch in ica_raw.ch_names if 'ICA' in ch]
assert (ica.n_components_ == len(ica_chans))
test_ica_fname = op.join(op.abspath(op.curdir), 'test-ica_raw.fif')
ica.n_components = np.int32(ica.n_components)
ica_raw.save(test_ica_fname, overwrite=True)
ica_raw2 = read_raw_fif(test_ica_fname, preload=True)
assert_allclose(ica_raw._data, ica_raw2._data, rtol=1e-5, atol=1e-4)
ica_raw2.close()
os.remove(test_ica_fname)
# Test ica epochs export
ica_epochs = ica.get_sources(epochs)
assert (ica_epochs.events.shape == epochs.events.shape)
ica_chans = [ch for ch in ica_epochs.ch_names if 'ICA' in ch]
assert (ica.n_components_ == len(ica_chans))
assert (ica.n_components_ == ica_epochs.get_data().shape[1])
assert (ica_epochs._raw is None)
assert (ica_epochs.preload is True)
# test float n pca components
ica.pca_explained_variance_ = np.array([0.2] * 5)
ica.n_components_ = 0
for ncomps, expected in [[0.3, 1], [0.9, 4], [1, 1]]:
ncomps_ = ica._check_n_pca_components(ncomps)
assert (ncomps_ == expected)
ica = ICA(method=method)
with pytest.warns(None): # sometimes does not converge
ica.fit(raw, picks=picks[:5])
with pytest.warns(RuntimeWarning, match='longer'):
ica.find_bads_ecg(raw)
ica.find_bads_eog(epochs, ch_name='MEG 0121')
assert_array_equal(raw_data, raw[:][0])
raw.drop_channels(['MEG 0122'])
pytest.raises(RuntimeError, ica.find_bads_eog, raw)
with pytest.warns(RuntimeWarning, match='longer'):
pytest.raises(RuntimeError, ica.find_bads_ecg, raw)
def preprocess_ICA_fif_to_ts(fif_file, ECG_ch_name, EoG_ch_name, l_freq, h_freq, down_sfreq, variance, is_sensor_space, data_type):
import os
import numpy as np
import mne
from mne.io import Raw
from mne.preprocessing import ICA, read_ica
from mne.preprocessing import create_ecg_epochs, create_eog_epochs
from mne.report import Report
from nipype.utils.filemanip import split_filename as split_f
report = Report()
subj_path, basename, ext = split_f(fif_file)
(data_path, sbj_name) = os.path.split(subj_path)
print data_path
# Read raw
# If None the compensation in the data is not modified.
# If set to n, e.g. 3, apply gradient compensation of grade n as for
# CTF systems (compensation=3)
raw = Raw(fif_file, preload=True)
# select sensors
select_sensors = mne.pick_types(raw.info, meg=True, ref_meg=False,
exclude='bads')
picks_meeg = mne.pick_types(raw.info, meg=True, eeg=True, exclude='bads')
# save electrode locations
sens_loc = [raw.info['chs'][i]['loc'][:3] for i in select_sensors]
sens_loc = np.array(sens_loc)
channel_coords_file = os.path.abspath("correct_channel_coords.txt")
print '*** ' + channel_coords_file + '***'
np.savetxt(channel_coords_file, sens_loc, fmt='%s')
# save electrode names
sens_names = np.array([raw.ch_names[pos] for pos in select_sensors],dtype = "str")
# AP 21032016
# channel_names_file = os.path.join(data_path, "correct_channel_names.txt")
channel_names_file = os.path.abspath("correct_channel_names.txt")
np.savetxt(channel_names_file,sens_names , fmt = '%s')
### filtering + downsampling
raw.filter(l_freq=l_freq, h_freq=h_freq, picks=picks_meeg,
method='iir', n_jobs=8)
# raw.filter(l_freq = l_freq, h_freq = h_freq, picks = picks_meeg,
# method='iir')
# raw.resample(sfreq=down_sfreq, npad=0)
### 1) Fit ICA model using the FastICA algorithm
# Other available choices are `infomax` or `extended-infomax`
# We pass a float value between 0 and 1 to select n_components based on the
# percentage of variance explained by the PCA components.
ICA_title = 'Sources related to %s artifacts (red)'
is_show = False # visualization
reject = dict(mag=4e-12, grad=4000e-13)
# check if we have an ICA, if yes, we load it
ica_filename = os.path.join(subj_path,basename + "-ica.fif")
if os.path.exists(ica_filename) is False:
ica = ICA(n_components=variance, method='fastica', max_iter=500) # , max_iter=500
ica.fit(raw, picks=select_sensors, reject=reject) # decim = 3,
has_ICA = False
else:
has_ICA = True
print ica_filename + ' exists!!!'
ica = read_ica(ica_filename)
ica.exclude = []
# 2) identify bad components by analyzing latent sources.
# generate ECG epochs use detection via phase statistics
# if we just have exclude channels we jump these steps
# if len(ica.exclude)==0:
n_max_ecg = 3
n_max_eog = 2
# check if ECG_ch_name is in the raw channels
if ECG_ch_name in raw.info['ch_names']:
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5,
picks=select_sensors,
ch_name=ECG_ch_name)
# if not a synthetic ECG channel is created from cross channel average
else:
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5,
picks=select_sensors)
# ICA for ECG artifact
# threshold=0.25 come default
ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
print scores
print '\n len ecg_inds *** ' + str(len(ecg_inds)) + '***\n'
if len(ecg_inds) > 0:
ecg_evoked = ecg_epochs.average()
fig1 = ica.plot_scores(scores, exclude=ecg_inds,
title=ICA_title % 'ecg', show=is_show)
show_picks = np.abs(scores).argsort()[::-1][:5] # Pick the five largest scores and plot them
# Plot estimated latent sources given the unmixing matrix.
#ica.plot_sources(raw, show_picks, exclude=ecg_inds, title=ICA_title % 'ecg', show=is_show)
t_start = 0
t_stop = 30 # take the fist 30s
fig2 = ica.plot_sources(raw, show_picks, exclude=ecg_inds, title=ICA_title % 'ecg' + ' in 30s'
,start = t_start, stop = t_stop, show=is_show)
# topoplot of unmixing matrix columns
fig3 = ica.plot_components(show_picks, title=ICA_title % 'ecg', colorbar=True, show=is_show)
ecg_inds = ecg_inds[:n_max_ecg]
ica.exclude += ecg_inds
fig4 = ica.plot_sources(ecg_evoked, exclude=ecg_inds, show=is_show) # plot ECG sources + selection
fig5 = ica.plot_overlay(ecg_evoked, exclude=ecg_inds, show=is_show) # plot ECG cleaning
fig = [fig1, fig2, fig3, fig4, fig5]
report.add_figs_to_section(fig, captions=['Scores of ICs related to ECG',
'Time Series plots of ICs (ECG)',
'TopoMap of ICs (ECG)',
'Time-locked ECG sources',
'ECG overlay'], section = 'ICA - ECG')
# check if EoG_ch_name is in the raw channels
# if EoG_ch_name is empty if data_type is fif, ICA routine automatically looks for EEG61, EEG62
# otherwise if data_type is ds we jump this step
if not EoG_ch_name and data_type=='ds':
eog_inds = []
else:
if EoG_ch_name in raw.info['ch_names']:
### ICA for eye blink artifact - detect EOG by correlation
eog_inds, scores = ica.find_bads_eog(raw, ch_name = EoG_ch_name)
else:
eog_inds, scores = ica.find_bads_eog(raw)
if len(eog_inds) > 0:
fig6 = ica.plot_scores(scores, exclude=eog_inds, title=ICA_title % 'eog', show=is_show)
report.add_figs_to_section(fig6, captions=['Scores of ICs related to EOG'],
section = 'ICA - EOG')
# check how many EoG ch we have
rs = np.shape(scores)
if len(rs)>1:
rr = rs[0]
show_picks = [np.abs(scores[i][:]).argsort()[::-1][:5] for i in range(rr)]
for i in range(rr):
fig7 = ica.plot_sources(raw, show_picks[i][:], exclude=eog_inds,
start = raw.times[0], stop = raw.times[-1],
title=ICA_title % 'eog',show=is_show)
fig8 = ica.plot_components(show_picks[i][:], title=ICA_title % 'eog', colorbar=True, show=is_show) # ICA nel tempo
fig = [fig7, fig8]
report.add_figs_to_section(fig, captions=['Scores of ICs related to EOG',
'Time Series plots of ICs (EOG)'],
section = 'ICA - EOG')
else:
show_picks = np.abs(scores).argsort()[::-1][:5]
fig7 = ica.plot_sources(raw, show_picks, exclude=eog_inds, title=ICA_title % 'eog', show=is_show)
fig8 = ica.plot_components(show_picks, title=ICA_title % 'eog', colorbar=True, show=is_show)
fig = [fig7, fig8]
report.add_figs_to_section(fig, captions=['Time Series plots of ICs (EOG)',
'TopoMap of ICs (EOG)',],
section = 'ICA - EOG')
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
if EoG_ch_name in raw.info['ch_names']:
eog_evoked = create_eog_epochs(raw, tmin=-.5, tmax=.5, picks=select_sensors,
ch_name=EoG_ch_name).average()
else:
eog_evoked = create_eog_epochs(raw, tmin=-.5, tmax=.5, picks=select_sensors).average()
fig9 = ica.plot_sources(eog_evoked, exclude=eog_inds, show=is_show) # plot EOG sources + selection
fig10 = ica.plot_overlay(eog_evoked, exclude=eog_inds, show=is_show) # plot EOG cleaning
fig = [fig9, fig10]
report.add_figs_to_section(fig, captions=['Time-locked EOG sources',
'EOG overlay'], section = 'ICA - EOG')
fig11 = ica.plot_overlay(raw, show=is_show)
report.add_figs_to_section(fig11, captions=['Signal'], section = 'Signal quality')
### plot all topographies and time seris of the ICA components
n_ica_components = ica.mixing_matrix_.shape[1]
n_topo = 10;
n_fig = n_ica_components/n_topo;
n_plot = n_ica_components%n_topo;
print '*************** n_fig = ' + str(n_fig) + ' n_plot = ' + str(n_plot) + '********************'
fig = []
t_start = 0
t_stop = None # 60 if we want to take the fist 60s
for n in range(0,n_fig):
fig_tmp = ica.plot_components(range(n_topo*n,n_topo*(n+1)),title='ICA components', show=is_show)
fig.append(fig_tmp)
fig_tmp = ica.plot_sources(raw, range(n_topo*n,n_topo*(n+1)),
start = t_start, stop = t_stop,
title='ICA components')
fig.append(fig_tmp)
# if n_plot > 0:
# fig_tmp = ica.plot_components(range(n_fig*n_topo,n_ica_components), title='ICA components', show=is_show)
# fig.append(fig_tmp)
# fig_tmp = ica.plot_sources(raw, range(n_fig*n_topo,n_ica_components),
# start = t_start, stop = t_stop,
# title='ICA components')
# fig.append(fig_tmp)
#
# for n in range(0, len(fig)):
# report.add_figs_to_section(fig[n], captions=['TOPO'], section = 'ICA Topo Maps')
if n_plot > 5:
n_fig_l = n_plot//5
print '*************************** ' + str(n_fig_l) + ' *********************************'
for n in range(0,n_fig_l):
print range(n_fig*n_topo+5*n, n_fig*n_topo+5*(n+1))
fig_tmp = ica.plot_components(range(n_fig*n_topo+5*n, n_fig*n_topo+5*(n+1)),title='ICA components')
fig.append(fig_tmp)
fig_tmp = ica.plot_sources(raw, range(n_fig*n_topo+5*n, n_fig*n_topo+5*(n+1)),
start = t_start, stop = t_stop,
title='ICA components')
fig.append(fig_tmp)
print range(n_fig*n_topo+5*(n+1),n_ica_components)
fig_tmp = ica.plot_components(range(n_fig*n_topo+5*(n+1),n_ica_components), title='ICA components')
fig.append(fig_tmp)
fig_tmp = ica.plot_sources(raw, range(n_fig*n_topo+5*(n+1),n_ica_components),
start = t_start, stop = t_stop,
title='ICA components')
fig.append(fig_tmp)
for n in range(0, len(fig)):
report.add_figs_to_section(fig[n], captions=['TOPO'], section = 'ICA Topo Maps')
report_filename = os.path.join(subj_path,basename + "-report.html")
print '******* ' + report_filename
report.save(report_filename, open_browser=False, overwrite=True)
# 3) apply ICA to raw data and save solution and report
# check the amplitudes do not change
# raw_ica_file = os.path.abspath(basename[:i_raw] + 'ica-raw.fif')
raw_ica_file = os.path.join(subj_path, basename + '-preproc-raw.fif')
raw_ica = ica.apply(raw)
raw_ica.resample(sfreq=down_sfreq, npad=0)
raw_ica.save(raw_ica_file, overwrite=True)
# save ICA solution
print ica_filename
if has_ICA is False:
ica.save(ica_filename)
# 4) save data
data_noIca, times = raw[select_sensors, :]
data, times = raw_ica[select_sensors, :]
print data.shape
print raw.info['sfreq']
ts_file = os.path.abspath(basename + "_ica.npy")
np.save(ts_file, data)
print '***** TS FILE ' + ts_file + '*****'
if is_sensor_space:
return ts_file, channel_coords_file, channel_names_file, raw.info['sfreq']
else:
# return raw_ica, channel_coords_file, channel_names_file, raw.info['sfreq']
return raw_ica_file, channel_coords_file, channel_names_file, raw.info['sfreq']
def test_ica_additional(method):
"""Test additional ICA functionality."""
_skip_check_picard(method)
tempdir = _TempDir()
stop2 = 500
raw = read_raw_fif(raw_fname).crop(1.5, stop).load_data()
raw.del_proj() # avoid warnings
raw.set_annotations(Annotations([0.5], [0.5], ['BAD']))
# XXX This breaks the tests :(
# raw.info['bads'] = [raw.ch_names[1]]
test_cov = read_cov(test_cov_name)
events = read_events(event_name)
picks = pick_types(raw.info,
meg=True,
stim=False,
ecg=False,
eog=False,
exclude='bads')[1::2]
epochs = Epochs(raw,
events,
None,
tmin,
tmax,
picks=picks,
baseline=(None, 0),
preload=True,
proj=False)
epochs.decimate(3, verbose='error')
assert len(epochs) == 4
# test if n_components=None works
ica = ICA(n_components=None,
max_pca_components=None,
n_pca_components=None,
random_state=0,
method=method,
max_iter=1)
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(epochs)
# for testing eog functionality
picks2 = np.concatenate([picks, pick_types(raw.info, False, eog=True)])
epochs_eog = Epochs(raw,
events[:4],
event_id,
tmin,
tmax,
picks=picks2,
baseline=(None, 0),
preload=True)
del picks2
test_cov2 = test_cov.copy()
ica = ICA(noise_cov=test_cov2,
n_components=3,
max_pca_components=4,
n_pca_components=4,
method=method)
assert (ica.info is None)
with pytest.warns(RuntimeWarning, match='normalize_proj'):
ica.fit(raw, picks[:5])
assert (isinstance(ica.info, Info))
assert (ica.n_components_ < 5)
ica = ICA(n_components=3,
max_pca_components=4,
method=method,
n_pca_components=4,
random_state=0)
pytest.raises(RuntimeError, ica.save, '')
ica.fit(raw, picks=[1, 2, 3, 4, 5], start=start, stop=stop2)
# check passing a ch_name to find_bads_ecg
with pytest.warns(RuntimeWarning, match='longer'):
_, scores_1 = ica.find_bads_ecg(raw)
_, scores_2 = ica.find_bads_ecg(raw, raw.ch_names[1])
assert scores_1[0] != scores_2[0]
# test corrmap
ica2 = ica.copy()
ica3 = ica.copy()
corrmap([ica, ica2], (0, 0),
threshold='auto',
label='blinks',
plot=True,
ch_type="mag")
corrmap([ica, ica2], (0, 0), threshold=2, plot=False, show=False)
assert (ica.labels_["blinks"] == ica2.labels_["blinks"])
assert (0 in ica.labels_["blinks"])
# test retrieval of component maps as arrays
components = ica.get_components()
template = components[:, 0]
EvokedArray(components, ica.info, tmin=0.).plot_topomap([0], time_unit='s')
corrmap([ica, ica3],
template,
threshold='auto',
label='blinks',
plot=True,
ch_type="mag")
assert (ica2.labels_["blinks"] == ica3.labels_["blinks"])
plt.close('all')
# make sure a single threshold in a list works
corrmap([ica, ica3],
template,
threshold=[0.5],
label='blinks',
plot=True,
ch_type="mag")
ica_different_channels = ICA(n_components=2,
random_state=0).fit(raw, picks=[2, 3, 4, 5])
pytest.raises(ValueError, corrmap, [ica_different_channels, ica], (0, 0))
# test warnings on bad filenames
ica_badname = op.join(op.dirname(tempdir), 'test-bad-name.fif.gz')
with pytest.warns(RuntimeWarning, match='-ica.fif'):
ica.save(ica_badname)
with pytest.warns(RuntimeWarning, match='-ica.fif'):
read_ica(ica_badname)
# test decim
ica = ICA(n_components=3,
max_pca_components=4,
n_pca_components=4,
method=method,
max_iter=1)
raw_ = raw.copy()
for _ in range(3):
raw_.append(raw_)
n_samples = raw_._data.shape[1]
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(raw, picks=picks[:5], decim=3)
assert raw_._data.shape[1] == n_samples
# test expl var
ica = ICA(n_components=1.0,
max_pca_components=4,
n_pca_components=4,
method=method,
max_iter=1)
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(raw, picks=None, decim=3)
assert (ica.n_components_ == 4)
ica_var = _ica_explained_variance(ica, raw, normalize=True)
assert (np.all(ica_var[:-1] >= ica_var[1:]))
# test ica sorting
ica.exclude = [0]
ica.labels_ = dict(blink=[0], think=[1])
ica_sorted = _sort_components(ica, [3, 2, 1, 0], copy=True)
assert_equal(ica_sorted.exclude, [3])
assert_equal(ica_sorted.labels_, dict(blink=[3], think=[2]))
# epochs extraction from raw fit
pytest.raises(RuntimeError, ica.get_sources, epochs)
# test reading and writing
test_ica_fname = op.join(op.dirname(tempdir), 'test-ica.fif')
for cov in (None, test_cov):
ica = ICA(noise_cov=cov,
n_components=2,
max_pca_components=4,
n_pca_components=4,
method=method,
max_iter=1)
with pytest.warns(None): # ICA does not converge
ica.fit(raw, picks=picks[:10], start=start, stop=stop2)
sources = ica.get_sources(epochs).get_data()
assert (ica.mixing_matrix_.shape == (2, 2))
assert (ica.unmixing_matrix_.shape == (2, 2))
assert (ica.pca_components_.shape == (4, 10))
assert (sources.shape[1] == ica.n_components_)
for exclude in [[], [0], np.array([1, 2, 3])]:
ica.exclude = exclude
ica.labels_ = {'foo': [0]}
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert (list(ica.exclude) == ica_read.exclude)
assert_equal(ica.labels_, ica_read.labels_)
ica.apply(raw)
ica.exclude = []
ica.apply(raw, exclude=[1])
assert (ica.exclude == [])
ica.exclude = [0, 1]
ica.apply(raw, exclude=[1])
assert (ica.exclude == [0, 1])
ica_raw = ica.get_sources(raw)
assert (ica.exclude == [
ica_raw.ch_names.index(e) for e in ica_raw.info['bads']
])
# test filtering
d1 = ica_raw._data[0].copy()
ica_raw.filter(4, 20, fir_design='firwin2')
assert_equal(ica_raw.info['lowpass'], 20.)
assert_equal(ica_raw.info['highpass'], 4.)
assert ((d1 != ica_raw._data[0]).any())
d1 = ica_raw._data[0].copy()
ica_raw.notch_filter([10], trans_bandwidth=10, fir_design='firwin')
assert ((d1 != ica_raw._data[0]).any())
ica.n_pca_components = 2
ica.method = 'fake'
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert (ica.n_pca_components == ica_read.n_pca_components)
assert_equal(ica.method, ica_read.method)
assert_equal(ica.labels_, ica_read.labels_)
# check type consistency
attrs = ('mixing_matrix_ unmixing_matrix_ pca_components_ '
'pca_explained_variance_ pre_whitener_')
def f(x, y):
return getattr(x, y).dtype
for attr in attrs.split():
assert_equal(f(ica_read, attr), f(ica, attr))
ica.n_pca_components = 4
ica_read.n_pca_components = 4
ica.exclude = []
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
for attr in [
'mixing_matrix_', 'unmixing_matrix_', 'pca_components_',
'pca_mean_', 'pca_explained_variance_', 'pre_whitener_'
]:
assert_array_almost_equal(getattr(ica, attr),
getattr(ica_read, attr))
assert (ica.ch_names == ica_read.ch_names)
assert (isinstance(ica_read.info, Info))
sources = ica.get_sources(raw)[:, :][0]
sources2 = ica_read.get_sources(raw)[:, :][0]
assert_array_almost_equal(sources, sources2)
_raw1 = ica.apply(raw, exclude=[1])
_raw2 = ica_read.apply(raw, exclude=[1])
assert_array_almost_equal(_raw1[:, :][0], _raw2[:, :][0])
os.remove(test_ica_fname)
# check score funcs
for name, func in get_score_funcs().items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(raw,
target='EOG 061',
score_func=func,
start=0,
stop=10)
assert (ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(raw, start=0, stop=50, score_func=stats.skew)
# check exception handling
pytest.raises(ValueError, ica.score_sources, raw, target=np.arange(1))
params = []
params += [(None, -1, slice(2), [0, 1])] # variance, kurtosis params
params += [(None, 'MEG 1531')] # ECG / EOG channel params
for idx, ch_name in product(*params):
ica.detect_artifacts(raw,
start_find=0,
stop_find=50,
ecg_ch=ch_name,
eog_ch=ch_name,
skew_criterion=idx,
var_criterion=idx,
kurt_criterion=idx)
# Make sure detect_artifacts marks the right components.
# For int criterion, the doc says "E.g. range(2) would return the two
# sources with the highest score". Assert that's what it does.
# Only test for skew, since it's always the same code.
ica.exclude = []
ica.detect_artifacts(raw,
start_find=0,
stop_find=50,
ecg_ch=None,
eog_ch=None,
skew_criterion=0,
var_criterion=None,
kurt_criterion=None)
assert np.abs(scores[ica.exclude]) == np.max(np.abs(scores))
evoked = epochs.average()
evoked_data = evoked.data.copy()
raw_data = raw[:][0].copy()
epochs_data = epochs.get_data().copy()
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_ecg(raw, method='ctps')
assert_equal(len(scores), ica.n_components_)
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_ecg(raw, method='correlation')
assert_equal(len(scores), ica.n_components_)
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_eog(raw)
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(epochs, method='ctps')
assert_equal(len(scores), ica.n_components_)
pytest.raises(ValueError,
ica.find_bads_ecg,
epochs.average(),
method='ctps')
pytest.raises(ValueError, ica.find_bads_ecg, raw, method='crazy-coupling')
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_eog(raw)
assert_equal(len(scores), ica.n_components_)
raw.info['chs'][raw.ch_names.index('EOG 061') - 1]['kind'] = 202
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_eog(raw)
assert (isinstance(scores, list))
assert_equal(len(scores[0]), ica.n_components_)
idx, scores = ica.find_bads_eog(evoked, ch_name='MEG 1441')
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(evoked, method='correlation')
assert_equal(len(scores), ica.n_components_)
assert_array_equal(raw_data, raw[:][0])
assert_array_equal(epochs_data, epochs.get_data())
assert_array_equal(evoked_data, evoked.data)
# check score funcs
for name, func in get_score_funcs().items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(epochs_eog,
target='EOG 061',
score_func=func)
assert (ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(epochs, score_func=stats.skew)
# check exception handling
pytest.raises(ValueError, ica.score_sources, epochs, target=np.arange(1))
# ecg functionality
ecg_scores = ica.score_sources(raw,
target='MEG 1531',
score_func='pearsonr')
with pytest.warns(RuntimeWarning, match='longer'):
ecg_events = ica_find_ecg_events(raw,
sources[np.abs(ecg_scores).argmax()])
assert (ecg_events.ndim == 2)
# eog functionality
eog_scores = ica.score_sources(raw,
target='EOG 061',
score_func='pearsonr')
with pytest.warns(RuntimeWarning, match='longer'):
eog_events = ica_find_eog_events(raw,
sources[np.abs(eog_scores).argmax()])
assert (eog_events.ndim == 2)
# Test ica fiff export
ica_raw = ica.get_sources(raw, start=0, stop=100)
assert (ica_raw.last_samp - ica_raw.first_samp == 100)
assert_equal(len(ica_raw._filenames), 1) # API consistency
ica_chans = [ch for ch in ica_raw.ch_names if 'ICA' in ch]
assert (ica.n_components_ == len(ica_chans))
test_ica_fname = op.join(op.abspath(op.curdir), 'test-ica_raw.fif')
ica.n_components = np.int32(ica.n_components)
ica_raw.save(test_ica_fname, overwrite=True)
ica_raw2 = read_raw_fif(test_ica_fname, preload=True)
assert_allclose(ica_raw._data, ica_raw2._data, rtol=1e-5, atol=1e-4)
ica_raw2.close()
os.remove(test_ica_fname)
# Test ica epochs export
ica_epochs = ica.get_sources(epochs)
assert (ica_epochs.events.shape == epochs.events.shape)
ica_chans = [ch for ch in ica_epochs.ch_names if 'ICA' in ch]
assert (ica.n_components_ == len(ica_chans))
assert (ica.n_components_ == ica_epochs.get_data().shape[1])
assert (ica_epochs._raw is None)
assert (ica_epochs.preload is True)
# test float n pca components
ica.pca_explained_variance_ = np.array([0.2] * 5)
ica.n_components_ = 0
for ncomps, expected in [[0.3, 1], [0.9, 4], [1, 1]]:
ncomps_ = ica._check_n_pca_components(ncomps)
assert (ncomps_ == expected)
ica = ICA(method=method)
with pytest.warns(None): # sometimes does not converge
ica.fit(raw, picks=picks[:5])
with pytest.warns(RuntimeWarning, match='longer'):
ica.find_bads_ecg(raw)
ica.find_bads_eog(epochs, ch_name='MEG 0121')
assert_array_equal(raw_data, raw[:][0])
raw.drop_channels(['MEG 0122'])
pytest.raises(RuntimeError, ica.find_bads_eog, raw)
with pytest.warns(RuntimeWarning, match='longer'):
pytest.raises(RuntimeError, ica.find_bads_ecg, raw)
def apply_ica(subject):
print("Processing subject: %s" % subject)
meg_subject_dir = op.join(config.meg_dir, subject)
# load epochs to reject ICA components
extension = '-epo'
fname_in = op.join(meg_subject_dir, config.base_fname.format(**locals()))
epochs = mne.read_epochs(fname_in, preload=True)
extension = '_cleaned-epo'
fname_out = op.join(meg_subject_dir, config.base_fname.format(**locals()))
print("Input: ", fname_in)
print("Output: ", fname_out)
# load first run of raw data for ecg /eog epochs
raw_list = list()
print(" Loading one run from raw data")
extension = config.runs[0] + '_sss_raw'
raw_fname_in = op.join(meg_subject_dir,
config.base_fname.format(**locals()))
raw = mne.io.read_raw_fif(raw_fname_in, preload=True)
# run ICA on MEG and EEG
picks_meg = mne.pick_types(raw.info,
meg=True,
eeg=False,
eog=False,
stim=False,
exclude='bads')
picks_eeg = mne.pick_types(raw.info,
meg=False,
eeg=True,
eog=False,
stim=False,
exclude='bads')
all_picks = {'meg': picks_meg, 'eeg': picks_eeg}
ch_types = []
if 'eeg' in config.ch_types:
ch_types.append('eeg')
if set(config.ch_types).intersection(('meg', 'grad', 'mag')):
ch_types.append('meg')
for ch_type in ch_types:
print(ch_type)
picks = all_picks[ch_type]
# Load ICA
fname_ica = op.join(
meg_subject_dir,
'{0}_{1}_{2}-ica.fif'.format(subject, config.study_name, ch_type))
print('Reading ICA: ' + fname_ica)
ica = read_ica(fname=fname_ica)
pick_ecg = mne.pick_types(raw.info,
meg=False,
eeg=False,
ecg=True,
eog=False)
# ECG
# either needs an ecg channel, or avg of the mags (i.e. MEG data)
if pick_ecg or ch_type == 'meg':
picks_ecg = np.concatenate([picks, pick_ecg])
# Create ecg epochs
if ch_type == 'meg':
reject = {
'mag': config.reject['mag'],
'grad': config.reject['grad']
}
elif ch_type == 'eeg':
reject = {'eeg': config.reject['eeg']}
ecg_epochs = create_ecg_epochs(raw,
picks=picks_ecg,
reject=reject,
baseline=(None, 0),
tmin=-0.5,
tmax=0.5)
ecg_average = ecg_epochs.average()
ecg_inds, scores = \
ica.find_bads_ecg(ecg_epochs, method='ctps',
threshold=config.ica_ctps_ecg_threshold)
del ecg_epochs
report_fname = \
'{0}_{1}_{2}-reject_ica.html'.format(subject,
config.study_name,
ch_type)
report_fname = op.join(meg_subject_dir, report_fname)
report = Report(report_fname, verbose=False)
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores,
exclude=ecg_inds,
show=config.plot),
captions=ch_type.upper() + ' - ECG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(ecg_average,
exclude=ecg_inds,
show=config.plot),
captions=ch_type.upper() + ' - ECG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(ecg_average,
exclude=ecg_inds,
show=config.plot),
captions=ch_type.upper() + ' - ECG - ' +
'Corrections')
else:
# XXX : to check when EEG only is processed
print('no ECG channel is present. Cannot automate ICAs component '
'detection for EOG!')
# EOG
pick_eog = mne.pick_types(raw.info,
meg=False,
eeg=False,
ecg=False,
eog=True)
if pick_eog.any():
print('using EOG channel')
picks_eog = np.concatenate([picks, pick_eog])
# Create eog epochs
eog_epochs = create_eog_epochs(raw,
picks=picks_eog,
reject=None,
baseline=(None, 0),
tmin=-0.5,
tmax=0.5)
eog_average = eog_epochs.average()
eog_inds, scores = ica.find_bads_eog(eog_epochs, threshold=3.0)
del eog_epochs
params = dict(exclude=eog_inds, show=config.plot)
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores, **params),
captions=ch_type.upper() + ' - EOG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(eog_average, **params),
captions=ch_type.upper() + ' - EOG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(eog_average, **params),
captions=ch_type.upper() + ' - EOG - ' +
'Corrections')
report.save(report_fname, overwrite=True, open_browser=False)
else:
print('no EOG channel is present. Cannot automate ICAs component '
'detection for EOG!')
ica_reject = (list(ecg_inds) + list(eog_inds) +
list(config.rejcomps_man[subject][ch_type]))
# now reject the components
print('Rejecting from %s: %s' % (ch_type, ica_reject))
epochs = ica.apply(epochs, exclude=ica_reject)
print('Saving cleaned epochs')
epochs.save(fname_out)
fig = ica.plot_overlay(raw, exclude=ica_reject, show=config.plot)
report.add_figs_to_section(fig,
captions=ch_type.upper() +
' - ALL(epochs) - Corrections')
if config.plot:
epochs.plot_image(combine='gfp',
group_by='type',
sigma=2.,
cmap="YlGnBu_r",
show=config.plot)
#noise_cov_er.save(empty_room_fname[:-4]+'-cov.fif')
###############################################################################
# 1) Fit ICA model using the FastICA algorithm
# Other available choices are `infomax` or `extended-infomax`
# We pass a float value between 0 and 1 to select n_components based on the
# percentage of variance explained by the PCA components.
raw = Raw(raw_fname, preload=True)
picks = mne.pick_types(raw.info, meg=True, eeg=False, eog=False, ecg=False,
stim=False, exclude='bads')
# maximum number of components to reject
n_max_ecg, n_max_eog = 2, 3 # here we expect horizontal EOG components
try:
ica = read_ica(raw_data_folder + '/ica_pre.fif')
except:
ica = ICA(n_components=0.95, max_pca_components = 64, method='fastica')
#noise_cov = noise_cov_er)
#ica.fit(raw, picks=picks, decim=3, reject=dict(mag=4e-12, grad=4000e-13))
ica.fit(raw, picks=picks, decim = 5, reject=dict(mag=4e-11, grad=4000e-12))
# To save an ICA solution you can say:
###############################################################################
# 2) identify bad components by analyzing latent sources.
title = 'Sources related to %s artifacts (red)'
# generate ECG epochs use detection via phase statistics
def apply_ica(subject, run, session):
deriv_path = config.get_subject_deriv_path(subject=subject,
session=session,
kind=config.get_kind())
bids_basename = make_bids_basename(subject=subject,
session=session,
task=config.get_task(),
acquisition=config.acq,
run=None,
processing=config.proc,
recording=config.rec,
space=config.space)
fname_in = op.join(deriv_path, bids_basename + '-epo.fif')
fname_out = op.join(deriv_path, bids_basename + '_cleaned-epo.fif')
# load epochs to reject ICA components
epochs = mne.read_epochs(fname_in, preload=True)
msg = f'Input: {fname_in}, Output: {fname_out}'
logger.info(
gen_log_message(message=msg, step=5, subject=subject, session=session))
# load first run of raw data for ecg / eog epochs
msg = 'Loading first run from raw data'
logger.debug(
gen_log_message(message=msg, step=5, subject=subject, session=session))
bids_basename = make_bids_basename(subject=subject,
session=session,
task=config.get_task(),
acquisition=config.acq,
run=config.get_runs()[0],
processing=config.proc,
recording=config.rec,
space=config.space)
if config.use_maxwell_filter:
raw_fname_in = op.join(deriv_path, bids_basename + '_sss_raw.fif')
else:
raw_fname_in = \
op.join(deriv_path, bids_basename + '_filt_raw.fif')
raw = mne.io.read_raw_fif(raw_fname_in, preload=True)
# run ICA on MEG and EEG
picks_meg = mne.pick_types(raw.info,
meg=True,
eeg=False,
eog=False,
stim=False,
exclude='bads')
picks_eeg = mne.pick_types(raw.info,
meg=False,
eeg=True,
eog=False,
stim=False,
exclude='bads')
all_picks = {'meg': picks_meg, 'eeg': picks_eeg}
for ch_type in config.ch_types:
report = None
picks = all_picks[ch_type]
# Load ICA
fname_ica = op.join(deriv_path, f'bids_basename_{ch_type}-ica.fif')
msg = f'Reading ICA: {fname_ica}'
logger.debug(
gen_log_message(message=msg,
step=5,
subject=subject,
session=session))
ica = read_ica(fname=fname_ica)
pick_ecg = mne.pick_types(raw.info,
meg=False,
eeg=False,
ecg=True,
eog=False)
# ECG
# either needs an ecg channel, or avg of the mags (i.e. MEG data)
ecg_inds = list()
if pick_ecg or ch_type == 'meg':
picks_ecg = np.concatenate([picks, pick_ecg])
# Create ecg epochs
if ch_type == 'meg':
reject = {
'mag': config.reject['mag'],
'grad': config.reject['grad']
}
elif ch_type == 'eeg':
reject = {'eeg': config.reject['eeg']}
ecg_epochs = create_ecg_epochs(raw,
picks=picks_ecg,
reject=reject,
baseline=(None, 0),
tmin=-0.5,
tmax=0.5)
ecg_average = ecg_epochs.average()
ecg_inds, scores = \
ica.find_bads_ecg(ecg_epochs, method='ctps',
threshold=config.ica_ctps_ecg_threshold)
del ecg_epochs
report_fname = \
op.join(deriv_path,
bids_basename + f'_{ch_type}-reject_ica.html')
report = Report(report_fname, verbose=False)
# Plot r score
report.add_figs_to_section(
ica.plot_scores(scores,
exclude=ecg_inds,
show=config.interactive),
captions=ch_type.upper() + ' - ECG - R scores')
# Plot source time course
report.add_figs_to_section(
ica.plot_sources(ecg_average,
exclude=ecg_inds,
show=config.interactive),
captions=ch_type.upper() + ' - ECG - Sources time course')
# Plot source time course
report.add_figs_to_section(
ica.plot_overlay(ecg_average,
exclude=ecg_inds,
show=config.interactive),
captions=ch_type.upper() + ' - ECG - Corrections')
else:
# XXX : to check when EEG only is processed
msg = ('No ECG channel is present. Cannot automate IC detection '
'for ECG')
logger.info(
gen_log_message(message=msg,
step=5,
subject=subject,
session=session))
# EOG
pick_eog = mne.pick_types(raw.info,
meg=False,
eeg=False,
ecg=False,
eog=True)
eog_inds = list()
if pick_eog.any():
msg = 'Using EOG channel'
logger.debug(
gen_log_message(message=msg,
step=5,
subject=subject,
session=session))
picks_eog = np.concatenate([picks, pick_eog])
# Create eog epochs
eog_epochs = create_eog_epochs(raw,
picks=picks_eog,
reject=None,
baseline=(None, 0),
tmin=-0.5,
tmax=0.5)
eog_average = eog_epochs.average()
eog_inds, scores = ica.find_bads_eog(eog_epochs, threshold=3.0)
del eog_epochs
params = dict(exclude=eog_inds, show=config.interactive)
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores, **params),
captions=ch_type.upper() + ' - EOG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(eog_average, **params),
captions=ch_type.upper() + ' - EOG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(eog_average, **params),
captions=ch_type.upper() + ' - EOG - ' +
'Corrections')
report.save(report_fname, overwrite=True, open_browser=False)
else:
msg = ('No EOG channel is present. Cannot automate IC detection '
'for EOG')
logger.info(
gen_log_message(message=msg,
step=5,
subject=subject,
session=session))
ica_reject = (list(ecg_inds) + list(eog_inds) +
list(config.rejcomps_man[subject][ch_type]))
# now reject the components
msg = f'Rejecting from {ch_type}: {ica_reject}'
logger.info(
gen_log_message(message=msg,
step=5,
subject=subject,
session=session))
epochs = ica.apply(epochs, exclude=ica_reject)
msg = 'Saving cleaned epochs'
logger.info(
gen_log_message(message=msg,
step=5,
subject=subject,
session=session))
epochs.save(fname_out)
if report is not None:
fig = ica.plot_overlay(raw,
exclude=ica_reject,
show=config.interactive)
report.add_figs_to_section(fig,
captions=ch_type.upper() +
' - ALL(epochs) - Corrections')
if config.interactive:
epochs.plot_image(combine='gfp',
group_by='type',
sigma=2.,
cmap="YlGnBu_r",
show=config.interactive)
parser = ArgumentParser(description=__doc__)
parser.add_argument("subject", help="subject id")
subj = parser.parse_args().subject
json_path = bp.root_json.fpath(subject=subj, task="rest", run=None)
with open(json_path, "r") as f:
er_relpath = json.load(f)["AssociatedEmptyRoom"]
er_path = dirs.bids_root / er_relpath
er_raw = read_raw_fif(er_path, preload=True)
# remove ICA components marked for 'quesitions' data from ER data
ica_sol_path = bp.ica_sol.fpath(subject=subj, task="questions")
ica_bads_path = bp.ica_bads.fpath(subject=subj, task="questions")
ica = read_ica(ica_sol_path)
ica.exclude = read_ica_bads(ica_bads_path, logger)
logger.info(f"Excluding ICs {ica.exclude} from ER data")
ica.apply(er_raw)
noise_cov = compute_raw_covariance(er_raw, method="auto")
del er_raw
info = read_info(bp.epochs.fpath(subject=subj))
fwd = read_forward_solution(bp.fwd.fpath(subject=subj))
inv_path = bp.inv.fpath(subject=subj)
inverse_operator = make_inverse_operator(info, fwd, noise_cov, rank="info")
write_inverse_operator(inv_path, inverse_operator)
def test_ica_additional():
"""Test additional functionality
"""
stop2 = 500
test_cov2 = deepcopy(test_cov)
ica = ICA(noise_cov=test_cov2, n_components=3, max_pca_components=4,
n_pca_components=4)
ica.decompose_raw(raw, picks[:5])
assert_true(ica.n_components_ < 5)
ica = ICA(n_components=3, max_pca_components=4,
n_pca_components=4)
assert_raises(RuntimeError, ica.save, '')
ica.decompose_raw(raw, picks=None, start=start, stop=stop2)
# epochs extraction from raw fit
assert_raises(RuntimeError, ica.get_sources_epochs, epochs)
# test reading and writing
test_ica_fname = op.join(op.dirname(tempdir), 'ica_test.fif')
for cov in (None, test_cov):
ica = ICA(noise_cov=cov, n_components=3, max_pca_components=4,
n_pca_components=4)
ica.decompose_raw(raw, picks=picks, start=start, stop=stop2)
sources = ica.get_sources_epochs(epochs)
assert_true(sources.shape[1] == ica.n_components_)
for exclude in [[], [0]]:
ica.exclude = [0]
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.exclude == ica_read.exclude)
# test pick merge -- add components
ica.pick_sources_raw(raw, exclude=[1])
assert_true(ica.exclude == [0, 1])
# -- only as arg
ica.exclude = []
ica.pick_sources_raw(raw, exclude=[0, 1])
assert_true(ica.exclude == [0, 1])
# -- remove duplicates
ica.exclude += [1]
ica.pick_sources_raw(raw, exclude=[0, 1])
assert_true(ica.exclude == [0, 1])
ica_raw = ica.sources_as_raw(raw)
assert_true(ica.exclude == [ica.ch_names.index(e) for e in
ica_raw.info['bads']])
ica.n_pca_components = 2
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.n_pca_components ==
ica_read.n_pca_components)
ica.n_pca_components = 4
ica_read.n_pca_components = 4
ica.exclude = []
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.ch_names == ica_read.ch_names)
assert_true(np.allclose(ica.mixing_matrix_, ica_read.mixing_matrix_,
rtol=1e-16, atol=1e-32))
assert_array_equal(ica.pca_components_,
ica_read.pca_components_)
assert_array_equal(ica.pca_mean_, ica_read.pca_mean_)
assert_array_equal(ica.pca_explained_variance_,
ica_read.pca_explained_variance_)
assert_array_equal(ica._pre_whitener, ica_read._pre_whitener)
# assert_raises(RuntimeError, ica_read.decompose_raw, raw)
sources = ica.get_sources_raw(raw)
sources2 = ica_read.get_sources_raw(raw)
assert_array_almost_equal(sources, sources2)
_raw1 = ica.pick_sources_raw(raw, exclude=[1])
_raw2 = ica_read.pick_sources_raw(raw, exclude=[1])
assert_array_almost_equal(_raw1[:, :][0], _raw2[:, :][0])
os.remove(test_ica_fname)
# score funcs raw, with catch since "ties preclude exact" warning
# XXX this should be fixed by a future PR...
with warnings.catch_warnings(True) as w:
sfunc_test = [ica.find_sources_raw(raw, target='EOG 061',
score_func=n, start=0, stop=10)
for n, f in score_funcs.items()]
# score funcs raw
# check lenght of scores
[assert_true(ica.n_components_ == len(scores)) for scores in sfunc_test]
# check univariate stats
scores = ica.find_sources_raw(raw, score_func=stats.skew)
# check exception handling
assert_raises(ValueError, ica.find_sources_raw, raw,
target=np.arange(1))
## score funcs epochs ##
# check lenght of scores
# XXX this needs to be fixed, some of the score funcs don't seem to be
# suited for the testing data.
with warnings.catch_warnings(True) as w:
sfunc_test = [ica.find_sources_epochs(epochs_eog, target='EOG 061',
score_func=n)
for n, f in score_funcs.items()]
# check lenght of scores
[assert_true(ica.n_components_ == len(scores)) for scores in sfunc_test]
# check univariat stats
scores = ica.find_sources_epochs(epochs, score_func=stats.skew)
# check exception handling
assert_raises(ValueError, ica.find_sources_epochs, epochs,
target=np.arange(1))
# ecg functionality
ecg_scores = ica.find_sources_raw(raw, target='MEG 1531',
score_func='pearsonr')
ecg_events = ica_find_ecg_events(raw, sources[np.abs(ecg_scores).argmax()])
assert_true(ecg_events.ndim == 2)
# eog functionality
eog_scores = ica.find_sources_raw(raw, target='EOG 061',
score_func='pearsonr')
eog_events = ica_find_eog_events(raw, sources[np.abs(eog_scores).argmax()])
assert_true(eog_events.ndim == 2)
# Test ica fiff export
ica_raw = ica.sources_as_raw(raw, start=0, stop=100)
assert_true(ica_raw.last_samp - ica_raw.first_samp == 100)
ica_chans = [ch for ch in ica_raw.ch_names if 'ICA' in ch]
assert_true(ica.n_components_ == len(ica_chans))
test_ica_fname = op.join(op.abspath(op.curdir), 'test_ica.fif')
ica_raw.save(test_ica_fname)
ica_raw2 = fiff.Raw(test_ica_fname, preload=True)
assert_array_almost_equal(ica_raw._data, ica_raw2._data)
ica_raw2.close()
os.remove(test_ica_fname)
# regression test for plot method
assert_raises(ValueError, ica.plot_sources_raw, raw,
order=np.arange(50))
assert_raises(ValueError, ica.plot_sources_epochs, epochs,
order=np.arange(50))
def test_ica_additional():
"""Test additional ICA functionality"""
tempdir = _TempDir()
stop2 = 500
raw = Raw(raw_fname).crop(1.5, stop, False)
raw.load_data()
picks = pick_types(raw.info, meg=True, stim=False, ecg=False,
eog=False, exclude='bads')
test_cov = read_cov(test_cov_name)
events = read_events(event_name)
picks = pick_types(raw.info, meg=True, stim=False, ecg=False,
eog=False, exclude='bads')
epochs = Epochs(raw, events[:4], event_id, tmin, tmax, picks=picks,
baseline=(None, 0), preload=True)
# test if n_components=None works
with warnings.catch_warnings(record=True):
ica = ICA(n_components=None,
max_pca_components=None,
n_pca_components=None, random_state=0)
ica.fit(epochs, picks=picks, decim=3)
# for testing eog functionality
picks2 = pick_types(raw.info, meg=True, stim=False, ecg=False,
eog=True, exclude='bads')
epochs_eog = Epochs(raw, events[:4], event_id, tmin, tmax, picks=picks2,
baseline=(None, 0), preload=True)
test_cov2 = test_cov.copy()
ica = ICA(noise_cov=test_cov2, n_components=3, max_pca_components=4,
n_pca_components=4)
assert_true(ica.info is None)
with warnings.catch_warnings(record=True):
ica.fit(raw, picks[:5])
assert_true(isinstance(ica.info, Info))
assert_true(ica.n_components_ < 5)
ica = ICA(n_components=3, max_pca_components=4,
n_pca_components=4)
assert_raises(RuntimeError, ica.save, '')
with warnings.catch_warnings(record=True):
ica.fit(raw, picks=[1, 2, 3, 4, 5], start=start, stop=stop2)
# test corrmap
ica2 = ica.copy()
corrmap([ica, ica2], (0, 0), threshold='auto', label='blinks', plot=True,
ch_type="mag")
corrmap([ica, ica2], (0, 0), threshold=2, plot=False, show=False)
assert_true(ica.labels_["blinks"] == ica2.labels_["blinks"])
assert_true(0 in ica.labels_["blinks"])
plt.close('all')
# test warnings on bad filenames
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
ica_badname = op.join(op.dirname(tempdir), 'test-bad-name.fif.gz')
ica.save(ica_badname)
read_ica(ica_badname)
assert_naming(w, 'test_ica.py', 2)
# test decim
ica = ICA(n_components=3, max_pca_components=4,
n_pca_components=4)
raw_ = raw.copy()
for _ in range(3):
raw_.append(raw_)
n_samples = raw_._data.shape[1]
with warnings.catch_warnings(record=True):
ica.fit(raw, picks=None, decim=3)
assert_true(raw_._data.shape[1], n_samples)
# test expl var
ica = ICA(n_components=1.0, max_pca_components=4,
n_pca_components=4)
with warnings.catch_warnings(record=True):
ica.fit(raw, picks=None, decim=3)
assert_true(ica.n_components_ == 4)
# epochs extraction from raw fit
assert_raises(RuntimeError, ica.get_sources, epochs)
# test reading and writing
test_ica_fname = op.join(op.dirname(tempdir), 'test-ica.fif')
for cov in (None, test_cov):
ica = ICA(noise_cov=cov, n_components=2, max_pca_components=4,
n_pca_components=4)
with warnings.catch_warnings(record=True): # ICA does not converge
ica.fit(raw, picks=picks, start=start, stop=stop2)
sources = ica.get_sources(epochs).get_data()
assert_true(ica.mixing_matrix_.shape == (2, 2))
assert_true(ica.unmixing_matrix_.shape == (2, 2))
assert_true(ica.pca_components_.shape == (4, len(picks)))
assert_true(sources.shape[1] == ica.n_components_)
for exclude in [[], [0]]:
ica.exclude = [0]
ica.labels_ = {'foo': [0]}
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.exclude == ica_read.exclude)
assert_equal(ica.labels_, ica_read.labels_)
ica.exclude = []
ica.apply(raw, exclude=[1])
assert_true(ica.exclude == [])
ica.exclude = [0, 1]
ica.apply(raw, exclude=[1])
assert_true(ica.exclude == [0, 1])
ica_raw = ica.get_sources(raw)
assert_true(ica.exclude == [ica_raw.ch_names.index(e) for e in
ica_raw.info['bads']])
# test filtering
d1 = ica_raw._data[0].copy()
with warnings.catch_warnings(record=True): # dB warning
ica_raw.filter(4, 20)
assert_true((d1 != ica_raw._data[0]).any())
d1 = ica_raw._data[0].copy()
with warnings.catch_warnings(record=True): # dB warning
ica_raw.notch_filter([10])
assert_true((d1 != ica_raw._data[0]).any())
ica.n_pca_components = 2
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.n_pca_components == ica_read.n_pca_components)
# check type consistency
attrs = ('mixing_matrix_ unmixing_matrix_ pca_components_ '
'pca_explained_variance_ _pre_whitener')
def f(x, y):
return getattr(x, y).dtype
for attr in attrs.split():
assert_equal(f(ica_read, attr), f(ica, attr))
ica.n_pca_components = 4
ica_read.n_pca_components = 4
ica.exclude = []
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
for attr in ['mixing_matrix_', 'unmixing_matrix_', 'pca_components_',
'pca_mean_', 'pca_explained_variance_',
'_pre_whitener']:
assert_array_almost_equal(getattr(ica, attr),
getattr(ica_read, attr))
assert_true(ica.ch_names == ica_read.ch_names)
assert_true(isinstance(ica_read.info, Info))
sources = ica.get_sources(raw)[:, :][0]
sources2 = ica_read.get_sources(raw)[:, :][0]
assert_array_almost_equal(sources, sources2)
_raw1 = ica.apply(raw, exclude=[1])
_raw2 = ica_read.apply(raw, exclude=[1])
assert_array_almost_equal(_raw1[:, :][0], _raw2[:, :][0])
os.remove(test_ica_fname)
# check scrore funcs
for name, func in get_score_funcs().items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(raw, target='EOG 061', score_func=func,
start=0, stop=10)
assert_true(ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(raw, score_func=stats.skew)
# check exception handling
assert_raises(ValueError, ica.score_sources, raw,
target=np.arange(1))
params = []
params += [(None, -1, slice(2), [0, 1])] # varicance, kurtosis idx params
params += [(None, 'MEG 1531')] # ECG / EOG channel params
for idx, ch_name in product(*params):
ica.detect_artifacts(raw, start_find=0, stop_find=50, ecg_ch=ch_name,
eog_ch=ch_name, skew_criterion=idx,
var_criterion=idx, kurt_criterion=idx)
with warnings.catch_warnings(record=True):
idx, scores = ica.find_bads_ecg(raw, method='ctps')
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(raw, method='correlation')
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(epochs, method='ctps')
assert_equal(len(scores), ica.n_components_)
assert_raises(ValueError, ica.find_bads_ecg, epochs.average(),
method='ctps')
assert_raises(ValueError, ica.find_bads_ecg, raw,
method='crazy-coupling')
idx, scores = ica.find_bads_eog(raw)
assert_equal(len(scores), ica.n_components_)
raw.info['chs'][raw.ch_names.index('EOG 061') - 1]['kind'] = 202
idx, scores = ica.find_bads_eog(raw)
assert_true(isinstance(scores, list))
assert_equal(len(scores[0]), ica.n_components_)
# check score funcs
for name, func in get_score_funcs().items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(epochs_eog, target='EOG 061',
score_func=func)
assert_true(ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(epochs, score_func=stats.skew)
# check exception handling
assert_raises(ValueError, ica.score_sources, epochs,
target=np.arange(1))
# ecg functionality
ecg_scores = ica.score_sources(raw, target='MEG 1531',
score_func='pearsonr')
with warnings.catch_warnings(record=True): # filter attenuation warning
ecg_events = ica_find_ecg_events(raw,
sources[np.abs(ecg_scores).argmax()])
assert_true(ecg_events.ndim == 2)
# eog functionality
eog_scores = ica.score_sources(raw, target='EOG 061',
score_func='pearsonr')
with warnings.catch_warnings(record=True): # filter attenuation warning
eog_events = ica_find_eog_events(raw,
sources[np.abs(eog_scores).argmax()])
assert_true(eog_events.ndim == 2)
# Test ica fiff export
ica_raw = ica.get_sources(raw, start=0, stop=100)
assert_true(ica_raw.last_samp - ica_raw.first_samp == 100)
assert_true(len(ica_raw._filenames) == 0) # API consistency
ica_chans = [ch for ch in ica_raw.ch_names if 'ICA' in ch]
assert_true(ica.n_components_ == len(ica_chans))
test_ica_fname = op.join(op.abspath(op.curdir), 'test-ica_raw.fif')
ica.n_components = np.int32(ica.n_components)
ica_raw.save(test_ica_fname, overwrite=True)
ica_raw2 = Raw(test_ica_fname, preload=True)
assert_allclose(ica_raw._data, ica_raw2._data, rtol=1e-5, atol=1e-4)
ica_raw2.close()
os.remove(test_ica_fname)
# Test ica epochs export
ica_epochs = ica.get_sources(epochs)
assert_true(ica_epochs.events.shape == epochs.events.shape)
ica_chans = [ch for ch in ica_epochs.ch_names if 'ICA' in ch]
assert_true(ica.n_components_ == len(ica_chans))
assert_true(ica.n_components_ == ica_epochs.get_data().shape[1])
assert_true(ica_epochs._raw is None)
assert_true(ica_epochs.preload is True)
# test float n pca components
ica.pca_explained_variance_ = np.array([0.2] * 5)
ica.n_components_ = 0
for ncomps, expected in [[0.3, 1], [0.9, 4], [1, 1]]:
ncomps_ = ica._check_n_pca_components(ncomps)
assert_true(ncomps_ == expected)
def run_evoked(subject):
print("Processing subject: %s" % subject)
meg_subject_dir = op.join(config.meg_dir, subject)
# load epochs to reject ICA components
extension = '-epo'
fname_in = op.join(meg_subject_dir, config.base_fname.format(**locals()))
epochs = mne.read_epochs(fname_in, preload=True)
extension = 'cleaned-epo'
fname_out = op.join(meg_subject_dir, config.base_fname.format(**locals()))
print("Input: ", fname_in)
print("Output: ", fname_out)
# load first run of raw data for ecg /eog epochs
raw_list = list()
print(" Loading one run from raw data")
extension = config.runs[0] + '_sss_raw'
raw_fname_in = op.join(meg_subject_dir,
config.base_fname.format(**locals()))
raw = mne.io.read_raw_fif(raw_fname_in, preload=True)
# run ICA on MEG and EEG
picks_meg = mne.pick_types(raw.info,
meg=True,
eeg=False,
eog=False,
stim=False,
exclude='bads')
picks_eeg = mne.pick_types(raw.info,
meg=False,
eeg=True,
eog=False,
stim=False,
exclude='bads')
all_picks = {'meg': picks_meg, 'eeg': picks_eeg}
if config.eeg:
ch_types = ['meg', 'eeg']
else:
ch_types = ['meg']
for ch_type in ch_types:
print(ch_type)
picks = all_picks[ch_type]
# Load ICA
fname_ica = op.join(
meg_subject_dir,
'{0}_{1}_{2}-ica.fif'.format(subject, config.study_name, ch_type))
print('Reading ICA: ' + fname_ica)
ica = read_ica(fname=fname_ica)
pick_ecg = mne.pick_types(raw.info,
meg=False,
eeg=False,
ecg=True,
eog=False)
# ECG
# either needs an ecg channel, or avg of the mags (i.e. MEG data)
if pick_ecg or ch_type == 'meg':
picks_ecg = np.concatenate([picks, pick_ecg])
# Create ecg epochs
if ch_type == 'meg':
reject = {
'mag': config.reject['mag'],
'grad': config.reject['grad']
}
elif ch_type == 'eeg':
reject = {'eeg': config.reject['eeg']}
ecg_epochs = create_ecg_epochs(raw,
picks=picks_ecg,
reject=reject,
baseline=(None, 0),
tmin=-0.5,
tmax=0.5)
ecg_average = ecg_epochs.average()
# XXX I had to lower the threshold for ctps (default 0.25), otherwise it does not
# find any components
# check how this behaves on other data
ecg_inds, scores = ica.find_bads_ecg(ecg_epochs,
method='ctps',
threshold=0.1)
del ecg_epochs
report_name = op.join(
meg_subject_dir,
'{0}_{1}_{2}-reject_ica.html'.format(subject,
config.study_name,
ch_type))
report = Report(report_name, verbose=False)
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores,
exclude=ecg_inds),
captions=ch_type.upper() + ' - ECG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(ecg_average,
exclude=ecg_inds),
captions=ch_type.upper() + ' - ECG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(ecg_average,
exclude=ecg_inds),
captions=ch_type.upper() + ' - ECG - ' +
'Corrections')
else:
print('no ECG channel!')
# EOG
pick_eog = mne.pick_types(raw.info,
meg=False,
eeg=False,
ecg=False,
eog=True)
if pick_eog.any():
print('using EOG channel')
picks_eog = np.concatenate([picks, pick_eog])
# Create eog epochs
eog_epochs = create_eog_epochs(raw,
picks=picks_eog,
reject=None,
baseline=(None, 0),
tmin=-0.5,
tmax=0.5)
eog_average = eog_epochs.average()
eog_inds, scores = ica.find_bads_eog(eog_epochs)
del eog_epochs
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores,
exclude=eog_inds),
captions=ch_type.upper() + ' - EOG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(eog_average,
exclude=eog_inds),
captions=ch_type.upper() + ' - EOG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(eog_average,
exclude=eog_inds),
captions=ch_type.upper() + ' - EOG - ' +
'Corrections')
report.save(report_name, overwrite=True, open_browser=False)
else:
print('no EOG channel!')
ica_reject = (list(ecg_inds) + list(eog_inds) +
list(config.rejcomps_man[subject][ch_type]))
# now reject the components
print('Rejecting from ' + ch_type + ': ' + str(ica_reject))
epochs = ica.apply(epochs, exclude=ica_reject)
print('Saving epochs')
epochs.save(fname_out)
report.add_figs_to_section(ica.plot_overlay(raw.copy(),
exclude=ica_reject),
captions=ch_type.upper() +
' - ALL(epochs) - ' + 'Corrections')
if config.plot:
epochs.plot_image(combine='gfp',
group_by='type',
sigma=2.,
cmap="YlGnBu_r")
