def test_ica_labels():
"""Test ICA labels."""
# The CTF data are uniquely well suited to testing the ICA.find_bads_
# methods
raw = read_raw_ctf(ctf_fname, preload=True)
# set the appropriate EEG channels to EOG and ECG
raw.set_channel_types({'EEG057': 'eog', 'EEG058': 'eog', 'EEG059': 'ecg'})
ica = ICA(n_components=4, random_state=0, max_iter=2, method='fastica',
allow_ref_meg=True)
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(raw)
_assert_ica_attributes(ica)
ica.find_bads_eog(raw, l_freq=None, h_freq=None)
picks = list(pick_types(raw.info, meg=False, eog=True))
for idx, ch in enumerate(picks):
assert '{}/{}/{}'.format('eog', idx, raw.ch_names[ch]) in ica.labels_
assert 'eog' in ica.labels_
for key in ('ecg', 'ref_meg', 'ecg/ECG-MAG'):
assert key not in ica.labels_
ica.find_bads_ecg(raw, l_freq=None, h_freq=None, method='correlation',
threshold='auto')
picks = list(pick_types(raw.info, meg=False, ecg=True))
for idx, ch in enumerate(picks):
assert '{}/{}/{}'.format('ecg', idx, raw.ch_names[ch]) in ica.labels_
for key in ('ecg', 'eog'):
assert key in ica.labels_
for key in ('ref_meg', 'ecg/ECG-MAG'):
assert key not in ica.labels_
# derive reference ICA components and append them to raw
ica_rf = ICA(n_components=2, random_state=0, max_iter=2,
allow_ref_meg=True)
with pytest.warns(UserWarning, match='did not converge'):
ica_rf.fit(raw.copy().pick_types(meg=False, ref_meg=True))
icacomps = ica_rf.get_sources(raw)
# rename components so they are auto-detected by find_bads_ref
icacomps.rename_channels({c: 'REF_' + c for c in icacomps.ch_names})
# and add them to raw
raw.add_channels([icacomps])
ica.find_bads_ref(raw, l_freq=None, h_freq=None, method="separate")
picks = pick_channels_regexp(raw.ch_names, 'REF_ICA*')
for idx, ch in enumerate(picks):
assert '{}/{}/{}'.format('ref_meg', idx,
raw.ch_names[ch]) in ica.labels_
ica.find_bads_ref(raw, l_freq=None, h_freq=None, method="together")
assert 'ref_meg' in ica.labels_
for key in ('ecg', 'eog', 'ref_meg'):
assert key in ica.labels_
assert 'ecg/ECG-MAG' not in ica.labels_
ica.find_bads_ecg(raw, l_freq=None, h_freq=None, threshold='auto')
for key in ('ecg', 'eog', 'ref_meg', 'ecg/ECG-MAG'):
assert key in ica.labels_
def test_ica_labels():
"""Test ICA labels."""
# The CTF data are uniquely well suited to testing the ICA.find_bads_
# methods
raw = read_raw_ctf(ctf_fname, preload=True)
# derive reference ICA components and append them to raw
icarf = ICA(n_components=2, random_state=0, max_iter=2, allow_ref_meg=True)
with pytest.warns(UserWarning, match='did not converge'):
icarf.fit(raw.copy().pick_types(meg=False, ref_meg=True))
icacomps = icarf.get_sources(raw)
# rename components so they are auto-detected by find_bads_ref
icacomps.rename_channels({c: 'REF_' + c for c in icacomps.ch_names})
# and add them to raw
raw.add_channels([icacomps])
# set the appropriate EEG channels to EOG and ECG
raw.set_channel_types({'EEG057': 'eog', 'EEG058': 'eog', 'EEG059': 'ecg'})
ica = ICA(n_components=4, random_state=0, max_iter=2, method='fastica')
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(raw)
ica.find_bads_eog(raw, l_freq=None, h_freq=None)
picks = list(pick_types(raw.info, meg=False, eog=True))
for idx, ch in enumerate(picks):
assert '{}/{}/{}'.format('eog', idx, raw.ch_names[ch]) in ica.labels_
assert 'eog' in ica.labels_
for key in ('ecg', 'ref_meg', 'ecg/ECG-MAG'):
assert key not in ica.labels_
ica.find_bads_ecg(raw, l_freq=None, h_freq=None, method='correlation')
picks = list(pick_types(raw.info, meg=False, ecg=True))
for idx, ch in enumerate(picks):
assert '{}/{}/{}'.format('ecg', idx, raw.ch_names[ch]) in ica.labels_
for key in ('ecg', 'eog'):
assert key in ica.labels_
for key in ('ref_meg', 'ecg/ECG-MAG'):
assert key not in ica.labels_
ica.find_bads_ref(raw, l_freq=None, h_freq=None)
picks = pick_channels_regexp(raw.ch_names, 'REF_ICA*')
for idx, ch in enumerate(picks):
assert '{}/{}/{}'.format('ref_meg', idx,
raw.ch_names[ch]) in ica.labels_
for key in ('ecg', 'eog', 'ref_meg'):
assert key in ica.labels_
assert 'ecg/ECG-MAG' not in ica.labels_
ica.find_bads_ecg(raw, l_freq=None, h_freq=None)
for key in ('ecg', 'eog', 'ref_meg', 'ecg/ECG-MAG'):
assert key in ica.labels_
def filter_with_ica(raw_data, event_id, thresh):
"""Use ICA to filter data for artifacts.
Args:
raw_data: Raw array of data
events: Event Array
event_id: A Dictionary of events
thresh: A float that indicates a threshold
Resturns:
raw: ICA and bandpass filtered data
"""
raw = raw_data.copy().crop(2, raw_data.times.max())
#raw = raw_data.copy()
raw.filter(3, 42, n_jobs=1, fir_design='firwin')
# Run ICA
method = 'fastica'
# Choose other parameters
n_components, decim, random_state = 25, 3, 42 # if float, select n_components by explained variance of PCA
ica = ICA(n_components=n_components,
method=method,
random_state=random_state)
ica.fit(raw,
picks=pick_types(raw.info,
eeg=True,
misc=False,
stim=False,
eog=False),
decim=decim)
eog_epochs = create_eog_epochs(raw) # get single EOG trials
eog_inds, scores = ica.find_bads_eog(
eog_epochs, threshold=thresh) # find via correlation
ica.exclude.extend(eog_inds)
ica.apply(raw)
return raw
def run_ica(raw, ica_fname=None, picks=None, n_max_ecg=3, n_max_eog=1):
raw_f = raw.copy().filter(1,
45,
n_jobs=1,
l_trans_bandwidth=0.5,
h_trans_bandwidth=0.5,
filter_length='10s',
phase='zero-double')
ica = ICA(method='fastica', random_state=42, n_components=0.98)
if picks is None:
picks = mne.pick_types(raw_f.info,
meg=True,
eeg=False,
eog=False,
stim=False,
exclude='bads')
ica.fit(raw_f, picks=picks, reject=dict(grad=4000e-13, mag=4e-12), decim=3)
ecg_epochs = create_ecg_epochs(raw_f, tmin=-.5, tmax=.5, picks=picks)
ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
ecg_inds = ecg_inds[:n_max_ecg]
ica.exclude += ecg_inds
eog_inds, scores = ica.find_bads_eog(raw_f)
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
if ica_fname is None:
ica_fname = raw_f._filenames[0][:-4] + '-pyimpress-ica.fif'
ica.save(ica_fname)
return ica, ica_fname
def run_ica(method, fit_params=None):
ica = ICA(n_components=14,
method=method,
fit_params=fit_params,
random_state=95)
filt_raw = raw.copy()
filt_raw.load_data().filter(l_freq=1., h_freq=None)
t0 = time()
ica.fit(filt_raw)
fit_time = time() - t0
title = ('ICA decomposition using %s (took %.1fs)' % (method, fit_time))
ica.plot_sources(raw)
#ica.plot_components(title=title)
ica.exclude = []
# find which ICs match the EOG pattern
eog_indices, eog_scores = ica.find_bads_eog(raw, ch_name='F4')
ica.exclude = eog_indices
# barplot of ICA component "EOG match" scores
ica.plot_scores(eog_scores)
eog_indices, eog_scores = ica.find_bads_eog(raw, ch_name='AF4')
ica.exclude = eog_indices
# barplot of ICA component "EOG match" scores
ica.plot_scores(eog_scores)
# plot diagnostics
ica.plot_properties(raw, picks=eog_indices)
# plot ICs applied to raw data, with EOG matches highlighted
ica.plot_sources(raw)
# plot ICs applied to the averaged EOG epochs, with EOG matches highlighted
#ica.plot_sources(eog_evoked)
ica.plot_overlay(raw)
ica.exclude = [1, 2] # indices chosen based on various plots above
# ica.apply() changes the Raw object in-place, so let's make a copy first:
print('copiando el original')
reconst_raw = raw.copy()
print('aplicando ICA')
ica.apply(reconst_raw)
raw.plot()
print('muestra reconstruida')
reconst_raw.plot()
plt.show()
def inscapesMEG_PP(fname, DATA_FOLDER, SAVE_FOLDER):
fpath = DATA_FOLDER + fname
raw = read_raw_ctf(fpath, preload=True)
picks = mne.pick_types(raw.info, meg=True, eog=True, exclude='bads')
raw.plot()
raw.plot_psd(average=False, picks=picks)
## Filtering
high_cutoff = 200
low_cutoff = 0.5
raw.filter(low_cutoff, high_cutoff, fir_design="firwin")
raw.notch_filter(np.arange(60, high_cutoff + 1, 60),
picks=picks,
filter_length='auto',
phase='zero',
fir_design="firwin")
raw.plot_psd(average=False, picks=picks)
## ICA
ica = ICA(n_components=20, random_state=0).fit(raw, decim=3)
ica.plot_sources(raw)
fmax = 40. ## correlation threshold for ICA components (maybe increase to 40. ?)
## FIND ECG COMPONENTS
ecg_epochs = create_ecg_epochs(raw, ch_name='EEG059')
ecg_inds, ecg_scores = ica.find_bads_ecg(ecg_epochs, ch_name='EEG059')
ica.plot_scores(ecg_scores, ecg_inds)
ica.plot_properties(ecg_epochs,
picks=ecg_inds,
psd_args={'fmax': fmax},
image_args={'sigma': 1.})
## FIND EOG COMPONENTS
eog_epochs = create_eog_epochs(raw, ch_name='EEG057')
eog_inds, eog_scores = ica.find_bads_eog(eog_epochs, ch_name='EEG057')
ica.plot_scores(eog_scores, eog_inds)
ica.plot_properties(eog_epochs,
picks=eog_inds,
psd_args={'fmax': fmax},
image_args={'sigma': 1.})
## EXCLUDE COMPONENTS
ica.exclude = ecg_inds
ica.apply(raw)
ica.exclude = eog_inds
ica.apply(raw)
raw.plot()
# Plot the clean signal.
## SAVE PREPROCESSED FILE
time.sleep(60)
raw.save(SAVE_FOLDER + fname + '_preprocessed.fif.gz', overwrite=True)
time.sleep(30)
filename = SAVE_FOLDER + fname + '_log.html'
#!jupyter nbconvert inscapesMEG_preproc.ipynb --output $filename
clear_output()
def perform_ICA(raw, eeg_reject=50.):
picks_eeg = mne.pick_types(raw.info, meg=False, eeg=True, eog=False, stim=False, exclude='bads')
ica = ICA(n_components=25, random_state=1) # using default method 'fastica'
reject = dict(eeg=eeg_reject)
ica.fit(raw, picks=picks_eeg, reject=reject)
eog_average = create_eog_epochs(raw, reject=reject, picks=picks_eeg).average()
eog_epochs = create_eog_epochs(raw, reject=reject) # get single EOG trials
eog_inds, scores = ica.find_bads_eog(eog_epochs) # find via correlation
ica.exclude.extend(eog_inds)
ica.apply(raw)
return ica
def ICA(self, ep):
#%% Artifact Correction with ICA
"""
ICA finds directions in the feature space corresponding to projections with high non-Gaussianity.
We obtain a decomposition into independent components, and the artifact's contribution
is localized in only a small number of components.These components have to be correctly identified and removed.
If EOG or ECG recordings are available, they can be used in ICA to
automatically select the corresponding artifact components from the
decomposition. To do so, you have to first build an Epoch object around
blink or heartbeat event.
"""
from mne.preprocessing import ICA
# ICA parameters:
n_components = 20 # if float, select n_components by explained variance of PCA
method = 'fastica'
decim = 3 # need sufficient statistics, not all time points -> saves time
# Set state of the random number generator - ICA is a
# non-deterministic algorithm, but we want to have the same decomposition
# and the same order of components each time
picks_eeg = mne.pick_types(ep.info, meg=False, eeg=True,
eog=False, stim=False, exclude='bads')
# Define the ICA object instance
ica = ICA(n_components=n_components, method=method, random_state = 23)
print(ica)
# avoid fitting ICA on crazy environmental artifacts that would
# dominate the variance and decomposition
reject = dict(eeg=40e-6)
ica.fit(ep, picks=picks_eeg, reject = reject, decim=decim)
if self.ICAplot:
ica.plot_components() # can you spot some potential bad guys?
#% Artifact detection
eog_inds, scores = ica.find_bads_eog(ep, ch_name = 'Fp1', threshold=1) # find via correlation
#ica.plot_scores(scores, exclude=eog_inds) # look at r scores of components
ica.exclude.extend(eog_inds)
# apply ICA
ep = ica.apply(ep, exclude=eog_inds)
if self.MRCP_bandpass: # this is basically for NIKUNJ data (by default it is bandpassed)
# Extract MRCP and return a *band-pass* filtered signal in the range .1 Hz - 4 Hz
ep.filter(None, 4., l_trans_bandwidth='auto', h_trans_bandwidth='auto',
filter_length='auto', phase='zero')
return ep
def ica_correction(raw, picks):
# ## Function "ica_correction" will correct artifacts in eeg data
# (eg. blink) using ICA and return an ICA array under "my-ica.fif"
## Will also plot ICA components
from mne.preprocessing import ICA
from mne.preprocessing import create_eog_epochs
#ICA parameters
n_components = 25 # if float, select n_components by explained variance of PCA
method = 'fastica' # for comparison with EEGLAB try "extended-infomax" here
decim = 3 # we need sufficient statistics, not all time points -> saves time
# we will also set state of the random number generator - ICA is a
# non-deterministic algorithm, but we want to have the same decomposition
# and the same order of components each time this tutorial is run
random_state = 23
ica = ICA(n_components=n_components,
method=method,
random_state=random_state)
ica.fit(raw, picks=picks, decim=decim, reject=dict(eeg=200e-6))
ica.plot_components()
eog_average = create_eog_epochs(raw, reject=dict(eeg=200e-6),
picks=picks).average()
n_max_eog = 1 # here we bet on finding the vertical EOG components
eog_epochs = create_eog_epochs(raw, reject=dict(eeg=200e-6),
picks=picks) # get single EOG trials
eog_inds, scores = ica.find_bads_eog(eog_epochs) # find via correlation
ica.plot_scores(scores, exclude=eog_inds) # look at r scores of components
# we can see that only one component is highly correlated and that this
# component got detected by our correlation analysis (red).
ica.plot_sources(eog_average,
exclude=eog_inds) # look at source time course
ica.plot_properties(eog_epochs,
picks=eog_inds,
psd_args={'fmax': 35.},
image_args={'sigma': 1.})
print(ica.labels_)
ica.plot_overlay(eog_average, exclude=eog_inds, show=False)
ica.exclude.extend(eog_inds)
def run_ica(method, fit_params=None):
chn=['AF3','F7','F3','FC5','T7','P7','O1','O2','P8','T8','FC6','F4','F8','AF4']
ica = ICA(n_components=14, method=method, fit_params=fit_params,
random_state=75)
filt_raw = raw.copy()
filt_raw.load_data().filter(l_freq=2.5, h_freq=None)
t0 = time()
ica.fit(filt_raw)
fit_time = time() - t0
title = ('ICA decomposition using %s (took %.1fs)' % (method, fit_time))
ica.plot_sources(raw)
ica.plot_components(title=title)
ica.exclude = []
# find which ICs match the EOG pattern
eog_indices, eog_scores = ica.find_bads_eog(raw,ch_name='AF4')
ica.exclude = eog_indices
# barplot of ICA component "EOG match" scores
ica.plot_scores(eog_scores)
print(eog_indices)
inpre=input("está de acuerdo con los indices?: (y/n)")
if(inpre=='y'):
print('ok, sigamos')
else:
eog_indices=[]
n=int(input("elija el numero de indices a incluir: "))
for i in range(0,n):
print("introduzca el indice ")
eog_indices.append(int(input(": ")))
ica.plot_properties(raw, picks=eog_indices)
ica.exclude = eog_indices # indices chosen based on various plots above
# plot ICs applied to raw data, with EOG matches highlighted
ica.plot_sources(raw)
# plot ICs applied to the averaged EOG epochs, with EOG matches highlighted
ica.plot_overlay(raw,exclude=eog_indices)
# ica.apply() changes the Raw object in-place, so let's make a copy first:
print('copiando el original')
reconst_raw = raw.copy()
print('aplicando ICA')
ica.apply(reconst_raw)
raw.plot(events=events,event_color=mapcol)
print('muestra reconstruida')
reconst_raw.plot(events=events,event_color=mapcol)
plt.show()
return reconst_raw
def ica_correction(raw, picks):
"""Function "ica_correction" will correct artifacts in eeg data
(eg. blink) using ICA and return an ICA array under "my-ica.fif"
Will also plot ICA components"""
from mne.preprocessing import ICA
from mne.preprocessing import create_eog_epochs
#ICA parameters
n_components = 25 # if float, select n_components by explained variance of PCA
method = 'fastica' # for comparison with EEGLAB try "extended-infomax" here
decim = 3 # we need sufficient statistics, not all time points -> saves time
# we will also set state of the random number generator - ICA is a
# non-deterministic algorithm, but we want to have the same decomposition
# and the same order of components each time this tutorial is run
random_state = 23
ica = ICA(n_components=n_components,
method=method,
random_state=random_state)
ica.fit(raw, picks=picks, decim=decim, reject=dict(eeg=200e-6))
ica.plot_components()
n_max_eog = 1 #don't expect to find horizontal eog components
title = 'Sources related to EOG components (red)'
eog_epochs = create_eog_epochs(raw, tmin=-.5, tmax=1, picks=picks)
eog_inds, scores = ica.find_bads_eog(eog_epochs)
ica.plot_scores(scores, exclude=eog_inds, title=title, labels='eog')
show_picks = np.abs(scores).argsort()[::-1][:5]
ica.plot_sources(raw, show_picks, exclude=eog_inds, title=title)
ica.plot_components(eog_inds, title=title, colorbar=True)
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
# uncomment this for reading and writing
#ica.save('my-ica.fif')
#ica = read_ica('my-ica.fif')
#apply to epochs
ica.apply(epochs)
def getPlot(path):
with open(path, "r") as f:
data = f.read().split('\n')[1:-1]
data = np.array([[i.split(';')[j] for j in range(len(i.split(';'))) if j in [2, 4, 10, 12]] for i in data]).T
ch_names = ['AF3', 'T8', 'AF4', 'T7'] # TODO: finish this list
info = mne.create_info(ch_names, sfreq=128, ch_types='eeg')
raw = mne.io.RawArray(data, info)
montage = mne.channels.make_standard_montage(kind='standard_1020', head_size=0.095)
raw.set_montage(montage, match_case=True)
raw.plot_psd(area_mode='range', average=False);
raw.info['bads'] = ['AF3']
raw.interpolate_bads(reset_bads=True, mode='accurate')
raw.plot(block=True);
eog_events = mne.preprocessing.find_eog_events(raw, ch_name='AF3')
n_blinks = len(eog_events)
onset = eog_events[:, 0] / raw.info['sfreq'] - 0.25
duration = np.repeat(0.5, n_blinks)
annot = mne.Annotations(onset, duration, ['bad blink'] * n_blinks, orig_time=raw.info['meas_date'])
raw.set_annotations(annot)
print(raw.annotations)
raw.plot(block=True, events=eog_events);
from mne.preprocessing import ICA, create_eog_epochs
decim = 3
reject = dict(eeg=40e-6)
#raw.filter(1., None, fir_design='firwin');
ica = ICA(n_components=3, method='fastica', random_state=1)
ica.fit(raw, decim=3)
print(ica)
#ica.plot_components(ch_type='eeg');
eog_average = create_eog_epochs(raw, ch_name='AF3', reject_by_annotation=False).average();
eog_epochs = create_eog_epochs(raw, ch_name='AF3', reject_by_annotation=False); # get single EOG trials
eog_inds, scores = ica.find_bads_eog(eog_epochs, ch_name='AF3', threshold=2.0, l_freq=1, h_freq=10,
reject_by_annotation=True);
# In[10]:
ica.plot_scores(scores, exclude=eog_inds);
ica.plot_sources(eog_average);
def remove_eog_ica(raw, n_components, ch_name, threshold):
"""Remove EOG artifacts by ICA.
Parameters
----------
raw : instance of Raw.
The raw data.
n_components : int
Number of principal components for ICA.
ch_name : str
The name of the channel to use for EOG peak detection.
threshold : int
The value above which a feature is classified as outlier.
Returns
-------
raw : instance of Raw.
The raw data.
Notes
-----
"""
ica = ICA(n_components=n_components, max_iter='auto')
ica.fit(raw, verbose=0)
while threshold > 1:
eog_inds, _ = ica.find_bads_eog(raw,
ch_name=ch_name,
threshold=threshold,
verbose=0)
if eog_inds:
break
threshold -= 0.3
if not eog_inds:
raise RuntimeError('Didn\'t find a EOG component.')
ica.plot_properties(raw, eog_inds)
ica.exclude = eog_inds
ica.apply(raw, verbose=0)
return raw
def preprocess(eeg_rawdata):
"""
:param eeg_rawdata: numpy array with the shape of (n_channels, n_samples)
:return: filtered EEG raw data
"""
assert eeg_rawdata.shape[0] == 62
eeg_rawdata = np.array(eeg_rawdata)
temp = 0 - (eeg_rawdata.shape[1] % 5)
if temp != 0:
eeg_rawdata = eeg_rawdata[:, :temp]
ch_names = [
'FP1', 'FPZ', 'FP2', 'AF3', 'AF4', 'F7', 'F5', 'F3', 'F1', 'FZ', 'F2',
'F4', 'F6', 'F8', 'FT7', 'FC5', 'FC3', 'FC1', 'FCZ', 'FC2', 'FC4',
'FC6', 'FT8', 'T7', 'C5', 'C3', 'C1', 'CZ', 'C2', 'C4', 'C6', 'T8',
'TP7', 'CP5', 'CP3', 'CP1', 'CPZ', 'CP2', 'CP4', 'CP6', 'TP8', 'P7',
'P5', 'P3', 'P1', 'PZ', 'P2', 'P4', 'P6', 'P8', 'PO7', 'PO5', 'PO3',
'POZ', 'PO4', 'PO6', 'PO8', 'CB1', 'O1', 'OZ', 'O2', 'CB2'
]
info = mne.create_info(
# 通道名
ch_names=ch_names,
# 通道类型
ch_types=['eeg' for _ in range(62)],
# 采样频率
sfreq=1000)
raw_data = mne.io.RawArray(eeg_rawdata, info)
raw_data.load_data().filter(l_freq=1., h_freq=75)
raw_data.resample(200)
ica = ICA(n_components=5, random_state=97)
ica.fit(raw_data)
eog_indices, eog_scores = ica.find_bads_eog(raw_data, ch_name='FP1')
a = abs(eog_scores).tolist()
ica.exclude = [a.index(max(a))]
ica.apply(raw_data)
filted_eeg_rawdata = np.array(raw_data.get_data())
return filted_eeg_rawdata
def run_ica(epochs: Epochs, fit_params: dict = None) -> ICA:
"""
Runs ICA decomposition on Epochs instance.
If there are no EOG channels found, it tries to use 'Fp1' and 'Fp2' as EOG
channels; if they are not found either, it chooses the first two channels
to identify EOG components with mne.preprocessing.ica.find_bads_eog().
Parameters
----------
epochs: the instance to be used for ICA decomposition
fit_params: parameters to be passed to ICA fit (e.g. orthogonal picard, extended infomax)
Returns
-------
ICA instance
"""
ica = ICA(
n_components=settings["ica"]["n_components"],
random_state=42,
method=settings["ica"]["method"],
fit_params=fit_params,
)
ica_epochs = epochs.copy()
ica.fit(ica_epochs, decim=settings["ica"]["decim"])
if "eog" not in epochs.get_channel_types():
if "Fp1" and "Fp2" in epochs.get_montage().ch_names:
eog_channels = ["Fp1", "Fp2"]
else:
eog_channels = epochs.get_montage().ch_names[:2]
logger.info("EOG channels are not found. Attempting to use "
f'{",".join(eog_channels)} channels as EOG channels.')
ica_epochs.set_channel_types({ch: "eog" for ch in eog_channels})
eog_indices, _ = ica.find_bads_eog(ica_epochs)
ica.exclude = eog_indices
return ica
fig.savefig(img_folder + '/ica_ecg_scores.png') show_picks = np.abs(scores).argsort()[::-1][:5] fig = ica.plot_sources(raw, show_picks, exclude=ecg_inds, title=title % 'ecg') fig.savefig(img_folder + '/ica_ecg_sources.png') fig = ica.plot_components(ecg_inds, title=title % 'ecg', colorbar=True) fig.set_size_inches(12.,8.) fig.savefig(img_folder + '/ica_ecg_components.png') ecg_inds = ecg_inds[:n_max_ecg] ica.exclude += ecg_inds # detect EOG by correlation # First do both together eog_inds, scores = ica.find_bads_eog(raw, ch_name='EOG001,EOG003') allscores = np.vstack((scores[0], scores[1])) mscores = np.max(np.abs(allscores), axis=0) # now scores is show_picks = mscores.argsort()[::-1][:5] eog_inds = list(show_picks[:n_max_eog]) fig = ica.plot_scores(scores, exclude=eog_inds, title=title % 'eog') fig.savefig(img_folder + '/ica_eog_scores.png') fig = ica.plot_sources(raw, show_picks, exclude=eog_inds, title=title % 'eog') fig.savefig(img_folder + '/ica_eog_sources.png') fig = ica.plot_components(show_picks, title=title % 'eog', colorbar=True) fig.set_size_inches(12.,8.) fig.savefig(img_folder + '/ica_eog_components.png')
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # # It may have seemed easy to review the plots and manually select which ICs to # exclude, but when processing dozens or hundreds of subjects this can become # a tedious, rate-limiting step in the analysis pipeline. One alternative is to # use dedicated EOG or ECG sensors as a "pattern" to check the ICs against, and # automatically mark for exclusion any ICs that match the EOG/ECG pattern. Here # we'll use :meth:`~mne.preprocessing.ICA.find_bads_eog` to automatically find # the ICs that best match the EOG signal, then use # :meth:`~mne.preprocessing.ICA.plot_scores` along with our other plotting # functions to see which ICs it picked. We'll start by resetting # ``ica.exclude`` back to an empty list: ica.exclude = [] # find which ICs match the EOG pattern eog_indices, eog_scores = ica.find_bads_eog(raw) ica.exclude = eog_indices # barplot of ICA component "EOG match" scores ica.plot_scores(eog_scores) # plot diagnostics ica.plot_properties(raw, picks=eog_indices) # plot ICs applied to raw data, with EOG matches highlighted ica.plot_sources(raw, show_scrollbars=False) # plot ICs applied to the averaged EOG epochs, with EOG matches highlighted ica.plot_sources(eog_evoked) ###############################################################################
stim=False,
exclude=[])
ica.fit(epochs, picks=picks, decim=decim, reject=None)
# maximum number of components to reject
n_max_eog = 1
##########################################################################
# 2) identify bad components by analyzing latent sources.
# DETECT EOG BY CORRELATION
title = "ICA: %s for %s"
# EOG
eog_inds, scores = ica.find_bads_eog(epochs)
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
if eog_inds:
fig = ica.plot_scores(
scores, exclude=eog_inds, title=title % ('eog', subject))
fig.savefig(ica_folder + "plots/%s_%s_eog_scores.png" % (subject,
condition))
fig = ica.plot_sources(epochs, exclude=eog_inds)
fig.savefig(ica_folder + "plots/%s_%s_eog_source.png" % (subject,
condition))
fig = ica.plot_components(
eog_inds, title=title % ('eog', subject), colorbar=True)
epochs = mne.Epochs(raw,
events,
event_ids,
tmin,
tmax,
picks=picks,
baseline=baseline,
preload=True,
reject=reject)
# Fit ICA, find and remove major artifacts
ica = ICA(n_components=0.95, random_state=0).fit(raw, decim=1, reject=reject)
# compute correlation scores, get bad indices sorted by score
eog_epochs = create_eog_epochs(raw, ch_name='MRT31-2908', reject=reject)
eog_inds, eog_scores = ica.find_bads_eog(eog_epochs, ch_name='MRT31-2908')
ica.plot_scores(eog_scores, eog_inds) # see scores the selection is based on
ica.plot_components(eog_inds) # view topographic sensitivity of components
ica.exclude += eog_inds[:1] # we saw the 2nd ECG component looked too dipolar
ica.plot_overlay(eog_epochs.average()) # inspect artifact removal
ica.apply(epochs) # clean data, default in place
evoked = [epochs[k].average() for k in event_ids]
contrast = combine_evoked(evoked, weights=[-1, 1]) # Faces - scrambled
evoked.append(contrast)
for e in evoked:
e.plot(ylim=dict(mag=[-400, 400]))
def main():
#################################################
## SETUP
## Get list of subject files
subj_files = listdir(DAT_PATH)
subj_files = [file for file in subj_files if EXT.lower() in file.lower()]
## Set up FOOOF Objects
# Initialize FOOOF settings & objects objects
fooof_settings = FOOOFSettings(peak_width_limits=PEAK_WIDTH_LIMITS, max_n_peaks=MAX_N_PEAKS,
min_peak_amplitude=MIN_PEAK_AMP, peak_threshold=PEAK_THRESHOLD,
aperiodic_mode=APERIODIC_MODE)
fm = FOOOF(*fooof_settings, verbose=False)
fg = FOOOFGroup(*fooof_settings, verbose=False)
# Save out a settings file
fg.save('0-FOOOF_Settings', pjoin(RES_PATH, 'FOOOF'), save_settings=True)
# Set up the dictionary to store all the FOOOF results
fg_dict = dict()
for load_label in LOAD_LABELS:
fg_dict[load_label] = dict()
for side_label in SIDE_LABELS:
fg_dict[load_label][side_label] = dict()
for seg_label in SEG_LABELS:
fg_dict[load_label][side_label][seg_label] = []
## Initialize group level data stores
n_subjs, n_conds, n_times = len(subj_files), 3, N_TIMES
group_fooofed_alpha_freqs = np.zeros(shape=[n_subjs])
dropped_components = np.ones(shape=[n_subjs, 50]) * 999
dropped_trials = np.ones(shape=[n_subjs, 1500]) * 999
canonical_group_avg_dat = np.zeros(shape=[n_subjs, n_conds, n_times])
fooofed_group_avg_dat = np.zeros(shape=[n_subjs, n_conds, n_times])
# Set channel types
ch_types = {'LHor' : 'eog', 'RHor' : 'eog', 'IVer' : 'eog', 'SVer' : 'eog',
'LMas' : 'misc', 'RMas' : 'misc', 'Nose' : 'misc', 'EXG8' : 'misc'}
#################################################
## RUN ACROSS ALL SUBJECTS
# Run analysis across each subject
for s_ind, subj_file in enumerate(subj_files):
# Get subject label and print status
subj_label = subj_file.split('.')[0]
print('\nCURRENTLY RUNNING SUBJECT: ', subj_label, '\n')
#################################################
## LOAD / ORGANIZE / SET-UP DATA
# Load subject of data, apply apply fixes for channels, etc
eeg_dat = mne.io.read_raw_edf(pjoin(DAT_PATH, subj_file),
preload=True, verbose=False)
# Fix channel name labels
eeg_dat.info['ch_names'] = [chl[2:] for chl in \
eeg_dat.ch_names[:-1]] + [eeg_dat.ch_names[-1]]
for ind, chi in enumerate(eeg_dat.info['chs']):
eeg_dat.info['chs'][ind]['ch_name'] = eeg_dat.info['ch_names'][ind]
# Update channel types
eeg_dat.set_channel_types(ch_types)
# Set reference - average reference
eeg_dat = eeg_dat.set_eeg_reference(ref_channels='average',
projection=False, verbose=False)
# Set channel montage
chs = mne.channels.read_montage('standard_1020', eeg_dat.ch_names)
eeg_dat.set_montage(chs)
# Get event information & check all used event codes
evs = mne.find_events(eeg_dat, shortest_event=1, verbose=False)
# Pull out sampling rate
srate = eeg_dat.info['sfreq']
#################################################
## Pre-Processing: ICA
# High-pass filter data for running ICA
eeg_dat.filter(l_freq=1., h_freq=None, fir_design='firwin')
if RUN_ICA:
print("\nICA: CALCULATING SOLUTION\n")
# ICA settings
method = 'fastica'
n_components = 0.99
random_state = 47
reject = {'eeg': 20e-4}
# Initialize ICA object
ica = ICA(n_components=n_components, method=method,
random_state=random_state)
# Fit ICA
ica.fit(eeg_dat, reject=reject)
# Save out ICA solution
ica.save(pjoin(RES_PATH, 'ICA', subj_label + '-ica.fif'))
# Otherwise: load previously saved ICA to apply
else:
print("\nICA: USING PRECOMPUTED\n")
ica = read_ica(pjoin(RES_PATH, 'ICA', subj_label + '-ica.fif'))
# Find components to drop, based on correlation with EOG channels
drop_inds = []
for chi in EOG_CHS:
inds, _ = ica.find_bads_eog(eeg_dat, ch_name=chi, threshold=2.5,
l_freq=1, h_freq=10, verbose=False)
drop_inds.extend(inds)
drop_inds = list(set(drop_inds))
# Set which components to drop, and collect record of this
ica.exclude = drop_inds
dropped_components[s_ind, 0:len(drop_inds)] = drop_inds
# Apply ICA to data
eeg_dat = ica.apply(eeg_dat)
#################################################
## SORT OUT EVENT CODES
# Extract a list of all the event labels
all_trials = [it for it2 in EV_DICT.values() for it in it2]
# Create list of new event codes to be used to label correct trials (300s)
all_trials_new = [it + 100 for it in all_trials]
# This is an annoying way to collapse across the doubled event markers from above
all_trials_new = [it - 1 if not ind%2 == 0 else it for ind, it in enumerate(all_trials_new)]
# Get labelled dictionary of new event names
ev_dict2 = {k:v for k, v in zip(EV_DICT.keys(), set(all_trials_new))}
# Initialize variables to store new event definitions
evs2 = np.empty(shape=[0, 3], dtype='int64')
lags = np.array([])
# Loop through, creating new events for all correct trials
t_min, t_max = -0.4, 3.0
for ref_id, targ_id, new_id in zip(all_trials, CORR_CODES * 6, all_trials_new):
t_evs, t_lags = mne.event.define_target_events(evs, ref_id, targ_id, srate,
t_min, t_max, new_id)
if len(t_evs) > 0:
evs2 = np.vstack([evs2, t_evs])
lags = np.concatenate([lags, t_lags])
#################################################
## FOOOF
# Set channel of interest
ch_ind = eeg_dat.ch_names.index(CHL)
# Calculate PSDs over ~ first 2 minutes of data, for specified channel
fmin, fmax = 1, 50
tmin, tmax = 5, 125
psds, freqs = mne.time_frequency.psd_welch(eeg_dat, fmin=fmin, fmax=fmax,
tmin=tmin, tmax=tmax,
n_fft=int(2*srate), n_overlap=int(srate),
n_per_seg=int(2*srate),
verbose=False)
# Fit FOOOF across all channels
fg.fit(freqs, psds, FREQ_RANGE, n_jobs=-1)
# Save out FOOOF results
fg.save(subj_label + '_fooof', pjoin(RES_PATH, 'FOOOF'), save_results=True)
# Extract individualized CF from specified channel, add to group collection
fm = fg.get_fooof(ch_ind, False)
fooof_freq, _, _ = get_band_peak(fm.peak_params_, [7, 14])
group_fooofed_alpha_freqs[s_ind] = fooof_freq
# If not FOOOF alpha extracted, reset to 10
if np.isnan(fooof_freq):
fooof_freq = 10
#################################################
## ALPHA FILTERING
# CANONICAL: Filter data to canonical alpha band: 8-12 Hz
alpha_dat = eeg_dat.copy()
alpha_dat.filter(8, 12, fir_design='firwin', verbose=False)
alpha_dat.apply_hilbert(envelope=True, verbose=False)
# FOOOF: Filter data to FOOOF derived alpha band
fooof_dat = eeg_dat.copy()
fooof_dat.filter(fooof_freq-2, fooof_freq+2, fir_design='firwin')
fooof_dat.apply_hilbert(envelope=True)
#################################################
## EPOCH TRIALS
# Set epoch timings
tmin, tmax = -0.85, 1.1
# Epoch trials - raw data for trial rejection
epochs = mne.Epochs(eeg_dat, evs2, ev_dict2, tmin=tmin, tmax=tmax,
baseline=None, preload=True, verbose=False)
# Epoch trials - filtered version
epochs_alpha = mne.Epochs(alpha_dat, evs2, ev_dict2, tmin=tmin, tmax=tmax,
baseline=(-0.5, -0.35), preload=True, verbose=False)
epochs_fooof = mne.Epochs(fooof_dat, evs2, ev_dict2, tmin=tmin, tmax=tmax,
baseline=(-0.5, -0.35), preload=True, verbose=False)
#################################################
## PRE-PROCESSING: AUTO-REJECT
if RUN_AUTOREJECT:
print('\nAUTOREJECT: CALCULATING SOLUTION\n')
# Initialize and run autoreject across epochs
ar = AutoReject(n_jobs=4, verbose=False)
ar.fit(epochs)
# Save out AR solution
ar.save(pjoin(RES_PATH, 'AR', subj_label + '-ar.hdf5'), overwrite=True)
# Otherwise: load & apply previously saved AR solution
else:
print('\nAUTOREJECT: USING PRECOMPUTED\n')
ar = read_auto_reject(pjoin(RES_PATH, 'AR', subj_label + '-ar.hdf5'))
ar.verbose = 'tqdm'
# Apply autoreject to the original epochs object it was learnt on
epochs, rej_log = ar.transform(epochs, return_log=True)
# Apply autoreject to the copies of the data - apply interpolation, then drop same epochs
_apply_interp(rej_log, epochs_alpha, ar.threshes_, ar.picks_, ar.verbose)
epochs_alpha.drop(rej_log.bad_epochs)
_apply_interp(rej_log, epochs_fooof, ar.threshes_, ar.picks_, ar.verbose)
epochs_fooof.drop(rej_log.bad_epochs)
# Collect which epochs were dropped
dropped_trials[s_ind, 0:sum(rej_log.bad_epochs)] = np.where(rej_log.bad_epochs)[0]
#################################################
## SET UP CHANNEL CLUSTERS
# Set channel clusters - take channels contralateral to stimulus presentation
# Note: channels will be used to extract data contralateral to stimulus presentation
le_chs = ['P3', 'P5', 'P7', 'P9', 'O1', 'PO3', 'PO7'] # Left Side Channels
le_inds = [epochs.ch_names.index(chn) for chn in le_chs]
ri_chs = ['P4', 'P6', 'P8', 'P10', 'O2', 'PO4', 'PO8'] # Right Side Channels
ri_inds = [epochs.ch_names.index(chn) for chn in ri_chs]
#################################################
## TRIAL-RELATED ANALYSIS: CANONICAL vs. FOOOF
## Pull out channels of interest for each load level
# Channels extracted are those contralateral to stimulus presentation
# Canonical Data
lo1_a = np.concatenate([epochs_alpha['LeLo1']._data[:, ri_inds, :],
epochs_alpha['RiLo1']._data[:, le_inds, :]], 0)
lo2_a = np.concatenate([epochs_alpha['LeLo2']._data[:, ri_inds, :],
epochs_alpha['RiLo2']._data[:, le_inds, :]], 0)
lo3_a = np.concatenate([epochs_alpha['LeLo3']._data[:, ri_inds, :],
epochs_alpha['RiLo3']._data[:, le_inds, :]], 0)
# FOOOFed data
lo1_f = np.concatenate([epochs_fooof['LeLo1']._data[:, ri_inds, :],
epochs_fooof['RiLo1']._data[:, le_inds, :]], 0)
lo2_f = np.concatenate([epochs_fooof['LeLo2']._data[:, ri_inds, :],
epochs_fooof['RiLo2']._data[:, le_inds, :]], 0)
lo3_f = np.concatenate([epochs_fooof['LeLo3']._data[:, ri_inds, :],
epochs_fooof['RiLo3']._data[:, le_inds, :]], 0)
## Calculate average across trials and channels - add to group data collection
# Canonical data
canonical_group_avg_dat[s_ind, 0, :] = np.mean(lo1_a, 1).mean(0)
canonical_group_avg_dat[s_ind, 1, :] = np.mean(lo2_a, 1).mean(0)
canonical_group_avg_dat[s_ind, 2, :] = np.mean(lo3_a, 1).mean(0)
# FOOOFed data
fooofed_group_avg_dat[s_ind, 0, :] = np.mean(lo1_f, 1).mean(0)
fooofed_group_avg_dat[s_ind, 1, :] = np.mean(lo2_f, 1).mean(0)
fooofed_group_avg_dat[s_ind, 2, :] = np.mean(lo3_f, 1).mean(0)
#################################################
## FOOOFING TRIAL AVERAGED DATA
# Loop loop loads & trials segments
for seg_label, seg_time in zip(SEG_LABELS, SEG_TIMES):
tmin, tmax = seg_time[0], seg_time[1]
# Calculate PSDs across trials, fit FOOOF models to averages
for le_label, ri_label, load_label in zip(['LeLo1', 'LeLo2', 'LeLo3'],
['RiLo1', 'RiLo2', 'RiLo3'],
LOAD_LABELS):
## Calculate trial wise PSDs for left & right side trials
trial_freqs, le_trial_psds = periodogram(
epochs[le_label]._data[:, :, _time_mask(epochs.times, tmin, tmax, srate)],
srate, window='hann', nfft=4*srate)
trial_freqs, ri_trial_psds = periodogram(
epochs[ri_label]._data[:, :, _time_mask(epochs.times, tmin, tmax, srate)],
srate, window='hann', nfft=4*srate)
## FIT ALL CHANNELS VERSION
if FIT_ALL_CHANNELS:
## Average spectra across trials within a given load & side
le_avg_psd_contra = avg_func(le_trial_psds[:, ri_inds, :], 0)
le_avg_psd_ipsi = avg_func(le_trial_psds[:, le_inds, :], 0)
ri_avg_psd_contra = avg_func(ri_trial_psds[:, le_inds, :], 0)
ri_avg_psd_ipsi = avg_func(ri_trial_psds[:, ri_inds, :], 0)
## Combine spectra across left & right trials for given load
ch_psd_contra = np.vstack([le_avg_psd_contra, ri_avg_psd_contra])
ch_psd_ipsi = np.vstack([le_avg_psd_ipsi, ri_avg_psd_ipsi])
## Fit FOOOFGroup to all channels, average & and collect results
fg.fit(trial_freqs, ch_psd_contra, FREQ_RANGE)
fm = avg_fg(fg)
fg_dict[load_label]['Contra'][seg_label].append(fm.copy())
fg.fit(trial_freqs, ch_psd_ipsi, FREQ_RANGE)
fm = avg_fg(fg)
fg_dict[load_label]['Ipsi'][seg_label].append(fm.copy())
## COLLAPSE ACROSS CHANNELS VERSION
else:
## Average spectra across trials and channels within a given load & side
le_avg_psd_contra = avg_func(avg_func(le_trial_psds[:, ri_inds, :], 0), 0)
le_avg_psd_ipsi = avg_func(avg_func(le_trial_psds[:, le_inds, :], 0), 0)
ri_avg_psd_contra = avg_func(avg_func(ri_trial_psds[:, le_inds, :], 0), 0)
ri_avg_psd_ipsi = avg_func(avg_func(ri_trial_psds[:, ri_inds, :], 0), 0)
## Collapse spectra across left & right trials for given load
avg_psd_contra = avg_func(np.vstack([le_avg_psd_contra, ri_avg_psd_contra]), 0)
avg_psd_ipsi = avg_func(np.vstack([le_avg_psd_ipsi, ri_avg_psd_ipsi]), 0)
## Fit FOOOF, and collect results
fm.fit(trial_freqs, avg_psd_contra, FREQ_RANGE)
fg_dict[load_label]['Contra'][seg_label].append(fm.copy())
fm.fit(trial_freqs, avg_psd_ipsi, FREQ_RANGE)
fg_dict[load_label]['Ipsi'][seg_label].append(fm.copy())
#################################################
## SAVE OUT RESULTS
# Save out group data
np.save(pjoin(RES_PATH, 'Group', 'alpha_freqs_group'), group_fooofed_alpha_freqs)
np.save(pjoin(RES_PATH, 'Group', 'canonical_group'), canonical_group_avg_dat)
np.save(pjoin(RES_PATH, 'Group', 'fooofed_group'), fooofed_group_avg_dat)
np.save(pjoin(RES_PATH, 'Group', 'dropped_trials'), dropped_trials)
np.save(pjoin(RES_PATH, 'Group', 'dropped_components'), dropped_components)
# Save out second round of FOOOFing
for load_label in LOAD_LABELS:
for side_label in SIDE_LABELS:
for seg_label in SEG_LABELS:
fg = combine_fooofs(fg_dict[load_label][side_label][seg_label])
fg.save('Group_' + load_label + '_' + side_label + '_' + seg_label,
pjoin(RES_PATH, 'FOOOF'), save_results=True)
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5, picks=picks) ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps') ica.plot_scores(scores, exclude=ecg_inds, title=title % 'ecg', labels='ecg') show_picks = np.abs(scores).argsort()[::-1][:5] ica.plot_sources(raw, show_picks, exclude=ecg_inds, title=title % 'ecg') ica.plot_components(ecg_inds, title=title % 'ecg', colorbar=True) ecg_inds = ecg_inds[:n_max_ecg] ica.exclude += ecg_inds # detect EOG by correlation eog_inds, scores = ica.find_bads_eog(raw) ica.plot_scores(scores, exclude=eog_inds, title=title % 'eog', labels='eog') show_picks = np.abs(scores).argsort()[::-1][:5] ica.plot_sources(raw, show_picks, exclude=eog_inds, title=title % 'eog') ica.plot_components(eog_inds, title=title % 'eog', colorbar=True) eog_inds = eog_inds[:n_max_eog] ica.exclude += eog_inds ############################################################################### # 3) Assess component selection and unmixing quality. # estimate average artifact ecg_evoked = ecg_epochs.average()
def compute_ica(fif_file, ecg_ch_name, eog_ch_name, n_components, reject):
"""Compute ica solution"""
import os
import mne
from mne.io import read_raw_fif
from mne.preprocessing import ICA
from mne.preprocessing import create_ecg_epochs, create_eog_epochs
from nipype.utils.filemanip import split_filename as split_f
subj_path, basename, ext = split_f(fif_file)
raw = read_raw_fif(fif_file, preload=True)
# select sensors
select_sensors = mne.pick_types(raw.info, meg=True,
ref_meg=False, exclude='bads')
# 1) Fit ICA model using the FastICA algorithm
# Other available choices are `infomax` or `extended-infomax`
# We pass a float value between 0 and 1 to select n_components based on the
# percentage of variance explained by the PCA components.
# reject = dict(mag=1e-1, grad=1e-9)
flat = dict(mag=1e-13, grad=1e-13)
ica = ICA(n_components=n_components, method='fastica', max_iter=500)
ica.fit(raw, picks=select_sensors, reject=reject, flat=flat)
# -------------------- Save ica timeseries ---------------------------- #
ica_ts_file = os.path.abspath(basename + "_ica-tseries.fif")
ica_src = ica.get_sources(raw)
ica_src.save(ica_ts_file)
ica_src = None
# --------------------------------------------------------------------- #
# 2) identify bad components by analyzing latent sources.
# generate ECG epochs use detection via phase statistics
# if we just have exclude channels we jump these steps
n_max_ecg = 3
n_max_eog = 2
# check if ecg_ch_name is in the raw channels
if ecg_ch_name in raw.info['ch_names']:
raw.set_channel_types({ecg_ch_name: 'ecg'})
else:
ecg_ch_name = None
ecg_epochs = create_ecg_epochs(raw, tmin=-0.5, tmax=0.5,
picks=select_sensors,
ch_name=ecg_ch_name)
ecg_inds, ecg_scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
ecg_evoked = ecg_epochs.average()
ecg_epochs = None
ecg_inds = ecg_inds[:n_max_ecg]
ica.exclude += ecg_inds
eog_ch_name = eog_ch_name.replace(' ', '')
if set(eog_ch_name.split(',')).issubset(set(raw.info['ch_names'])):
print('*** EOG CHANNELS FOUND ***')
eog_inds, eog_scores = ica.find_bads_eog(raw, ch_name=eog_ch_name)
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
eog_evoked = create_eog_epochs(raw, tmin=-0.5, tmax=0.5,
picks=select_sensors,
ch_name=eog_ch_name).average()
else:
print('*** NO EOG CHANNELS FOUND!!! ***')
eog_inds = eog_scores = eog_evoked = None
report_file = generate_report(raw=raw, ica=ica, subj_name=fif_file,
basename=basename,
ecg_evoked=ecg_evoked, ecg_scores=ecg_scores,
ecg_inds=ecg_inds, ecg_ch_name=ecg_ch_name,
eog_evoked=eog_evoked, eog_scores=eog_scores,
eog_inds=eog_inds, eog_ch_name=eog_ch_name)
report_file = os.path.abspath(report_file)
ica_sol_file = os.path.abspath(basename + '_ica_solution.fif')
ica.save(ica_sol_file)
raw_ica = ica.apply(raw)
raw_ica_file = os.path.abspath(basename + '_ica' + ext)
raw_ica.save(raw_ica_file)
return raw_ica_file, ica_sol_file, ica_ts_file, report_file
def preprocess_ICA_fif_to_ts(fif_file, ECG_ch_name, EoG_ch_name, l_freq, h_freq, down_sfreq, variance, is_sensor_space, data_type):
import os
import numpy as np
import mne
from mne.io import Raw
from mne.preprocessing import ICA, read_ica
from mne.preprocessing import create_ecg_epochs, create_eog_epochs
from mne.report import Report
from nipype.utils.filemanip import split_filename as split_f
report = Report()
subj_path, basename, ext = split_f(fif_file)
(data_path, sbj_name) = os.path.split(subj_path)
print data_path
# Read raw
# If None the compensation in the data is not modified.
# If set to n, e.g. 3, apply gradient compensation of grade n as for
# CTF systems (compensation=3)
raw = Raw(fif_file, preload=True)
# select sensors
select_sensors = mne.pick_types(raw.info, meg=True, ref_meg=False,
exclude='bads')
picks_meeg = mne.pick_types(raw.info, meg=True, eeg=True, exclude='bads')
# save electrode locations
sens_loc = [raw.info['chs'][i]['loc'][:3] for i in select_sensors]
sens_loc = np.array(sens_loc)
channel_coords_file = os.path.abspath("correct_channel_coords.txt")
print '*** ' + channel_coords_file + '***'
np.savetxt(channel_coords_file, sens_loc, fmt='%s')
# save electrode names
sens_names = np.array([raw.ch_names[pos] for pos in select_sensors],dtype = "str")
# AP 21032016
# channel_names_file = os.path.join(data_path, "correct_channel_names.txt")
channel_names_file = os.path.abspath("correct_channel_names.txt")
np.savetxt(channel_names_file,sens_names , fmt = '%s')
### filtering + downsampling
raw.filter(l_freq=l_freq, h_freq=h_freq, picks=picks_meeg,
method='iir', n_jobs=8)
# raw.filter(l_freq = l_freq, h_freq = h_freq, picks = picks_meeg,
# method='iir')
# raw.resample(sfreq=down_sfreq, npad=0)
### 1) Fit ICA model using the FastICA algorithm
# Other available choices are `infomax` or `extended-infomax`
# We pass a float value between 0 and 1 to select n_components based on the
# percentage of variance explained by the PCA components.
ICA_title = 'Sources related to %s artifacts (red)'
is_show = False # visualization
reject = dict(mag=4e-12, grad=4000e-13)
# check if we have an ICA, if yes, we load it
ica_filename = os.path.join(subj_path,basename + "-ica.fif")
if os.path.exists(ica_filename) is False:
ica = ICA(n_components=variance, method='fastica', max_iter=500) # , max_iter=500
ica.fit(raw, picks=select_sensors, reject=reject) # decim = 3,
has_ICA = False
else:
has_ICA = True
print ica_filename + ' exists!!!'
ica = read_ica(ica_filename)
ica.exclude = []
# 2) identify bad components by analyzing latent sources.
# generate ECG epochs use detection via phase statistics
# if we just have exclude channels we jump these steps
# if len(ica.exclude)==0:
n_max_ecg = 3
n_max_eog = 2
# check if ECG_ch_name is in the raw channels
if ECG_ch_name in raw.info['ch_names']:
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5,
picks=select_sensors,
ch_name=ECG_ch_name)
# if not a synthetic ECG channel is created from cross channel average
else:
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5,
picks=select_sensors)
# ICA for ECG artifact
# threshold=0.25 come default
ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
print scores
print '\n len ecg_inds *** ' + str(len(ecg_inds)) + '***\n'
if len(ecg_inds) > 0:
ecg_evoked = ecg_epochs.average()
fig1 = ica.plot_scores(scores, exclude=ecg_inds,
title=ICA_title % 'ecg', show=is_show)
show_picks = np.abs(scores).argsort()[::-1][:5] # Pick the five largest scores and plot them
# Plot estimated latent sources given the unmixing matrix.
#ica.plot_sources(raw, show_picks, exclude=ecg_inds, title=ICA_title % 'ecg', show=is_show)
t_start = 0
t_stop = 30 # take the fist 30s
fig2 = ica.plot_sources(raw, show_picks, exclude=ecg_inds, title=ICA_title % 'ecg' + ' in 30s'
,start = t_start, stop = t_stop, show=is_show)
# topoplot of unmixing matrix columns
fig3 = ica.plot_components(show_picks, title=ICA_title % 'ecg', colorbar=True, show=is_show)
ecg_inds = ecg_inds[:n_max_ecg]
ica.exclude += ecg_inds
fig4 = ica.plot_sources(ecg_evoked, exclude=ecg_inds, show=is_show) # plot ECG sources + selection
fig5 = ica.plot_overlay(ecg_evoked, exclude=ecg_inds, show=is_show) # plot ECG cleaning
fig = [fig1, fig2, fig3, fig4, fig5]
report.add_figs_to_section(fig, captions=['Scores of ICs related to ECG',
'Time Series plots of ICs (ECG)',
'TopoMap of ICs (ECG)',
'Time-locked ECG sources',
'ECG overlay'], section = 'ICA - ECG')
# check if EoG_ch_name is in the raw channels
# if EoG_ch_name is empty if data_type is fif, ICA routine automatically looks for EEG61, EEG62
# otherwise if data_type is ds we jump this step
if not EoG_ch_name and data_type=='ds':
eog_inds = []
else:
if EoG_ch_name in raw.info['ch_names']:
### ICA for eye blink artifact - detect EOG by correlation
eog_inds, scores = ica.find_bads_eog(raw, ch_name = EoG_ch_name)
else:
eog_inds, scores = ica.find_bads_eog(raw)
if len(eog_inds) > 0:
fig6 = ica.plot_scores(scores, exclude=eog_inds, title=ICA_title % 'eog', show=is_show)
report.add_figs_to_section(fig6, captions=['Scores of ICs related to EOG'],
section = 'ICA - EOG')
# check how many EoG ch we have
rs = np.shape(scores)
if len(rs)>1:
rr = rs[0]
show_picks = [np.abs(scores[i][:]).argsort()[::-1][:5] for i in range(rr)]
for i in range(rr):
fig7 = ica.plot_sources(raw, show_picks[i][:], exclude=eog_inds,
start = raw.times[0], stop = raw.times[-1],
title=ICA_title % 'eog',show=is_show)
fig8 = ica.plot_components(show_picks[i][:], title=ICA_title % 'eog', colorbar=True, show=is_show) # ICA nel tempo
fig = [fig7, fig8]
report.add_figs_to_section(fig, captions=['Scores of ICs related to EOG',
'Time Series plots of ICs (EOG)'],
section = 'ICA - EOG')
else:
show_picks = np.abs(scores).argsort()[::-1][:5]
fig7 = ica.plot_sources(raw, show_picks, exclude=eog_inds, title=ICA_title % 'eog', show=is_show)
fig8 = ica.plot_components(show_picks, title=ICA_title % 'eog', colorbar=True, show=is_show)
fig = [fig7, fig8]
report.add_figs_to_section(fig, captions=['Time Series plots of ICs (EOG)',
'TopoMap of ICs (EOG)',],
section = 'ICA - EOG')
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
if EoG_ch_name in raw.info['ch_names']:
eog_evoked = create_eog_epochs(raw, tmin=-.5, tmax=.5, picks=select_sensors,
ch_name=EoG_ch_name).average()
else:
eog_evoked = create_eog_epochs(raw, tmin=-.5, tmax=.5, picks=select_sensors).average()
fig9 = ica.plot_sources(eog_evoked, exclude=eog_inds, show=is_show) # plot EOG sources + selection
fig10 = ica.plot_overlay(eog_evoked, exclude=eog_inds, show=is_show) # plot EOG cleaning
fig = [fig9, fig10]
report.add_figs_to_section(fig, captions=['Time-locked EOG sources',
'EOG overlay'], section = 'ICA - EOG')
fig11 = ica.plot_overlay(raw, show=is_show)
report.add_figs_to_section(fig11, captions=['Signal'], section = 'Signal quality')
### plot all topographies and time seris of the ICA components
n_ica_components = ica.mixing_matrix_.shape[1]
n_topo = 10;
n_fig = n_ica_components/n_topo;
n_plot = n_ica_components%n_topo;
print '*************** n_fig = ' + str(n_fig) + ' n_plot = ' + str(n_plot) + '********************'
fig = []
t_start = 0
t_stop = None # 60 if we want to take the fist 60s
for n in range(0,n_fig):
fig_tmp = ica.plot_components(range(n_topo*n,n_topo*(n+1)),title='ICA components', show=is_show)
fig.append(fig_tmp)
fig_tmp = ica.plot_sources(raw, range(n_topo*n,n_topo*(n+1)),
start = t_start, stop = t_stop,
title='ICA components')
fig.append(fig_tmp)
# if n_plot > 0:
# fig_tmp = ica.plot_components(range(n_fig*n_topo,n_ica_components), title='ICA components', show=is_show)
# fig.append(fig_tmp)
# fig_tmp = ica.plot_sources(raw, range(n_fig*n_topo,n_ica_components),
# start = t_start, stop = t_stop,
# title='ICA components')
# fig.append(fig_tmp)
#
# for n in range(0, len(fig)):
# report.add_figs_to_section(fig[n], captions=['TOPO'], section = 'ICA Topo Maps')
if n_plot > 5:
n_fig_l = n_plot//5
print '*************************** ' + str(n_fig_l) + ' *********************************'
for n in range(0,n_fig_l):
print range(n_fig*n_topo+5*n, n_fig*n_topo+5*(n+1))
fig_tmp = ica.plot_components(range(n_fig*n_topo+5*n, n_fig*n_topo+5*(n+1)),title='ICA components')
fig.append(fig_tmp)
fig_tmp = ica.plot_sources(raw, range(n_fig*n_topo+5*n, n_fig*n_topo+5*(n+1)),
start = t_start, stop = t_stop,
title='ICA components')
fig.append(fig_tmp)
print range(n_fig*n_topo+5*(n+1),n_ica_components)
fig_tmp = ica.plot_components(range(n_fig*n_topo+5*(n+1),n_ica_components), title='ICA components')
fig.append(fig_tmp)
fig_tmp = ica.plot_sources(raw, range(n_fig*n_topo+5*(n+1),n_ica_components),
start = t_start, stop = t_stop,
title='ICA components')
fig.append(fig_tmp)
for n in range(0, len(fig)):
report.add_figs_to_section(fig[n], captions=['TOPO'], section = 'ICA Topo Maps')
report_filename = os.path.join(subj_path,basename + "-report.html")
print '******* ' + report_filename
report.save(report_filename, open_browser=False, overwrite=True)
# 3) apply ICA to raw data and save solution and report
# check the amplitudes do not change
# raw_ica_file = os.path.abspath(basename[:i_raw] + 'ica-raw.fif')
raw_ica_file = os.path.join(subj_path, basename + '-preproc-raw.fif')
raw_ica = ica.apply(raw)
raw_ica.resample(sfreq=down_sfreq, npad=0)
raw_ica.save(raw_ica_file, overwrite=True)
# save ICA solution
print ica_filename
if has_ICA is False:
ica.save(ica_filename)
# 4) save data
data_noIca, times = raw[select_sensors, :]
data, times = raw_ica[select_sensors, :]
print data.shape
print raw.info['sfreq']
ts_file = os.path.abspath(basename + "_ica.npy")
np.save(ts_file, data)
print '***** TS FILE ' + ts_file + '*****'
if is_sensor_space:
return ts_file, channel_coords_file, channel_names_file, raw.info['sfreq']
else:
# return raw_ica, channel_coords_file, channel_names_file, raw.info['sfreq']
return raw_ica_file, channel_coords_file, channel_names_file, raw.info['sfreq']
def compute_ica(raw, subject, n_components=0.99, picks=None, decim=None,
reject=None, ecg_tmin=-0.5, ecg_tmax=0.5, eog_tmin=-0.5,
eog_tmax=0.5, n_max_ecg=3, n_max_eog=1,
n_max_ecg_epochs=200, show=True, img_scale=1.0,
random_state=None, report=None, artifact_stats=None):
"""Run ICA in raw data
Parameters
----------,
raw : instance of Raw
Raw measurements to be decomposed.
subject : str
The name of the subject.
picks : array-like of int, shape(n_channels, ) | None
Channels to be included. This selection remains throughout the
initialized ICA solution. If None only good data channels are used.
Defaults to None.
n_components : int | float | None | 'rank'
The number of components used for ICA decomposition. If int, it must be
smaller then max_pca_components. If None, all PCA components will be
used. If float between 0 and 1 components can will be selected by the
cumulative percentage of explained variance.
If 'rank', the number of components equals the rank estimate.
Defaults to 0.99.
decim : int | None
Increment for selecting each nth time slice. If None, all samples
within ``start`` and ``stop`` are used. Defalts to None.
reject : dict | None
Rejection parameters based on peak to peak amplitude.
Valid keys are 'grad' | 'mag' | 'eeg' | 'eog' | 'ecg'.
If reject is None then no rejection is done. You should
use such parameters to reject big measurement artifacts
and not EOG for example. It only applies if `inst` is of type Raw.
Defaults to {'mag': 5e-12}
ecg_tmin : float
Start time before ECG event. Defaults to -0.5.
ecg_tmax : float
End time after ECG event. Defaults to 0.5.
eog_tmin : float
Start time before rog event. Defaults to -0.5.
eog_tmax : float
End time after rog event. Defaults to 0.5.
n_max_ecg : int | None
The maximum number of ECG components to exclude. Defaults to 3.
n_max_eog : int | None
The maximum number of EOG components to exclude. Defaults to 1.
n_max_ecg_epochs : int
The maximum number of ECG epochs to use for phase-consistency
estimation. Defaults to 200.
show : bool
Show figure if True
scale_img : float
The scaling factor for the report. Defaults to 1.0.
random_state : None | int | instance of np.random.RandomState
np.random.RandomState to initialize the FastICA estimation.
As the estimation is non-deterministic it can be useful to
fix the seed to have reproducible results. Defaults to None.
report : instance of Report | None
The report object. If None, a new report will be generated.
artifact_stats : None | dict
A dict that contains info on amplitude ranges of artifacts and
numbers of events, etc. by channel type.
Returns
-------
ica : instance of ICA
The ICA solution.
report : dict
A dict with an html report ('html') and artifact statistics ('stats').
"""
if report is None:
report = Report(subject=subject, title='ICA preprocessing')
if n_components == 'rank':
n_components = raw.estimate_rank(picks=picks)
ica = ICA(n_components=n_components, max_pca_components=None,
random_state=random_state, max_iter=256)
ica.fit(raw, picks=picks, decim=decim, reject=reject)
comment = []
for ch in ('mag', 'grad', 'eeg'):
if ch in ica:
comment += [ch.upper()]
if len(comment) > 0:
comment = '+'.join(comment) + ' '
else:
comment = ''
topo_ch_type = 'mag'
if 'GRAD' in comment and 'MAG' not in comment:
topo_ch_type = 'grad'
elif 'EEG' in comment:
topo_ch_type = 'eeg'
###########################################################################
# 2) identify bad components by analyzing latent sources.
title = '%s related to %s artifacts (red) ({})'.format(subject)
# generate ECG epochs use detection via phase statistics
reject_ = {'mag': 5e-12, 'grad': 5000e-13, 'eeg': 300e-6}
if reject is not None:
reject_.update(reject)
for ch_type in ['mag', 'grad', 'eeg']:
if ch_type not in ica:
reject_.pop(ch_type)
picks_ = np.array([raw.ch_names.index(k) for k in ica.ch_names])
if 'eeg' in ica:
if 'ecg' in raw:
picks_ = np.append(picks_,
pick_types(raw.info, meg=False, ecg=True)[0])
else:
logger.info('There is no ECG channel, trying to guess ECG from '
'magnetormeters')
if artifact_stats is None:
artifact_stats = dict()
ecg_epochs = create_ecg_epochs(raw, tmin=ecg_tmin, tmax=ecg_tmax,
keep_ecg=True, picks=picks_, reject=reject_)
n_ecg_epochs_found = len(ecg_epochs.events)
artifact_stats['ecg_n_events'] = n_ecg_epochs_found
n_max_ecg_epochs = min(n_max_ecg_epochs, n_ecg_epochs_found)
artifact_stats['ecg_n_used'] = n_max_ecg_epochs
sel_ecg_epochs = np.arange(n_ecg_epochs_found)
rng = np.random.RandomState(42)
rng.shuffle(sel_ecg_epochs)
ecg_ave = ecg_epochs.average()
report.add_figs_to_section(ecg_ave.plot(), 'ECG-full', 'artifacts')
ecg_epochs = ecg_epochs[sel_ecg_epochs[:n_max_ecg_epochs]]
ecg_ave = ecg_epochs.average()
report.add_figs_to_section(ecg_ave.plot(), 'ECG-used', 'artifacts')
_put_artifact_range(artifact_stats, ecg_ave, kind='ecg')
ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
if len(ecg_inds) > 0:
ecg_evoked = ecg_epochs.average()
del ecg_epochs
fig = ica.plot_scores(scores, exclude=ecg_inds, labels='ecg',
title='', show=show)
report.add_figs_to_section(fig, 'scores ({})'.format(subject),
section=comment + 'ECG',
scale=img_scale)
current_exclude = [e for e in ica.exclude] # issue #2608 MNE
fig = ica.plot_sources(raw, ecg_inds, exclude=ecg_inds,
title=title % ('components', 'ecg'), show=show)
report.add_figs_to_section(fig, 'sources ({})'.format(subject),
section=comment + 'ECG',
scale=img_scale)
ica.exclude = current_exclude
fig = ica.plot_components(ecg_inds, ch_type=topo_ch_type,
title='', colorbar=True, show=show)
report.add_figs_to_section(fig, title % ('sources', 'ecg'),
section=comment + 'ECG', scale=img_scale)
ica.exclude = current_exclude
ecg_inds = ecg_inds[:n_max_ecg]
ica.exclude += ecg_inds
fig = ica.plot_sources(ecg_evoked, exclude=ecg_inds, show=show)
report.add_figs_to_section(fig, 'evoked sources ({})'.format(subject),
section=comment + 'ECG',
scale=img_scale)
fig = ica.plot_overlay(ecg_evoked, exclude=ecg_inds, show=show)
report.add_figs_to_section(fig,
'rejection overlay ({})'.format(subject),
section=comment + 'ECG',
scale=img_scale)
# detect EOG by correlation
picks_eog = np.concatenate(
[picks_, pick_types(raw.info, meg=False, eeg=False, ecg=False,
eog=True)])
eog_epochs = create_eog_epochs(raw, tmin=eog_tmin, tmax=eog_tmax,
picks=picks_eog, reject=reject_)
artifact_stats['eog_n_events'] = len(eog_epochs.events)
artifact_stats['eog_n_used'] = artifact_stats['eog_n_events']
eog_ave = eog_epochs.average()
report.add_figs_to_section(eog_ave.plot(), 'EOG-used', 'artifacts')
_put_artifact_range(artifact_stats, eog_ave, kind='eog')
eog_inds = None
if len(eog_epochs.events) > 0:
eog_inds, scores = ica.find_bads_eog(eog_epochs)
if eog_inds is not None and len(eog_epochs.events) > 0:
fig = ica.plot_scores(scores, exclude=eog_inds, labels='eog',
show=show, title='')
report.add_figs_to_section(fig, 'scores ({})'.format(subject),
section=comment + 'EOG',
scale=img_scale)
current_exclude = [e for e in ica.exclude] # issue #2608 MNE
fig = ica.plot_sources(raw, eog_inds, exclude=ecg_inds,
title=title % ('sources', 'eog'), show=show)
report.add_figs_to_section(fig, 'sources', section=comment + 'EOG',
scale=img_scale)
ica.exclude = current_exclude
fig = ica.plot_components(eog_inds, ch_type=topo_ch_type,
title='', colorbar=True, show=show)
report.add_figs_to_section(fig, title % ('components', 'eog'),
section=comment + 'EOG', scale=img_scale)
ica.exclude = current_exclude
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
eog_evoked = eog_epochs.average()
fig = ica.plot_sources(eog_evoked, exclude=eog_inds, show=show)
report.add_figs_to_section(
fig, 'evoked sources ({})'.format(subject),
section=comment + 'EOG', scale=img_scale)
fig = ica.plot_overlay(eog_evoked, exclude=eog_inds, show=show)
report.add_figs_to_section(
fig, 'rejection overlay({})'.format(subject),
section=comment + 'EOG', scale=img_scale)
else:
del eog_epochs
# check the amplitudes do not change
if len(ica.exclude) > 0:
fig = ica.plot_overlay(raw, show=show) # EOG artifacts remain
html = _render_components_table(ica)
report.add_htmls_to_section(
html, captions='excluded components',
section='ICA rejection summary (%s)' % ch_type)
report.add_figs_to_section(
fig, 'rejection overlay({})'.format(subject),
section=comment + 'RAW', scale=img_scale)
return ica, dict(html=report, stats=artifact_stats)
def compute_ica(subject):
"""Function will compute ICA on raw and apply the ICA.
params:
subject : str
the subject id to be loaded
"""
raw = Raw(save_folder + "%s_filtered_data_mc_raw_tsss.fif" % subject,
preload=True)
# ICA Part
ica = ICA(n_components=0.95, method='fastica', max_iter=256)
picks = mne.pick_types(raw.info, meg=True, eeg=True,
stim=False, exclude='bads')
ica.fit(raw, picks=picks, decim=decim, reject=reject)
# maximum number of components to reject
n_max_ecg, n_max_eog = 3, 1
##########################################################################
# 2) identify bad components by analyzing latent sources.
title = 'Sources related to %s artifacts (red) for sub: %s'
# generate ECG epochs use detection via phase statistics
ecg_epochs = create_ecg_epochs(raw, ch_name="ECG002",
tmin=-.5, tmax=.5, picks=picks)
n_ecg_epochs_found = len(ecg_epochs.events)
sel_ecg_epochs = np.arange(0, n_ecg_epochs_found, 10)
ecg_epochs = ecg_epochs[sel_ecg_epochs]
ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
fig = ica.plot_scores(scores, exclude=ecg_inds,
title=title % ('ecg', subject))
fig.savefig(save_folder + "pics/%s_ecg_scores.png" % subject)
if ecg_inds:
show_picks = np.abs(scores).argsort()[::-1][:5]
fig = ica.plot_sources(raw, show_picks, exclude=ecg_inds,
title=title % ('ecg', subject), show=False)
fig.savefig(save_folder + "pics/%s_ecg_sources.png" % subject)
fig = ica.plot_components(ecg_inds, title=title % ('ecg', subject),
colorbar=True)
fig.savefig(save_folder + "pics/%s_ecg_component.png" % subject)
ecg_inds = ecg_inds[:n_max_ecg]
ica.exclude += ecg_inds
# estimate average artifact
ecg_evoked = ecg_epochs.average()
del ecg_epochs
# plot ECG sources + selection
fig = ica.plot_sources(ecg_evoked, exclude=ecg_inds)
fig.savefig(save_folder + "pics/%s_ecg_sources_ave.png" % subject)
# plot ECG cleaning
ica.plot_overlay(ecg_evoked, exclude=ecg_inds)
fig.savefig(save_folder + "pics/%s_ecg_sources_clean_ave.png" % subject)
# DETECT EOG BY CORRELATION
# HORIZONTAL EOG
eog_epochs = create_eog_epochs(raw, ch_name="EOG001")
eog_inds, scores = ica.find_bads_eog(raw)
fig = ica.plot_scores(scores, exclude=eog_inds,
title=title % ('eog', subject))
fig.savefig(save_folder + "pics/%s_eog_scores.png" % subject)
fig = ica.plot_components(eog_inds, title=title % ('eog', subject),
colorbar=True)
fig.savefig(save_folder + "pics/%s_eog_component.png" % subject)
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
del eog_epochs
##########################################################################
# Apply the solution to Raw, Epochs or Evoked like this:
raw_ica = ica.apply(raw, copy=False)
ica.save(save_folder + "%s-ica.fif" % subject) # save ICA componenets
# Save raw with ICA removed
raw_ica.save(save_folder + "%s_filtered_ica_mc_raw_tsss.fif" % subject,
overwrite=True)
plt.close("all")
#ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5, picks=picks) #ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps') #ica.plot_scores(scores, exclude=ecg_inds, title=title % 'ecg', labels='ecg') #show_picks = np.abs(scores).argsort()[::-1][:5] #ica.plot_sources(raw, show_picks, exclude=ecg_inds, title=title % 'ecg') #ica.plot_components(ecg_inds, title=title % 'ecg', colorbar=True) #ecg_inds = ecg_inds[:n_max_ecg] #ica.exclude += ecg_inds # detect EOG by correlation eog_inds, scores = ica.find_bads_eog(raw, threshold=2.0) ica.plot_scores(scores, exclude=eog_inds, title=title % 'eog', labels='eog') show_picks = np.abs(scores).argsort()[::-1][:5] ica.plot_sources(raw, show_picks, exclude=eog_inds, title=title % 'eog') ica.plot_components(eog_inds, title=title % 'eog', colorbar=True) eog_inds = eog_inds[:n_max_eog] ica.exclude += eog_inds ############################################################################### # 3) Assess component selection and unmixing quality # estimate average artifact #ecg_evoked = ecg_epochs.average()
def runICA(raw,saveRoot,name):
saveRoot = saveRoot
icaList = []
ica = []
n_max_ecg = 3 # max number of ecg components
# n_max_eog_1 = 2 # max number of vert eog comps
# n_max_eog_2 = 2 # max number of horiz eog comps
ecg_source_idx, ecg_scores, ecg_exclude = [], [], []
eog_source_idx, eog_scores, eog_exclude = [], [], []
#horiz = 1 # will later be modified to horiz = 0 if no horizontal EOG components are identified
ica = ICA(n_components=0.90,n_pca_components=64,max_pca_components=100,noise_cov=None)
ica.fit(raw)
#*************
eog_picks = mne.pick_types(raw.info, meg=False, eeg=False, stim=False, eog=True, ecg=False, emg=False)[0]
ecg_picks = mne.pick_types(raw.info, meg=False, eeg=False, stim=False, ecg=True, eog=False, emg=False)[0]
ica_picks = mne.pick_types(raw.info, meg=True, eeg=False, eog=False, ecg=False,
stim=False, exclude='bads')
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5, picks=ica_picks)
ecg_evoked = ecg_epochs.average()
eog_evoked = create_eog_epochs(raw, tmin=-.5, tmax=.5, picks=ica_picks).average()
ecg_source_idx, ecg_scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
eog_source_idx, eog_scores = ica.find_bads_eog(raw,ch_name=raw.ch_names[eog_picks].encode('ascii', 'ignore'))
# defining a title-frame for later use
title = 'Sources related to %s artifacts (red)'
# extracting number of ica-components and plotting their topographies
source_idx = range(0, ica.n_components_)
ica_plot = ica.plot_components(source_idx, ch_type="mag")
# select ICA sources and reconstruct MEG signals, compute clean ERFs
# Add detected artefact sources to exclusion list
# We now add the eog artefacts to the ica.exclusion list
if not ecg_source_idx:
print("No ECG components above threshold were identified for subject " + name +
" - selecting the component with the highest score under threshold")
ecg_exclude = [np.absolute(ecg_scores).argmax()]
ecg_source_idx=[np.absolute(ecg_scores).argmax()]
elif ecg_source_idx:
ecg_exclude += ecg_source_idx[:n_max_ecg]
ica.exclude += ecg_exclude
if not eog_source_idx:
if np.absolute(eog_scores).any>0.3:
eog_exclude=[np.absolute(eog_scores).argmax()]
eog_source_idx=[np.absolute(eog_scores).argmax()]
print("No EOG components above threshold were identified " + name +
" - selecting the component with the highest score under threshold above 0.3")
elif not np.absolute(eog_scores).any>0.3:
eog_exclude=[]
print("No EOG components above threshold were identified" + name)
elif eog_source_idx:
eog_exclude += eog_source_idx
ica.exclude += eog_exclude
print('########## saving')
if len(eog_exclude) == 0:
if len(ecg_exclude) == 0:
ica_plot.savefig(saveRoot + name + '_comps_eog_none-ecg_none' + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 1:
ica_plot.savefig(saveRoot + name + '_comps_eog_none-ecg' + map(str, ecg_exclude)[0] + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 2:
ica_plot.savefig(saveRoot + name + '_comps_eog_none-ecg' + map(str, ecg_exclude)[0] + '_' + map(str, ecg_exclude)[1] + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 3:
ica_plot.savefig(saveRoot + name + '_comps_eog_none-ecg' + map(str, ecg_exclude)[0] + '_' + map(str, ecg_exclude)[1] + '_' + map(str, ecg_exclude)[2] + '.pdf', format = 'pdf')
elif len(eog_exclude) == 1:
if len(ecg_exclude) == 0:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] +
'-ecg_none' + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 1:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] +
'-ecg' + map(str, ecg_exclude)[0] + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 2:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] +
'-ecg' + map(str, ecg_exclude)[0] + '_' + map(str, ecg_exclude)[1] + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 3:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] +
'-ecg' + map(str, ecg_exclude)[0] + '_' + map(str, ecg_exclude)[1] + '_' + map(str, ecg_exclude)[2] + '.pdf', format = 'pdf')
elif len(eog_exclude) == 2:
if len(ecg_exclude) == 0:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] + '_' + map(str, eog_exclude)[1] +
'-ecg_none' + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 1:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] + '_' + map(str, eog_exclude)[1] +
'-ecg' + map(str, ecg_exclude)[0] + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 2:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] + '_' + map(str, eog_exclude)[1] +
'-ecg' + map(str, ecg_exclude)[0] + '_' + map(str, ecg_exclude)[1] + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 3:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] + '_' + map(str, eog_exclude)[1] +
'-ecg' + map(str, ecg_exclude)[0] + '_' + map(str, ecg_exclude)[1] + '_' + map(str, ecg_exclude)[2] + '.pdf', format = 'pdf')
# plot the scores for the different components highlighting in red that/those related to ECG
scores_plots_ecg=ica.plot_scores(ecg_scores, exclude=ecg_source_idx, title=title % 'ecg')
scores_plots_ecg.savefig(saveRoot + name + '_ecg_scores.pdf', format = 'pdf')
scores_plots_eog=ica.plot_scores(eog_scores, exclude=eog_source_idx, title=title % 'eog')
scores_plots_eog.savefig(saveRoot + name + '_eog_scores.pdf', format = 'pdf')
source_source_ecg=ica.plot_sources(ecg_evoked, exclude=ecg_source_idx)
source_source_ecg.savefig(saveRoot + name + '_ecg_source.pdf', format = 'pdf')
#ax = plt.subplot(2,1,2)
source_clean_ecg=ica.plot_overlay(ecg_evoked, exclude=ecg_source_idx)
source_clean_ecg.savefig(saveRoot + name + '_ecg_clean.pdf', format = 'pdf')
#clean_plot.savefig(saveRoot + name + '_ecg_clean.pdf', format = 'pdf')
#if len(eog_exclude) > 0:
source_source_eog=ica.plot_sources(eog_evoked, exclude=eog_source_idx)
source_source_eog.savefig(saveRoot + name + '_eog_source.pdf', format = 'pdf')
source_clean_eog=ica.plot_overlay(eog_evoked, exclude=eog_source_idx)
source_clean_eog.savefig(saveRoot + name + '_eog_clean.pdf', format = 'pdf')
overl_plot = ica.plot_overlay(raw)
overl_plot.savefig(saveRoot + name + '_overl.pdf', format = 'pdf')
event_id = 999
ecg_events, _, _ = mne.preprocessing.find_ecg_events(raw, event_id,
ch_name=raw.ch_names[ecg_picks].encode('UTF8'))
picks = mne.pick_types(raw.info, meg=False, eeg=False, stim=False, ecg=True)
tmin, tmax = -0.1, 0.1
epochs_ecg = mne.Epochs(raw, ecg_events, event_id, tmin, tmax,picks=picks)
data_ecg = epochs_ecg.get_data()
event_id = 998
eog_events = mne.preprocessing.find_eog_events(raw, event_id,
ch_name=raw.ch_names[eog_picks].encode('UTF8'))
picks = mne.pick_types(raw.info, meg=False, eeg=False, stim=False, eog=True,
exclude='bads')
tmin, tmax = -0.5, 0.5
epochs_eog = mne.Epochs(raw, eog_events, event_id, tmin, tmax,picks=[picks[0]])#,
data_eog = epochs_eog.get_data()
plSize = 1
pltRes = 128
ecg_eogAvg=plt.figure(figsize=(20,10))
ax = plt.subplot(2,3,1)
plt.plot(1e3 * epochs_ecg.times, np.squeeze(data_ecg).T)
plt.xlabel('Times (ms)')
plt.title('ECG')
plt.ylabel('ECG')
plt.show()
ax2 = plt.subplot(2,3,2)
plot_evoked_topomap(ecg_evoked, times=0, average=0.02, ch_type='mag',colorbar=False,
size=plSize, res=pltRes,
axes=ax2)
ax3= plt.subplot(2,3,3)
plot_evoked_topomap(ecg_evoked, times=0, average=0.02, ch_type='grad',colorbar=False,
size=plSize, res=pltRes,
axes=ax3)
ax = plt.subplot(2,3,4)
plt.plot(1e3 * epochs_eog.times, np.squeeze(data_eog).T)
plt.xlabel('Times (ms)')
plt.title('EOG')
plt.ylabel('EOG')
ax = plt.subplot(2,3,5)
plot_evoked_topomap(eog_evoked, times=0, average=0.05, ch_type='mag',colorbar=False,
size=plSize, res=pltRes,
axes=ax)
ax = plt.subplot(2,3,6)
plot_evoked_topomap(eog_evoked, times=0, average=0.05, ch_type='grad',colorbar=False,
size=plSize, res=pltRes, show=False,
axes=ax)
plt.tight_layout()
ecg_eogAvg.savefig(saveRoot + name +'_ecg_eog_Avg.pdf',format = 'pdf')
plt.close('all')
## restore sensor space data
icaList = ica.apply(raw)
return(icaList, ica)
def runICA(raw,saveRoot,name):
saveRoot = saveRoot
icaList = []
ica = []
n_max_ecg = 3 # max number of ecg components
# n_max_eog_1 = 2 # max number of vert eog comps
# n_max_eog_2 = 2 # max number of horiz eog comps
ecg_source_idx, ecg_scores, ecg_exclude = [], [], []
eog_source_idx, eog_scores, eog_exclude = [], [], []
#horiz = 1 # will later be modified to horiz = 0 if no horizontal EOG components are identified
ica = ICA(n_components=0.90,n_pca_components=64,max_pca_components=100,noise_cov=None)
fit_picks = mne.pick_types(raw.info, meg=True, eeg=True, eog=False, ecg=False,
stim=False, exclude='bads')
ica.fit(raw, picks=fit_picks)
#ica.fit(raw)
#*************
eog_picks = mne.pick_types(raw.info, meg=False, eeg=False, stim=False, eog=True, ecg=False, emg=False)[0]
ecg_picks = mne.pick_types(raw.info, meg=False, eeg=False, stim=False, ecg=True, eog=False, emg=False)[0]
ica_picks = mne.pick_types(raw.info, meg=True, eeg=True, eog=False, ecg=False,
stim=False, exclude='bads')
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5, picks=ica_picks)
ecg_evoked = ecg_epochs.average()
eog_evoked = create_eog_epochs(raw, tmin=-.5, tmax=.5, picks=ica_picks).average()
ecg_source_idx, ecg_scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
eog_source_idx, eog_scores = ica.find_bads_eog(raw,ch_name=raw.ch_names[eog_picks].encode('ascii', 'ignore'))
#eog_source_idx_2, eog_scores_2 = ica.find_bads_eog(raw,ch_name='EOG002')
#if not eog_source_idx_2:
# horiz = 0
#show_picks = np.abs(scores).argsort()[::-1][:5]
#ica.plot_sources(raw, show_picks, exclude=ecg_inds, title=title % 'ecg')
# defining a title-frame for later use
title = 'Sources related to %s artifacts (red)'
# extracting number of ica-components and plotting their topographies
source_idx = range(0, ica.n_components_)
#ica_plot = ica.plot_components(source_idx, ch_type="mag")
ica_plot = ica.plot_components(source_idx)
#ica_plot = ica.plot_components(source_idx)
# select ICA sources and reconstruct MEG signals, compute clean ERFs
# Add detected artefact sources to exclusion list
# We now add the eog artefacts to the ica.exclusion list
if not ecg_source_idx:
print("No ECG components above threshold were identified for subject " + name +
" - selecting the component with the highest score under threshold")
ecg_exclude = [np.absolute(ecg_scores).argmax()]
ecg_source_idx=[np.absolute(ecg_scores).argmax()]
elif ecg_source_idx:
ecg_exclude += ecg_source_idx[:n_max_ecg]
ica.exclude += ecg_exclude
if not eog_source_idx:
if np.absolute(eog_scores).any>0.3:
eog_exclude=[np.absolute(eog_scores).argmax()]
eog_source_idx=[np.absolute(eog_scores).argmax()]
print("No EOG components above threshold were identified " + name +
" - selecting the component with the highest score under threshold above 0.3")
elif not np.absolute(eog_scores).any>0.3:
eog_exclude=[]
print("No EOG components above threshold were identified" + name)
elif eog_source_idx:
eog_exclude += eog_source_idx
ica.exclude += eog_exclude
print('########## saving')
if len(eog_exclude) == 0:
if len(ecg_exclude) == 0:
ica_plot.savefig(saveRoot + name + '_comps_eog_none-ecg_none' + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 1:
ica_plot.savefig(saveRoot + name + '_comps_eog_none-ecg' + map(str, ecg_exclude)[0] + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 2:
ica_plot.savefig(saveRoot + name + '_comps_eog_none-ecg' + map(str, ecg_exclude)[0] + '_' + map(str, ecg_exclude)[1] + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 3:
ica_plot.savefig(saveRoot + name + '_comps_eog_none-ecg' + map(str, ecg_exclude)[0] + '_' + map(str, ecg_exclude)[1] + '_' + map(str, ecg_exclude)[2] + '.pdf', format = 'pdf')
elif len(eog_exclude) == 1:
if len(ecg_exclude) == 0:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] +
'-ecg_none' + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 1:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] +
'-ecg' + map(str, ecg_exclude)[0] + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 2:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] +
'-ecg' + map(str, ecg_exclude)[0] + '_' + map(str, ecg_exclude)[1] + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 3:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] +
'-ecg' + map(str, ecg_exclude)[0] + '_' + map(str, ecg_exclude)[1] + '_' + map(str, ecg_exclude)[2] + '.pdf', format = 'pdf')
elif len(eog_exclude) == 2:
if len(ecg_exclude) == 0:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] + '_' + map(str, eog_exclude)[1] +
'-ecg_none' + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 1:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] + '_' + map(str, eog_exclude)[1] +
'-ecg' + map(str, ecg_exclude)[0] + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 2:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] + '_' + map(str, eog_exclude)[1] +
'-ecg' + map(str, ecg_exclude)[0] + '_' + map(str, ecg_exclude)[1] + '.pdf', format = 'pdf')
elif len(ecg_exclude) == 3:
ica_plot.savefig(saveRoot + name + '_comps_eog' + map(str, eog_exclude)[0] + '_' + map(str, eog_exclude)[1] +
'-ecg' + map(str, ecg_exclude)[0] + '_' + map(str, ecg_exclude)[1] + '_' + map(str, ecg_exclude)[2] + '.pdf', format = 'pdf')
# plot the scores for the different components highlighting in red that/those related to ECG
#scores_plots=plt.figure()
#ax = plt.subplot(2,1,1)
scores_plots_ecg=ica.plot_scores(ecg_scores, exclude=ecg_source_idx, title=title % 'ecg')
scores_plots_ecg.savefig(saveRoot + name + '_ecg_scores.pdf', format = 'pdf')
#ax = plt.subplot(2,1,2)
scores_plots_eog=ica.plot_scores(eog_scores, exclude=eog_source_idx, title=title % 'eog')
scores_plots_eog.savefig(saveRoot + name + '_eog_scores.pdf', format = 'pdf')
#if len(ecg_exclude) > 0:
# estimate average artifact
#source_clean_ecg=plt.figure()
#ax = plt.subplot(2,1,1)
source_source_ecg=ica.plot_sources(ecg_evoked, exclude=ecg_source_idx)
source_source_ecg.savefig(saveRoot + name + '_ecg_source.pdf', format = 'pdf')
#ax = plt.subplot(2,1,2)
source_clean_ecg=ica.plot_overlay(ecg_evoked, exclude=ecg_source_idx)
source_clean_ecg.savefig(saveRoot + name + '_ecg_clean.pdf', format = 'pdf')
#clean_plot.savefig(saveRoot + name + '_ecg_clean.pdf', format = 'pdf')
#if len(eog_exclude) > 0:
source_source_eog=ica.plot_sources(eog_evoked, exclude=eog_source_idx)
source_source_eog.savefig(saveRoot + name + '_eog_source.pdf', format = 'pdf')
source_clean_eog=ica.plot_overlay(eog_evoked, exclude=eog_source_idx)
source_clean_eog.savefig(saveRoot + name + '_eog_clean.pdf', format = 'pdf')
overl_plot = ica.plot_overlay(raw)
overl_plot.savefig(saveRoot + name + '_overl.pdf', format = 'pdf')
plt.close('all')
## restore sensor space data
icaList = ica.apply(raw)
return(icaList, ica)
stim=False,
exclude=[])
ica.fit(epochs, picks=picks, decim=decim, reject=None)
# maximum number of components to reject
n_max_eog = 1
##########################################################################
# 2) identify bad components by analyzing latent sources.
# DETECT EOG BY CORRELATION
title = "ICA: %s for %s"
# EOG
eog_inds, scores = ica.find_bads_eog(epochs)
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
if eog_inds:
fig = ica.plot_scores(scores,
exclude=eog_inds,
title=title % ('eog', subject))
fig.savefig(ica_folder + "plots/%s_%s_eog_scores.png" %
(subject, condition))
fig = ica.plot_sources(epochs, exclude=eog_inds)
fig.savefig(ica_folder + "plots/%s_%s_eog_source.png" %
(subject, condition))
fig = ica.plot_components(eog_inds,
def preprocess_eeg(id_num, random_seed=None):
# Set important variables
bids_path = BIDSPath(id_num, task=task, datatype=datatype, root=bids_root)
plot_path = os.path.join(plotdir, "sub_{0}".format(id_num))
if os.path.exists(plot_path):
shutil.rmtree(plot_path)
os.mkdir(plot_path)
if not random_seed:
random_seed = int(binascii.b2a_hex(os.urandom(4)), 16)
random.seed(random_seed)
id_info = {"id": id_num, "random_seed": random_seed}
### Load and prepare EEG data #############################################
header = "### Processing sub-{0} (seed: {1}) ###".format(
id_num, random_seed)
print("\n" + "#" * len(header))
print(header)
print("#" * len(header) + "\n")
# Load EEG data
raw = read_raw_bids(bids_path, verbose=True)
# Check if recording is complete
complete = len(raw.annotations) >= 600
# Add a montage to the data
montage_kind = "standard_1005"
montage = mne.channels.make_standard_montage(montage_kind)
mne.datasets.eegbci.standardize(raw)
raw.set_montage(montage)
# Extract some info
eeg_index = mne.pick_types(raw.info, eeg=True, eog=False, meg=False)
ch_names = raw.info["ch_names"]
ch_names_eeg = list(np.asarray(ch_names)[eeg_index])
sample_rate = raw.info["sfreq"]
# Make a copy of the data
raw_copy = raw.copy()
raw_copy.load_data()
# Trim duplicated data (only needed for sub-005)
annot = raw_copy.annotations
file_starts = [a for a in annot if a['description'] == "file start"]
if len(file_starts):
duplicate_start = file_starts[0]['onset']
raw_copy.crop(tmax=duplicate_start)
# Make backup of EOG and EMG channels to re-append after PREP
raw_other = raw_copy.copy()
raw_other.pick_types(eog=True, emg=True, stim=False)
# Prepare copy of raw data for PREP
raw_copy.pick_types(eeg=True)
# Plot data prior to any processing
if complete:
save_psd_plot(id_num, "psd_0_raw", plot_path, raw_copy)
save_channel_plot(id_num, "ch_0_raw", plot_path, raw_copy)
### Clean up events #######################################################
print("\n\n=== Processing Event Annotations... ===\n")
event_names = [
"stim_on", "red_on", "trace_start", "trace_end", "accuracy_submit",
"vividness_submit"
]
doubled = []
wrong_label = []
new_onsets = []
new_durations = []
new_descriptions = []
# Find and flag any duplicate triggers
annot = raw_copy.annotations
trigger_count = len(annot)
for i in range(1, trigger_count - 1):
a = annot[i]
on_last = i + 1 == trigger_count
prev_trigger = annot[i - 1]['description']
next_onset = annot[i + 1]['onset'] if not on_last else a['onset'] + 100
# Determine whether duplicates are doubles or mislabeled
if a['description'] == prev_trigger:
if (next_onset - a['onset']) < 0.002:
doubled.append(a)
else:
wrong_label.append(a)
# Rename annotations to have meaningful names & fix duplicates
for a in raw_copy.annotations:
if a in doubled or a['description'] not in event_names:
continue
if a in wrong_label:
index = event_names.index(a['description'])
a['description'] = event_names[index + 1]
new_onsets.append(a['onset'])
new_durations.append(a['duration'])
new_descriptions.append(a['description'])
# Replace old annotations with new fixed ones
if len(annot):
new_annot = mne.Annotations(
new_onsets,
new_durations,
new_descriptions,
orig_time=raw_copy.annotations[0]['orig_time'])
raw_copy.set_annotations(new_annot)
# Check annotations to verify we have equal numbers of each
orig_counts = Counter(annot.description)
counts = Counter(raw_copy.annotations.description)
print("Updated Annotation Counts:")
for a in event_names:
out = " - '{0}': {1} -> {2}"
print(out.format(a, orig_counts[a], counts[a]))
# Get info
id_info['annot_doubled'] = len(doubled)
id_info['annot_wrong'] = len(wrong_label)
count_vals = [
n for n in counts.values() if n != counts['vividness_submit']
]
id_info['equal_triggers'] = all(x == count_vals[0] for x in count_vals)
id_info['stim_on'] = counts['stim_on']
id_info['red_on'] = counts['red_on']
id_info['trace_start'] = counts['trace_start']
id_info['trace_end'] = counts['trace_end']
id_info['acc_submit'] = counts['accuracy_submit']
id_info['vivid_submit'] = counts['vividness_submit']
if not complete:
remaining_info = {
'initial_bad': "NA",
'num_initial_bad': "NA",
'interpolated': "NA",
'num_interpolated': "NA",
'remaining_bad': "NA",
'num_remaining_bad': "NA"
}
id_info.update(remaining_info)
e = "\n\n### Incomplete recording for sub-{0}, skipping... ###\n\n"
print(e.format(id_num))
return id_info
### Run components of PREP manually #######################################
print("\n\n=== Performing CleanLine... ===")
# Try to remove line noise using CleanLine approach
linenoise = np.arange(60, sample_rate / 2, 60)
EEG_raw = raw_copy.get_data() * 1e6
EEG_new = removeTrend(EEG_raw, sample_rate=raw.info["sfreq"])
EEG_clean = mne.filter.notch_filter(
EEG_new,
Fs=raw.info["sfreq"],
freqs=linenoise,
filter_length="10s",
method="spectrum_fit",
mt_bandwidth=2,
p_value=0.01,
)
EEG_final = EEG_raw - EEG_new + EEG_clean
raw_copy._data = EEG_final * 1e-6
del linenoise, EEG_raw, EEG_new, EEG_clean, EEG_final
# Plot data following cleanline
save_psd_plot(id_num, "psd_1_cleanline", plot_path, raw_copy)
save_channel_plot(id_num, "ch_1_cleanline", plot_path, raw_copy)
# Perform robust re-referencing
prep_params = {"ref_chs": ch_names_eeg, "reref_chs": ch_names_eeg}
reference = Reference(raw_copy,
prep_params,
ransac=True,
random_state=random_seed)
print("\n\n=== Performing Robust Re-referencing... ===\n")
reference.perform_reference()
# If not interpolating bad channels, use pre-interpolation channel data
if not interpolate_bads:
reference.raw._data = reference.EEG_before_interpolation * 1e-6
reference.interpolated_channels = []
reference.still_noisy_channels = reference.bad_before_interpolation
reference.raw.info["bads"] = reference.bad_before_interpolation
# Plot data following robust re-reference
save_psd_plot(id_num, "psd_2_reref", plot_path, reference.raw)
save_channel_plot(id_num, "ch_2_reref", plot_path, reference.raw)
# Re-append removed EMG/EOG/trigger channels
raw_prepped = reference.raw.add_channels([raw_other])
# Get info
initial_bad = reference.noisy_channels_original["bad_all"]
id_info['initial_bad'] = " ".join(initial_bad)
id_info['num_initial_bad'] = len(initial_bad)
interpolated = reference.interpolated_channels
id_info['interpolated'] = " ".join(interpolated)
id_info['num_interpolated'] = len(interpolated)
remaining_bad = reference.still_noisy_channels
id_info['remaining_bad'] = " ".join(remaining_bad)
id_info['num_remaining_bad'] = len(remaining_bad)
# Print re-referencing info
print("\nRe-Referencing Info:")
print(" - Bad channels original: {0}".format(initial_bad))
if interpolate_bads:
print(" - Bad channels after re-referencing: {0}".format(interpolated))
print(" - Bad channels after interpolation: {0}".format(remaining_bad))
else:
print(
" - Bad channels after re-referencing: {0}".format(remaining_bad))
# Check if too many channels were interpolated for the participant
prop_interpolated = len(
reference.interpolated_channels) / len(ch_names_eeg)
e = "### NOTE: Too many interpolated channels for sub-{0} ({1}) ###"
if max_interpolated < prop_interpolated:
print("\n")
print(e.format(id_num, len(reference.interpolated_channels)))
print("\n")
### Filter data and apply ICA to remove blinks ############################
# Apply highpass & lowpass filters
print("\n\n=== Applying Highpass & Lowpass Filters... ===")
raw_prepped.filter(1.0, 50.0, fir_design='firwin')
# Plot data following frequency filters
save_psd_plot(id_num, "psd_3_filtered", plot_path, raw_prepped)
save_channel_plot(id_num, "ch_3_filtered", plot_path, raw_prepped)
# Perform ICA using EOG data on eye blinks
print("\n\n=== Removing Blinks Using ICA... ===\n")
ica = ICA(n_components=20, random_state=random_seed, method='picard')
ica.fit(raw_prepped, decim=5)
eog_indices, eog_scores = ica.find_bads_eog(raw_prepped)
ica.exclude = eog_indices
if not len(ica.exclude):
err = " - Encountered an ICA error for sub-{0}, skipping for now..."
print("\n")
print(err.format(id_num))
print("\n")
save_bad_fif(raw_prepped, id_num, ica_err_dir)
return id_info
# Plot ICA info & diagnostics before removing from signal
save_ica_plots(id_num, plot_path, raw_prepped, ica, eog_scores)
# Remove eye blink independent components based on ICA
ica.apply(raw_prepped)
# Plot data following ICA
save_psd_plot(id_num, "psd_4_ica", plot_path, raw_prepped)
save_channel_plot(id_num, "ch_4_ica", plot_path, raw_prepped)
### Compute Current Source Density (CSD) estimates ########################
if perform_csd:
print("\n")
print("=== Computing Current Source Density (CSD) Estimates... ===\n")
raw_prepped = mne.preprocessing.compute_current_source_density(
raw_prepped.drop_channels(remaining_bad))
# Plot data following CSD
save_psd_plot(id_num, "psd_5_csd", plot_path, raw_prepped)
save_channel_plot(id_num, "ch_5_csd", plot_path, raw_prepped)
### Write preprocessed data to new EDF ####################################
if max_interpolated < prop_interpolated:
if not os.path.isdir(noisy_bad_dir):
os.makedirs(noisy_bad_dir)
outpath = os.path.join(noisy_bad_dir, outfile_fmt.format(id_num))
else:
outpath = os.path.join(outdir, outfile_fmt.format(id_num))
write_mne_edf(outpath, raw_prepped)
print("\n\n### sub-{0} complete! ###\n\n".format(id_num))
return id_info
# uncomment the code below to test the inteactive mode of plot_components:
# ica.plot_components(picks=range(10), inst=raw)
###############################################################################
# Advanced artifact detection
# ---------------------------
#
# Let's use a more efficient way to find artefacts
eog_average = create_eog_epochs(raw, reject=dict(mag=5e-12, grad=4000e-13),
picks=picks_meg).average()
# We simplify things by setting the maximum number of components to reject
n_max_eog = 1 # here we bet on finding the vertical EOG components
eog_epochs = create_eog_epochs(raw, reject=reject) # get single EOG trials
eog_inds, scores = ica.find_bads_eog(eog_epochs) # find via correlation
ica.plot_scores(scores, exclude=eog_inds) # look at r scores of components
# we can see that only one component is highly correlated and that this
# component got detected by our correlation analysis (red).
ica.plot_sources(eog_average, exclude=eog_inds) # look at source time course
###############################################################################
# We can take a look at the properties of that component, now using the
# data epoched with respect to EOG events.
# We will also use a little bit of smoothing along the trials axis in the
# epochs image:
ica.plot_properties(eog_epochs, picks=eog_inds, psd_args={'fmax': 35.},
image_args={'sigma': 1.})
def preprocess_ICA_fif_to_ts(fif_file, ECG_ch_name, EoG_ch_name, l_freq, h_freq):
# ------------------------ Import stuff ------------------------ #
import os
import mne
import sys
from mne.io import Raw
from mne.preprocessing import ICA
from mne.preprocessing import create_ecg_epochs, create_eog_epochs
from nipype.utils.filemanip import split_filename as split_f
from reportGen import generateReport
import pickle
subj_path, basename, ext = split_f(fif_file)
# -------------------- Delete later ------------------- #
subj_name = subj_path[-5:]
results_dir = subj_path[:-6]
# results_dir += '2016'
subj_path = results_dir + '/' + subj_name
if not os.path.exists(subj_path):
try:
os.makedirs(subj_path)
except OSError:
sys.stderr.write('ica_preproc: problem creating directory: ' + subj_path)
########################################################
# Read raw
# If None the compensation in the data is not modified. If set to n, e.g. 3, apply
# gradient compensation of grade n as for CTF systems (compensation=3)
print(fif_file)
print(EoG_ch_name)
# ----------------------------- end Import stuff ----------------- #
# EoG_ch_name = "EOG061, EOG062"
# ------------- Load raw ------------- #
raw = Raw(fif_file, preload=True)
# select sensors
select_sensors = mne.pick_types(raw.info, meg=True, ref_meg=False, exclude='bads')
picks_meeg = mne.pick_types(raw.info, meg=True, eeg=True, exclude='bads')
# filtering
raw.filter(l_freq=l_freq, h_freq=h_freq, picks=picks_meeg, method='iir', n_jobs=1)
# if ECG_ch_name == 'EMG063':
if ECG_ch_name in raw.info['ch_names']:
raw.set_channel_types({ECG_ch_name: 'ecg'}) # Without this files with ECG_ch_name = 'EMG063' fail
# ECG_ch_name = 'ECG063'
if EoG_ch_name == 'EMG065,EMG066,EMG067,EMG068': # Because ica.find_bads_eog... can process max 2 EoG channels
EoG_ch_name = 'EMG065,EMG067' # it won't fail if I specify 4 channels, but it'll use only first
# EMG065 and EMG066 are for vertical eye movements and
# EMG067 and EMG068 are for horizontal
# print rnk
rnk = 'N/A'
# 1) Fit ICA model using the FastICA algorithm
# Other available choices are `infomax` or `extended-infomax`
# We pass a float value between 0 and 1 to select n_components based on the
# percentage of variance explained by the PCA components.
reject = dict(mag=10e-12, grad=10000e-13)
flat = dict(mag=0.1e-12, grad=1e-13)
# check if we have an ICA, if yes, we load it
ica_filename = os.path.join(subj_path, basename + "-ica.fif")
raw_ica_filename = os.path.join(subj_path, basename + "_ica_raw.fif")
info_filename = os.path.join(subj_path, basename + "_info.pickle")
# if os.path.exists(ica_filename) == False:
ica = ICA(n_components=0.99, method='fastica') # , max_iter=500
ica.fit(raw, picks=select_sensors, reject=reject, flat=flat) # decim = 3,
# has_ICA = False
# else:
# has_ICA = True
# ica = read_ica(ica_filename)
# ica.exclude = []
# ica.labels_ = dict() # to avoid bug; Otherwise it'll throw an exception
ica_sources_filename = subj_path + '/' + basename + '_ica_timecourse.fif'
# if not os.path.isfile(ica_sources_filename):
icaSrc = ica.get_sources(raw, add_channels=None, start=None, stop=None)
icaSrc.save(ica_sources_filename, picks=None, tmin=0, tmax=None, buffer_size_sec=10,
drop_small_buffer=False, proj=False, fmt='single', overwrite=True, split_size='2GB', verbose=None)
icaSrc = None
# if has_ICA == False:
# ica.save(ica_filename)
# return
# 2) identify bad components by analyzing latent sources.
# generate ECG epochs use detection via phase statistics
# check if ECG_ch_name is in the raw channels
# import ipdb; ipdb.set_trace()
if ECG_ch_name in raw.info['ch_names']:
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5, picks=select_sensors, ch_name=ECG_ch_name)
# if not a synthetic ECG channel is created from cross channel average
else:
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5, picks=select_sensors)
# ICA for ECG artifact
# threshold=0.25 come defualt
ecg_inds, ecg_scores = ica.find_bads_ecg(ecg_epochs, method='ctps', threshold=0.25)
# if len(ecg_inds) > 0:
ecg_evoked = ecg_epochs.average()
ecg_epochs = None # ecg_epochs use too much memory
n_max_ecg = 3
ecg_inds = ecg_inds[:n_max_ecg]
ica.exclude += ecg_inds
n_max_eog = 4
# import pdb; pdb.set_trace()
if set(EoG_ch_name.split(',')).issubset(set(raw.info['ch_names'])):
eog_inds, eog_scores = ica.find_bads_eog(raw, ch_name=EoG_ch_name)
eog_inds = eog_inds[:n_max_eog]
eog_evoked = create_eog_epochs(raw, tmin=-.5, tmax=.5, picks=select_sensors, ch_name=EoG_ch_name).average()
else:
eog_inds = eog_scores = eog_evoked = []
# ------------------------------------------------------ #
# This is necessary. Otherwise line
# will through an exception
if not eog_inds:
eog_inds = None
# ------------------------------------------------------ #
generateReport(raw=raw, ica=ica, subj_name=subj_name, subj_path=subj_path, basename=basename,
ecg_evoked=ecg_evoked, ecg_scores=ecg_scores, ecg_inds=ecg_inds, ECG_ch_name=ECG_ch_name,
eog_evoked=eog_evoked, eog_scores=eog_scores, eog_inds=eog_inds, EoG_ch_name=EoG_ch_name)
sfreq = raw.info['sfreq']
raw = None # To save memory
# save ICA solution
print ica_filename
# ------------------------- Generate log ------------------------ #
f = open(subj_path + '/' + basename + '_ica.log', 'w')
f.write('Data rank after SSS: ' + str(rnk) + '\n')
f.write('Sampling freq: ' + str(sfreq) + '\n')
f.write('ECG exclude suggested: ' + str(ecg_inds) + '\n')
f.write('EOG exclude suggested: ' + str(eog_inds) + '\n')
f.write('\n')
f.write('ECG exclude final: ' + str(ecg_inds) + '\n')
f.write('EOG exclude final: ' + str(eog_inds) + '\n')
f.write('Muscles exclude: []' + '\n')
f.close()
# ------------------------ end generate log ---------------------- #
with open(info_filename, 'wb') as f:
pickle.dump(ecg_inds, f)
pickle.dump(ecg_scores, f)
pickle.dump(eog_inds, f)
pickle.dump(eog_scores, f)
# -------------------- Save ICA solution ------------------------- #
ica.save(ica_filename)
return raw_ica_filename, sfreq
def test_ica_additional():
"""Test additional ICA functionality"""
stop2 = 500
raw = io.Raw(raw_fname, preload=True).crop(0, stop, False).crop(1.5)
picks = pick_types(raw.info, meg=True, stim=False, ecg=False, eog=False, exclude="bads")
test_cov = read_cov(test_cov_name)
events = read_events(event_name)
picks = pick_types(raw.info, meg=True, stim=False, ecg=False, eog=False, exclude="bads")
epochs = Epochs(raw, events[:4], event_id, tmin, tmax, picks=picks, baseline=(None, 0), preload=True)
# for testing eog functionality
picks2 = pick_types(raw.info, meg=True, stim=False, ecg=False, eog=True, exclude="bads")
epochs_eog = Epochs(raw, events[:4], event_id, tmin, tmax, picks=picks2, baseline=(None, 0), preload=True)
test_cov2 = deepcopy(test_cov)
ica = ICA(noise_cov=test_cov2, n_components=3, max_pca_components=4, n_pca_components=4)
assert_true(ica.info is None)
with warnings.catch_warnings(record=True):
ica.fit(raw, picks[:5])
assert_true(isinstance(ica.info, Info))
assert_true(ica.n_components_ < 5)
ica = ICA(n_components=3, max_pca_components=4, n_pca_components=4)
assert_raises(RuntimeError, ica.save, "")
with warnings.catch_warnings(record=True):
ica.fit(raw, picks=None, start=start, stop=stop2)
# test warnings on bad filenames
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
ica_badname = op.join(op.dirname(tempdir), "test-bad-name.fif.gz")
ica.save(ica_badname)
read_ica(ica_badname)
assert_true(len(w) == 2)
# test decim
ica = ICA(n_components=3, max_pca_components=4, n_pca_components=4)
raw_ = raw.copy()
for _ in range(3):
raw_.append(raw_)
n_samples = raw_._data.shape[1]
with warnings.catch_warnings(record=True):
ica.fit(raw, picks=None, decim=3)
assert_true(raw_._data.shape[1], n_samples)
# test expl var
ica = ICA(n_components=1.0, max_pca_components=4, n_pca_components=4)
with warnings.catch_warnings(record=True):
ica.fit(raw, picks=None, decim=3)
assert_true(ica.n_components_ == 4)
# epochs extraction from raw fit
assert_raises(RuntimeError, ica.get_sources, epochs)
# test reading and writing
test_ica_fname = op.join(op.dirname(tempdir), "test-ica.fif")
for cov in (None, test_cov):
ica = ICA(noise_cov=cov, n_components=2, max_pca_components=4, n_pca_components=4)
with warnings.catch_warnings(record=True): # ICA does not converge
ica.fit(raw, picks=picks, start=start, stop=stop2)
sources = ica.get_sources(epochs).get_data()
assert_true(ica.mixing_matrix_.shape == (2, 2))
assert_true(ica.unmixing_matrix_.shape == (2, 2))
assert_true(ica.pca_components_.shape == (4, len(picks)))
assert_true(sources.shape[1] == ica.n_components_)
for exclude in [[], [0]]:
ica.exclude = [0]
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.exclude == ica_read.exclude)
ica.exclude = []
ica.apply(raw, exclude=[1])
assert_true(ica.exclude == [])
ica.exclude = [0, 1]
ica.apply(raw, exclude=[1])
assert_true(ica.exclude == [0, 1])
ica_raw = ica.get_sources(raw)
assert_true(ica.exclude == [ica_raw.ch_names.index(e) for e in ica_raw.info["bads"]])
# test filtering
d1 = ica_raw._data[0].copy()
with warnings.catch_warnings(record=True): # dB warning
ica_raw.filter(4, 20)
assert_true((d1 != ica_raw._data[0]).any())
d1 = ica_raw._data[0].copy()
with warnings.catch_warnings(record=True): # dB warning
ica_raw.notch_filter([10])
assert_true((d1 != ica_raw._data[0]).any())
ica.n_pca_components = 2
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.n_pca_components == ica_read.n_pca_components)
# check type consistency
attrs = "mixing_matrix_ unmixing_matrix_ pca_components_ " "pca_explained_variance_ _pre_whitener"
f = lambda x, y: getattr(x, y).dtype
for attr in attrs.split():
assert_equal(f(ica_read, attr), f(ica, attr))
ica.n_pca_components = 4
ica_read.n_pca_components = 4
ica.exclude = []
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
for attr in [
"mixing_matrix_",
"unmixing_matrix_",
"pca_components_",
"pca_mean_",
"pca_explained_variance_",
"_pre_whitener",
]:
assert_array_almost_equal(getattr(ica, attr), getattr(ica_read, attr))
assert_true(ica.ch_names == ica_read.ch_names)
assert_true(isinstance(ica_read.info, Info))
sources = ica.get_sources(raw)[:, :][0]
sources2 = ica_read.get_sources(raw)[:, :][0]
assert_array_almost_equal(sources, sources2)
_raw1 = ica.apply(raw, exclude=[1])
_raw2 = ica_read.apply(raw, exclude=[1])
assert_array_almost_equal(_raw1[:, :][0], _raw2[:, :][0])
os.remove(test_ica_fname)
# check scrore funcs
for name, func in score_funcs.items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(raw, target="EOG 061", score_func=func, start=0, stop=10)
assert_true(ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(raw, score_func=stats.skew)
# check exception handling
assert_raises(ValueError, ica.score_sources, raw, target=np.arange(1))
params = []
params += [(None, -1, slice(2), [0, 1])] # varicance, kurtosis idx params
params += [(None, "MEG 1531")] # ECG / EOG channel params
for idx, ch_name in product(*params):
ica.detect_artifacts(
raw,
start_find=0,
stop_find=50,
ecg_ch=ch_name,
eog_ch=ch_name,
skew_criterion=idx,
var_criterion=idx,
kurt_criterion=idx,
)
idx, scores = ica.find_bads_ecg(raw, method="ctps")
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(raw, method="correlation")
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(epochs, method="ctps")
assert_equal(len(scores), ica.n_components_)
assert_raises(ValueError, ica.find_bads_ecg, epochs.average(), method="ctps")
assert_raises(ValueError, ica.find_bads_ecg, raw, method="crazy-coupling")
idx, scores = ica.find_bads_eog(raw)
assert_equal(len(scores), ica.n_components_)
raw.info["chs"][raw.ch_names.index("EOG 061") - 1]["kind"] = 202
idx, scores = ica.find_bads_eog(raw)
assert_true(isinstance(scores, list))
assert_equal(len(scores[0]), ica.n_components_)
# check score funcs
for name, func in score_funcs.items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(epochs_eog, target="EOG 061", score_func=func)
assert_true(ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(epochs, score_func=stats.skew)
# check exception handling
assert_raises(ValueError, ica.score_sources, epochs, target=np.arange(1))
# ecg functionality
ecg_scores = ica.score_sources(raw, target="MEG 1531", score_func="pearsonr")
with warnings.catch_warnings(record=True): # filter attenuation warning
ecg_events = ica_find_ecg_events(raw, sources[np.abs(ecg_scores).argmax()])
assert_true(ecg_events.ndim == 2)
# eog functionality
eog_scores = ica.score_sources(raw, target="EOG 061", score_func="pearsonr")
with warnings.catch_warnings(record=True): # filter attenuation warning
eog_events = ica_find_eog_events(raw, sources[np.abs(eog_scores).argmax()])
assert_true(eog_events.ndim == 2)
# Test ica fiff export
ica_raw = ica.get_sources(raw, start=0, stop=100)
assert_true(ica_raw.last_samp - ica_raw.first_samp == 100)
assert_true(len(ica_raw._filenames) == 0) # API consistency
ica_chans = [ch for ch in ica_raw.ch_names if "ICA" in ch]
assert_true(ica.n_components_ == len(ica_chans))
test_ica_fname = op.join(op.abspath(op.curdir), "test-ica_raw.fif")
ica.n_components = np.int32(ica.n_components)
ica_raw.save(test_ica_fname, overwrite=True)
ica_raw2 = io.Raw(test_ica_fname, preload=True)
assert_allclose(ica_raw._data, ica_raw2._data, rtol=1e-5, atol=1e-4)
ica_raw2.close()
os.remove(test_ica_fname)
# Test ica epochs export
ica_epochs = ica.get_sources(epochs)
assert_true(ica_epochs.events.shape == epochs.events.shape)
ica_chans = [ch for ch in ica_epochs.ch_names if "ICA" in ch]
assert_true(ica.n_components_ == len(ica_chans))
assert_true(ica.n_components_ == ica_epochs.get_data().shape[1])
assert_true(ica_epochs.raw is None)
assert_true(ica_epochs.preload is True)
# test float n pca components
ica.pca_explained_variance_ = np.array([0.2] * 5)
ica.n_components_ = 0
for ncomps, expected in [[0.3, 1], [0.9, 4], [1, 1]]:
ncomps_ = _check_n_pca_components(ica, ncomps)
assert_true(ncomps_ == expected)
for trial_type in ['VS']:
epochs = mne.Epochs(raw, eve_dict[trial_type], id_dict,
tmin, tmax, picks=picks, verbose=False,
baseline=baseline, reject=reject, preload=True,
reject_tmin=rej_tmin, reject_tmax=rej_tmax) # Check rejection settings
picks = pick_types(epochs.info, meg=True, eog=False)
# cannot compute EOG covariance?!?!!!
#baseline_cov = compute_covariance(epochs, tmin=None, tmax=0)
#ica = ICA(n_components=n_components, max_pca_components=None,max_iter=256, noise_cov=baseline_cov)
ica = ICA(n_components=n_components, max_pca_components=None,max_iter=256)
ica.fit(epochs, picks=picks, decim = 5, reject=ica_reject)
eog_inds, scores = ica.find_bads_eog(epochs, ch_name='EOG001,EOG003')
allscores = np.vstack((scores[0], scores[1]))
mscores = np.max(np.abs(allscores), axis=0)
# now scores is
show_picks = mscores.argsort()[::-1][:5]
eog_inds = list(show_picks[:n_max_eog])
ica.exclude += eog_inds
fig = ica.plot_scores(scores, exclude=eog_inds, title=title % 'eog')
fig.savefig(img_folder + '/ica_eog_scores.png')
fig = ica.plot_sources(epochs, show_picks, exclude=eog_inds, title=title % 'eog')
fig.savefig(img_folder + '/ica_eog_sources.png')
fig = ica.plot_components(show_picks, title=title % 'eog', colorbar=True)
fig.set_size_inches(12.,8.)
fig.savefig(img_folder + '/ica_eog_components.png')
def run(self):
eog = self.info['channel_info']['EOG']
misc = self.info['channel_info']['Misc']
stim = self.info['channel_info']['Stim']
try:
ext_files = glob.glob(self.info['ext_file_folder'] + '/' +
self.participant + '/*axis0.dat')
except:
pass
tmin = self.t_epoch[0]
tmax = self.t_epoch[1]
raw = read_raw_edf(self.file, eog=eog, misc=misc)
self.raw = cp.deepcopy(raw)
raw.load_data()
# marker detection (one marker continous trial)
if self.info['marker_detection'] == True:
starts = find_trialstart(raw,
stim_channel=raw.ch_names[stim[0]],
new_samplin_rate=self.sr_new)
try:
starts[1] = starts[0] + 30 * 200
except:
starts = np.r_[starts, (starts[0] + 30 * 200)]
events = np.zeros((len(starts), 3))
events[:, 0] = starts
events[:, 2] = list(self.info['event_dict'].values())
events = events.astype(np.int)
# event detection (one marker regular events)
if self.info['event_detection'] == True:
starts = find_trialstart(raw,
stim_channel=raw.ch_names[stim[0]],
new_samplin_rate=self.sr_new)
events = force_events(ext_files, self.info['event_dict'],
self.sr_new, self.info['trial_length'],
self.info['trials'],
starts[:len(self.info['event_dict'])])
if self.info['ICA'] == True:
ica = ICA(method='fastica')
if self.info['Autoreject'] == True:
ar = AutoReject()
## EEG preprocessing options will applied if parameters are set in object
#read montage
try:
montage = make_standard_montage(self.montage)
raw.set_montage(montage)
except:
pass
#resampling
try:
raw.resample(sfreq=self.sr_new)
except:
pass
#rereferencing
try:
raw, _ = mne.set_eeg_reference(raw, ref_channels=['EXG5', 'EXG6'])
except:
pass
#filter
try:
low = self.filter_freqs[0]
high = self.filter_freqs[1]
raw.filter(low, high, fir_design='firwin')
except:
pass
# occular correction
try:
ica.fit(raw)
ica.exclude = []
eog_indices, eog_scores = ica.find_bads_eog(raw)
ica.exclude = eog_indices
ica.apply(raw)
self.ica = ica
except:
pass
picks = mne.pick_types(raw.info,
meg=False,
eeg=True,
stim=False,
eog=False,
exclude='bads')
event_id = self.info['event_dict']
epochs = mne.Epochs(raw,
events,
event_id,
tmin,
tmax,
proj=True,
baseline=None,
preload=True,
picks=picks)
#epoch rejection
try:
epochs = epochs.drop(indices=self.bads)
except:
pass
try:
epochs, self.autoreject_log = ar.fit_transform(epochs,
return_log=True)
except:
pass
bads = np.asarray(
[l == ['USER'] or l == ['AUTOREJECT'] for l in epochs.drop_log])
self.bads = np.where(bads == True)
self.epochs = epochs
return (self)
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5, picks=picks) ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps') ica.plot_scores(scores, exclude=ecg_inds, title=title % 'ecg', labels='ecg') show_picks = np.abs(scores).argsort()[::-1][:5] ica.plot_sources(raw, show_picks, exclude=ecg_inds, title=title % 'ecg') ica.plot_components(ecg_inds, title=title % 'ecg', colorbar=True) ecg_inds = ecg_inds[:n_max_ecg] ica.exclude += ecg_inds # detect EOG by correlation eog_inds, scores = ica.find_bads_eog(raw) ica.plot_scores(scores, exclude=eog_inds, title=title % 'eog', labels='eog') show_picks = np.abs(scores).argsort()[::-1][:5] ica.plot_sources(raw, show_picks, exclude=eog_inds, title=title % 'eog') ica.plot_components(eog_inds, title=title % 'eog', colorbar=True) eog_inds = eog_inds[:n_max_eog] ica.exclude += eog_inds ############################################################################### # 3) Assess component selection and unmixing quality # estimate average artifact ecg_evoked = ecg_epochs.average()
def run_preprocessing_eog(raw_eeg,
tmin=0,
tmax=60,
remove_artefact=True,
show=True):
'''Function to remove occular artefact by removing the ICA component(s) that are closest to a reference channel (here Fp2)
Parameters
----------
raw_eeg : Raw file
tmin : default 0
tmax : default 60
remove_artefact : wether to remove the ICA component from raw (default True)
show : show scores (default : true)
Return
------
Return cleaned raw or original raw if remove_artefact = True or no matching components were found.
'''
raw_inter = raw_eeg.copy().crop(tmin=tmin, tmax=tmax)
filtered_raw = raw_inter.filter(l_freq=1., h_freq=40.)
ica = ICA(n_components=raw_eeg.info.get('nchan') - 2, random_state=9)
print(raw_eeg.info)
ica.fit(filtered_raw, verbose=False)
ica.exclude = []
# find which ICs match the EOG pattern
eog_indices, eog_scores = ica.find_bads_eog(raw_inter,
ch_name='Fp2',
threshold=1.4,
verbose=False)
ica.exclude = eog_indices
if show == True:
ica.plot_sources(filtered_raw)
ica.plot_components()
# barplot of ICA component "EOG match" scores
ica.plot_scores(eog_scores)
#ica.plot_overlay(raw_inter)
#ica.plot_properties(raw_inter)
if eog_indices != []:
print('matching ICA component found')
# plot diagnostics
#ica.plot_properties(filtered_raw, picks=eog_indices)
# plot ICs applied to raw data, with EOG matches highlighted
if show == True:
ica.plot_sources(raw_inter)
if remove_artefact == True:
reconst_raw = filtered_raw.copy()
ica.apply(reconst_raw)
return reconst_raw
else:
return filtered_raw
else:
print('No matching ICA component')
return filtered_raw
def test_ica_additional(method):
"""Test additional ICA functionality."""
_skip_check_picard(method)
tempdir = _TempDir()
stop2 = 500
raw = read_raw_fif(raw_fname).crop(1.5, stop).load_data()
raw.del_proj() # avoid warnings
raw.set_annotations(Annotations([0.5], [0.5], ['BAD']))
# XXX This breaks the tests :(
# raw.info['bads'] = [raw.ch_names[1]]
test_cov = read_cov(test_cov_name)
events = read_events(event_name)
picks = pick_types(raw.info, meg=True, stim=False, ecg=False,
eog=False, exclude='bads')[1::2]
epochs = Epochs(raw, events, None, tmin, tmax, picks=picks,
baseline=(None, 0), preload=True, proj=False)
epochs.decimate(3, verbose='error')
assert len(epochs) == 4
# test if n_components=None works
ica = ICA(n_components=None, max_pca_components=None,
n_pca_components=None, random_state=0, method=method, max_iter=1)
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(epochs)
# for testing eog functionality
picks2 = np.concatenate([picks, pick_types(raw.info, False, eog=True)])
epochs_eog = Epochs(raw, events[:4], event_id, tmin, tmax, picks=picks2,
baseline=(None, 0), preload=True)
del picks2
test_cov2 = test_cov.copy()
ica = ICA(noise_cov=test_cov2, n_components=3, max_pca_components=4,
n_pca_components=4, method=method)
assert (ica.info is None)
with pytest.warns(RuntimeWarning, match='normalize_proj'):
ica.fit(raw, picks[:5])
assert (isinstance(ica.info, Info))
assert (ica.n_components_ < 5)
ica = ICA(n_components=3, max_pca_components=4, method=method,
n_pca_components=4, random_state=0)
pytest.raises(RuntimeError, ica.save, '')
ica.fit(raw, picks=[1, 2, 3, 4, 5], start=start, stop=stop2)
# check passing a ch_name to find_bads_ecg
with pytest.warns(RuntimeWarning, match='longer'):
_, scores_1 = ica.find_bads_ecg(raw)
_, scores_2 = ica.find_bads_ecg(raw, raw.ch_names[1])
assert scores_1[0] != scores_2[0]
# test corrmap
ica2 = ica.copy()
ica3 = ica.copy()
corrmap([ica, ica2], (0, 0), threshold='auto', label='blinks', plot=True,
ch_type="mag")
corrmap([ica, ica2], (0, 0), threshold=2, plot=False, show=False)
assert (ica.labels_["blinks"] == ica2.labels_["blinks"])
assert (0 in ica.labels_["blinks"])
# test retrieval of component maps as arrays
components = ica.get_components()
template = components[:, 0]
EvokedArray(components, ica.info, tmin=0.).plot_topomap([0], time_unit='s')
corrmap([ica, ica3], template, threshold='auto', label='blinks', plot=True,
ch_type="mag")
assert (ica2.labels_["blinks"] == ica3.labels_["blinks"])
plt.close('all')
ica_different_channels = ICA(n_components=2, random_state=0).fit(
raw, picks=[2, 3, 4, 5])
pytest.raises(ValueError, corrmap, [ica_different_channels, ica], (0, 0))
# test warnings on bad filenames
ica_badname = op.join(op.dirname(tempdir), 'test-bad-name.fif.gz')
with pytest.warns(RuntimeWarning, match='-ica.fif'):
ica.save(ica_badname)
with pytest.warns(RuntimeWarning, match='-ica.fif'):
read_ica(ica_badname)
# test decim
ica = ICA(n_components=3, max_pca_components=4,
n_pca_components=4, method=method, max_iter=1)
raw_ = raw.copy()
for _ in range(3):
raw_.append(raw_)
n_samples = raw_._data.shape[1]
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(raw, picks=picks[:5], decim=3)
assert raw_._data.shape[1] == n_samples
# test expl var
ica = ICA(n_components=1.0, max_pca_components=4,
n_pca_components=4, method=method, max_iter=1)
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(raw, picks=None, decim=3)
assert (ica.n_components_ == 4)
ica_var = _ica_explained_variance(ica, raw, normalize=True)
assert (np.all(ica_var[:-1] >= ica_var[1:]))
# test ica sorting
ica.exclude = [0]
ica.labels_ = dict(blink=[0], think=[1])
ica_sorted = _sort_components(ica, [3, 2, 1, 0], copy=True)
assert_equal(ica_sorted.exclude, [3])
assert_equal(ica_sorted.labels_, dict(blink=[3], think=[2]))
# epochs extraction from raw fit
pytest.raises(RuntimeError, ica.get_sources, epochs)
# test reading and writing
test_ica_fname = op.join(op.dirname(tempdir), 'test-ica.fif')
for cov in (None, test_cov):
ica = ICA(noise_cov=cov, n_components=2, max_pca_components=4,
n_pca_components=4, method=method, max_iter=1)
with pytest.warns(None): # ICA does not converge
ica.fit(raw, picks=picks[:10], start=start, stop=stop2)
sources = ica.get_sources(epochs).get_data()
assert (ica.mixing_matrix_.shape == (2, 2))
assert (ica.unmixing_matrix_.shape == (2, 2))
assert (ica.pca_components_.shape == (4, 10))
assert (sources.shape[1] == ica.n_components_)
for exclude in [[], [0], np.array([1, 2, 3])]:
ica.exclude = exclude
ica.labels_ = {'foo': [0]}
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert (list(ica.exclude) == ica_read.exclude)
assert_equal(ica.labels_, ica_read.labels_)
ica.apply(raw)
ica.exclude = []
ica.apply(raw, exclude=[1])
assert (ica.exclude == [])
ica.exclude = [0, 1]
ica.apply(raw, exclude=[1])
assert (ica.exclude == [0, 1])
ica_raw = ica.get_sources(raw)
assert (ica.exclude == [ica_raw.ch_names.index(e) for e in
ica_raw.info['bads']])
# test filtering
d1 = ica_raw._data[0].copy()
ica_raw.filter(4, 20, fir_design='firwin2')
assert_equal(ica_raw.info['lowpass'], 20.)
assert_equal(ica_raw.info['highpass'], 4.)
assert ((d1 != ica_raw._data[0]).any())
d1 = ica_raw._data[0].copy()
ica_raw.notch_filter([10], trans_bandwidth=10, fir_design='firwin')
assert ((d1 != ica_raw._data[0]).any())
ica.n_pca_components = 2
ica.method = 'fake'
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert (ica.n_pca_components == ica_read.n_pca_components)
assert_equal(ica.method, ica_read.method)
assert_equal(ica.labels_, ica_read.labels_)
# check type consistency
attrs = ('mixing_matrix_ unmixing_matrix_ pca_components_ '
'pca_explained_variance_ pre_whitener_')
def f(x, y):
return getattr(x, y).dtype
for attr in attrs.split():
assert_equal(f(ica_read, attr), f(ica, attr))
ica.n_pca_components = 4
ica_read.n_pca_components = 4
ica.exclude = []
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
for attr in ['mixing_matrix_', 'unmixing_matrix_', 'pca_components_',
'pca_mean_', 'pca_explained_variance_',
'pre_whitener_']:
assert_array_almost_equal(getattr(ica, attr),
getattr(ica_read, attr))
assert (ica.ch_names == ica_read.ch_names)
assert (isinstance(ica_read.info, Info))
sources = ica.get_sources(raw)[:, :][0]
sources2 = ica_read.get_sources(raw)[:, :][0]
assert_array_almost_equal(sources, sources2)
_raw1 = ica.apply(raw, exclude=[1])
_raw2 = ica_read.apply(raw, exclude=[1])
assert_array_almost_equal(_raw1[:, :][0], _raw2[:, :][0])
os.remove(test_ica_fname)
# check score funcs
for name, func in get_score_funcs().items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(raw, target='EOG 061', score_func=func,
start=0, stop=10)
assert (ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(raw, start=0, stop=50, score_func=stats.skew)
# check exception handling
pytest.raises(ValueError, ica.score_sources, raw,
target=np.arange(1))
params = []
params += [(None, -1, slice(2), [0, 1])] # variance, kurtosis params
params += [(None, 'MEG 1531')] # ECG / EOG channel params
for idx, ch_name in product(*params):
ica.detect_artifacts(raw, start_find=0, stop_find=50, ecg_ch=ch_name,
eog_ch=ch_name, skew_criterion=idx,
var_criterion=idx, kurt_criterion=idx)
# Make sure detect_artifacts marks the right components.
# For int criterion, the doc says "E.g. range(2) would return the two
# sources with the highest score". Assert that's what it does.
# Only test for skew, since it's always the same code.
ica.exclude = []
ica.detect_artifacts(raw, start_find=0, stop_find=50, ecg_ch=None,
eog_ch=None, skew_criterion=0,
var_criterion=None, kurt_criterion=None)
assert np.abs(scores[ica.exclude]) == np.max(np.abs(scores))
evoked = epochs.average()
evoked_data = evoked.data.copy()
raw_data = raw[:][0].copy()
epochs_data = epochs.get_data().copy()
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_ecg(raw, method='ctps')
assert_equal(len(scores), ica.n_components_)
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_ecg(raw, method='correlation')
assert_equal(len(scores), ica.n_components_)
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_eog(raw)
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(epochs, method='ctps')
assert_equal(len(scores), ica.n_components_)
pytest.raises(ValueError, ica.find_bads_ecg, epochs.average(),
method='ctps')
pytest.raises(ValueError, ica.find_bads_ecg, raw,
method='crazy-coupling')
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_eog(raw)
assert_equal(len(scores), ica.n_components_)
raw.info['chs'][raw.ch_names.index('EOG 061') - 1]['kind'] = 202
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_eog(raw)
assert (isinstance(scores, list))
assert_equal(len(scores[0]), ica.n_components_)
idx, scores = ica.find_bads_eog(evoked, ch_name='MEG 1441')
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(evoked, method='correlation')
assert_equal(len(scores), ica.n_components_)
assert_array_equal(raw_data, raw[:][0])
assert_array_equal(epochs_data, epochs.get_data())
assert_array_equal(evoked_data, evoked.data)
# check score funcs
for name, func in get_score_funcs().items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(epochs_eog, target='EOG 061',
score_func=func)
assert (ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(epochs, score_func=stats.skew)
# check exception handling
pytest.raises(ValueError, ica.score_sources, epochs,
target=np.arange(1))
# ecg functionality
ecg_scores = ica.score_sources(raw, target='MEG 1531',
score_func='pearsonr')
with pytest.warns(RuntimeWarning, match='longer'):
ecg_events = ica_find_ecg_events(
raw, sources[np.abs(ecg_scores).argmax()])
assert (ecg_events.ndim == 2)
# eog functionality
eog_scores = ica.score_sources(raw, target='EOG 061',
score_func='pearsonr')
with pytest.warns(RuntimeWarning, match='longer'):
eog_events = ica_find_eog_events(
raw, sources[np.abs(eog_scores).argmax()])
assert (eog_events.ndim == 2)
# Test ica fiff export
ica_raw = ica.get_sources(raw, start=0, stop=100)
assert (ica_raw.last_samp - ica_raw.first_samp == 100)
assert_equal(len(ica_raw._filenames), 1) # API consistency
ica_chans = [ch for ch in ica_raw.ch_names if 'ICA' in ch]
assert (ica.n_components_ == len(ica_chans))
test_ica_fname = op.join(op.abspath(op.curdir), 'test-ica_raw.fif')
ica.n_components = np.int32(ica.n_components)
ica_raw.save(test_ica_fname, overwrite=True)
ica_raw2 = read_raw_fif(test_ica_fname, preload=True)
assert_allclose(ica_raw._data, ica_raw2._data, rtol=1e-5, atol=1e-4)
ica_raw2.close()
os.remove(test_ica_fname)
# Test ica epochs export
ica_epochs = ica.get_sources(epochs)
assert (ica_epochs.events.shape == epochs.events.shape)
ica_chans = [ch for ch in ica_epochs.ch_names if 'ICA' in ch]
assert (ica.n_components_ == len(ica_chans))
assert (ica.n_components_ == ica_epochs.get_data().shape[1])
assert (ica_epochs._raw is None)
assert (ica_epochs.preload is True)
# test float n pca components
ica.pca_explained_variance_ = np.array([0.2] * 5)
ica.n_components_ = 0
for ncomps, expected in [[0.3, 1], [0.9, 4], [1, 1]]:
ncomps_ = ica._check_n_pca_components(ncomps)
assert (ncomps_ == expected)
ica = ICA(method=method)
with pytest.warns(None): # sometimes does not converge
ica.fit(raw, picks=picks[:5])
with pytest.warns(RuntimeWarning, match='longer'):
ica.find_bads_ecg(raw)
ica.find_bads_eog(epochs, ch_name='MEG 0121')
assert_array_equal(raw_data, raw[:][0])
raw.drop_channels(['MEG 0122'])
pytest.raises(RuntimeError, ica.find_bads_eog, raw)
with pytest.warns(RuntimeWarning, match='longer'):
pytest.raises(RuntimeError, ica.find_bads_ecg, raw)
ica = ICA(n_components=0.90, max_pca_components=None)
###############################################################################
# 1) Fit ICA model and identify bad sources
picks = mne.pick_types(raw.info, meg=True, eeg=False, eog=False,
stim=False, exclude='bads')
ica.fit(raw, picks=picks, decim=3, reject=dict(mag=4e-12, grad=4000e-13))
# create EOG epochs to improve detection by correlation
picks = mne.pick_types(raw.info, meg=True, eog=True)
eog_epochs = create_eog_epochs(raw, picks=picks)
eog_inds, scores = ica.find_bads_eog(eog_epochs) # inds sorted!
ica.plot_scores(scores, exclude=eog_inds) # inspect metrics used
show_picks = np.abs(scores).argsort()[::-1][:5] # indices of top five scores
# detected artifacts drawn in red (via exclude)
ica.plot_sources(raw, show_picks, exclude=eog_inds, start=0., stop=3.0)
ica.plot_components(eog_inds, colorbar=False) # show component sensitivites
ica.exclude += eog_inds[:1] # mark first for exclusion
###############################################################################
# 3) check detection and visualize artifact rejection
# estimate average artifact
def test_ica_additional(method):
"""Test additional ICA functionality."""
_skip_check_picard(method)
tempdir = _TempDir()
stop2 = 500
raw = read_raw_fif(raw_fname).crop(1.5, stop).load_data()
raw.del_proj() # avoid warnings
raw.set_annotations(Annotations([0.5], [0.5], ['BAD']))
# XXX This breaks the tests :(
# raw.info['bads'] = [raw.ch_names[1]]
test_cov = read_cov(test_cov_name)
events = read_events(event_name)
picks = pick_types(raw.info,
meg=True,
stim=False,
ecg=False,
eog=False,
exclude='bads')[1::2]
epochs = Epochs(raw,
events,
None,
tmin,
tmax,
picks=picks,
baseline=(None, 0),
preload=True,
proj=False)
epochs.decimate(3, verbose='error')
assert len(epochs) == 4
# test if n_components=None works
ica = ICA(n_components=None,
max_pca_components=None,
n_pca_components=None,
random_state=0,
method=method,
max_iter=1)
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(epochs)
# for testing eog functionality
picks2 = np.concatenate([picks, pick_types(raw.info, False, eog=True)])
epochs_eog = Epochs(raw,
events[:4],
event_id,
tmin,
tmax,
picks=picks2,
baseline=(None, 0),
preload=True)
del picks2
test_cov2 = test_cov.copy()
ica = ICA(noise_cov=test_cov2,
n_components=3,
max_pca_components=4,
n_pca_components=4,
method=method)
assert (ica.info is None)
with pytest.warns(RuntimeWarning, match='normalize_proj'):
ica.fit(raw, picks[:5])
assert (isinstance(ica.info, Info))
assert (ica.n_components_ < 5)
ica = ICA(n_components=3,
max_pca_components=4,
method=method,
n_pca_components=4,
random_state=0)
pytest.raises(RuntimeError, ica.save, '')
ica.fit(raw, picks=[1, 2, 3, 4, 5], start=start, stop=stop2)
# check passing a ch_name to find_bads_ecg
with pytest.warns(RuntimeWarning, match='longer'):
_, scores_1 = ica.find_bads_ecg(raw)
_, scores_2 = ica.find_bads_ecg(raw, raw.ch_names[1])
assert scores_1[0] != scores_2[0]
# test corrmap
ica2 = ica.copy()
ica3 = ica.copy()
corrmap([ica, ica2], (0, 0),
threshold='auto',
label='blinks',
plot=True,
ch_type="mag")
corrmap([ica, ica2], (0, 0), threshold=2, plot=False, show=False)
assert (ica.labels_["blinks"] == ica2.labels_["blinks"])
assert (0 in ica.labels_["blinks"])
# test retrieval of component maps as arrays
components = ica.get_components()
template = components[:, 0]
EvokedArray(components, ica.info, tmin=0.).plot_topomap([0], time_unit='s')
corrmap([ica, ica3],
template,
threshold='auto',
label='blinks',
plot=True,
ch_type="mag")
assert (ica2.labels_["blinks"] == ica3.labels_["blinks"])
plt.close('all')
# make sure a single threshold in a list works
corrmap([ica, ica3],
template,
threshold=[0.5],
label='blinks',
plot=True,
ch_type="mag")
ica_different_channels = ICA(n_components=2,
random_state=0).fit(raw, picks=[2, 3, 4, 5])
pytest.raises(ValueError, corrmap, [ica_different_channels, ica], (0, 0))
# test warnings on bad filenames
ica_badname = op.join(op.dirname(tempdir), 'test-bad-name.fif.gz')
with pytest.warns(RuntimeWarning, match='-ica.fif'):
ica.save(ica_badname)
with pytest.warns(RuntimeWarning, match='-ica.fif'):
read_ica(ica_badname)
# test decim
ica = ICA(n_components=3,
max_pca_components=4,
n_pca_components=4,
method=method,
max_iter=1)
raw_ = raw.copy()
for _ in range(3):
raw_.append(raw_)
n_samples = raw_._data.shape[1]
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(raw, picks=picks[:5], decim=3)
assert raw_._data.shape[1] == n_samples
# test expl var
ica = ICA(n_components=1.0,
max_pca_components=4,
n_pca_components=4,
method=method,
max_iter=1)
with pytest.warns(UserWarning, match='did not converge'):
ica.fit(raw, picks=None, decim=3)
assert (ica.n_components_ == 4)
ica_var = _ica_explained_variance(ica, raw, normalize=True)
assert (np.all(ica_var[:-1] >= ica_var[1:]))
# test ica sorting
ica.exclude = [0]
ica.labels_ = dict(blink=[0], think=[1])
ica_sorted = _sort_components(ica, [3, 2, 1, 0], copy=True)
assert_equal(ica_sorted.exclude, [3])
assert_equal(ica_sorted.labels_, dict(blink=[3], think=[2]))
# epochs extraction from raw fit
pytest.raises(RuntimeError, ica.get_sources, epochs)
# test reading and writing
test_ica_fname = op.join(op.dirname(tempdir), 'test-ica.fif')
for cov in (None, test_cov):
ica = ICA(noise_cov=cov,
n_components=2,
max_pca_components=4,
n_pca_components=4,
method=method,
max_iter=1)
with pytest.warns(None): # ICA does not converge
ica.fit(raw, picks=picks[:10], start=start, stop=stop2)
sources = ica.get_sources(epochs).get_data()
assert (ica.mixing_matrix_.shape == (2, 2))
assert (ica.unmixing_matrix_.shape == (2, 2))
assert (ica.pca_components_.shape == (4, 10))
assert (sources.shape[1] == ica.n_components_)
for exclude in [[], [0], np.array([1, 2, 3])]:
ica.exclude = exclude
ica.labels_ = {'foo': [0]}
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert (list(ica.exclude) == ica_read.exclude)
assert_equal(ica.labels_, ica_read.labels_)
ica.apply(raw)
ica.exclude = []
ica.apply(raw, exclude=[1])
assert (ica.exclude == [])
ica.exclude = [0, 1]
ica.apply(raw, exclude=[1])
assert (ica.exclude == [0, 1])
ica_raw = ica.get_sources(raw)
assert (ica.exclude == [
ica_raw.ch_names.index(e) for e in ica_raw.info['bads']
])
# test filtering
d1 = ica_raw._data[0].copy()
ica_raw.filter(4, 20, fir_design='firwin2')
assert_equal(ica_raw.info['lowpass'], 20.)
assert_equal(ica_raw.info['highpass'], 4.)
assert ((d1 != ica_raw._data[0]).any())
d1 = ica_raw._data[0].copy()
ica_raw.notch_filter([10], trans_bandwidth=10, fir_design='firwin')
assert ((d1 != ica_raw._data[0]).any())
ica.n_pca_components = 2
ica.method = 'fake'
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert (ica.n_pca_components == ica_read.n_pca_components)
assert_equal(ica.method, ica_read.method)
assert_equal(ica.labels_, ica_read.labels_)
# check type consistency
attrs = ('mixing_matrix_ unmixing_matrix_ pca_components_ '
'pca_explained_variance_ pre_whitener_')
def f(x, y):
return getattr(x, y).dtype
for attr in attrs.split():
assert_equal(f(ica_read, attr), f(ica, attr))
ica.n_pca_components = 4
ica_read.n_pca_components = 4
ica.exclude = []
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
for attr in [
'mixing_matrix_', 'unmixing_matrix_', 'pca_components_',
'pca_mean_', 'pca_explained_variance_', 'pre_whitener_'
]:
assert_array_almost_equal(getattr(ica, attr),
getattr(ica_read, attr))
assert (ica.ch_names == ica_read.ch_names)
assert (isinstance(ica_read.info, Info))
sources = ica.get_sources(raw)[:, :][0]
sources2 = ica_read.get_sources(raw)[:, :][0]
assert_array_almost_equal(sources, sources2)
_raw1 = ica.apply(raw, exclude=[1])
_raw2 = ica_read.apply(raw, exclude=[1])
assert_array_almost_equal(_raw1[:, :][0], _raw2[:, :][0])
os.remove(test_ica_fname)
# check score funcs
for name, func in get_score_funcs().items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(raw,
target='EOG 061',
score_func=func,
start=0,
stop=10)
assert (ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(raw, start=0, stop=50, score_func=stats.skew)
# check exception handling
pytest.raises(ValueError, ica.score_sources, raw, target=np.arange(1))
params = []
params += [(None, -1, slice(2), [0, 1])] # variance, kurtosis params
params += [(None, 'MEG 1531')] # ECG / EOG channel params
for idx, ch_name in product(*params):
ica.detect_artifacts(raw,
start_find=0,
stop_find=50,
ecg_ch=ch_name,
eog_ch=ch_name,
skew_criterion=idx,
var_criterion=idx,
kurt_criterion=idx)
# Make sure detect_artifacts marks the right components.
# For int criterion, the doc says "E.g. range(2) would return the two
# sources with the highest score". Assert that's what it does.
# Only test for skew, since it's always the same code.
ica.exclude = []
ica.detect_artifacts(raw,
start_find=0,
stop_find=50,
ecg_ch=None,
eog_ch=None,
skew_criterion=0,
var_criterion=None,
kurt_criterion=None)
assert np.abs(scores[ica.exclude]) == np.max(np.abs(scores))
evoked = epochs.average()
evoked_data = evoked.data.copy()
raw_data = raw[:][0].copy()
epochs_data = epochs.get_data().copy()
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_ecg(raw, method='ctps')
assert_equal(len(scores), ica.n_components_)
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_ecg(raw, method='correlation')
assert_equal(len(scores), ica.n_components_)
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_eog(raw)
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(epochs, method='ctps')
assert_equal(len(scores), ica.n_components_)
pytest.raises(ValueError,
ica.find_bads_ecg,
epochs.average(),
method='ctps')
pytest.raises(ValueError, ica.find_bads_ecg, raw, method='crazy-coupling')
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_eog(raw)
assert_equal(len(scores), ica.n_components_)
raw.info['chs'][raw.ch_names.index('EOG 061') - 1]['kind'] = 202
with pytest.warns(RuntimeWarning, match='longer'):
idx, scores = ica.find_bads_eog(raw)
assert (isinstance(scores, list))
assert_equal(len(scores[0]), ica.n_components_)
idx, scores = ica.find_bads_eog(evoked, ch_name='MEG 1441')
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(evoked, method='correlation')
assert_equal(len(scores), ica.n_components_)
assert_array_equal(raw_data, raw[:][0])
assert_array_equal(epochs_data, epochs.get_data())
assert_array_equal(evoked_data, evoked.data)
# check score funcs
for name, func in get_score_funcs().items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(epochs_eog,
target='EOG 061',
score_func=func)
assert (ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(epochs, score_func=stats.skew)
# check exception handling
pytest.raises(ValueError, ica.score_sources, epochs, target=np.arange(1))
# ecg functionality
ecg_scores = ica.score_sources(raw,
target='MEG 1531',
score_func='pearsonr')
with pytest.warns(RuntimeWarning, match='longer'):
ecg_events = ica_find_ecg_events(raw,
sources[np.abs(ecg_scores).argmax()])
assert (ecg_events.ndim == 2)
# eog functionality
eog_scores = ica.score_sources(raw,
target='EOG 061',
score_func='pearsonr')
with pytest.warns(RuntimeWarning, match='longer'):
eog_events = ica_find_eog_events(raw,
sources[np.abs(eog_scores).argmax()])
assert (eog_events.ndim == 2)
# Test ica fiff export
ica_raw = ica.get_sources(raw, start=0, stop=100)
assert (ica_raw.last_samp - ica_raw.first_samp == 100)
assert_equal(len(ica_raw._filenames), 1) # API consistency
ica_chans = [ch for ch in ica_raw.ch_names if 'ICA' in ch]
assert (ica.n_components_ == len(ica_chans))
test_ica_fname = op.join(op.abspath(op.curdir), 'test-ica_raw.fif')
ica.n_components = np.int32(ica.n_components)
ica_raw.save(test_ica_fname, overwrite=True)
ica_raw2 = read_raw_fif(test_ica_fname, preload=True)
assert_allclose(ica_raw._data, ica_raw2._data, rtol=1e-5, atol=1e-4)
ica_raw2.close()
os.remove(test_ica_fname)
# Test ica epochs export
ica_epochs = ica.get_sources(epochs)
assert (ica_epochs.events.shape == epochs.events.shape)
ica_chans = [ch for ch in ica_epochs.ch_names if 'ICA' in ch]
assert (ica.n_components_ == len(ica_chans))
assert (ica.n_components_ == ica_epochs.get_data().shape[1])
assert (ica_epochs._raw is None)
assert (ica_epochs.preload is True)
# test float n pca components
ica.pca_explained_variance_ = np.array([0.2] * 5)
ica.n_components_ = 0
for ncomps, expected in [[0.3, 1], [0.9, 4], [1, 1]]:
ncomps_ = ica._check_n_pca_components(ncomps)
assert (ncomps_ == expected)
ica = ICA(method=method)
with pytest.warns(None): # sometimes does not converge
ica.fit(raw, picks=picks[:5])
with pytest.warns(RuntimeWarning, match='longer'):
ica.find_bads_ecg(raw)
ica.find_bads_eog(epochs, ch_name='MEG 0121')
assert_array_equal(raw_data, raw[:][0])
raw.drop_channels(['MEG 0122'])
pytest.raises(RuntimeError, ica.find_bads_eog, raw)
with pytest.warns(RuntimeWarning, match='longer'):
pytest.raises(RuntimeError, ica.find_bads_ecg, raw)
event_ids = {"faces": 1, "scrambled": 2}
tmin, tmax = -0.2, 0.6
baseline = None # no baseline as high-pass is applied
reject = dict(mag=5e-12)
epochs = mne.Epochs(raw, events, event_ids, tmin, tmax, picks=picks,
baseline=baseline, preload=True, reject=reject)
# Fit ICA, find and remove major artifacts
ica = ICA(n_components=0.95, random_state=0).fit(raw, decim=1, reject=reject)
# compute correlation scores, get bad indices sorted by score
eog_epochs = create_eog_epochs(raw, ch_name='MRT31-2908', reject=reject)
eog_inds, eog_scores = ica.find_bads_eog(eog_epochs, ch_name='MRT31-2908')
ica.plot_scores(eog_scores, eog_inds) # see scores the selection is based on
ica.plot_components(eog_inds) # view topographic sensitivity of components
ica.exclude += eog_inds[:1] # we saw the 2nd ECG component looked too dipolar
ica.plot_overlay(eog_epochs.average()) # inspect artifact removal
ica.apply(epochs) # clean data, default in place
evoked = [epochs[k].average() for k in event_ids]
contrast = combine_evoked(evoked, weights=[-1, 1]) # Faces - scrambled
evoked.append(contrast)
for e in evoked:
e.plot(ylim=dict(mag=[-400, 400]))
print(ica)
reject = dict(
grad=4000e-13, # T / m (gradiometers)
mag=4e-12) # T (magnetometers)
ica.fit(raw_sss, picks=picks_meg, decim=decim, reject=reject)
print(ica)
raw_sss.plot_psd(fmax=100)
eog_average = create_eog_epochs(raw_sss,
reject=dict(mag=5e-12, grad=4000e-13),
picks=picks_meg).average()
n_max_eog = 1 # here we bet on finding the vertical EOG components
eog_epochs = create_eog_epochs(raw_sss, reject=reject) # get single EOG trials
eog_inds, scores = ica.find_bads_eog(eog_epochs) # find via correlation
print(ica)
raw_sss.plot_psd(fmax=100)
#ica.plot_scores(scores, exclude=eog_inds)
#ica.plot_sources(eog_average, exclude=eog_inds) # look at source time course
ica.apply(raw_sss, exclude=eog_inds)
print(ica)
raw_sss.plot_psd(fmax=100)
ecg_epochs = create_ecg_epochs(raw_sss, tmin=-.5, tmax=.5)
ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
ica.apply(raw_sss, exclude=ecg_inds)
raw_sss.plot_psd(fmax=100)
def preproc(subject):
global report
from mne.preprocessing import ICA, create_ecg_epochs, create_eog_epochs
from mne.io import read_raw_fif
import numpy as np
raw_path = op.join(rest_dir, subject, 'meg_clean', 'nose-pts-out_raw.fif')
clean_raw_path = op.join(rest_dir, subject, 'meg_clean', 'clean_raw.fif')
if not os.path.exists(
raw_path): # skip subjects that have T1 but do not have MEG data
return
raw = read_raw_fif(raw_path, preload=True)
pick_meg = mne.pick_types(raw.info,
meg=True,
eeg=False,
stim=False,
ref_meg=False)
pick_eog1 = mne.pick_channels(raw.ch_names, ['EOG061'])
pick_eog2 = mne.pick_channels(raw.ch_names, ['EOG062'])
pick_ecg = mne.pick_channels(raw.ch_names, ['ECG063'])
if pick_eog1.size == 0 or pick_eog2.size == 0 or pick_ecg.size == 0:
logFile.write("Subject " + subject +
": EOG and/or ECG channels not found.\n")
return
##-- Filtering, PSD --##
fig = raw.plot_psd(area_mode='range', average=False, picks=pick_meg)
report.add_figs_to_section(fig,
captions=subject + ' => PSD before filtering',
section='Filtering')
raw.notch_filter((50, 100),
filter_length='auto',
fir_design='firwin',
phase='zero',
picks=np.append(pick_meg,
(pick_ecg, pick_eog1, pick_eog2)))
raw.filter(1., None, fir_design='firwin',
phase='zero') # high-pass filtering at 1 Hz
raw.resample(
200, npad="auto"
) # set sampling frequency to 200Hz (antialiasing filter at 100 Hz)
##-- Artifact removal --##
random_state = 23 # we want to have the same decomposition and the same order of components each time it is run for reproducibility
ica = ICA(n_components=25, method='fastica', random_state=random_state)
reject = dict(
mag=5e-12, grad=4000e-13
) # avoid fitting ICA on large environmental artifacts that would dominate the variance and decomposition
try:
ica.fit(raw, picks=pick_meg, reject=reject)
except:
logFile.write("Subject " + subject +
": ICA could not run. Large environmental artifacts.\n")
return
fig = ica.plot_components(inst=raw)
report.add_figs_to_section(fig,
captions=[subject + ' => ICA components'] *
len(fig),
section='Artifact removal')
# EOG artifact
eog_epochs = create_eog_epochs(raw, reject=reject)
if eog_epochs.events.size != 0:
eog_inds, scores = ica.find_bads_eog(eog_epochs)
if len(eog_inds) != 0:
fig = ica.plot_properties(eog_epochs,
picks=eog_inds,
psd_args={'fmax': 35.},
image_args={'sigma': 1.})
report.add_figs_to_section(
fig,
captions=[subject + ' => ICA correlated with EOG'] * len(fig),
section='Artifact removal')
ica.exclude.extend(eog_inds)
else:
logFile.write("Subject " + subject +
": No ICA component correlated with EOG\n")
else:
logFile.write("Subject " + subject + ": No EOG events found\n")
# ECG artifact
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5)
if ecg_epochs.events.size != 0:
ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
if len(ecg_inds) != 0:
fig = ica.plot_properties(ecg_epochs,
picks=ecg_inds,
psd_args={'fmax': 35.})
report.add_figs_to_section(
fig,
captions=[subject + ' => ICA correlated with ECG'] * len(fig),
section='Artifact removal')
ica.exclude.extend(ecg_inds)
else:
logFile.write("Subject " + subject +
": No ICA component correlated with ECG\n")
else:
logFile.write("Subject " + subject + ": No ECG events found\n")
raw_clean = raw.copy()
ica.apply(raw_clean)
fig = raw_clean.plot_psd(area_mode='range', picks=pick_meg)
report.add_figs_to_section(
fig,
captions=subject + ' => PSD after filtering and artifact correction',
section='Filtering')
raw_clean.save(clean_raw_path, overwrite=True)
def remove_eyeblinks_and_heartbeat(raw, subject, figdir, events, eventid, rng):
"""
Find and repair eyeblink and heartbeat artifacts in the data.
Data should be filtered.
Importantly, ICA is fitted on artificially epoched data with reject
criteria estimates via the autoreject package - this is done to reject high-
amplitude artifacts to influence the ICA solution.
The ICA fit is then applied to the raw data.
:param raw: Raw data
:param subject: str, subject identifier, e.g., '001'
:param figdir:
:param rng: random state instance
"""
# prior to an ICA, it is recommended to high-pass filter the data
# as low frequency artifacts can alter the ICA solution. We fit the ICA
# to high-pass filtered (1Hz) data, and apply it to non-highpass-filtered
# data
logging.info("Applying a temporary high-pass filtering prior to ICA")
filt_raw = raw.copy()
filt_raw.load_data().filter(l_freq=1., h_freq=None)
# evoked eyeblinks and heartbeats for diagnostic plots
eog_evoked = create_eog_epochs(filt_raw).average()
eog_evoked.apply_baseline(baseline=(None, -0.2))
if subject == '008':
# subject 008's ECG channel is flat. It will not find any heartbeats by
# default. We let it estimate heartbeat from magnetometers. For this,
# we'll drop the ECG channel
filt_raw.drop_channels('ECG003')
ecg_evoked = create_ecg_epochs(filt_raw).average()
ecg_evoked.apply_baseline(baseline=(None, -0.2))
# make sure that we actually found sensible artifacts here
eog_fig = eog_evoked.plot_joint()
for i, fig in enumerate(eog_fig):
fname = _construct_path([
Path(figdir),
f"sub-{subject}",
"meg",
f"evoked-artifact_eog_sub-{subject}_{i}.png",
])
fig.savefig(fname)
ecg_fig = ecg_evoked.plot_joint()
for i, fig in enumerate(ecg_fig):
fname = _construct_path([
Path(figdir),
f"sub-{subject}",
"meg",
f"evoked-artifact_ecg_sub-{subject}_{i}.png",
])
fig.savefig(fname)
# Chunk raw data into epochs to fit the ICA
# No baseline correction as it would interfere with ICA.
logging.info("Epoching filtered data")
epochs = mne.Epochs(filt_raw,
events,
event_id=eventid,
tmin=0,
tmax=3,
picks='meg',
baseline=None)
## First, estimate rejection criteria for high-amplitude artifacts. This is
## done via autoreject
#logging.info('Estimating bad epochs quick-and-dirty, to improve ICA')
#ar = AutoReject(random_state=rng)
# fit on first 200 epochs to save (a bit of) time
#epochs.load_data()
#ar.fit(epochs[:200])
#epochs_ar, reject_log = ar.transform(epochs, return_log=True)
# run an ICA to capture heartbeat and eyeblink artifacts.
# set a seed for reproducibility.
# When left to figure out the component number by itself, it ends up with
# about 80. I'm setting n_components to 45 to have a chance at checking them
# by hand.
# We fit it on a set of epochs excluding the initial bad epochs following
# https://github.com/autoreject/autoreject/blob/dfbc64f49eddeda53c5868290a6792b5233843c6/examples/plot_autoreject_workflow.py
logging.info('Fitting the ICA')
ica = ICA(max_iter='auto', n_components=45, random_state=rng)
ica.fit(epochs) #[~reject_log.bad_epochs])
logging.info("Searching for eyeblink and heartbeat artifacts in the data")
# get ICA components for the given subject
if subject == '008':
eog_indices = [10]
ecg_indices = [29]
#eog_indices = ica_comps[subject]['eog']
#ecg_indices = ica_comps[subject]['ecg']
# An initially manual component detection did not reproduce after a software
# update - for now, we have to do the automatic detection for all but sub 8
else:
eog_indices, eog_scores = ica.find_bads_eog(filt_raw)
ecg_indices, ecg_scores = ica.find_bads_ecg(filt_raw)
logging.info(f"Identified the following EOG components: {eog_indices}")
logging.info(f"Identified the following ECG components: {ecg_indices}")
# visualize the components
components = ica.plot_components()
for i, fig in enumerate(components):
fname = _construct_path([
Path(figdir),
f"sub-{subject}",
"meg",
f"ica-components_sub-{subject}_{i}.png",
])
fig.savefig(fname)
# visualize the time series of components and save it
plt.rcParams['figure.figsize'] = 30, 20
comp_sources = ica.plot_sources(epochs)
fname = _construct_path([
Path(figdir),
f"sub-{subject}",
"meg",
f"ica-components_sources_sub-{subject}.png",
])
comp_sources.savefig(fname)
# reset plotting params
plt.rcParams['figure.figsize'] = plt.rcParamsDefault['figure.figsize']
# plot EOG components
overlay_eog = ica.plot_overlay(eog_evoked,
exclude=ica_comps[subject]['eog'])
fname = _construct_path([
Path(figdir),
f"sub-{subject}",
"meg",
f"ica-eog-components_over-avg-epochs_sub-{subject}.png",
])
overlay_eog.savefig(fname)
# plot ECG components
overlay_ecg = ica.plot_overlay(ecg_evoked,
exclude=ica_comps[subject]['ecg'])
fname = _construct_path([
Path(figdir),
f"sub-{subject}",
"meg",
f"ica-ecg-components_over-avg-epochs_sub-{subject}.png",
])
overlay_ecg.savefig(fname)
# plot EOG component properties
figs = ica.plot_properties(filt_raw, picks=eog_indices)
for i, fig in enumerate(figs):
fname = _construct_path([
Path(figdir),
f"sub-{subject}",
"meg",
f"ica-property{i}_artifact-eog_sub-{subject}.png",
])
fig.savefig(fname)
# plot ECG component properties
figs = ica.plot_properties(filt_raw, picks=ecg_indices)
for i, fig in enumerate(figs):
fname = _construct_path([
Path(figdir),
f"sub-{subject}",
"meg",
f"ica-property{i}_artifact-ecg_sub-{subject}.png",
])
fig.savefig(fname)
# Set the indices to be excluded
ica.exclude = eog_indices
ica.exclude.extend(ecg_indices)
# plot ICs applied to the averaged EOG epochs, with EOG matches highlighted
sources = ica.plot_sources(eog_evoked)
fname = _construct_path([
Path(figdir),
f"sub-{subject}",
"meg",
f"ica-sources_artifact-eog_sub-{subject}.png",
])
sources.savefig(fname)
# plot ICs applied to the averaged ECG epochs, with ECG matches highlighted
sources = ica.plot_sources(ecg_evoked)
fname = _construct_path([
Path(figdir),
f"sub-{subject}",
"meg",
f"ica-sources_artifact-ecg_sub-{subject}.png",
])
sources.savefig(fname)
# apply the ICA to the raw data
logging.info('Applying ICA to the raw data.')
raw.load_data()
ica.apply(raw)
def compute_ica(subject, data_folder):
"""Function will compute ICA on raw and apply the ICA.
Parameters
----------
subject : string
the subject id to be loaded
"""
raw = mne.io.Raw(data_folder + "%s_bp-raw.fif" % subject, preload=True)
raw.set_montage = montage
raw.apply_proj()
# raw.resample(512, n_jobs=2)
# ICA Part
ica = ICA(n_components=None,
max_pca_components=40,
method='fastica',
max_iter=256)
picks = mne.pick_types(
raw.info, meg=False, eeg=True, stim=False, exclude='bads')
ica.fit(raw, picks=picks, decim=decim, reject=reject)
# maximum number of components to reject
n_max_eog = 1
##########################################################################
# 2) identify bad components by analyzing latent sources.
title = 'Sources related to %s artifacts (red) for sub: %s'
#
# # generate ECG epochs use detection via phase statistics
# ecg_epochs = create_ecg_epochs(raw, ch_name="Ext4",
# tmin=-.5, tmax=.5, picks=picks)
# n_ecg_epochs_found = len(ecg_epochs.events)
# sel_ecg_epochs = np.arange(0, n_ecg_epochs_found, 10)
# ecg_epochs = ecg_epochs[sel_ecg_epochs]
#
# ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
# fig = ica.plot_scores(scores, exclude=ecg_inds,
# title=title % ('ecg', subject))
# fig.savefig(data_folder + "pics/%s_ecg_scores.png" % subject)
#
# if ecg_inds:
# show_picks = np.abs(scores).argsort()[::-1][:5]
#
# fig = ica.plot_sources(raw, show_picks, exclude=ecg_inds,
# title=title % ('ecg', subject), show=False)
# fig.savefig(data_folder + "pics/%s_ecg_sources.png" % subject)
# fig = ica.plot_components(ecg_inds, title=title % ('ecg', subject),
# colorbar=True)
# fig.savefig(data_folder + "pics/%s_ecg_component.png" % subject)
#
# ecg_inds = ecg_inds[:n_max_ecg]
# ica.exclude += ecg_inds
#
# # estimate average artifact
# ecg_evoked = ecg_epochs.average()
# del ecg_epochs
#
# # plot ECG sources + selection
# fig = ica.plot_sources(ecg_evoked, exclude=ecg_inds)
# fig.savefig(data_folder + "pics/%s_ecg_sources_ave.png" % subject)
#
# # plot ECG cleaning
# ica.plot_overlay(ecg_evoked, exclude=ecg_inds)
# fig.savefig(data_folder + "pics/%s_ecg_sources_clean_ave.png" % subject)
# DETECT EOG BY CORRELATION
# HORIZONTAL EOG
eog_epochs = create_eog_epochs(raw, ch_name="EXG4")
eog_indices, scores = ica.find_bads_eog(raw, ch_name="EXG4")
fig = ica.plot_scores(
scores, exclude=eog_indices, title=title % ('eog', subject))
fig.savefig(data_folder + "pics/%s_eog_scores.png" % subject)
fig = ica.plot_components(
eog_indices, title=title % ('eog', subject), colorbar=True)
fig.savefig(data_folder + "pics/%s_eog_component.png" % subject)
eog_indices = eog_indices[:n_max_eog]
ica.exclude += eog_indices
del eog_epochs
##########################################################################
# Apply the solution to Raw, Epochs or Evoked like this:
raw_ica = ica.apply(raw)
ica.save(data_folder + "%s-ica.fif" % subject) # save ICA componenets
# Save raw with ICA removed
raw_ica.save(data_folder + "%s_bp_ica-raw.fif" % subject, overwrite=True)
plt.close("all")
def preproc_meg(data_dir, session_id, task=True, n_loc=6, n_stim=11):
data = os.listdir(data_dir)
data = sorted([i for i in data if '.ds' in i and str(session_id) in i])
output_dir = os.path.join('/data/twise/', str(session_id))
if not os.path.exists(output_dir):
os.mkdir(output_dir)
print(data)
localiser_raws = []
task_raws = []
localiser_idx = range(0, n_loc)
for i in localiser_idx:
localiser_raws.append(
read_raw_ctf(os.path.join(data_dir, data[i]), preload=True))
if task:
task_idx = range(n_loc, n_loc + 5)
for i in task_idx:
task_raws.append(
read_raw_ctf(os.path.join(data_dir, data[i]), preload=True))
localiser_raw = mne.concatenate_raws(localiser_raws)
if task:
task_raw = mne.concatenate_raws(task_raws)
localiser_raw.apply_gradient_compensation(0)
if task:
task_raw.apply_gradient_compensation(0)
mf_kwargs = dict(origin=(0., 0., 0.), st_duration=10.)
localiser_raw = mne.preprocessing.maxwell_filter(localiser_raw,
**mf_kwargs)
localiser_raw.plot()
plt.savefig(os.path.join(output_dir, 'localiser_raw.pdf'))
if task:
task_raw = mne.preprocessing.maxwell_filter(task_raw, **mf_kwargs)
task_raw.plot()
plt.savefig(os.path.join(output_dir, 'task_raw.pdf'))
task_raw.set_channel_types({
'UADC001-2910': 'eog',
'UADC002-2901': 'eog',
'UADC003-2901': 'eog'
})
localiser_raw.set_channel_types({
'UADC001-2910': 'eog',
'UADC002-2901': 'eog',
'UADC003-2901': 'eog'
})
# localiser_raws.info['bads'] = ['MLT54']
# picks = mne.pick_types(localiser_raw.info, meg='mag', eog=True)
# REJECT BAD CHANNELS
# eyes = mne.pick_channels(raw.info['ch_names'], ['UADC001-2910', 'UADC002-2901'])
print("FILTERING")
localiser_raw.filter(0.5, 45, method='fir', fir_design='firwin')
if task:
task_raw.filter(0.5, 45, method='fir', fir_design='firwin')
print("FINDING EVENTS")
localiser_events = mne.find_events(localiser_raw,
stim_channel='UPPT001',
shortest_event=1)
if task:
task_events = mne.find_events(task_raw,
stim_channel='UPPT001',
shortest_event=1)
reject = dict(mag=5e-9, eog=20)
print("EPOCHING")
localiser_epochs = mne.Epochs(
localiser_raw,
localiser_events,
tmin=-0.1,
tmax=0.8,
preload=True,
event_id=[i for i in list(range(2, n_stim * 2 + 1, 2))] + [99],
reject=None)
if task:
task_dropped = {'trial': [], 'event_id': []}
task_epochs = mne.Epochs(task_raw,
task_events,
tmin=0,
tmax=8,
preload=True,
reject=None,
event_id=[30])
print(task_epochs)
for n, i in enumerate(task_epochs.drop_log):
if len(i) and i != ['IGNORED']:
task_dropped['trial'].append(
np.where(task_events[task_events[:, 2] == task_events[n]
[2]][:,
0] == task_events[n][0])[0][0])
task_dropped['event_id'].append(task_events[n][2])
task_dropped = pd.DataFrame(task_dropped)
print(task_dropped)
task_dropped.to_csv(
os.path.join(output_dir, '{0}_task_dropped_trials.csv'.format(
str(session_id))))
# Downsample to 100 Hz
print("DOWNSAMPLING")
print('Original sampling rate:', localiser_epochs.info['sfreq'], 'Hz')
localiser_epochs = localiser_epochs.copy().resample(100, npad='auto')
if task: task_epochs = task_epochs.copy().resample(100, npad='auto')
print('New sampling rate:', localiser_epochs.info['sfreq'], 'Hz')
print("ICA")
ica = ICA(n_components=0.95,
method='fastica',
random_state=0,
max_iter=100).fit(task_epochs, decim=1, reject=None)
ica.plot_components()
pp = PdfPages(os.path.join(output_dir, 'ICA.pdf'))
figs = [plt.figure(n) for n in plt.get_fignums()]
for fig in figs:
fig.savefig(pp, format='pdf')
pp.close()
ica.save(os.path.join(output_dir, 'meg-ica.fif.gz'))
# detect EOG by correlation
eog_inds, scores = ica.find_bads_eog(task_epochs)
print(eog_inds)
print("SAVING")
localiser_epochs.save(
os.path.join(output_dir, 'pre_ICA_localiser-epo.fif.gz'))
if task:
task_epochs.save(os.path.join(output_dir, 'pre_ICA_task-epo.fif.gz'))
print("DONE")
def ica_method(raw, picks, plot='n', save ='n'):
###############################################################################
# 1) Fit ICA model using the FastICA algorithm
# Other available choices are `infomax` or `extended-infomax`
# We pass a float value between 0 and 1 to select n_components based on the
# percentage of variance explained by the PCA components.
ica = ICA(n_components=0.95, method='fastica')
picks = mne.pick_types(raw.info, meg=False, eeg=True, eog=False,
stim=False, exclude='bads')
ica.fit(raw, picks=picks, decim=3)
# maximum number of components to reject
n_max_eog = 1 # here we don't expect horizontal EOG components
###############################################################################
# 2) identify bad components by analyzing latent sources.
# detect EOG by correlation
eog_inds, scores = ica.find_bads_eog(raw, threshold=2.5)
show_picks = np.abs(scores).argsort()[::-1][:5]
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
###############################################################################
# 3) Assess component selection and unmixing quality
eog_evoked = create_eog_epochs(raw, tmin=-.5, tmax=.5, picks=picks).average()
if plot=='y':
title = 'Sources related to %s artifacts (red)'
ica.plot_scores(scores, exclude=eog_inds, title=title % 'eog', labels='eog')
if save=='y':
pylab.savefig('2.png')
ica.plot_sources(raw, show_picks, exclude=eog_inds, title=title % 'eog')
if save=='y':
pylab.savefig('3.png')
ica.plot_components(eog_inds, title=title % 'eog', colorbar=True)
if save=='y':
pylab.savefig('4.png')
ica.plot_overlay(raw) # EOG artifacts remain
if save=='y':
pylab.savefig('5.png')
ica.plot_sources(eog_evoked, exclude=eog_inds) # plot EOG sources + selection
if save=='y':
pylab.savefig('6.png')
ica.plot_overlay(eog_evoked, exclude=eog_inds) # plot EOG cleaning
if save=='y':
pylab.savefig('7.png')
ica.apply(raw, exclude=eog_inds)
eeg_only_after=raw.pick_types(meg=False, eeg=True)
return eeg_only_after
def test_ica_additional():
"""Test additional ICA functionality"""
tempdir = _TempDir()
stop2 = 500
raw = Raw(raw_fname).crop(1.5, stop, False)
raw.load_data()
picks = pick_types(raw.info, meg=True, stim=False, ecg=False,
eog=False, exclude='bads')
test_cov = read_cov(test_cov_name)
events = read_events(event_name)
picks = pick_types(raw.info, meg=True, stim=False, ecg=False,
eog=False, exclude='bads')
epochs = Epochs(raw, events[:4], event_id, tmin, tmax, picks=picks,
baseline=(None, 0), preload=True)
# test if n_components=None works
with warnings.catch_warnings(record=True):
ica = ICA(n_components=None,
max_pca_components=None,
n_pca_components=None, random_state=0)
ica.fit(epochs, picks=picks, decim=3)
# for testing eog functionality
picks2 = pick_types(raw.info, meg=True, stim=False, ecg=False,
eog=True, exclude='bads')
epochs_eog = Epochs(raw, events[:4], event_id, tmin, tmax, picks=picks2,
baseline=(None, 0), preload=True)
test_cov2 = test_cov.copy()
ica = ICA(noise_cov=test_cov2, n_components=3, max_pca_components=4,
n_pca_components=4)
assert_true(ica.info is None)
with warnings.catch_warnings(record=True):
ica.fit(raw, picks[:5])
assert_true(isinstance(ica.info, Info))
assert_true(ica.n_components_ < 5)
ica = ICA(n_components=3, max_pca_components=4,
n_pca_components=4)
assert_raises(RuntimeError, ica.save, '')
with warnings.catch_warnings(record=True):
ica.fit(raw, picks=[1, 2, 3, 4, 5], start=start, stop=stop2)
# test corrmap
ica2 = ica.copy()
corrmap([ica, ica2], (0, 0), threshold='auto', label='blinks', plot=True,
ch_type="mag")
corrmap([ica, ica2], (0, 0), threshold=2, plot=False, show=False)
assert_true(ica.labels_["blinks"] == ica2.labels_["blinks"])
assert_true(0 in ica.labels_["blinks"])
plt.close('all')
# test warnings on bad filenames
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
ica_badname = op.join(op.dirname(tempdir), 'test-bad-name.fif.gz')
ica.save(ica_badname)
read_ica(ica_badname)
assert_naming(w, 'test_ica.py', 2)
# test decim
ica = ICA(n_components=3, max_pca_components=4,
n_pca_components=4)
raw_ = raw.copy()
for _ in range(3):
raw_.append(raw_)
n_samples = raw_._data.shape[1]
with warnings.catch_warnings(record=True):
ica.fit(raw, picks=None, decim=3)
assert_true(raw_._data.shape[1], n_samples)
# test expl var
ica = ICA(n_components=1.0, max_pca_components=4,
n_pca_components=4)
with warnings.catch_warnings(record=True):
ica.fit(raw, picks=None, decim=3)
assert_true(ica.n_components_ == 4)
# epochs extraction from raw fit
assert_raises(RuntimeError, ica.get_sources, epochs)
# test reading and writing
test_ica_fname = op.join(op.dirname(tempdir), 'test-ica.fif')
for cov in (None, test_cov):
ica = ICA(noise_cov=cov, n_components=2, max_pca_components=4,
n_pca_components=4)
with warnings.catch_warnings(record=True): # ICA does not converge
ica.fit(raw, picks=picks, start=start, stop=stop2)
sources = ica.get_sources(epochs).get_data()
assert_true(ica.mixing_matrix_.shape == (2, 2))
assert_true(ica.unmixing_matrix_.shape == (2, 2))
assert_true(ica.pca_components_.shape == (4, len(picks)))
assert_true(sources.shape[1] == ica.n_components_)
for exclude in [[], [0]]:
ica.exclude = [0]
ica.labels_ = {'foo': [0]}
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.exclude == ica_read.exclude)
assert_equal(ica.labels_, ica_read.labels_)
ica.exclude = []
ica.apply(raw, exclude=[1])
assert_true(ica.exclude == [])
ica.exclude = [0, 1]
ica.apply(raw, exclude=[1])
assert_true(ica.exclude == [0, 1])
ica_raw = ica.get_sources(raw)
assert_true(ica.exclude == [ica_raw.ch_names.index(e) for e in
ica_raw.info['bads']])
# test filtering
d1 = ica_raw._data[0].copy()
with warnings.catch_warnings(record=True): # dB warning
ica_raw.filter(4, 20)
assert_true((d1 != ica_raw._data[0]).any())
d1 = ica_raw._data[0].copy()
with warnings.catch_warnings(record=True): # dB warning
ica_raw.notch_filter([10])
assert_true((d1 != ica_raw._data[0]).any())
ica.n_pca_components = 2
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
assert_true(ica.n_pca_components == ica_read.n_pca_components)
# check type consistency
attrs = ('mixing_matrix_ unmixing_matrix_ pca_components_ '
'pca_explained_variance_ _pre_whitener')
def f(x, y):
return getattr(x, y).dtype
for attr in attrs.split():
assert_equal(f(ica_read, attr), f(ica, attr))
ica.n_pca_components = 4
ica_read.n_pca_components = 4
ica.exclude = []
ica.save(test_ica_fname)
ica_read = read_ica(test_ica_fname)
for attr in ['mixing_matrix_', 'unmixing_matrix_', 'pca_components_',
'pca_mean_', 'pca_explained_variance_',
'_pre_whitener']:
assert_array_almost_equal(getattr(ica, attr),
getattr(ica_read, attr))
assert_true(ica.ch_names == ica_read.ch_names)
assert_true(isinstance(ica_read.info, Info))
sources = ica.get_sources(raw)[:, :][0]
sources2 = ica_read.get_sources(raw)[:, :][0]
assert_array_almost_equal(sources, sources2)
_raw1 = ica.apply(raw, exclude=[1])
_raw2 = ica_read.apply(raw, exclude=[1])
assert_array_almost_equal(_raw1[:, :][0], _raw2[:, :][0])
os.remove(test_ica_fname)
# check scrore funcs
for name, func in get_score_funcs().items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(raw, target='EOG 061', score_func=func,
start=0, stop=10)
assert_true(ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(raw, score_func=stats.skew)
# check exception handling
assert_raises(ValueError, ica.score_sources, raw,
target=np.arange(1))
params = []
params += [(None, -1, slice(2), [0, 1])] # varicance, kurtosis idx params
params += [(None, 'MEG 1531')] # ECG / EOG channel params
for idx, ch_name in product(*params):
ica.detect_artifacts(raw, start_find=0, stop_find=50, ecg_ch=ch_name,
eog_ch=ch_name, skew_criterion=idx,
var_criterion=idx, kurt_criterion=idx)
with warnings.catch_warnings(record=True):
idx, scores = ica.find_bads_ecg(raw, method='ctps')
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(raw, method='correlation')
assert_equal(len(scores), ica.n_components_)
idx, scores = ica.find_bads_ecg(epochs, method='ctps')
assert_equal(len(scores), ica.n_components_)
assert_raises(ValueError, ica.find_bads_ecg, epochs.average(),
method='ctps')
assert_raises(ValueError, ica.find_bads_ecg, raw,
method='crazy-coupling')
idx, scores = ica.find_bads_eog(raw)
assert_equal(len(scores), ica.n_components_)
raw.info['chs'][raw.ch_names.index('EOG 061') - 1]['kind'] = 202
idx, scores = ica.find_bads_eog(raw)
assert_true(isinstance(scores, list))
assert_equal(len(scores[0]), ica.n_components_)
# check score funcs
for name, func in get_score_funcs().items():
if name in score_funcs_unsuited:
continue
scores = ica.score_sources(epochs_eog, target='EOG 061',
score_func=func)
assert_true(ica.n_components_ == len(scores))
# check univariate stats
scores = ica.score_sources(epochs, score_func=stats.skew)
# check exception handling
assert_raises(ValueError, ica.score_sources, epochs,
target=np.arange(1))
# ecg functionality
ecg_scores = ica.score_sources(raw, target='MEG 1531',
score_func='pearsonr')
with warnings.catch_warnings(record=True): # filter attenuation warning
ecg_events = ica_find_ecg_events(raw,
sources[np.abs(ecg_scores).argmax()])
assert_true(ecg_events.ndim == 2)
# eog functionality
eog_scores = ica.score_sources(raw, target='EOG 061',
score_func='pearsonr')
with warnings.catch_warnings(record=True): # filter attenuation warning
eog_events = ica_find_eog_events(raw,
sources[np.abs(eog_scores).argmax()])
assert_true(eog_events.ndim == 2)
# Test ica fiff export
ica_raw = ica.get_sources(raw, start=0, stop=100)
assert_true(ica_raw.last_samp - ica_raw.first_samp == 100)
assert_true(len(ica_raw._filenames) == 0) # API consistency
ica_chans = [ch for ch in ica_raw.ch_names if 'ICA' in ch]
assert_true(ica.n_components_ == len(ica_chans))
test_ica_fname = op.join(op.abspath(op.curdir), 'test-ica_raw.fif')
ica.n_components = np.int32(ica.n_components)
ica_raw.save(test_ica_fname, overwrite=True)
ica_raw2 = Raw(test_ica_fname, preload=True)
assert_allclose(ica_raw._data, ica_raw2._data, rtol=1e-5, atol=1e-4)
ica_raw2.close()
os.remove(test_ica_fname)
# Test ica epochs export
ica_epochs = ica.get_sources(epochs)
assert_true(ica_epochs.events.shape == epochs.events.shape)
ica_chans = [ch for ch in ica_epochs.ch_names if 'ICA' in ch]
assert_true(ica.n_components_ == len(ica_chans))
assert_true(ica.n_components_ == ica_epochs.get_data().shape[1])
assert_true(ica_epochs._raw is None)
assert_true(ica_epochs.preload is True)
# test float n pca components
ica.pca_explained_variance_ = np.array([0.2] * 5)
ica.n_components_ = 0
for ncomps, expected in [[0.3, 1], [0.9, 4], [1, 1]]:
ncomps_ = ica._check_n_pca_components(ncomps)
assert_true(ncomps_ == expected)
proc_dir = root_dir+"proc/"
proclist = listdir(proc_dir)
l_freq = 0.3
h_freq = 200
n_jobs = "cuda"
max_thresh = 2e-3
for filename in proclist:
this_match = re.match("c_EPI_(.*)-raw.fif", filename)
if not this_match:
continue
file_id = this_match.groups(1)[0]
raw = mne.io.Raw(proc_dir+filename, preload=True)
raw_sel = raw.copy().pick_channels(["Cz", "Fz", "F3", "F4", "C3", "C4",
"Pz", "Fp1", "Fp2", "O1", "O2",
"VEOG", "HEOG", "ECG1+"])
raw_sel.filter(l_freq=0.3, h_freq=30, n_jobs=4)
raw_sel.resample(100, n_jobs="cuda")
ica = ICA(method="picard")
ica.fit(raw_sel)
ica.save("{}sel_c_EPI_{}-ica.fif".format(proc_dir, file_id))
bads_eog, scores = ica.find_bads_eog(raw_sel, ch_name="VEOG", threshold=2.3)
bads_ecg ,scores = ica.find_bads_ecg(raw_sel, ch_name="ECG1+")
bad_comps = bads_eog + bads_ecg
raw_sel = ica.apply(raw_sel, exclude=bad_comps)
raw_sel.save("{}csel_{}".format(proc_dir, filename), overwrite=True)
