def test_manual_report_2d(tmpdir, invisible_fig):
"""Simulate user manually creating report by adding one file at a time."""
from sklearn.exceptions import ConvergenceWarning
r = Report(title='My Report')
raw = read_raw_fif(raw_fname)
raw.pick_channels(raw.ch_names[:6]).crop(10, None)
raw.info.normalize_proj()
cov = read_cov(cov_fname)
cov = pick_channels_cov(cov, raw.ch_names)
events = read_events(events_fname)
epochs = Epochs(raw=raw, events=events, baseline=None)
evokeds = read_evokeds(evoked_fname)
evoked = evokeds[0].pick('eeg')
with pytest.warns(ConvergenceWarning, match='did not converge'):
ica = (ICA(n_components=2, max_iter=1, random_state=42)
.fit(inst=raw.copy().crop(tmax=1)))
ica_ecg_scores = ica_eog_scores = np.array([3, 0])
ica_ecg_evoked = ica_eog_evoked = epochs.average()
r.add_raw(raw=raw, title='my raw data', tags=('raw',), psd=True,
projs=False)
r.add_events(events=events_fname, title='my events',
sfreq=raw.info['sfreq'])
r.add_epochs(epochs=epochs, title='my epochs', tags=('epochs',), psd=False,
projs=False)
r.add_evokeds(evokeds=evoked, noise_cov=cov_fname,
titles=['my evoked 1'], tags=('evoked',), projs=False,
n_time_points=2)
r.add_projs(info=raw_fname, projs=ecg_proj_fname, title='my proj',
tags=('ssp', 'ecg'))
r.add_ica(ica=ica, title='my ica', inst=None)
with pytest.raises(RuntimeError, match='not preloaded'):
r.add_ica(ica=ica, title='ica', inst=raw)
r.add_ica(
ica=ica, title='my ica with inst',
inst=raw.copy().load_data(),
picks=[0],
ecg_evoked=ica_ecg_evoked,
eog_evoked=ica_eog_evoked,
ecg_scores=ica_ecg_scores,
eog_scores=ica_eog_scores
)
r.add_covariance(cov=cov, info=raw_fname, title='my cov')
r.add_forward(forward=fwd_fname, title='my forward', subject='sample',
subjects_dir=subjects_dir)
r.add_html(html='<strong>Hello</strong>', title='Bold')
r.add_code(code=__file__, title='my code')
r.add_sys_info(title='my sysinfo')
fname = op.join(tmpdir, 'report.html')
r.save(fname=fname, open_browser=False)
def apply_ica(*, cfg, subject, session):
bids_basename = BIDSPath(subject=subject,
session=session,
task=cfg.task,
acquisition=cfg.acq,
run=None,
recording=cfg.rec,
space=cfg.space,
datatype=cfg.datatype,
root=cfg.deriv_root,
check=False)
fname_epo_in = bids_basename.copy().update(suffix='epo', extension='.fif')
fname_epo_out = bids_basename.copy().update(processing='ica',
suffix='epo',
extension='.fif')
fname_ica = bids_basename.copy().update(suffix='ica', extension='.fif')
fname_ica_components = bids_basename.copy().update(processing='ica',
suffix='components',
extension='.tsv')
report_fname = (bids_basename.copy().update(processing='ica',
suffix='report',
extension='.html'))
title = f'ICA artifact removal – sub-{subject}'
if session is not None:
title += f', ses-{session}'
if cfg.task is not None:
title += f', task-{cfg.task}'
# Load ICA.
msg = f'Reading ICA: {fname_ica}'
logger.debug(
**gen_log_kwargs(message=msg, subject=subject, session=session))
ica = read_ica(fname=fname_ica)
# Select ICs to remove.
tsv_data = pd.read_csv(fname_ica_components, sep='\t')
ica.exclude = (tsv_data.loc[tsv_data['status'] == 'bad',
'component'].to_list())
# Load epochs to reject ICA components.
msg = f'Input: {fname_epo_in}, Output: {fname_epo_out}'
logger.info(
**gen_log_kwargs(message=msg, subject=subject, session=session))
epochs = mne.read_epochs(fname_epo_in, preload=True)
epochs.drop_bad(cfg.ica_reject)
# Now actually reject the components.
msg = f'Rejecting ICs: {", ".join([str(ic) for ic in ica.exclude])}'
logger.info(
**gen_log_kwargs(message=msg, subject=subject, session=session))
epochs_cleaned = ica.apply(epochs.copy()) # Copy b/c works in-place!
msg = 'Saving reconstructed epochs after ICA.'
logger.info(
**gen_log_kwargs(message=msg, subject=subject, session=session))
epochs_cleaned.save(fname_epo_out, overwrite=True, split_naming='bids')
# Compare ERP/ERF before and after ICA artifact rejection. The evoked
# response is calculated across ALL epochs, just like ICA was run on
# all epochs, regardless of their respective experimental condition.
#
# We apply baseline correction here to (hopefully!) make the effects of
# ICA easier to see. Otherwise, individual channels might just have
# arbitrary DC shifts, and we wouldn't be able to easily decipher what's
# going on!
report = Report(report_fname, title=title, verbose=False)
picks = ica.exclude if ica.exclude else None
report.add_ica(ica=ica,
title='Effects of ICA cleaning',
inst=epochs.copy().apply_baseline(cfg.baseline),
picks=picks)
report.save(report_fname, overwrite=True, open_browser=cfg.interactive)
def test_manual_report_2d(tmp_path, invisible_fig):
"""Simulate user manually creating report by adding one file at a time."""
from sklearn.exceptions import ConvergenceWarning
r = Report(title='My Report')
raw = read_raw_fif(raw_fname)
raw.pick_channels(raw.ch_names[:6]).crop(10, None)
raw.info.normalize_proj()
cov = read_cov(cov_fname)
cov = pick_channels_cov(cov, raw.ch_names)
events = read_events(events_fname)
event_id = {
'auditory/left': 1,
'auditory/right': 2,
'visual/left': 3,
'visual/right': 4,
'face': 5,
'buttonpress': 32
}
metadata, metadata_events, metadata_event_id = make_metadata(
events=events,
event_id=event_id,
tmin=-0.2,
tmax=0.5,
sfreq=raw.info['sfreq'])
epochs_without_metadata = Epochs(raw=raw,
events=events,
event_id=event_id,
baseline=None)
epochs_with_metadata = Epochs(raw=raw,
events=metadata_events,
event_id=metadata_event_id,
baseline=None,
metadata=metadata)
evokeds = read_evokeds(evoked_fname)
evoked = evokeds[0].pick('eeg')
with pytest.warns(ConvergenceWarning, match='did not converge'):
ica = (ICA(n_components=2, max_iter=1,
random_state=42).fit(inst=raw.copy().crop(tmax=1)))
ica_ecg_scores = ica_eog_scores = np.array([3, 0])
ica_ecg_evoked = ica_eog_evoked = epochs_without_metadata.average()
r.add_raw(raw=raw,
title='my raw data',
tags=('raw', ),
psd=True,
projs=False)
r.add_raw(raw=raw,
title='my raw data 2',
psd=False,
projs=False,
butterfly=1)
r.add_events(events=events_fname,
title='my events',
sfreq=raw.info['sfreq'])
r.add_epochs(epochs=epochs_without_metadata,
title='my epochs',
tags=('epochs', ),
psd=False,
projs=False)
r.add_epochs(epochs=epochs_without_metadata,
title='my epochs 2',
psd=1,
projs=False)
r.add_epochs(epochs=epochs_without_metadata,
title='my epochs 2',
psd=True,
projs=False)
assert 'Metadata' not in r.html[-1]
# Try with metadata
r.add_epochs(epochs=epochs_with_metadata,
title='my epochs with metadata',
psd=False,
projs=False)
assert 'Metadata' in r.html[-1]
with pytest.raises(ValueError,
match='requested to calculate PSD on a duration'):
r.add_epochs(epochs=epochs_with_metadata,
title='my epochs 2',
psd=100000000,
projs=False)
r.add_evokeds(evokeds=evoked,
noise_cov=cov_fname,
titles=['my evoked 1'],
tags=('evoked', ),
projs=False,
n_time_points=2)
r.add_projs(info=raw_fname,
projs=ecg_proj_fname,
title='my proj',
tags=('ssp', 'ecg'))
r.add_ica(ica=ica, title='my ica', inst=None)
with pytest.raises(RuntimeError, match='not preloaded'):
r.add_ica(ica=ica, title='ica', inst=raw)
r.add_ica(ica=ica,
title='my ica with inst',
inst=raw.copy().load_data(),
picks=[0],
ecg_evoked=ica_ecg_evoked,
eog_evoked=ica_eog_evoked,
ecg_scores=ica_ecg_scores,
eog_scores=ica_eog_scores)
r.add_covariance(cov=cov, info=raw_fname, title='my cov')
r.add_forward(forward=fwd_fname,
title='my forward',
subject='sample',
subjects_dir=subjects_dir)
r.add_html(html='<strong>Hello</strong>', title='Bold')
r.add_code(code=__file__, title='my code')
r.add_sys_info(title='my sysinfo')
# drop locations (only EEG channels in `evoked`)
evoked_no_ch_locs = evoked.copy()
for ch in evoked_no_ch_locs.info['chs']:
ch['loc'][:3] = np.nan
with pytest.warns(RuntimeWarning, match='No EEG channel locations'):
r.add_evokeds(evokeds=evoked_no_ch_locs,
titles=['evoked no chan locs'],
tags=('evoked', ),
projs=True,
n_time_points=1)
assert 'Time course' not in r._content[-1].html
assert 'Topographies' not in r._content[-1].html
assert evoked.info['projs'] # only then the following test makes sense
assert 'SSP' not in r._content[-1].html
assert 'Global field power' in r._content[-1].html
# Drop locations from Info used for projs
info_no_ch_locs = raw.info.copy()
for ch in info_no_ch_locs['chs']:
ch['loc'][:3] = np.nan
with pytest.warns(RuntimeWarning, match='No channel locations found'):
r.add_projs(info=info_no_ch_locs, title='Projs no chan locs')
# Drop locations from ICA
ica_no_ch_locs = ica.copy()
for ch in ica_no_ch_locs.info['chs']:
ch['loc'][:3] = np.nan
with pytest.warns(RuntimeWarning,
match='No Magnetometers channel locations'):
r.add_ica(ica=ica_no_ch_locs,
picks=[0],
inst=raw.copy().load_data(),
title='ICA')
assert 'ICA component properties' not in r._content[-1].html
assert 'ICA component topographies' not in r._content[-1].html
assert 'Original and cleaned signal' in r._content[-1].html
fname = op.join(tmp_path, 'report.html')
r.save(fname=fname, open_browser=False)
def run_ica(*, cfg, subject, session=None):
"""Run ICA."""
bids_basename = BIDSPath(subject=subject,
session=session,
task=cfg.task,
acquisition=cfg.acq,
recording=cfg.rec,
space=cfg.space,
datatype=cfg.datatype,
root=cfg.deriv_root,
check=False)
raw_fname = bids_basename.copy().update(processing='filt', suffix='raw')
ica_fname = bids_basename.copy().update(suffix='ica', extension='.fif')
ica_components_fname = bids_basename.copy().update(processing='ica',
suffix='components',
extension='.tsv')
report_fname = bids_basename.copy().update(processing='ica+components',
suffix='report',
extension='.html')
# Generate a list of raw data paths (i.e., paths of individual runs)
# we want to create epochs from.
raw_fnames = []
for run in cfg.runs:
raw_fname.update(run=run)
if raw_fname.copy().update(split='01').fpath.exists():
raw_fname.update(split='01')
raw_fnames.append(raw_fname.copy())
# Generate a unique event name -> event code mapping that can be used
# across all runs.
event_name_to_code_map = annotations_to_events(raw_paths=raw_fnames)
# Now, generate epochs from each individual run
eog_epochs_all_runs = None
ecg_epochs_all_runs = None
for idx, (run, raw_fname) in enumerate(zip(cfg.runs, raw_fnames)):
msg = f'Loading filtered raw data from {raw_fname} and creating epochs'
logger.info(**gen_log_kwargs(
message=msg, subject=subject, session=session, run=run))
# ECG epochs
ecg_epochs = make_ecg_epochs(cfg=cfg,
raw_path=raw_fname,
subject=subject,
session=session,
run=run,
n_runs=len(cfg.runs))
if ecg_epochs is not None:
if idx == 0:
ecg_epochs_all_runs = ecg_epochs
else:
ecg_epochs_all_runs = mne.concatenate_epochs(
[ecg_epochs_all_runs, ecg_epochs], on_mismatch='warn')
del ecg_epochs
# EOG epochs
raw = mne.io.read_raw_fif(raw_fname, preload=True)
eog_epochs = make_eog_epochs(raw=raw,
eog_channels=cfg.eog_channels,
subject=subject,
session=session,
run=run)
if eog_epochs is not None:
if idx == 0:
eog_epochs_all_runs = eog_epochs
else:
eog_epochs_all_runs = mne.concatenate_epochs(
[eog_epochs_all_runs, eog_epochs], on_mismatch='warn')
del eog_epochs
# Produce high-pass filtered version of the data for ICA.
# Sanity check – make sure we're using the correct data!
if cfg.resample_sfreq is not None:
assert np.allclose(raw.info['sfreq'], cfg.resample_sfreq)
if cfg.l_freq is not None:
assert np.allclose(raw.info['highpass'], cfg.l_freq)
filter_for_ica(cfg=cfg,
raw=raw,
subject=subject,
session=session,
run=run)
# Only keep the subset of the mapping that applies to the current run
event_id = event_name_to_code_map.copy()
for event_name in event_id.copy().keys():
if event_name not in raw.annotations.description:
del event_id[event_name]
msg = 'Creating task-related epochs …'
logger.info(**gen_log_kwargs(
message=msg, subject=subject, session=session, run=run))
epochs = make_epochs(raw=raw,
event_id=event_id,
tmin=cfg.epochs_tmin,
tmax=cfg.epochs_tmax,
event_repeated=cfg.event_repeated,
decim=cfg.decim)
# Only keep epochs that will be analyzed -> Keeps ICA in sync with
# epochs generated in the make_epochs script (save preserves memory)!
if cfg.task != 'rest':
if isinstance(cfg.conditions, dict):
conditions = list(cfg.conditions.keys())
else:
conditions = cfg.conditions
epochs = epochs[conditions]
epochs.load_data() # Remove reference to raw
del raw # free memory
if idx == 0:
epochs_all_runs = epochs
else:
epochs_all_runs = mne.concatenate_epochs([epochs_all_runs, epochs],
on_mismatch='warn')
del epochs
# Clean up namespace
epochs = epochs_all_runs
epochs_ecg = ecg_epochs_all_runs
epochs_eog = eog_epochs_all_runs
del epochs_all_runs, eog_epochs_all_runs, ecg_epochs_all_runs
# Set an EEG reference
if 'eeg' in cfg.ch_types:
projection = True if cfg.eeg_reference == 'average' else False
epochs.set_eeg_reference(cfg.eeg_reference, projection=projection)
# Reject epochs based on peak-to-peak rejection thresholds
msg = f'Using PTP rejection thresholds: {cfg.ica_reject}'
logger.info(
**gen_log_kwargs(message=msg, subject=subject, session=session))
# Reject epochs based on peak-to-peak amplitude
epochs.drop_bad(reject=cfg.ica_reject)
if epochs_eog is not None:
epochs_eog.drop_bad(reject=cfg.ica_reject)
if epochs_ecg is not None:
epochs_ecg.drop_bad(reject=cfg.ica_reject)
# Now actually perform ICA.
msg = 'Calculating ICA solution.'
logger.info(
**gen_log_kwargs(message=msg, subject=subject, session=session))
ica = fit_ica(cfg=cfg, epochs=epochs, subject=subject, session=session)
# Start a report
title = f'ICA – sub-{subject}'
if session is not None:
title += f', ses-{session}'
if cfg.task is not None:
title += f', task-{cfg.task}'
# ECG and EOG component detection
if epochs_ecg:
ecg_ics, ecg_scores = detect_bad_components(cfg=cfg,
which='ecg',
epochs=epochs_ecg,
ica=ica,
subject=subject,
session=session)
else:
ecg_ics = ecg_scores = []
if epochs_eog:
eog_ics, eog_scores = detect_bad_components(cfg=cfg,
which='eog',
epochs=epochs_eog,
ica=ica,
subject=subject,
session=session)
else:
eog_ics = eog_scores = []
# Save ICA to disk.
# We also store the automatically identified ECG- and EOG-related ICs.
msg = 'Saving ICA solution and detected artifacts to disk.'
logger.info(
**gen_log_kwargs(message=msg, subject=subject, session=session))
ica.exclude = sorted(set(ecg_ics + eog_ics))
ica.save(ica_fname, overwrite=True)
# Create TSV.
tsv_data = pd.DataFrame(
dict(component=list(range(ica.n_components_)),
type=['ica'] * ica.n_components_,
description=['Independent Component'] * ica.n_components_,
status=['good'] * ica.n_components_,
status_description=['n/a'] * ica.n_components_))
for component in ecg_ics:
row_idx = tsv_data['component'] == component
tsv_data.loc[row_idx, 'status'] = 'bad'
tsv_data.loc[row_idx,
'status_description'] = 'Auto-detected ECG artifact'
for component in eog_ics:
row_idx = tsv_data['component'] == component
tsv_data.loc[row_idx, 'status'] = 'bad'
tsv_data.loc[row_idx,
'status_description'] = 'Auto-detected EOG artifact'
tsv_data.to_csv(ica_components_fname, sep='\t', index=False)
# Lastly, add info about the epochs used for the ICA fit, and plot all ICs
# for manual inspection.
msg = 'Adding diagnostic plots for all ICA components to the HTML report …'
logger.info(
**gen_log_kwargs(message=msg, subject=subject, session=session))
report = Report(info_fname=epochs, title=title, verbose=False)
report.add_epochs(epochs=epochs, title='Epochs used for ICA fitting')
ecg_evoked = None if epochs_ecg is None else epochs_ecg.average()
eog_evoked = None if epochs_eog is None else epochs_eog.average()
ecg_scores = None if len(ecg_scores) == 0 else ecg_scores
eog_scores = None if len(eog_scores) == 0 else eog_scores
report.add_ica(ica=ica,
title='ICA cleaning',
inst=epochs,
ecg_evoked=ecg_evoked,
eog_evoked=eog_evoked,
ecg_scores=ecg_scores,
eog_scores=eog_scores)
msg = (f"ICA completed. Please carefully review the extracted ICs in the "
f"report {report_fname.basename}, and mark all components you wish "
f"to reject as 'bad' in {ica_components_fname.basename}")
logger.info(
**gen_log_kwargs(message=msg, subject=subject, session=session))
report.save(report_fname, overwrite=True, open_browser=cfg.interactive)
