def test_render_add_sections():
"""Test adding figures/images to section.
"""
tempdir = _TempDir()
import matplotlib.pyplot as plt
report = Report(subjects_dir=subjects_dir)
# Check add_figs_to_section functionality
fig = plt.plot([1, 2], [1, 2])[0].figure
report.add_figs_to_section(figs=fig, # test non-list input
captions=['evoked response'], scale=1.2,
image_format='svg')
assert_raises(ValueError, report.add_figs_to_section, figs=[fig, fig],
captions='H')
assert_raises(ValueError, report.add_figs_to_section, figs=fig,
captions=['foo'], scale=0, image_format='svg')
assert_raises(ValueError, report.add_figs_to_section, figs=fig,
captions=['foo'], scale=1e-10, image_format='svg')
# need to recreate because calls above change size
fig = plt.plot([1, 2], [1, 2])[0].figure
# Check add_images_to_section
img_fname = op.join(tempdir, 'testimage.png')
fig.savefig(img_fname)
report.add_images_to_section(fnames=[img_fname],
captions=['evoked response'])
assert_raises(ValueError, report.add_images_to_section,
fnames=[img_fname, img_fname], captions='H')
evoked = read_evokeds(evoked_fname, condition='Left Auditory',
baseline=(-0.2, 0.0))
fig = plot_trans(evoked.info, trans_fname, subject='sample',
subjects_dir=subjects_dir)
report.add_figs_to_section(figs=fig, # test non-list input
captions='random image', scale=1.2)
def test_render_add_sections():
"""Test adding figures/images to section.
"""
tempdir = _TempDir()
import matplotlib.pyplot as plt
report = Report(subjects_dir=subjects_dir)
# Check add_figs_to_section functionality
fig = plt.plot([1, 2], [1, 2])[0].figure
report.add_figs_to_section(
figs=fig, # test non-list input
captions=['evoked response'])
assert_raises(ValueError,
report.add_figs_to_section,
figs=[fig, fig],
captions='H')
# Check add_images_to_section
img_fname = op.join(tempdir, 'testimage.png')
fig.savefig(img_fname)
report.add_images_to_section(fnames=[img_fname],
captions=['evoked response'])
assert_raises(ValueError,
report.add_images_to_section,
fnames=[img_fname, img_fname],
captions='H')
# Check deprecation of add_section
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
report.add_section(figs=fig, captions=['evoked response'])
assert_true(w[0].category == DeprecationWarning)
def test_remove():
"""Test removing figures from a report."""
r = Report()
fig1, fig2 = _get_example_figures()
r.add_figs_to_section(fig1, 'figure1', 'mysection')
r.add_slider_to_section([fig1, fig2], title='figure1',
section='othersection')
r.add_figs_to_section(fig2, 'figure1', 'mysection')
r.add_figs_to_section(fig2, 'figure2', 'mysection')
# Test removal by caption
r2 = copy.deepcopy(r)
removed_index = r2.remove(caption='figure1')
assert removed_index == 2
assert len(r2.html) == 3
assert r2.html[0] == r.html[0]
assert r2.html[1] == r.html[1]
assert r2.html[2] == r.html[3]
# Test restricting to section
r2 = copy.deepcopy(r)
removed_index = r2.remove(caption='figure1', section='othersection')
assert removed_index == 1
assert len(r2.html) == 3
assert r2.html[0] == r.html[0]
assert r2.html[1] == r.html[2]
assert r2.html[2] == r.html[3]
# Test removal of empty sections
r2 = copy.deepcopy(r)
r2.remove(caption='figure1', section='othersection')
assert r2.sections == ['mysection']
assert r2._sectionvars == {'mysection': 'report_mysection'}
def test_render_add_sections():
"""Test adding figures/images to section.
"""
tempdir = _TempDir()
import matplotlib.pyplot as plt
report = Report(subjects_dir=subjects_dir)
# Check add_figs_to_section functionality
fig = plt.plot([1, 2], [1, 2])[0].figure
report.add_figs_to_section(figs=fig, # test non-list input
captions=['evoked response'], scale=1.2,
image_format='svg')
assert_raises(ValueError, report.add_figs_to_section, figs=[fig, fig],
captions='H')
assert_raises(ValueError, report.add_figs_to_section, figs=fig,
captions=['foo'], scale=0, image_format='svg')
assert_raises(ValueError, report.add_figs_to_section, figs=fig,
captions=['foo'], scale=1e-10, image_format='svg')
# need to recreate because calls above change size
fig = plt.plot([1, 2], [1, 2])[0].figure
# Check add_images_to_section
img_fname = op.join(tempdir, 'testimage.png')
fig.savefig(img_fname)
report.add_images_to_section(fnames=[img_fname],
captions=['evoked response'])
assert_raises(ValueError, report.add_images_to_section,
fnames=[img_fname, img_fname], captions='H')
evoked = read_evokeds(evoked_fname, condition='Left Auditory',
baseline=(-0.2, 0.0))
fig = plot_trans(evoked.info, trans_fname, subject='sample',
subjects_dir=subjects_dir)
report.add_figs_to_section(figs=fig, # test non-list input
captions='random image', scale=1.2)
def test_render_add_sections():
"""Test adding figures/images to section.
"""
tempdir = _TempDir()
import matplotlib.pyplot as plt
report = Report(subjects_dir=subjects_dir)
# Check add_figs_to_section functionality
fig = plt.plot([1, 2], [1, 2])[0].figure
report.add_figs_to_section(figs=fig, # test non-list input
captions=['evoked response'], scale=1.2,
image_format='svg')
assert_raises(ValueError, report.add_figs_to_section, figs=[fig, fig],
captions='H')
# Check add_images_to_section
img_fname = op.join(tempdir, 'testimage.png')
fig.savefig(img_fname)
report.add_images_to_section(fnames=[img_fname],
captions=['evoked response'])
assert_raises(ValueError, report.add_images_to_section,
fnames=[img_fname, img_fname], captions='H')
# Check deprecation of add_section
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
report.add_section(figs=fig,
captions=['evoked response'])
assert_true(w[0].category == DeprecationWarning)
def test_remove():
"""Test removing figures from a report."""
r = Report()
fig1, fig2 = _get_example_figures()
r.add_figs_to_section(fig1, 'figure1', 'mysection')
r.add_slider_to_section([fig1, fig2],
title='figure1',
section='othersection')
r.add_figs_to_section(fig2, 'figure1', 'mysection')
r.add_figs_to_section(fig2, 'figure2', 'mysection')
# Test removal by caption
r2 = copy.deepcopy(r)
removed_index = r2.remove(caption='figure1')
assert removed_index == 2
assert len(r2.html) == 3
assert r2.html[0] == r.html[0]
assert r2.html[1] == r.html[1]
assert r2.html[2] == r.html[3]
# Test restricting to section
r2 = copy.deepcopy(r)
removed_index = r2.remove(caption='figure1', section='othersection')
assert removed_index == 1
assert len(r2.html) == 3
assert r2.html[0] == r.html[0]
assert r2.html[1] == r.html[2]
assert r2.html[2] == r.html[3]
# Test removal of empty sections
r2 = copy.deepcopy(r)
r2.remove(caption='figure1', section='othersection')
assert r2.sections == ['mysection']
assert r2._sectionvars == {'mysection': 'report_mysection'}
def plotICA_comp(subject,run):
# subject parameters
data_path = '/neurospin/meg/meg_tmp/MTT_MEG_Baptiste/MEG/'
raw_fname = data_path + subject + '/' + run + '_trans_sss.fif'
# ICA
report = Report(subject)
raw = Raw(raw_fname, preload=True)
raw.filter(None, 30, n_jobs=4)
picks = mne.pick_types(raw.info, meg=True, eeg=False, eog=False,
stim=False, exclude='bads')
ica = ICA(n_components=0.95, method='fastica')
ica.fit(raw, picks=picks, decim=4)
# Create the results directory if it doesn't exist
results_dir = op.join(data_path, subject,'artefactICA')
if not op.exists(results_dir):
os.makedirs(results_dir)
report = Report(subject)
fig = ica.plot_components(title= 'ICA comp', colorbar=True)
for i in range(np.shape(fig)[0]):
report.add_figs_to_section(fig[i], captions='ICA comp', section='ICA_allcomp')
report.save((results_dir + '/'+ run + '_allcomp.html'), open_browser=False,overwrite=True)
def browse_components(subject, runlist, badEEG):
import numpy as np
import mne
import os.path as op
import os
from mne.report import Report
from mne.io.pick import _picks_by_type as picks_by_type
from mne.preprocessing import read_ica
from mne.io import Raw
data_path = '/neurospin/meg/meg_tmp/MTT_MEG_Baptiste/MEG/'
report = Report(subject)
raw_fnames = []
for run in runlist:
raw_fnames.append(data_path + subject + '/' + run + '_trans_sss.fif')
# read raw data
raw = Raw(raw_fnames, preload=True)
raw.info['bads'] = badEEG
# Highpass filter 1Hz on EOG/ECG channels
#picks = mne.pick_types(raw.info, meg=False,eeg=False, eog=True, ecg=True)
#raw.filter(l_freq = 1, h_freq=30, picks = picks)
#picks = mne.pick_types(raw.info, meg=True,eeg=True, eog=True, ecg=True)
#raw.filter(l_freq = None, h_freq=30, picks = picks,n_jobs = 4)
picks = mne.pick_types(raw.info, meg=False, eeg=False, eog=False, ecg=True)
raw.filter(l_freq=1, h_freq=30, picks=picks)
picks = mne.pick_types(raw.info, meg=False, eeg=False, eog=True, ecg=False)
raw.filter(l_freq=1, h_freq=5, picks=picks)
picks = mne.pick_types(raw.info, meg=True, eeg=True, eog=True, ecg=True)
raw.filter(l_freq=None, h_freq=30, picks=picks, n_jobs=4)
results_dir = op.join(data_path, subject, 'artefactICA')
if not op.exists(results_dir):
os.makedirs(results_dir)
# Plot all ICA component
for ch_type, picks in picks_by_type(
raw.info, meg_combined=True): # bad EEG channels are excluded
# Read ICA file for each data type (MEG and EEG) - created in preprocessing_ica
ica = read_ica(data_path + subject + '/mne_python/ICA/ICA_' + ch_type +
'_allRuns')
print subject + ' ' + ch_type
#fig = ica.plot_sources(raw, block = True) # the viewer will open and ICA component can be screened
#report.add_figs_to_section(fig, 'All ICA sources (%s)' % ch_type,'ICA decomposition')
fig = ica.plot_components(colorbar=True)
for i in range(np.shape(fig)[0]):
report.add_figs_to_section(fig[i],
'All ICA components (%s)' % ch_type,
'ICA decomposition')
report.save((results_dir + '/AllRuns_checkcomponents.html'),
open_browser=False,
overwrite=True)
del raw
def test_deprecated_methods(tmpdir):
"""Test methods that are scheduled for removal after 0.24."""
r = Report()
r.add_projs(info=raw_fname, title='SSP Projectors', tags=('mytag',))
fig = plt.figure() # Empty figure
img_fname = op.join(tmpdir, 'testimage.png')
fig.savefig(img_fname)
with pytest.warns(DeprecationWarning, match='Report.fnames'):
assert len(r.fnames) == 1
with pytest.warns(DeprecationWarning, match='Report.sections'):
assert len(r.sections) == 1
with pytest.warns(DeprecationWarning, match='use "title" instead'):
r.remove(caption='SSP Projectors')
with pytest.warns(DeprecationWarning, match='use .* instead'):
r.remove(caption='SSP Projectors', tags=('mytag',))
with pytest.warns(DeprecationWarning, match='Use.*Report.add_figure'):
with pytest.raises(TypeError, match='It seems you passed a path'):
r.add_figs_to_section(['foo'], 'caption', 'section')
with pytest.raises(
ValueError,
match='Number of "captions" and report items must be equal'
):
with pytest.warns(DeprecationWarning, match='Use.*Report.add_figure'):
r.add_figs_to_section(figs=[fig, fig], captions='H')
# Passing lists should work
with pytest.warns(DeprecationWarning, match='Use.*Report.add_image'):
r.add_images_to_section(fnames=[img_fname],
captions=['evoked response'])
with pytest.raises(
ValueError,
match='Number of "captions" and report items must be equal'
):
with pytest.warns(DeprecationWarning, match='Use.*Report.add_image'):
r.add_images_to_section(fnames=[img_fname, img_fname],
captions='H')
with pytest.warns(DeprecationWarning, match='Use.*Report.add_bem'):
r.add_bem_to_section(
subject='sample', subjects_dir=subjects_dir, decim=100
)
def test_add_custom_css(tmpdir):
"""Test adding custom CSS rules to the report."""
tempdir = str(tmpdir)
fname = op.join(tempdir, 'report.html')
fig = plt.figure() # Empty figure
report = Report()
report.add_figs_to_section(figs=fig, captions='Test section')
custom_css = '.report_custom { color: red; }'
report.add_custom_css(css=custom_css)
assert custom_css in report.include
report.save(fname, open_browser=False)
with open(fname, 'rb') as fid:
html = fid.read().decode('utf-8')
assert custom_css in html
def test_add_custom_js(tmpdir):
"""Test adding custom JavaScript to the report."""
tempdir = str(tmpdir)
fname = op.join(tempdir, 'report.html')
fig = plt.figure() # Empty figure
report = Report()
report.add_figs_to_section(figs=fig, captions='Test section')
custom_js = ('function hello() {\n' ' alert("Hello, report!");\n' '}')
report.add_custom_js(js=custom_js)
assert custom_js in report.include
report.save(fname, open_browser=False)
with open(fname, 'rb') as fid:
html = fid.read().decode('utf-8')
assert custom_js in html
def test_render_add_sections(renderer, tmpdir):
"""Test adding figures/images to section."""
tempdir = str(tmpdir)
report = Report(subjects_dir=subjects_dir)
# Check add_figs_to_section functionality
fig = plt.plot([1, 2], [1, 2])[0].figure
report.add_figs_to_section(figs=fig, # test non-list input
captions=['evoked response'], scale=1.2,
image_format='svg')
pytest.raises(ValueError, report.add_figs_to_section, figs=[fig, fig],
captions='H')
pytest.raises(ValueError, report.add_figs_to_section, figs=fig,
captions=['foo'], scale=0, image_format='svg')
pytest.raises(ValueError, report.add_figs_to_section, figs=fig,
captions=['foo'], scale=1e-10, image_format='svg')
# need to recreate because calls above change size
fig = plt.plot([1, 2], [1, 2])[0].figure
# Check add_images_to_section with png
img_fname = op.join(tempdir, 'testimage.png')
fig.savefig(img_fname)
report.add_images_to_section(fnames=[img_fname],
captions=['evoked response'])
report.add_images_to_section(fnames=[img_fname],
captions=['evoked response'])
pytest.raises(ValueError, report.add_images_to_section,
fnames=[img_fname, img_fname], captions='H')
pytest.raises(ValueError, report.add_images_to_section,
fnames=['foobar.xxx'], captions='H')
evoked = read_evokeds(evoked_fname, condition='Left Auditory',
baseline=(-0.2, 0.0))
fig = plot_alignment(evoked.info, trans_fname, subject='sample',
subjects_dir=subjects_dir)
report.add_figs_to_section(figs=fig, # test non-list input
captions='random image', scale=1.2)
assert (repr(report))
fname = op.join(str(tmpdir), 'test.html')
report.save(fname, open_browser=False)
with open(fname, 'r') as fid:
html = fid.read()
assert html.count('<li class="report_custom"') == 8 # several
def test_render_add_sections():
"""Test adding figures/images to section.
"""
tempdir = _TempDir()
import matplotlib.pyplot as plt
report = Report(subjects_dir=subjects_dir)
# Check add_figs_to_section functionality
fig = plt.plot([1, 2], [1, 2])[0].figure
report.add_figs_to_section(
figs=fig, # test non-list input
captions=['evoked response'],
scale=1.2,
image_format='svg')
assert_raises(ValueError,
report.add_figs_to_section,
figs=[fig, fig],
captions='H')
# Check add_images_to_section
img_fname = op.join(tempdir, 'testimage.png')
fig.savefig(img_fname)
report.add_images_to_section(fnames=[img_fname],
captions=['evoked response'])
assert_raises(ValueError,
report.add_images_to_section,
fnames=[img_fname, img_fname],
captions='H')
# Check deprecation of add_section
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
report.add_section(figs=fig, captions=['evoked response'])
assert_true(w[0].category == DeprecationWarning)
evoked = read_evokeds(evoked_fname,
condition='Left Auditory',
baseline=(-0.2, 0.0))
fig = plot_trans(evoked.info,
trans_fname=trans_fname,
subject='sample',
subjects_dir=subjects_dir)
report.add_figs_to_section(
figs=fig, # test non-list input
captions='random image',
scale=1.2)
def test_scraper(tmpdir):
"""Test report scraping."""
r = Report()
fig1, fig2 = _get_example_figures()
r.add_figs_to_section(fig1, 'a', 'mysection')
r.add_figs_to_section(fig2, 'b', 'mysection')
# Mock a Sphinx + sphinx_gallery config
app = Bunch(builder=Bunch(srcdir=str(tmpdir),
outdir=op.join(str(tmpdir), '_build', 'html')))
scraper = _ReportScraper()
scraper.app = app
gallery_conf = dict(src_dir=app.builder.srcdir, builder_name='html')
img_fname = op.join(app.builder.srcdir, 'auto_examples', 'images',
'sg_img.png')
target_file = op.join(app.builder.srcdir, 'auto_examples', 'sg.py')
os.makedirs(op.dirname(img_fname))
os.makedirs(app.builder.outdir)
block_vars = dict(image_path_iterator=(img for img in [img_fname]),
example_globals=dict(a=1),
target_file=target_file)
# Nothing yet
block = None
rst = scraper(block, block_vars, gallery_conf)
assert rst == ''
# Still nothing
block_vars['example_globals']['r'] = r
rst = scraper(block, block_vars, gallery_conf)
# Once it's saved, add it
assert rst == ''
fname = op.join(str(tmpdir), 'my_html.html')
r.save(fname, open_browser=False)
rst = scraper(block, block_vars, gallery_conf)
out_html = op.join(app.builder.outdir, 'auto_examples', 'my_html.html')
assert not op.isfile(out_html)
os.makedirs(op.join(app.builder.outdir, 'auto_examples'))
scraper.copyfiles()
assert op.isfile(out_html)
assert rst.count('"') == 6
assert "<iframe" in rst
assert op.isfile(img_fname.replace('png', 'svg'))
def test_render_add_sections():
"""Test adding figures/images to section.
"""
from PIL import Image
tempdir = _TempDir()
import matplotlib.pyplot as plt
report = Report(subjects_dir=subjects_dir)
# Check add_figs_to_section functionality
fig = plt.plot([1, 2], [1, 2])[0].figure
report.add_figs_to_section(
figs=fig, captions=["evoked response"], scale=1.2, image_format="svg" # test non-list input
)
assert_raises(ValueError, report.add_figs_to_section, figs=[fig, fig], captions="H")
assert_raises(ValueError, report.add_figs_to_section, figs=fig, captions=["foo"], scale=0, image_format="svg")
assert_raises(ValueError, report.add_figs_to_section, figs=fig, captions=["foo"], scale=1e-10, image_format="svg")
# need to recreate because calls above change size
fig = plt.plot([1, 2], [1, 2])[0].figure
# Check add_images_to_section with png and then gif
img_fname = op.join(tempdir, "testimage.png")
fig.savefig(img_fname)
report.add_images_to_section(fnames=[img_fname], captions=["evoked response"])
im = Image.open(img_fname)
op.join(tempdir, "testimage.gif")
im.save(img_fname) # matplotlib does not support gif
report.add_images_to_section(fnames=[img_fname], captions=["evoked response"])
assert_raises(ValueError, report.add_images_to_section, fnames=[img_fname, img_fname], captions="H")
assert_raises(ValueError, report.add_images_to_section, fnames=["foobar.xxx"], captions="H")
evoked = read_evokeds(evoked_fname, condition="Left Auditory", baseline=(-0.2, 0.0))
fig = plot_trans(evoked.info, trans_fname, subject="sample", subjects_dir=subjects_dir)
report.add_figs_to_section(figs=fig, captions="random image", scale=1.2) # test non-list input
def test_add_or_replace():
"""Test replacing existing figures in a report."""
r = Report()
r.add_figs_to_section(fig1, 'duplicate', 'mysection')
r.add_figs_to_section(fig1, 'duplicate', 'mysection')
r.add_figs_to_section(fig1, 'duplicate', 'othersection')
r.add_figs_to_section(fig2, 'nonduplicate', 'mysection')
# By default, replace=False, so all figures should be there
assert len(r.html) == 4
old_r = copy.deepcopy(r)
# Re-add fig1 with replace=True, it should overwrite the last occurrence of
# fig1 in section 'mysection'.
r.add_figs_to_section(fig2, 'duplicate', 'mysection', replace=True)
assert len(r.html) == 4
assert r.html[1] != old_r.html[1] # This figure should have changed
# All other figures should be the same
assert r.html[0] == old_r.html[0]
assert r.html[2] == old_r.html[2]
assert r.html[3] == old_r.html[3]
def test_add_or_replace():
"""Test replacing existing figures in a report."""
r = Report()
fig1, fig2 = _get_example_figures()
r.add_figs_to_section(fig1, 'duplicate', 'mysection')
r.add_figs_to_section(fig1, 'duplicate', 'mysection')
r.add_figs_to_section(fig1, 'duplicate', 'othersection')
r.add_figs_to_section(fig2, 'nonduplicate', 'mysection')
# By default, replace=False, so all figures should be there
assert len(r.html) == 4
old_r = copy.deepcopy(r)
# Re-add fig1 with replace=True, it should overwrite the last occurrence of
# fig1 in section 'mysection'.
r.add_figs_to_section(fig2, 'duplicate', 'mysection', replace=True)
assert len(r.html) == 4
assert r.html[1] != old_r.html[1] # This figure should have changed
# All other figures should be the same
assert r.html[0] == old_r.html[0]
assert r.html[2] == old_r.html[2]
assert r.html[3] == old_r.html[3]
def apply_ica(subject):
print("Processing subject: %s" % subject)
meg_subject_dir = op.join(config.meg_dir, subject)
# load epochs to reject ICA components
extension = '-epo'
fname_in = op.join(meg_subject_dir, config.base_fname.format(**locals()))
epochs = mne.read_epochs(fname_in, preload=True)
extension = '_cleaned-epo'
fname_out = op.join(meg_subject_dir, config.base_fname.format(**locals()))
print("Input: ", fname_in)
print("Output: ", fname_out)
# load first run of raw data for ecg /eog epochs
raw_list = list()
print(" Loading one run from raw data")
extension = config.runs[0] + '_sss_raw'
raw_fname_in = op.join(meg_subject_dir,
config.base_fname.format(**locals()))
raw = mne.io.read_raw_fif(raw_fname_in, preload=True)
# run ICA on MEG and EEG
picks_meg = mne.pick_types(raw.info,
meg=True,
eeg=False,
eog=False,
stim=False,
exclude='bads')
picks_eeg = mne.pick_types(raw.info,
meg=False,
eeg=True,
eog=False,
stim=False,
exclude='bads')
all_picks = {'meg': picks_meg, 'eeg': picks_eeg}
ch_types = []
if 'eeg' in config.ch_types:
ch_types.append('eeg')
if set(config.ch_types).intersection(('meg', 'grad', 'mag')):
ch_types.append('meg')
for ch_type in ch_types:
print(ch_type)
picks = all_picks[ch_type]
# Load ICA
fname_ica = op.join(
meg_subject_dir,
'{0}_{1}_{2}-ica.fif'.format(subject, config.study_name, ch_type))
print('Reading ICA: ' + fname_ica)
ica = read_ica(fname=fname_ica)
pick_ecg = mne.pick_types(raw.info,
meg=False,
eeg=False,
ecg=True,
eog=False)
# ECG
# either needs an ecg channel, or avg of the mags (i.e. MEG data)
if pick_ecg or ch_type == 'meg':
picks_ecg = np.concatenate([picks, pick_ecg])
# Create ecg epochs
if ch_type == 'meg':
reject = {
'mag': config.reject['mag'],
'grad': config.reject['grad']
}
elif ch_type == 'eeg':
reject = {'eeg': config.reject['eeg']}
ecg_epochs = create_ecg_epochs(raw,
picks=picks_ecg,
reject=reject,
baseline=(None, 0),
tmin=-0.5,
tmax=0.5)
ecg_average = ecg_epochs.average()
ecg_inds, scores = \
ica.find_bads_ecg(ecg_epochs, method='ctps',
threshold=config.ica_ctps_ecg_threshold)
del ecg_epochs
report_fname = \
'{0}_{1}_{2}-reject_ica.html'.format(subject,
config.study_name,
ch_type)
report_fname = op.join(meg_subject_dir, report_fname)
report = Report(report_fname, verbose=False)
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores,
exclude=ecg_inds,
show=config.plot),
captions=ch_type.upper() + ' - ECG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(ecg_average,
exclude=ecg_inds,
show=config.plot),
captions=ch_type.upper() + ' - ECG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(ecg_average,
exclude=ecg_inds,
show=config.plot),
captions=ch_type.upper() + ' - ECG - ' +
'Corrections')
else:
# XXX : to check when EEG only is processed
print('no ECG channel is present. Cannot automate ICAs component '
'detection for EOG!')
# EOG
pick_eog = mne.pick_types(raw.info,
meg=False,
eeg=False,
ecg=False,
eog=True)
if pick_eog.any():
print('using EOG channel')
picks_eog = np.concatenate([picks, pick_eog])
# Create eog epochs
eog_epochs = create_eog_epochs(raw,
picks=picks_eog,
reject=None,
baseline=(None, 0),
tmin=-0.5,
tmax=0.5)
eog_average = eog_epochs.average()
eog_inds, scores = ica.find_bads_eog(eog_epochs, threshold=3.0)
del eog_epochs
params = dict(exclude=eog_inds, show=config.plot)
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores, **params),
captions=ch_type.upper() + ' - EOG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(eog_average, **params),
captions=ch_type.upper() + ' - EOG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(eog_average, **params),
captions=ch_type.upper() + ' - EOG - ' +
'Corrections')
report.save(report_fname, overwrite=True, open_browser=False)
else:
print('no EOG channel is present. Cannot automate ICAs component '
'detection for EOG!')
ica_reject = (list(ecg_inds) + list(eog_inds) +
list(config.rejcomps_man[subject][ch_type]))
# now reject the components
print('Rejecting from %s: %s' % (ch_type, ica_reject))
epochs = ica.apply(epochs, exclude=ica_reject)
print('Saving cleaned epochs')
epochs.save(fname_out)
fig = ica.plot_overlay(raw, exclude=ica_reject, show=config.plot)
report.add_figs_to_section(fig,
captions=ch_type.upper() +
' - ALL(epochs) - Corrections')
if config.plot:
epochs.plot_image(combine='gfp',
group_by='type',
sigma=2.,
cmap="YlGnBu_r",
show=config.plot)
print("#" * 9 + f"\n# {subject} #\n" + "#" * 9)
# define filenames
path = op.join(bids_root, f"sub-{subject}", 'meg')
fname_raw = op.join(path, f"sub-{subject}_task-{task}_meg.fif")
fname_mp_raw = op.join(path, f"sub-{subject}_task-{task}_split-01_meg.fif")
fname_trans = op.join(path, f"sub-{subject}_task-{task}_{derivative}.fif")
try:
raw = mne.io.read_raw_fif(fname_raw)
except FileNotFoundError:
raw = mne.io.read_raw_fif(fname_mp_raw)
fname_raw = fname_mp_raw
if not op.exists(fname_trans) or redo:
mne.viz.set_3d_backend('mayavi')
mne.gui.coregistration(inst=fname_raw, subjects_dir=mri_subjects_dir)
mne.viz.set_3d_backend('pyvista')
p = mne.viz.plot_alignment(raw.info,
trans=fname_trans,
subject=f'sub-{subject}',
subjects_dir=mri_subjects_dir,
surfaces='head',
dig=True,
eeg=[],
meg='sensors',
coord_frame='meg')
rep_group.add_figs_to_section(p, f"{subject}", 'Coreg')
rep_group.save(fname_rep_group, overwrite=True, open_browser=False)
def check_apply_filter(raw, subject, filter_params=None,
notch_filter_params=None, plot_fmin=None,
plot_fmax=None, n_jobs=1, figsize=None, show=True,
report=None, img_scale=1.0):
"""Apply filtering and save diagnostic plots
Parameters
----------
raw : instance of Raw
Raw measurements to be decomposed.
subject : str
The name of the subject.
filter_params : dict | list of dict | None
The parametrs passed to raw.filter. If list, raw.filter will be
invoked len(filter_params) times. Defaults to None. If None, expands
to:
dict(l_freq=0.5, h_freq=200, n_jobs=n_jobs,
method='fft', l_trans_bandwidth=0.1, h_trans_bandwidth=0.5)
notch_filter_params : dict | list of dict | None
The parametrs passed to raw.notch_filter. Defaults to None.
If None, expands to:
n_jobs : int
The number of CPUs to use in parallel.
figsize : tuple of int
The figsize in inches. See matplotlib documentation.
show : bool
Show figure if True
scale_img : float
The scaling factor for the report. Defaults to 1.0.
report : instance of Report | None
The report object. If None, a new report will be generated.
"""
_default_filter_params = dict(l_freq=0.5, h_freq=200, n_jobs=n_jobs,
method='fft',
l_trans_bandwidth=0.1, h_trans_bandwidth=0.5)
if filter_params is None:
filter_params = _default_filter_params
if not isinstance(filter_params, (list, tuple)):
filter_params = [filter_params]
if notch_filter_params is None:
notch_filter_params = dict(freqs=(50, 100, 150, 200, 250,),
method='fft')
if report is None:
report = Report(subject)
notch_filter_params.update(n_jobs=n_jobs)
picks_list, n_rows, fig, axes = _prepare_filter_plot(raw, figsize)
iter_plot = zip(axes, picks_list)
fmin, fmax = plot_fmin or 0, plot_fmax or raw.info['lowpass'] + 20
###########################################################################
# plot before filter
for ax, (picks, ch_type) in iter_plot:
raw.plot_psd(fmin=fmin, fmax=fmax, ax=ax,
picks=picks, color='black', show=show)
first_line = ax.get_lines()[0]
first_line.set_label('{} - raw'.format(ch_type))
ax.set_ylabel('Power (dB)')
ax.grid(True)
ax.set_title(ch_type)
###########################################################################
# filter
# Note. It turns out to be safer to first run the notch filter.
# Ohterwise crazy notch resonance with some filter settings.
raw.notch_filter(**notch_filter_params)
for filter_params_ in filter_params:
final_filter_params_ = deepcopy(_default_filter_params)
final_filter_params_.update(filter_params_)
final_filter_params_.update({'n_jobs': n_jobs})
raw.filter(**final_filter_params_)
###########################################################################
# plot after filter
for ax, (picks, ch_type) in iter_plot:
raw.plot_psd(fmin=fmin, fmax=fmax, ax=ax,
picks=picks, color='red', show=show)
second_line = ax.get_lines()[1]
second_line.set_label('{} - filtered'.format(ch_type))
ax.legend(loc='best')
fig.suptitle('Multitaper PSD')
report.add_figs_to_section(fig, 'filter PSD spectra {}'.format(subject),
'FILTER', scale=img_scale)
return fig, report
def run_ica(subject, tsss=config.mf_st_duration):
print("Processing subject: %s" % subject)
meg_subject_dir = op.join(config.meg_dir, subject)
raw_list = list()
events_list = list()
print(" Loading raw data")
for run in config.runs:
extension = run + '_filt_raw'
raw_fname_in = op.join(meg_subject_dir,
config.base_fname.format(**locals()))
eve_fname = op.splitext(raw_fname_in)[0] + '-eve.fif'
print("Input: ", raw_fname_in, eve_fname)
raw = mne.io.read_raw_fif(raw_fname_in, preload=True)
events = mne.read_events(eve_fname)
events_list.append(events)
raw_list.append(raw)
print(' Concatenating runs')
raw, events = mne.concatenate_raws(raw_list, events_list=events_list)
if "eeg" in config.ch_types:
raw.set_eeg_reference(projection=True)
del raw_list
# don't reject based on EOG to keep blink artifacts
# in the ICA computation.
reject_ica = config.reject
if reject_ica and 'eog' in reject_ica:
reject_ica = dict(reject_ica)
del reject_ica['eog']
# produce high-pass filtered version of the data for ICA
raw_ica = raw.copy().filter(l_freq=1., h_freq=None)
print(" Running ICA...")
epochs_for_ica = mne.Epochs(raw_ica,
events,
config.event_id,
config.tmin,
config.tmax,
proj=True,
baseline=config.baseline,
preload=True,
decim=config.decim,
reject=reject_ica)
# run ICA on MEG and EEG
picks_meg = mne.pick_types(epochs_for_ica.info,
meg=True,
eeg=False,
eog=False,
stim=False,
exclude='bads')
picks_eeg = mne.pick_types(epochs_for_ica.info,
meg=False,
eeg=True,
eog=False,
stim=False,
exclude='bads')
all_picks = {'meg': picks_meg, 'eeg': picks_eeg}
# get number of components for ICA
# compute_rank requires 0.18
# n_components_meg = (mne.compute_rank(epochs_for_ica.copy()
# .pick_types(meg=True)))['meg']
n_components_meg = 0.999
n_components = {'meg': n_components_meg, 'eeg': 0.999}
ch_types = []
if 'eeg' in config.ch_types:
ch_types.append('eeg')
if set(config.ch_types).intersection(('meg', 'grad', 'mag')):
ch_types.append('meg')
for ch_type in ch_types:
print('Running ICA for ' + ch_type)
ica = ICA(method='fastica',
random_state=config.random_state,
n_components=n_components[ch_type])
picks = all_picks[ch_type]
ica.fit(epochs_for_ica, picks=picks, decim=config.ica_decim)
print(' Fit %d components (explaining at least %0.1f%% of the'
' variance)' % (ica.n_components_, 100 * n_components[ch_type]))
ica_fname = \
'{0}_{1}_{2}-ica.fif'.format(subject, config.study_name, ch_type)
ica_fname = op.join(meg_subject_dir, ica_fname)
ica.save(ica_fname)
if config.plot:
# plot ICA components to html report
report_fname = \
'{0}_{1}_{2}-ica.html'.format(subject, config.study_name,
ch_type)
report_fname = op.join(meg_subject_dir, report_fname)
report = Report(report_fname, verbose=False)
for idx in range(0, ica.n_components_):
figure = ica.plot_properties(epochs_for_ica,
picks=idx,
psd_args={'fmax': 60},
show=False)
report.add_figs_to_section(figure,
section=subject,
captions=(ch_type.upper() +
' - ICA Components'))
report.save(report_fname, overwrite=True, open_browser=False)
def run_ica(subject, session=None):
"""Run ICA."""
bids_basename = BIDSPath(subject=subject,
session=session,
task=config.get_task(),
acquisition=config.acq,
recording=config.rec,
space=config.space,
datatype=config.get_datatype(),
root=config.deriv_root,
check=False)
ica_fname = bids_basename.copy().update(suffix='ica', extension='.fif')
ica_components_fname = bids_basename.copy().update(processing='ica',
suffix='components',
extension='.tsv')
report_fname = bids_basename.copy().update(processing='ica',
suffix='report',
extension='.html')
msg = 'Loading and concatenating filtered continuous "raw" data'
logger.info(
gen_log_message(message=msg, step=4, subject=subject, session=session))
raw = load_and_concatenate_raws(bids_basename.copy().update(
suffix='raw', extension='.fif'))
# Produce high-pass filtered version of the data for ICA.
# filter_for_ica will concatenate all runs of our raw data.
# We don't have to worry about edge artifacts due to raw concatenation as
# we'll be epoching the data in the next step.
raw = filter_for_ica(raw, subject=subject, session=session)
epochs = make_epochs_for_ica(raw, subject=subject, session=session)
# Now actually perform ICA.
msg = 'Calculating ICA solution.'
logger.info(
gen_log_message(message=msg, step=4, subject=subject, session=session))
report = Report(info_fname=raw,
title='Independent Component Analysis (ICA)',
verbose=False)
ica = fit_ica(epochs, subject=subject, session=session)
ecg_ics = detect_ecg_artifacts(ica=ica,
raw=raw,
subject=subject,
session=session,
report=report)
eog_ics = detect_eog_artifacts(ica=ica,
raw=raw,
subject=subject,
session=session,
report=report)
# Save ICA to disk.
# We also store the automatically identified ECG- and EOG-related ICs.
msg = 'Saving ICA solution and detected artifacts to disk.'
logger.info(
gen_log_message(message=msg, step=4, subject=subject, session=session))
ica.exclude = sorted(set(ecg_ics + eog_ics))
ica.save(ica_fname)
# Create TSV.
tsv_data = pd.DataFrame(
dict(component=list(range(ica.n_components_)),
type=['ica'] * ica.n_components_,
description=['Independent Component'] * ica.n_components_,
status=['good'] * ica.n_components_,
status_description=['n/a'] * ica.n_components_))
for component in ecg_ics:
row_idx = tsv_data['component'] == component
tsv_data.loc[row_idx, 'status'] = 'bad'
tsv_data.loc[row_idx,
'status_description'] = 'Auto-detected ECG artifact'
for component in eog_ics:
row_idx = tsv_data['component'] == component
tsv_data.loc[row_idx, 'status'] = 'bad'
tsv_data.loc[row_idx,
'status_description'] = 'Auto-detected EOG artifact'
tsv_data.to_csv(ica_components_fname, sep='\t', index=False)
# Lastly, plot all ICs, and add them to the report for manual inspection.
msg = ('Adding diagnostic plots for all ICs the HTML report …')
logger.info(
gen_log_message(message=msg, step=4, subject=subject, session=session))
for component_num in tqdm(range(ica.n_components_)):
fig = ica.plot_properties(epochs,
picks=component_num,
psd_args={'fmax': 60},
show=False)
caption = f'IC {component_num}'
if component_num in eog_ics and component_num in ecg_ics:
caption += ' (EOG & ECG)'
elif component_num in eog_ics:
caption += ' (EOG)'
elif component_num in ecg_ics:
caption += ' (ECG)'
report.add_figs_to_section(fig,
section=f'sub-{subject}',
captions=caption)
open_browser = True if config.interactive else False
report.save(report_fname, overwrite=True, open_browser=open_browser)
msg = (f"ICA completed. Please carefully review the extracted ICs in the "
f"report {report_fname.basename}, and mark all components you wish "
f"to reject as 'bad' in {ica_components_fname.basename}")
logger.info(
gen_log_message(message=msg, step=4, subject=subject, session=session))
picks = mne.pick_types(raw.info, meg=True, eog=True)
for trial_type in trial_types:
epochs = mne.Epochs(raw, eve_dict[trial_type],
id_dict[trial_type],
tmin, tmax, picks=picks, verbose=False,
baseline=baseline, reject=reject,
preload=True, reject_tmin=rej_tmin,
reject_tmax=rej_tmax) # Check rejection settings
ica_check_evoked = \
epochs[ica_check_eves[trial_type]].average()
fig = ica.plot_overlay(ica_check_evoked, exclude=ica_excludes) # plot EOG cleaning
#fig.savefig(ica_check_img_folder + '/' +trial_type + session_no + '-savgol.png')
report.add_figs_to_section(fig, trial_type + session_no,
section=filter_string, scale=None, image_format='png')
plt.close(fig)
ica.exclude = ica_excludes
ica.apply(epochs, copy=False)
print('Resampling epochs...')
epochs.resample(epoch_params['rsl'],
n_jobs=1, verbose=False)
# Trust the defaults here
epochs.save(opj(epochs_folder,
trial_type + session_no + '-epo.fif'))
# Try if deleting the raw object helps here!
def test_render_report():
"""Test rendering -*.fif files for mne report."""
tempdir = _TempDir()
raw_fname_new = op.join(tempdir, 'temp_raw.fif')
ms_fname_new = op.join(tempdir, 'temp_ms_raw.fif')
event_fname_new = op.join(tempdir, 'temp_raw-eve.fif')
cov_fname_new = op.join(tempdir, 'temp_raw-cov.fif')
fwd_fname_new = op.join(tempdir, 'temp_raw-fwd.fif')
inv_fname_new = op.join(tempdir, 'temp_raw-inv.fif')
for a, b in [[raw_fname, raw_fname_new],
[ms_fname, ms_fname_new],
[event_fname, event_fname_new],
[cov_fname, cov_fname_new],
[fwd_fname, fwd_fname_new],
[inv_fname, inv_fname_new]]:
shutil.copyfile(a, b)
# create and add -epo.fif and -ave.fif files
epochs_fname = op.join(tempdir, 'temp-epo.fif')
evoked_fname = op.join(tempdir, 'temp-ave.fif')
# Speed it up by picking channels
raw = read_raw_fif(raw_fname_new, preload=True)
raw.pick_channels(['MEG 0111', 'MEG 0121'])
raw.del_proj()
epochs = Epochs(raw, read_events(event_fname), 1, -0.2, 0.2)
epochs.save(epochs_fname)
# This can take forever (stall Travis), so let's make it fast
# Also, make sure crop range is wide enough to avoid rendering bug
epochs.average().crop(0.1, 0.2).save(evoked_fname)
report = Report(info_fname=raw_fname_new, subjects_dir=subjects_dir)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
report.parse_folder(data_path=tempdir, on_error='raise')
assert_true(len(w) >= 1)
assert_true(repr(report))
# Check correct paths and filenames
fnames = glob.glob(op.join(tempdir, '*.fif'))
for fname in fnames:
assert_true(op.basename(fname) in
[op.basename(x) for x in report.fnames])
assert_true(''.join(report.html).find(op.basename(fname)) != -1)
assert_equal(len(report.fnames), len(fnames))
assert_equal(len(report.html), len(report.fnames))
assert_equal(len(report.fnames), len(report))
# Check saving functionality
report.data_path = tempdir
fname = op.join(tempdir, 'report.html')
report.save(fname=fname, open_browser=False)
assert_true(op.isfile(fname))
with open(fname, 'rb') as fid:
html = fid.read().decode('utf-8')
assert '(MaxShield on)' in html
assert_equal(len(report.html), len(fnames))
assert_equal(len(report.html), len(report.fnames))
# Check saving same report to new filename
report.save(fname=op.join(tempdir, 'report2.html'), open_browser=False)
assert_true(op.isfile(op.join(tempdir, 'report2.html')))
# Check overwriting file
report.save(fname=op.join(tempdir, 'report.html'), open_browser=False,
overwrite=True)
assert_true(op.isfile(op.join(tempdir, 'report.html')))
# Check pattern matching with multiple patterns
pattern = ['*raw.fif', '*eve.fif']
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
report.parse_folder(data_path=tempdir, pattern=pattern)
assert_true(len(w) >= 1)
assert_true(repr(report))
fnames = glob.glob(op.join(tempdir, '*.raw')) + \
glob.glob(op.join(tempdir, '*.raw'))
for fname in fnames:
assert_true(op.basename(fname) in
[op.basename(x) for x in report.fnames])
assert_true(''.join(report.html).find(op.basename(fname)) != -1)
assert_raises(ValueError, Report, image_format='foo')
assert_raises(ValueError, Report, image_format=None)
# SVG rendering
report = Report(info_fname=raw_fname_new, subjects_dir=subjects_dir,
image_format='svg')
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
report.parse_folder(data_path=tempdir, on_error='raise')
# ndarray support smoke test
report.add_figs_to_section(np.zeros((2, 3, 3)), 'caption', 'section')
def run_ica(subject, tsss=config.mf_st_duration):
print("Processing subject: %s" % subject)
meg_subject_dir = op.join(config.meg_dir, subject)
raw_list = list()
events_list = list()
print(" Loading raw data")
for run in config.runs:
extension = run + '_sss_raw'
raw_fname_in = op.join(meg_subject_dir,
config.base_fname.format(**locals()))
eve_fname = op.splitext(raw_fname_in)[0] + '-eve.fif'
print("Input: ", raw_fname_in, eve_fname)
raw = mne.io.read_raw_fif(raw_fname_in, preload=True)
events = mne.read_events(eve_fname)
events_list.append(events)
# XXX mark bads from any run – is it a problem for ICA
# if we just exclude the bads shared by all runs ?
if run:
bads = set(chain(*config.bads[subject].values()))
else:
bads = config.bads[subject]
raw.info['bads'] = bads
print("added bads: ", raw.info['bads'])
raw_list.append(raw)
print(' Concatenating runs')
raw, events = mne.concatenate_raws(raw_list, events_list=events_list)
raw.set_eeg_reference(projection=True)
del raw_list
# produce high-pass filtered version of the data for ICA
epochs_for_ica = mne.Epochs(raw.copy().filter(l_freq=1., h_freq=None),
events,
config.event_id,
config.tmin,
config.tmax,
proj=True,
baseline=config.baseline,
preload=True,
decim=config.decim,
reject=config.reject)
# run ICA on MEG and EEG
picks_meg = mne.pick_types(epochs_for_ica.info,
meg=True,
eeg=False,
eog=False,
stim=False,
exclude='bads')
picks_eeg = mne.pick_types(epochs_for_ica.info,
meg=False,
eeg=True,
eog=False,
stim=False,
exclude='bads')
all_picks = {'meg': picks_meg, 'eeg': picks_eeg}
# get number of components for ICA
# compute_rank requires 0.18
# n_components_meg = (mne.compute_rank(epochs_for_ica.copy()
# .pick_types(meg=True)))['meg']
n_components_meg = 0.999
n_components = {'meg': n_components_meg, 'eeg': 0.999}
for ch_type in ['meg', 'eeg']:
print('Running ICA for ' + ch_type)
ica = ICA(method='fastica',
random_state=config.random_state,
n_components=n_components[ch_type])
picks = all_picks[ch_type]
ica.fit(epochs_for_ica, picks=picks, decim=decim)
print(
' Fit %d components (explaining at least %0.1f%% of the variance)'
% (ica.n_components_, 100 * n_components[ch_type]))
ica_name = op.join(
meg_subject_dir,
'{0}_{1}_{2}-ica.fif'.format(subject, config.study_name, ch_type))
ica.save(ica_name)
if config.plot:
# plot ICA components to html report
from mne.report import Report
report_name = op.join(
meg_subject_dir,
'{0}_{1}_{2}-ica.html'.format(subject, config.study_name,
ch_type))
report = Report(report_name, verbose=False)
for figure in ica.plot_properties(epochs_for_ica,
picks=list(
range(0, ica.n_components_)),
psd_args={'fmax': 60},
show=False):
report.add_figs_to_section(figure,
section=subject,
captions=(ch_type.upper() +
' - ICA Components'))
# XXX how to close each figure within the loop to avoid
# runtime error: > 20 figures opened
report.save(report_name, overwrite=True, open_browser=False)
###########################################################################
####################### Cycle across channel type #########################
###########################################################################
for ch_type, picks in picks_by_type(raw.info, meg_combined=True):
# Apply ICA
ica = ICA(n_components=n_components,
method=method,
random_state=random_state)
# Fit it to the data we have
ica.fit(raw, picks=picks, decim=decim, reject=reject[ch_type])
# Plot ICA components location
for figure in ica.plot_components():
report.add_figs_to_section(figure,
captions=(ch_type.upper() +
' - ICA Components'),
section=nip)
# Save ICA
ica.save(data_ica_directory + nip + '/' + nip + '_' + ch_type +
'-ica.fif')
report.save(os.getcwd() + '/2_computeICA.html',
overwrite=True,
open_browser=False)
del ica, raw
# Save report
report.save(os.getcwd() + '/2_computeICA.html',
overwrite=True,
open_browser=False)
def apply_ica(subject, run, session):
deriv_path = config.get_subject_deriv_path(subject=subject,
session=session,
kind=config.get_kind())
bids_basename = make_bids_basename(subject=subject,
session=session,
task=config.get_task(),
acquisition=config.acq,
run=None,
processing=config.proc,
recording=config.rec,
space=config.space)
fname_in = op.join(deriv_path, bids_basename + '-epo.fif')
fname_out = op.join(deriv_path, bids_basename + '_cleaned-epo.fif')
# load epochs to reject ICA components
epochs = mne.read_epochs(fname_in, preload=True)
msg = f'Input: {fname_in}, Output: {fname_out}'
logger.info(
gen_log_message(message=msg, step=5, subject=subject, session=session))
# load first run of raw data for ecg / eog epochs
msg = 'Loading first run from raw data'
logger.debug(
gen_log_message(message=msg, step=5, subject=subject, session=session))
bids_basename = make_bids_basename(subject=subject,
session=session,
task=config.get_task(),
acquisition=config.acq,
run=config.get_runs()[0],
processing=config.proc,
recording=config.rec,
space=config.space)
if config.use_maxwell_filter:
raw_fname_in = op.join(deriv_path, bids_basename + '_sss_raw.fif')
else:
raw_fname_in = \
op.join(deriv_path, bids_basename + '_filt_raw.fif')
raw = mne.io.read_raw_fif(raw_fname_in, preload=True)
# run ICA on MEG and EEG
picks_meg = mne.pick_types(raw.info,
meg=True,
eeg=False,
eog=False,
stim=False,
exclude='bads')
picks_eeg = mne.pick_types(raw.info,
meg=False,
eeg=True,
eog=False,
stim=False,
exclude='bads')
all_picks = {'meg': picks_meg, 'eeg': picks_eeg}
for ch_type in config.ch_types:
report = None
picks = all_picks[ch_type]
# Load ICA
fname_ica = op.join(deriv_path, f'bids_basename_{ch_type}-ica.fif')
msg = f'Reading ICA: {fname_ica}'
logger.debug(
gen_log_message(message=msg,
step=5,
subject=subject,
session=session))
ica = read_ica(fname=fname_ica)
pick_ecg = mne.pick_types(raw.info,
meg=False,
eeg=False,
ecg=True,
eog=False)
# ECG
# either needs an ecg channel, or avg of the mags (i.e. MEG data)
ecg_inds = list()
if pick_ecg or ch_type == 'meg':
picks_ecg = np.concatenate([picks, pick_ecg])
# Create ecg epochs
if ch_type == 'meg':
reject = {
'mag': config.reject['mag'],
'grad': config.reject['grad']
}
elif ch_type == 'eeg':
reject = {'eeg': config.reject['eeg']}
ecg_epochs = create_ecg_epochs(raw,
picks=picks_ecg,
reject=reject,
baseline=(None, 0),
tmin=-0.5,
tmax=0.5)
ecg_average = ecg_epochs.average()
ecg_inds, scores = \
ica.find_bads_ecg(ecg_epochs, method='ctps',
threshold=config.ica_ctps_ecg_threshold)
del ecg_epochs
report_fname = \
op.join(deriv_path,
bids_basename + f'_{ch_type}-reject_ica.html')
report = Report(report_fname, verbose=False)
# Plot r score
report.add_figs_to_section(
ica.plot_scores(scores,
exclude=ecg_inds,
show=config.interactive),
captions=ch_type.upper() + ' - ECG - R scores')
# Plot source time course
report.add_figs_to_section(
ica.plot_sources(ecg_average,
exclude=ecg_inds,
show=config.interactive),
captions=ch_type.upper() + ' - ECG - Sources time course')
# Plot source time course
report.add_figs_to_section(
ica.plot_overlay(ecg_average,
exclude=ecg_inds,
show=config.interactive),
captions=ch_type.upper() + ' - ECG - Corrections')
else:
# XXX : to check when EEG only is processed
msg = ('No ECG channel is present. Cannot automate IC detection '
'for ECG')
logger.info(
gen_log_message(message=msg,
step=5,
subject=subject,
session=session))
# EOG
pick_eog = mne.pick_types(raw.info,
meg=False,
eeg=False,
ecg=False,
eog=True)
eog_inds = list()
if pick_eog.any():
msg = 'Using EOG channel'
logger.debug(
gen_log_message(message=msg,
step=5,
subject=subject,
session=session))
picks_eog = np.concatenate([picks, pick_eog])
# Create eog epochs
eog_epochs = create_eog_epochs(raw,
picks=picks_eog,
reject=None,
baseline=(None, 0),
tmin=-0.5,
tmax=0.5)
eog_average = eog_epochs.average()
eog_inds, scores = ica.find_bads_eog(eog_epochs, threshold=3.0)
del eog_epochs
params = dict(exclude=eog_inds, show=config.interactive)
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores, **params),
captions=ch_type.upper() + ' - EOG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(eog_average, **params),
captions=ch_type.upper() + ' - EOG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(eog_average, **params),
captions=ch_type.upper() + ' - EOG - ' +
'Corrections')
report.save(report_fname, overwrite=True, open_browser=False)
else:
msg = ('No EOG channel is present. Cannot automate IC detection '
'for EOG')
logger.info(
gen_log_message(message=msg,
step=5,
subject=subject,
session=session))
ica_reject = (list(ecg_inds) + list(eog_inds) +
list(config.rejcomps_man[subject][ch_type]))
# now reject the components
msg = f'Rejecting from {ch_type}: {ica_reject}'
logger.info(
gen_log_message(message=msg,
step=5,
subject=subject,
session=session))
epochs = ica.apply(epochs, exclude=ica_reject)
msg = 'Saving cleaned epochs'
logger.info(
gen_log_message(message=msg,
step=5,
subject=subject,
session=session))
epochs.save(fname_out)
if report is not None:
fig = ica.plot_overlay(raw,
exclude=ica_reject,
show=config.interactive)
report.add_figs_to_section(fig,
captions=ch_type.upper() +
' - ALL(epochs) - Corrections')
if config.interactive:
epochs.plot_image(combine='gfp',
group_by='type',
sigma=2.,
cmap="YlGnBu_r",
show=config.interactive)
def test_render_report():
"""Test rendering -*.fif files for mne report."""
tempdir = _TempDir()
raw_fname_new = op.join(tempdir, 'temp_raw.fif')
ms_fname_new = op.join(tempdir, 'temp_ms_raw.fif')
event_fname_new = op.join(tempdir, 'temp_raw-eve.fif')
cov_fname_new = op.join(tempdir, 'temp_raw-cov.fif')
fwd_fname_new = op.join(tempdir, 'temp_raw-fwd.fif')
inv_fname_new = op.join(tempdir, 'temp_raw-inv.fif')
for a, b in [[raw_fname, raw_fname_new], [ms_fname, ms_fname_new],
[event_fname, event_fname_new], [cov_fname, cov_fname_new],
[fwd_fname, fwd_fname_new], [inv_fname, inv_fname_new]]:
shutil.copyfile(a, b)
# create and add -epo.fif and -ave.fif files
epochs_fname = op.join(tempdir, 'temp-epo.fif')
evoked_fname = op.join(tempdir, 'temp-ave.fif')
# Speed it up by picking channels
raw = read_raw_fif(raw_fname_new, preload=True)
raw.pick_channels(['MEG 0111', 'MEG 0121'])
raw.del_proj()
epochs = Epochs(raw, read_events(event_fname), 1, -0.2, 0.2)
epochs.save(epochs_fname, overwrite=True)
# This can take forever (stall Travis), so let's make it fast
# Also, make sure crop range is wide enough to avoid rendering bug
epochs.average().crop(0.1, 0.2).save(evoked_fname)
report = Report(info_fname=raw_fname_new, subjects_dir=subjects_dir)
with pytest.warns(RuntimeWarning, match='Cannot render MRI'):
report.parse_folder(data_path=tempdir, on_error='raise')
assert repr(report)
# Check correct paths and filenames
fnames = glob.glob(op.join(tempdir, '*.fif'))
for fname in fnames:
assert (op.basename(fname) in [op.basename(x) for x in report.fnames])
assert (''.join(report.html).find(op.basename(fname)) != -1)
assert_equal(len(report.fnames), len(fnames))
assert_equal(len(report.html), len(report.fnames))
assert_equal(len(report.fnames), len(report))
# Check saving functionality
report.data_path = tempdir
fname = op.join(tempdir, 'report.html')
report.save(fname=fname, open_browser=False)
assert (op.isfile(fname))
with open(fname, 'rb') as fid:
html = fid.read().decode('utf-8')
assert '(MaxShield on)' in html
assert_equal(len(report.html), len(fnames))
assert_equal(len(report.html), len(report.fnames))
# Check saving same report to new filename
report.save(fname=op.join(tempdir, 'report2.html'), open_browser=False)
assert (op.isfile(op.join(tempdir, 'report2.html')))
# Check overwriting file
report.save(fname=op.join(tempdir, 'report.html'),
open_browser=False,
overwrite=True)
assert (op.isfile(op.join(tempdir, 'report.html')))
# Check pattern matching with multiple patterns
pattern = ['*raw.fif', '*eve.fif']
with pytest.warns(RuntimeWarning, match='Cannot render MRI'):
report.parse_folder(data_path=tempdir, pattern=pattern)
assert (repr(report))
fnames = glob.glob(op.join(tempdir, '*.raw')) + \
glob.glob(op.join(tempdir, '*.raw'))
for fname in fnames:
assert (op.basename(fname) in [op.basename(x) for x in report.fnames])
assert (''.join(report.html).find(op.basename(fname)) != -1)
pytest.raises(ValueError, Report, image_format='foo')
pytest.raises(ValueError, Report, image_format=None)
# SVG rendering
report = Report(info_fname=raw_fname_new,
subjects_dir=subjects_dir,
image_format='svg')
with pytest.warns(RuntimeWarning, match='Cannot render MRI'):
report.parse_folder(data_path=tempdir, on_error='raise')
# ndarray support smoke test
report.add_figs_to_section(np.zeros((2, 3, 3)), 'caption', 'section')
with pytest.raises(TypeError, match='Each fig must be a'):
report.add_figs_to_section('foo', 'caption', 'section')
with pytest.raises(TypeError, match='Each fig must be a'):
report.add_figs_to_section(['foo'], 'caption', 'section')
def test_render_report(renderer, tmpdir):
"""Test rendering *.fif files for mne report."""
tempdir = str(tmpdir)
raw_fname_new = op.join(tempdir, 'temp_raw.fif')
raw_fname_new_bids = op.join(tempdir, 'temp_meg.fif')
ms_fname_new = op.join(tempdir, 'temp_ms_raw.fif')
event_fname_new = op.join(tempdir, 'temp_raw-eve.fif')
cov_fname_new = op.join(tempdir, 'temp_raw-cov.fif')
proj_fname_new = op.join(tempdir, 'temp_ecg-proj.fif')
fwd_fname_new = op.join(tempdir, 'temp_raw-fwd.fif')
inv_fname_new = op.join(tempdir, 'temp_raw-inv.fif')
nirs_fname_new = op.join(tempdir, 'temp_raw-nirs.snirf')
for a, b in [[raw_fname, raw_fname_new], [raw_fname, raw_fname_new_bids],
[ms_fname, ms_fname_new], [event_fname, event_fname_new],
[cov_fname, cov_fname_new], [proj_fname, proj_fname_new],
[fwd_fname, fwd_fname_new], [inv_fname, inv_fname_new],
[nirs_fname, nirs_fname_new]]:
shutil.copyfile(a, b)
# create and add -epo.fif and -ave.fif files
epochs_fname = op.join(tempdir, 'temp-epo.fif')
evoked_fname = op.join(tempdir, 'temp-ave.fif')
# Speed it up by picking channels
raw = read_raw_fif(raw_fname_new, preload=True)
raw.pick_channels(['MEG 0111', 'MEG 0121', 'EEG 001', 'EEG 002'])
raw.del_proj()
raw.set_eeg_reference(projection=True)
epochs = Epochs(raw, read_events(event_fname), 1, -0.2, 0.2)
epochs.save(epochs_fname, overwrite=True)
# This can take forever, so let's make it fast
# Also, make sure crop range is wide enough to avoid rendering bug
evoked = epochs.average()
with pytest.warns(RuntimeWarning, match='tmax is not in Evoked'):
evoked.crop(0.1, 0.2)
evoked.save(evoked_fname)
report = Report(info_fname=raw_fname_new,
subjects_dir=subjects_dir,
projs=True)
with pytest.warns(RuntimeWarning, match='Cannot render MRI'):
report.parse_folder(data_path=tempdir, on_error='raise')
assert repr(report)
# Check correct paths and filenames
fnames = glob.glob(op.join(tempdir, '*.fif'))
fnames.extend(glob.glob(op.join(tempdir, '*.snirf')))
for fname in fnames:
assert (op.basename(fname) in [op.basename(x) for x in report.fnames])
assert (''.join(report.html).find(op.basename(fname)) != -1)
assert len(report.fnames) == len(fnames)
assert len(report.html) == len(report.fnames)
assert len(report.fnames) == len(report)
# Check saving functionality
report.data_path = tempdir
fname = op.join(tempdir, 'report.html')
report.save(fname=fname, open_browser=False)
assert (op.isfile(fname))
with open(fname, 'rb') as fid:
html = fid.read().decode('utf-8')
assert '(MaxShield on)' in html
# Projectors in Raw.info
assert '<h4>SSP Projectors</h4>' in html
# Projectors in `proj_fname_new`
assert f'SSP Projectors: {op.basename(proj_fname_new)}' in html
# Evoked in `evoked_fname`
assert f'Evoked: {op.basename(evoked_fname)} ({evoked.comment})' in html
assert 'Topomap (ch_type =' in html
assert f'Evoked: {op.basename(evoked_fname)} (GFPs)' in html
assert len(report.html) == len(fnames)
assert len(report.html) == len(report.fnames)
# Check saving same report to new filename
report.save(fname=op.join(tempdir, 'report2.html'), open_browser=False)
assert (op.isfile(op.join(tempdir, 'report2.html')))
# Check overwriting file
report.save(fname=op.join(tempdir, 'report.html'),
open_browser=False,
overwrite=True)
assert (op.isfile(op.join(tempdir, 'report.html')))
# Check pattern matching with multiple patterns
pattern = ['*raw.fif', '*eve.fif']
with pytest.warns(RuntimeWarning, match='Cannot render MRI'):
report.parse_folder(data_path=tempdir, pattern=pattern)
assert (repr(report))
fnames = glob.glob(op.join(tempdir, '*.raw')) + \
glob.glob(op.join(tempdir, '*.raw'))
for fname in fnames:
assert (op.basename(fname) in [op.basename(x) for x in report.fnames])
assert (''.join(report.html).find(op.basename(fname)) != -1)
pytest.raises(ValueError, Report, image_format='foo')
pytest.raises(ValueError, Report, image_format=None)
# SVG rendering
report = Report(info_fname=raw_fname_new,
subjects_dir=subjects_dir,
image_format='svg')
tempdir = pathlib.Path(tempdir) # test using pathlib.Path
with pytest.warns(RuntimeWarning, match='Cannot render MRI'):
report.parse_folder(data_path=tempdir, on_error='raise')
# ndarray support smoke test
report.add_figs_to_section(np.zeros((2, 3, 3)), 'caption', 'section')
with pytest.raises(TypeError, match='figure must be a'):
report.add_figs_to_section('foo', 'caption', 'section')
with pytest.raises(TypeError, match='figure must be a'):
report.add_figs_to_section(['foo'], 'caption', 'section')
def _generate_report(raw, ica, subj_name, basename, ecg_evoked, ecg_scores,
ecg_inds, ecg_ch_name, eog_evoked, eog_scores, eog_inds,
eog_ch_name):
"""Generate report for ica solution."""
import matplotlib.pyplot as plt
report = Report()
ica_title = 'Sources related to %s artifacts (red)'
is_show = False
# ------------------- Generate report for ECG ------------------------ #
fig_ecg_scores = ica.plot_scores(ecg_scores,
exclude=ecg_inds,
title=ica_title % 'ecg',
show=is_show)
# Pick the five largest ecg_scores and plot them
show_picks = np.abs(ecg_scores).argsort()[::-1][:5]
# Plot estimated latent sources given the unmixing matrix.
fig_ecg_ts = ica.plot_sources(raw,
show_picks,
exclude=ecg_inds,
title=ica_title % 'ecg' + ' in 30s',
start=0,
stop=30,
show=is_show)
# topoplot of unmixing matrix columns
fig_ecg_comp = ica.plot_components(show_picks,
title=ica_title % 'ecg',
colorbar=True,
show=is_show)
# plot ECG sources + selection
fig_ecg_src = ica.plot_sources(ecg_evoked, exclude=ecg_inds, show=is_show)
fig = [fig_ecg_scores, fig_ecg_ts, fig_ecg_comp, fig_ecg_src]
report.add_figs_to_section(fig,
captions=[
'Scores of ICs related to ECG',
'Time Series plots of ICs (ECG)',
'TopoMap of ICs (ECG)',
'Time-locked ECG sources'
],
section='ICA - ECG')
# -------------------- end generate report for ECG ---------------------- #
# -------------------------- Generate report for EoG -------------------- #
# check how many EoG ch we have
if set(eog_ch_name.split(',')).issubset(set(raw.info['ch_names'])):
fig_eog_scores = ica.plot_scores(eog_scores,
exclude=eog_inds,
title=ica_title % 'eog',
show=is_show)
report.add_figs_to_section(fig_eog_scores,
captions=['Scores of ICs related to EOG'],
section='ICA - EOG')
n_eogs = np.shape(eog_scores)
if len(n_eogs) > 1:
n_eog0 = n_eogs[0]
show_picks = [
np.abs(eog_scores[i][:]).argsort()[::-1][:5]
for i in range(n_eog0)
]
for i in range(n_eog0):
fig_eog_comp = ica.plot_components(show_picks[i][:],
title=ica_title % 'eog',
colorbar=True,
show=is_show)
fig = [fig_eog_comp]
report.add_figs_to_section(fig,
captions=['Scores of EoG ICs'],
section='ICA - EOG')
else:
show_picks = np.abs(eog_scores).argsort()[::-1][:5]
fig_eog_comp = ica.plot_components(show_picks,
title=ica_title % 'eog',
colorbar=True,
show=is_show)
fig = [fig_eog_comp]
report.add_figs_to_section(fig,
captions=['TopoMap of ICs (EOG)'],
section='ICA - EOG')
fig_eog_src = ica.plot_sources(eog_evoked,
exclude=eog_inds,
show=is_show)
fig = [fig_eog_src]
report.add_figs_to_section(fig,
captions=['Time-locked EOG sources'],
section='ICA - EOG')
# ----------------- end generate report for EoG ---------- #
ic_nums = list(range(ica.n_components_))
fig = ica.plot_components(picks=ic_nums, show=False)
report.add_figs_to_section(fig,
captions=['All IC topographies'],
section='ICA - muscles')
fig = ica.plot_sources(raw,
start=0,
stop=None,
show=False,
title='All IC time series')
report.add_figs_to_section(fig,
captions=['All IC time series'],
section='ICA - muscles')
psds_fig = []
captions_psd = []
ica_src = ica.get_sources(raw)
for i_ic in ic_nums:
psds, freqs = psd_multitaper(ica_src, picks=i_ic, fmax=140, tmax=60)
psds = np.squeeze(psds)
f, ax = plt.subplots()
psds = 10 * np.log10(psds)
ax.plot(freqs, psds, color='k')
ax.set(title='PSD',
xlabel='Frequency',
ylabel='Power Spectral Density (dB)')
psds_fig.append(f)
captions_psd.append('IC #' + str(i_ic))
report.add_figs_to_section(figs=psds_fig,
captions=captions_psd,
section='ICA - muscles')
report_filename = os.path.join(basename + "-report.html")
print(('******* ' + report_filename))
report.save(report_filename, open_browser=False, overwrite=True)
return report_filename
def gen_html_report(p, subjects, structurals, run_indices=None):
"""Generates HTML reports"""
import matplotlib.pyplot as plt
from ._mnefun import (_load_trans_to, plot_good_coils, _head_pos_annot,
_get_bem_src_trans, safe_inserter, _prebad,
_load_meg_bads, mlab_offscreen, _fix_raw_eog_cals,
_handle_dict, _get_t_window, plot_chpi_snr_raw)
if run_indices is None:
run_indices = [None] * len(subjects)
style = {'axes.spines.right': 'off', 'axes.spines.top': 'off',
'axes.grid': True}
time_kwargs = dict()
if 'time_unit' in mne.fixes._get_args(mne.viz.plot_evoked):
time_kwargs['time_unit'] = 's'
for si, subj in enumerate(subjects):
struc = structurals[si]
report = Report(verbose=False)
print(' Processing subject %s/%s (%s)'
% (si + 1, len(subjects), subj))
# raw
fnames = get_raw_fnames(p, subj, 'raw', erm=False, add_splits=False,
run_indices=run_indices[si])
for fname in fnames:
if not op.isfile(fname):
raise RuntimeError('Cannot create reports until raw data '
'exist, missing:\n%s' % fname)
raw = [read_raw_fif(fname, allow_maxshield='yes')
for fname in fnames]
_fix_raw_eog_cals(raw)
prebad_file = _prebad(p, subj)
for r in raw:
_load_meg_bads(r, prebad_file, disp=False)
raw = mne.concatenate_raws(raw)
# sss
sss_fnames = get_raw_fnames(p, subj, 'sss', False, False,
run_indices[si])
has_sss = all(op.isfile(fname) for fname in sss_fnames)
sss_info = mne.io.read_raw_fif(sss_fnames[0]) if has_sss else None
bad_file = get_bad_fname(p, subj)
if bad_file is not None:
sss_info.load_bad_channels(bad_file)
sss_info = sss_info.info
# pca
pca_fnames = get_raw_fnames(p, subj, 'pca', False, False,
run_indices[si])
has_pca = all(op.isfile(fname) for fname in pca_fnames)
# whitening and source localization
inv_dir = op.join(p.work_dir, subj, p.inverse_dir)
has_fwd = op.isfile(op.join(p.work_dir, subj, p.forward_dir,
subj + p.inv_tag + '-fwd.fif'))
with plt.style.context(style):
ljust = 25
#
# Head coils
#
section = 'Good HPI count'
if p.report_params.get('good_hpi_count', True) and p.movecomp:
t0 = time.time()
print((' %s ... ' % section).ljust(ljust), end='')
figs = list()
captions = list()
for fname in fnames:
_, _, fit_data = _head_pos_annot(p, fname, prefix=' ')
assert fit_data is not None
fig = plot_good_coils(fit_data, show=False)
fig.set_size_inches(10, 2)
fig.tight_layout()
figs.append(fig)
captions.append('%s: %s' % (section, op.split(fname)[-1]))
report.add_figs_to_section(figs, captions, section,
image_format='svg')
print('%5.1f sec' % ((time.time() - t0),))
else:
print(' %s skipped' % section)
#
# cHPI SNR
#
section = 'cHPI SNR'
if p.report_params.get('chpi_snr', True) and p.movecomp:
t0 = time.time()
print((' %s ... ' % section).ljust(ljust), end='')
figs = list()
captions = list()
for fname in fnames:
raw = mne.io.read_raw_fif(fname, allow_maxshield='yes')
t_window = _get_t_window(p, raw)
fig = plot_chpi_snr_raw(raw, t_window, show=False,
verbose=False)
fig.set_size_inches(10, 5)
fig.subplots_adjust(0.1, 0.1, 0.8, 0.95,
wspace=0, hspace=0.5)
figs.append(fig)
captions.append('%s: %s' % (section, op.split(fname)[-1]))
report.add_figs_to_section(figs, captions, section,
image_format='png') # svd too slow
print('%5.1f sec' % ((time.time() - t0),))
else:
print(' %s skipped' % section)
#
# Head movement
#
section = 'Head movement'
if p.report_params.get('head_movement', True) and p.movecomp:
print((' %s ... ' % section).ljust(ljust), end='')
t0 = time.time()
trans_to = _load_trans_to(p, subj, run_indices[si], raw)
figs = list()
captions = list()
for fname in fnames:
pos, _, _ = _head_pos_annot(p, fname, prefix=' ')
fig = plot_head_positions(pos=pos, destination=trans_to,
info=raw.info, show=False)
for ax in fig.axes[::2]:
"""
# tighten to the sensor limits
assert ax.lines[0].get_color() == (0., 0., 0., 1.)
mn, mx = np.inf, -np.inf
for line in ax.lines:
ydata = line.get_ydata()
if np.isfinite(ydata).any():
mn = min(np.nanmin(ydata), mn)
mx = max(np.nanmax(line.get_ydata()), mx)
"""
# always show at least 10cm span, and use tight limits
# if greater than that
coord = ax.lines[0].get_ydata()
for line in ax.lines:
if line.get_color() == 'r':
extra = line.get_ydata()[0]
mn, mx = coord.min(), coord.max()
md = (mn + mx) / 2.
mn = min([mn, md - 50., extra])
mx = max([mx, md + 50., extra])
assert (mn <= coord).all()
assert (mx >= coord).all()
ax.set_ylim(mn, mx)
fig.set_size_inches(10, 6)
fig.tight_layout()
figs.append(fig)
captions.append('%s: %s' % (section, op.split(fname)[-1]))
del trans_to
report.add_figs_to_section(figs, captions, section,
image_format='svg')
print('%5.1f sec' % ((time.time() - t0),))
else:
print(' %s skipped' % section)
#
# Raw segments
#
if op.isfile(pca_fnames[0]):
raw_pca = mne.concatenate_raws(
[mne.io.read_raw_fif(fname) for fname in pca_fnames])
section = 'Raw segments'
if p.report_params.get('raw_segments', True) and has_pca:
times = np.linspace(raw.times[0], raw.times[-1], 12)[1:-1]
raw_plot = list()
for t in times:
this_raw = raw_pca.copy().crop(t - 0.5, t + 0.5)
this_raw.load_data()
this_raw._data[:] -= np.mean(this_raw._data, axis=-1,
keepdims=True)
raw_plot.append(this_raw)
raw_plot = mne.concatenate_raws(raw_plot)
for key in ('BAD boundary', 'EDGE boundary'):
raw_plot.annotations.delete(
np.where(raw_plot.annotations.description == key)[0])
new_events = np.linspace(
0, int(round(10 * raw.info['sfreq'])) - 1, 11).astype(int)
new_events += raw_plot.first_samp
new_events = np.array([new_events,
np.zeros_like(new_events),
np.ones_like(new_events)]).T
fig = raw_plot.plot(group_by='selection', butterfly=True,
events=new_events)
fig.axes[0].lines[-1].set_zorder(10) # events
fig.axes[0].set(xticks=np.arange(0, len(times)) + 0.5)
xticklabels = ['%0.1f' % t for t in times]
fig.axes[0].set(xticklabels=xticklabels)
fig.axes[0].set(xlabel='Center of 1-second segments')
fig.axes[0].grid(False)
for _ in range(len(fig.axes) - 1):
fig.delaxes(fig.axes[-1])
fig.set(figheight=(fig.axes[0].get_yticks() != 0).sum(),
figwidth=12)
fig.subplots_adjust(0.0, 0.0, 1, 1, 0, 0)
report.add_figs_to_section(fig, 'Processed', section,
image_format='png') # svg too slow
#
# PSD
#
section = 'PSD'
if p.report_params.get('psd', True) and has_pca:
t0 = time.time()
print((' %s ... ' % section).ljust(ljust), end='')
if p.lp_trans == 'auto':
lp_trans = 0.25 * p.lp_cut
else:
lp_trans = p.lp_trans
n_fft = 8192
fmax = raw.info['lowpass']
figs = [raw.plot_psd(fmax=fmax, n_fft=n_fft, show=False)]
captions = ['%s: Raw' % section]
fmax = p.lp_cut + 2 * lp_trans
figs.append(raw.plot_psd(fmax=fmax, n_fft=n_fft, show=False))
captions.append('%s: Raw (zoomed)' % section)
if op.isfile(pca_fnames[0]):
figs.append(raw_pca.plot_psd(fmax=fmax, n_fft=n_fft,
show=False))
captions.append('%s: Processed' % section)
# shared y limits
n = len(figs[0].axes) // 2
for ai, axes in enumerate(list(zip(
*[f.axes for f in figs]))[:n]):
ylims = np.array([ax.get_ylim() for ax in axes])
ylims = [np.min(ylims[:, 0]), np.max(ylims[:, 1])]
for ax in axes:
ax.set_ylim(ylims)
ax.set(title='')
for fig in figs:
fig.set_size_inches(8, 8)
with warnings.catch_warnings(record=True):
fig.tight_layout()
report.add_figs_to_section(figs, captions, section,
image_format='svg')
print('%5.1f sec' % ((time.time() - t0),))
else:
print(' %s skipped' % section)
#
# SSP
#
section = 'SSP topomaps'
proj_nums = _handle_dict(p.proj_nums, subj)
if p.report_params.get('ssp_topomaps', True) and has_pca and \
np.sum(proj_nums) > 0:
assert sss_info is not None
t0 = time.time()
print((' %s ... ' % section).ljust(ljust), end='')
figs = []
comments = []
proj_files = get_proj_fnames(p, subj)
if p.proj_extra is not None:
comments.append('Custom')
projs = read_proj(op.join(p.work_dir, subj, p.pca_dir,
p.proj_extra))
figs.append(plot_projs_topomap(projs, info=sss_info,
show=False))
if any(proj_nums[0]): # ECG
if 'preproc_ecg-proj.fif' in proj_files:
comments.append('ECG')
figs.append(_proj_fig(op.join(
p.work_dir, subj, p.pca_dir,
'preproc_ecg-proj.fif'), sss_info,
proj_nums[0], p.proj_meg, 'ECG'))
if any(proj_nums[1]): # EOG
if 'preproc_blink-proj.fif' in proj_files:
comments.append('Blink')
figs.append(_proj_fig(op.join(
p.work_dir, subj, p.pca_dir,
'preproc_blink-proj.fif'), sss_info,
proj_nums[1], p.proj_meg, 'EOG'))
if any(proj_nums[2]): # ERM
if 'preproc_cont-proj.fif' in proj_files:
comments.append('Continuous')
figs.append(_proj_fig(op.join(
p.work_dir, subj, p.pca_dir,
'preproc_cont-proj.fif'), sss_info,
proj_nums[2], p.proj_meg, 'ERM'))
captions = [section] + [None] * (len(comments) - 1)
report.add_figs_to_section(
figs, captions, section, image_format='svg',
comments=comments)
print('%5.1f sec' % ((time.time() - t0),))
else:
print(' %s skipped' % section)
#
# Source alignment
#
section = 'Source alignment'
source_alignment = p.report_params.get('source_alignment', True)
if source_alignment is True or isinstance(source_alignment, dict) \
and has_sss and has_fwd:
assert sss_info is not None
kwargs = source_alignment
if isinstance(source_alignment, dict):
kwargs = dict(**source_alignment)
else:
assert source_alignment is True
kwargs = dict()
t0 = time.time()
print((' %s ... ' % section).ljust(ljust), end='')
captions = [section]
try:
from mayavi import mlab
except ImportError:
warnings.warn('Cannot plot alignment in Report, mayavi '
'could not be imported')
else:
subjects_dir = mne.utils.get_subjects_dir(
p.subjects_dir, raise_error=True)
bem, src, trans, _ = _get_bem_src_trans(
p, sss_info, subj, struc)
if len(mne.pick_types(sss_info)):
coord_frame = 'meg'
else:
coord_frame = 'head'
with mlab_offscreen():
fig = mlab.figure(bgcolor=(0., 0., 0.),
size=(1000, 1000))
for key, val in (
('info', sss_info),
('subjects_dir', subjects_dir), ('bem', bem),
('dig', True), ('coord_frame', coord_frame),
('show_axes', True), ('fig', fig),
('trans', trans), ('src', src)):
kwargs[key] = kwargs.get(key, val)
try_surfs = ['head-dense', 'head', 'inner_skull']
for surf in try_surfs:
try:
mne.viz.plot_alignment(surfaces=surf, **kwargs)
except Exception:
pass
else:
break
else:
raise RuntimeError('Could not plot any surface '
'for alignment:\n%s'
% (try_surfs,))
fig.scene.parallel_projection = True
view = list()
for ai, angle in enumerate([180, 90, 0]):
mlab.view(angle, 90, focalpoint=(0., 0., 0.),
distance=0.6, figure=fig)
view.append(mlab.screenshot(figure=fig))
mlab.close(fig)
view = trim_bg(np.concatenate(view, axis=1), 0)
report.add_figs_to_section(view, captions, section)
print('%5.1f sec' % ((time.time() - t0),))
else:
print(' %s skipped' % section)
#
# SNR
#
section = 'SNR'
if p.report_params.get('snr', None) is not None:
t0 = time.time()
print((' %s ... ' % section).ljust(ljust), end='')
snrs = p.report_params['snr']
if not isinstance(snrs, (list, tuple)):
snrs = [snrs]
for snr in snrs:
assert isinstance(snr, dict)
analysis = snr['analysis']
name = snr['name']
times = snr.get('times', [0.1])
inv_dir = op.join(p.work_dir, subj, p.inverse_dir)
fname_inv = op.join(inv_dir,
safe_inserter(snr['inv'], subj))
fname_evoked = op.join(inv_dir, '%s_%d%s_%s_%s-ave.fif'
% (analysis, p.lp_cut, p.inv_tag,
p.eq_tag, subj))
if not op.isfile(fname_inv):
print(' Missing inv: %s'
% op.basename(fname_inv), end='')
elif not op.isfile(fname_evoked):
print(' Missing evoked: %s'
% op.basename(fname_evoked), end='')
else:
inv = mne.minimum_norm.read_inverse_operator(fname_inv)
this_evoked = mne.read_evokeds(fname_evoked, name)
title = ('%s<br>%s["%s"] (N=%d)'
% (section, analysis, name, this_evoked.nave))
figs = plot_snr_estimate(this_evoked, inv,
verbose=False)
figs.axes[0].set_ylim(auto=True)
captions = ('%s<br>%s["%s"] (N=%d)'
% (section, analysis, name,
this_evoked.nave))
report.add_figs_to_section(
figs, captions, section=section,
image_format='svg')
print('%5.1f sec' % ((time.time() - t0),))
#
# BEM
#
section = 'BEM'
if p.report_params.get('bem', True) and has_fwd:
caption = '%s<br>%s' % (section, struc)
bem, src, trans, _ = _get_bem_src_trans(
p, raw.info, subj, struc)
if not bem['is_sphere']:
subjects_dir = mne.utils.get_subjects_dir(
p.subjects_dir, raise_error=True)
mri_fname = op.join(subjects_dir, struc, 'mri', 'T1.mgz')
if not op.isfile(mri_fname):
warnings.warn(
'Could not find MRI:\n%s\nIf using surrogate '
'subjects, use '
'params.report_params["bem"] = False to avoid '
'this warning', stacklevel=2)
else:
t0 = time.time()
print((' %s ... ' % section).ljust(ljust), end='')
report.add_bem_to_section(struc, caption, section,
decim=10, n_jobs=1,
subjects_dir=subjects_dir)
print('%5.1f sec' % ((time.time() - t0),))
else:
print(' %s skipped (sphere)' % section)
else:
print(' %s skipped' % section)
#
# Whitening
#
section = 'Whitening'
if p.report_params.get('whitening', False):
t0 = time.time()
print((' %s ... ' % section).ljust(ljust), end='')
whitenings = p.report_params['whitening']
if not isinstance(whitenings, (list, tuple)):
whitenings = [whitenings]
for whitening in whitenings:
assert isinstance(whitening, dict)
analysis = whitening['analysis']
name = whitening['name']
cov_name = op.join(p.work_dir, subj, p.cov_dir,
safe_inserter(whitening['cov'], subj))
# Load the inverse
fname_evoked = op.join(inv_dir, '%s_%d%s_%s_%s-ave.fif'
% (analysis, p.lp_cut, p.inv_tag,
p.eq_tag, subj))
if not op.isfile(cov_name):
print(' Missing cov: %s'
% op.basename(cov_name), end='')
elif not op.isfile(fname_evoked):
print(' Missing evoked: %s'
% op.basename(fname_evoked), end='')
else:
noise_cov = mne.read_cov(cov_name)
evo = mne.read_evokeds(fname_evoked, name)
captions = ('%s<br>%s["%s"] (N=%d)'
% (section, analysis, name, evo.nave))
fig = evo.plot_white(noise_cov, **time_kwargs)
report.add_figs_to_section(
fig, captions, section=section, image_format='png')
print('%5.1f sec' % ((time.time() - t0),))
else:
print(' %s skipped' % section)
#
# Sensor space plots
#
section = 'Responses'
if p.report_params.get('sensor', False):
t0 = time.time()
print((' %s ... ' % section).ljust(ljust), end='')
sensors = p.report_params['sensor']
if not isinstance(sensors, (list, tuple)):
sensors = [sensors]
for sensor in sensors:
assert isinstance(sensor, dict)
analysis = sensor['analysis']
name = sensor['name']
times = sensor.get('times', [0.1, 0.2])
fname_evoked = op.join(inv_dir, '%s_%d%s_%s_%s-ave.fif'
% (analysis, p.lp_cut, p.inv_tag,
p.eq_tag, subj))
if not op.isfile(fname_evoked):
print(' Missing evoked: %s'
% op.basename(fname_evoked), end='')
else:
this_evoked = mne.read_evokeds(fname_evoked, name)
figs = this_evoked.plot_joint(
times, show=False, ts_args=dict(**time_kwargs),
topomap_args=dict(outlines='head', **time_kwargs))
if not isinstance(figs, (list, tuple)):
figs = [figs]
captions = ('%s<br>%s["%s"] (N=%d)'
% (section, analysis, name,
this_evoked.nave))
captions = [captions] + [None] * (len(figs) - 1)
report.add_figs_to_section(
figs, captions, section=section,
image_format='png')
print('%5.1f sec' % ((time.time() - t0),))
#
# Source estimation
#
section = 'Source estimation'
if p.report_params.get('source', False):
t0 = time.time()
print((' %s ... ' % section).ljust(ljust), end='')
sources = p.report_params['source']
if not isinstance(sources, (list, tuple)):
sources = [sources]
for source in sources:
assert isinstance(source, dict)
analysis = source['analysis']
name = source['name']
times = source.get('times', [0.1, 0.2])
# Load the inverse
inv_dir = op.join(p.work_dir, subj, p.inverse_dir)
fname_inv = op.join(inv_dir,
safe_inserter(source['inv'], subj))
fname_evoked = op.join(inv_dir, '%s_%d%s_%s_%s-ave.fif'
% (analysis, p.lp_cut, p.inv_tag,
p.eq_tag, subj))
if not op.isfile(fname_inv):
print(' Missing inv: %s'
% op.basename(fname_inv), end='')
elif not op.isfile(fname_evoked):
print(' Missing evoked: %s'
% op.basename(fname_evoked), end='')
else:
inv = mne.minimum_norm.read_inverse_operator(fname_inv)
this_evoked = mne.read_evokeds(fname_evoked, name)
title = ('%s<br>%s["%s"] (N=%d)'
% (section, analysis, name, this_evoked.nave))
stc = mne.minimum_norm.apply_inverse(
this_evoked, inv,
lambda2=source.get('lambda2', 1. / 9.),
method=source.get('method', 'dSPM'))
stc = abs(stc)
# get clim using the reject_tmin <->reject_tmax
stc_crop = stc.copy().crop(
p.reject_tmin, p.reject_tmax)
clim = source.get('clim', dict(kind='percent',
lims=[82, 90, 98]))
out = mne.viz._3d._limits_to_control_points(
clim, stc_crop.data, 'viridis',
transparent=True) # dummy cmap
if isinstance(out[0], (list, tuple, np.ndarray)):
clim = out[0] # old MNE
else:
clim = out[1] # new MNE (0.17+)
clim = dict(kind='value', lims=clim)
if not isinstance(stc, mne.SourceEstimate):
print('Only surface source estimates currently '
'supported')
else:
subjects_dir = mne.utils.get_subjects_dir(
p.subjects_dir, raise_error=True)
with mlab_offscreen():
brain = stc.plot(
hemi=source.get('hemi', 'split'),
views=source.get('views', ['lat', 'med']),
size=source.get('size', (800, 600)),
colormap=source.get('colormap', 'viridis'),
transparent=source.get('transparent',
True),
foreground='k', background='w',
clim=clim, subjects_dir=subjects_dir,
)
imgs = list()
for t in times:
brain.set_time(t)
imgs.append(
trim_bg(brain.screenshot(), 255))
brain.close()
captions = ['%2.3f sec' % t for t in times]
report.add_slider_to_section(
imgs, captions=captions, section=section,
title=title, image_format='png')
print('%5.1f sec' % ((time.time() - t0),))
else:
print(' %s skipped' % section)
report_fname = get_report_fnames(p, subj)[0]
report.save(report_fname, open_browser=False, overwrite=True)
def run_ica(subject, session=None):
"""Run ICA."""
print("Processing subject: %s" % subject)
# Construct the search path for the data file. `sub` is mandatory
subject_path = op.join('sub-{}'.format(subject))
# `session` is optional
if session is not None:
subject_path = op.join(subject_path, 'ses-{}'.format(session))
subject_path = op.join(subject_path, config.kind)
fpath_deriv = op.join(config.bids_root, 'derivatives',
config.PIPELINE_NAME, subject_path)
raw_list = list()
print(" Loading raw data")
for run in config.runs:
# load first run of raw data for ecg /eog epochs
print(" Loading one run from raw data")
bids_basename = make_bids_basename(subject=subject,
session=session,
task=config.task,
acquisition=config.acq,
run=run,
processing=config.proc,
recording=config.rec,
space=config.space)
raw_fname_in = \
op.join(fpath_deriv, bids_basename + '_filt_raw.fif')
eve_fname = \
op.join(fpath_deriv, bids_basename + '-eve.fif')
print("Input: ", raw_fname_in, eve_fname)
raw = mne.io.read_raw_fif(raw_fname_in, preload=True)
raw_list.append(raw)
print(' Concatenating runs')
raw = mne.concatenate_raws(raw_list)
events, event_id = mne.events_from_annotations(raw)
if "eeg" in config.ch_types or config.kind == 'eeg':
raw.set_eeg_reference(projection=True)
del raw_list
# don't reject based on EOG to keep blink artifacts
# in the ICA computation.
reject_ica = config.reject
if reject_ica and 'eog' in reject_ica:
reject_ica = dict(reject_ica)
del reject_ica['eog']
# produce high-pass filtered version of the data for ICA
raw_ica = raw.copy().filter(l_freq=1., h_freq=None)
print(" Running ICA...")
epochs_for_ica = mne.Epochs(raw_ica,
events,
event_id,
config.tmin,
config.tmax,
proj=True,
baseline=config.baseline,
preload=True,
decim=config.decim,
reject=reject_ica)
# run ICA on MEG and EEG
picks_meg = mne.pick_types(epochs_for_ica.info,
meg=True,
eeg=False,
eog=False,
stim=False,
exclude='bads')
picks_eeg = mne.pick_types(epochs_for_ica.info,
meg=False,
eeg=True,
eog=False,
stim=False,
exclude='bads')
all_picks = {'meg': picks_meg, 'eeg': picks_eeg}
# get number of components for ICA
# compute_rank requires 0.18
# n_components_meg = (mne.compute_rank(epochs_for_ica.copy()
# .pick_types(meg=True)))['meg']
n_components_meg = 0.999
n_components = {'meg': n_components_meg, 'eeg': 0.999}
ch_types = []
if 'eeg' in config.ch_types or config.kind == 'eeg':
ch_types.append('eeg')
if set(config.ch_types).intersection(('meg', 'grad', 'mag')):
ch_types.append('meg')
for ch_type in ch_types:
print('Running ICA for ' + ch_type)
ica = ICA(method='fastica',
random_state=config.random_state,
n_components=n_components[ch_type])
picks = all_picks[ch_type]
if picks.size == 0:
ica.fit(epochs_for_ica, decim=config.ica_decim)
else:
ica.fit(epochs_for_ica, picks=picks, decim=config.ica_decim)
print(' Fit %d components (explaining at least %0.1f%% of the'
' variance)' % (ica.n_components_, 100 * n_components[ch_type]))
# Load ICA
ica_fname = \
op.join(fpath_deriv, bids_basename + '_%s-ica.fif' % ch_type)
ica.save(ica_fname)
if config.plot:
# plot ICA components to html report
report_fname = \
op.join(fpath_deriv,
bids_basename + '_%s-ica.html' % ch_type)
report = Report(report_fname, verbose=False)
for idx in range(0, ica.n_components_):
figure = ica.plot_properties(epochs_for_ica,
picks=idx,
psd_args={'fmax': 60},
show=False)
report.add_figs_to_section(figure,
section=subject,
captions=(ch_type.upper() +
' - ICA Components'))
report.save(report_fname, overwrite=True, open_browser=False)
#######################################################################
##################### Reject head-speaker artifacts ###################
#######################################################################
# Choose good channels (ch_types + misc)
picks_misc = np.concatenate([picks, mne.pick_types(raw.info, misc=True, meg=False)])
# Plot before transformation
misc_epochs = create_misc_epochs(raw, reject=reject, picks=picks_misc,
baseline=(None, 0), ch_name='MISC004',
tmax=0.5)
misc_average = misc_epochs.average()
report.add_figs_to_section(misc_average.plot(),
captions='MISC004 - Before time interpolation',
section=nip)
# Find MISC events
events = find_misc_events(raw, ch_name='MISC004', first_samp=0)
# Interpolate points in time around MISC event
raw.apply_function(interpolateAroundTimePoints, picks=picks, dtype=None,
n_jobs=1, events=events)
# Create misc-based epochs
misc_epochs = create_misc_epochs(raw, reject=reject, picks=picks_misc,
baseline=(None, 0), ch_name='MISC004',
tmax=0.5)
def preprocess_set_ICA_comp_fif_to_ts(fif_file, n_comp_exclude, l_freq, h_freq,
down_sfreq, is_sensor_space):
import os
import numpy as np
import sys
import mne
from mne.io import Raw
from mne.preprocessing import read_ica
from mne.report import Report
from nipype.utils.filemanip import split_filename as split_f
report = Report()
subj_path, basename, ext = split_f(fif_file)
(data_path, sbj_name) = os.path.split(subj_path)
print '*** SBJ %s' % sbj_name + '***'
# n_session = int(filter(str.isdigit, basename))
# print '*** n session = %d' % n_session + '***'
# Read raw
raw = Raw(fif_file, preload=True)
# select sensors
select_sensors = mne.pick_types(raw.info, meg=True, ref_meg=False,
exclude='bads')
picks_meeg = mne.pick_types(raw.info, meg=True, eeg=True,
exclude='bads')
# save electrode locations
sens_loc = [raw.info['chs'][i]['loc'][:3] for i in select_sensors]
sens_loc = np.array(sens_loc)
channel_coords_file = os.path.abspath("correct_channel_coords.txt")
np.savetxt(channel_coords_file, sens_loc, fmt='%s')
# save electrode names
sens_names = np.array([raw.ch_names[pos] for pos in select_sensors],
dtype="str")
channel_names_file = os.path.abspath("correct_channel_names.txt")
np.savetxt(channel_names_file, sens_names, fmt='%s')
# filtering + downsampling
# TODO n_jobs=8
raw.filter(l_freq=l_freq, h_freq=h_freq, picks=picks_meeg,
method='iir',n_jobs=8)
# raw.resample(sfreq=down_sfreq, npad=0)
# load ICA
is_show = False # visualization
ica_filename = os.path.join(subj_path, basename + '-ica.fif')
if os.path.exists(ica_filename) is False:
print "$$$ Warning, no %s found" % ica_filename
sys.exit()
else:
ica = read_ica(ica_filename)
# AP 210316
'''
print '*** ica.exclude before set components= ', ica.exclude
if n_comp_exclude.has_key(sbj_name):
print '*** ICA to be excluded for sbj %s ' % sbj_name + ' ' + str(n_comp_exclude[sbj_name]) + '***'
matrix_c_ICA = n_comp_exclude[sbj_name]
if not matrix_c_ICA[n_session-1]:
print 'no ICA'
else:
print '*** ICA to be excluded for session %d ' %n_session + ' ' + str(matrix_c_ICA[n_session-1]) + '***'
ica.exclude = matrix_c_ICA[n_session-1]
'''
# AP new dict
print '*** ica.exclude before set components= ', ica.exclude
if n_comp_exclude.has_key(sbj_name):
print '*** ICA to be excluded for sbj %s ' % sbj_name + ' ' + str(n_comp_exclude[sbj_name]) + '***'
session_dict = n_comp_exclude[sbj_name]
session_names = session_dict.keys()
componentes = []
for s in session_names:
if basename.find(s) > -1:
componentes = session_dict[s]
break
if len(componentes) == 0:
print '\n no ICA to be excluded \n'
else:
print '\n *** ICA to be excluded for session %s ' % s + \
' ' + str(componentes) + ' *** \n'
ica.exclude = componentes
print '\n *** ica.exclude after set components = ', ica.exclude
fig1 = ica.plot_overlay(raw, show=is_show)
report.add_figs_to_section(fig1, captions=['Signal'],
section='Signal quality')
report_filename = os.path.join(subj_path, basename + "-report_NEW.html")
print report_filename
report.save(report_filename, open_browser=False, overwrite=True)
# 3) apply ICA to raw data and save solution and report
# check the amplitudes do not change
# raw_ica_file = os.path.abspath(basename[:i_raw] + 'ica-raw.fif')
raw_ica_file = os.path.join(subj_path, basename + '-preproc-raw.fif')
raw_ica = ica.apply(raw)
raw_ica.resample(sfreq=down_sfreq, npad=0)
raw_ica.save(raw_ica_file, overwrite=True)
# save ICA solution
print ica_filename
ica.save(ica_filename)
# 4) save data
data_noIca, times = raw[select_sensors, :]
data, times = raw_ica[select_sensors, :]
print data.shape
print raw.info['sfreq']
ts_file = os.path.abspath(basename + '_ica.npy')
np.save(ts_file, data)
print '*** TS FILE ' + ts_file + '***'
print '*** raw.info[sfreq] = ' + str(raw.info['sfreq'])
if is_sensor_space:
return ts_file, channel_coords_file, channel_names_file, raw.info['sfreq']
else:
return raw_ica_file, channel_coords_file, channel_names_file, raw.info['sfreq']
# ++++++++++++++++++++++++++++++++++++++++++++++
#
# loop across files to process
#
#++++++++++++++++++++++++++++++++++++++++++++++
for fname in fnames:
# init object with config details
dcnn.load_gdcnn(fname)
name = op.basename(fname[:-4])
# print('>>> working on %s' % name)
# check IC labels (and apply corrections)
figs, captions = dcnn.plot_ica_traces(fname)
report.add_figs_to_section(figs,
captions=captions,
section='ICA components',
replace=True)
figs, captions = dcnn.plot_artifact_performance()
report.add_figs_to_section(figs,
captions=captions,
section='AR performance',
replace=True)
if do_save_h5:
report.save(fnreport + '.h5', overwrite=True)
report.save(fnreport + '.html', overwrite=True, open_browser=True)
logger.info("\nDONE MNE report: {}".format(fnreport))
def preprocess_ICA_fif_to_ts(fif_file, ECG_ch_name, EoG_ch_name, l_freq, h_freq, down_sfreq, variance, is_sensor_space, data_type):
import os
import numpy as np
import mne
from mne.io import Raw
from mne.preprocessing import ICA, read_ica
from mne.preprocessing import create_ecg_epochs, create_eog_epochs
from mne.report import Report
from nipype.utils.filemanip import split_filename as split_f
report = Report()
subj_path, basename, ext = split_f(fif_file)
(data_path, sbj_name) = os.path.split(subj_path)
print data_path
# Read raw
# If None the compensation in the data is not modified.
# If set to n, e.g. 3, apply gradient compensation of grade n as for
# CTF systems (compensation=3)
raw = Raw(fif_file, preload=True)
# select sensors
select_sensors = mne.pick_types(raw.info, meg=True, ref_meg=False,
exclude='bads')
picks_meeg = mne.pick_types(raw.info, meg=True, eeg=True, exclude='bads')
# save electrode locations
sens_loc = [raw.info['chs'][i]['loc'][:3] for i in select_sensors]
sens_loc = np.array(sens_loc)
channel_coords_file = os.path.abspath("correct_channel_coords.txt")
print '*** ' + channel_coords_file + '***'
np.savetxt(channel_coords_file, sens_loc, fmt='%s')
# save electrode names
sens_names = np.array([raw.ch_names[pos] for pos in select_sensors],dtype = "str")
# AP 21032016
# channel_names_file = os.path.join(data_path, "correct_channel_names.txt")
channel_names_file = os.path.abspath("correct_channel_names.txt")
np.savetxt(channel_names_file,sens_names , fmt = '%s')
### filtering + downsampling
raw.filter(l_freq=l_freq, h_freq=h_freq, picks=picks_meeg,
method='iir', n_jobs=8)
# raw.filter(l_freq = l_freq, h_freq = h_freq, picks = picks_meeg,
# method='iir')
# raw.resample(sfreq=down_sfreq, npad=0)
### 1) Fit ICA model using the FastICA algorithm
# Other available choices are `infomax` or `extended-infomax`
# We pass a float value between 0 and 1 to select n_components based on the
# percentage of variance explained by the PCA components.
ICA_title = 'Sources related to %s artifacts (red)'
is_show = False # visualization
reject = dict(mag=4e-12, grad=4000e-13)
# check if we have an ICA, if yes, we load it
ica_filename = os.path.join(subj_path,basename + "-ica.fif")
if os.path.exists(ica_filename) is False:
ica = ICA(n_components=variance, method='fastica', max_iter=500) # , max_iter=500
ica.fit(raw, picks=select_sensors, reject=reject) # decim = 3,
has_ICA = False
else:
has_ICA = True
print ica_filename + ' exists!!!'
ica = read_ica(ica_filename)
ica.exclude = []
# 2) identify bad components by analyzing latent sources.
# generate ECG epochs use detection via phase statistics
# if we just have exclude channels we jump these steps
# if len(ica.exclude)==0:
n_max_ecg = 3
n_max_eog = 2
# check if ECG_ch_name is in the raw channels
if ECG_ch_name in raw.info['ch_names']:
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5,
picks=select_sensors,
ch_name=ECG_ch_name)
# if not a synthetic ECG channel is created from cross channel average
else:
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5,
picks=select_sensors)
# ICA for ECG artifact
# threshold=0.25 come default
ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
print scores
print '\n len ecg_inds *** ' + str(len(ecg_inds)) + '***\n'
if len(ecg_inds) > 0:
ecg_evoked = ecg_epochs.average()
fig1 = ica.plot_scores(scores, exclude=ecg_inds,
title=ICA_title % 'ecg', show=is_show)
show_picks = np.abs(scores).argsort()[::-1][:5] # Pick the five largest scores and plot them
# Plot estimated latent sources given the unmixing matrix.
#ica.plot_sources(raw, show_picks, exclude=ecg_inds, title=ICA_title % 'ecg', show=is_show)
t_start = 0
t_stop = 30 # take the fist 30s
fig2 = ica.plot_sources(raw, show_picks, exclude=ecg_inds, title=ICA_title % 'ecg' + ' in 30s'
,start = t_start, stop = t_stop, show=is_show)
# topoplot of unmixing matrix columns
fig3 = ica.plot_components(show_picks, title=ICA_title % 'ecg', colorbar=True, show=is_show)
ecg_inds = ecg_inds[:n_max_ecg]
ica.exclude += ecg_inds
fig4 = ica.plot_sources(ecg_evoked, exclude=ecg_inds, show=is_show) # plot ECG sources + selection
fig5 = ica.plot_overlay(ecg_evoked, exclude=ecg_inds, show=is_show) # plot ECG cleaning
fig = [fig1, fig2, fig3, fig4, fig5]
report.add_figs_to_section(fig, captions=['Scores of ICs related to ECG',
'Time Series plots of ICs (ECG)',
'TopoMap of ICs (ECG)',
'Time-locked ECG sources',
'ECG overlay'], section = 'ICA - ECG')
# check if EoG_ch_name is in the raw channels
# if EoG_ch_name is empty if data_type is fif, ICA routine automatically looks for EEG61, EEG62
# otherwise if data_type is ds we jump this step
if not EoG_ch_name and data_type=='ds':
eog_inds = []
else:
if EoG_ch_name in raw.info['ch_names']:
### ICA for eye blink artifact - detect EOG by correlation
eog_inds, scores = ica.find_bads_eog(raw, ch_name = EoG_ch_name)
else:
eog_inds, scores = ica.find_bads_eog(raw)
if len(eog_inds) > 0:
fig6 = ica.plot_scores(scores, exclude=eog_inds, title=ICA_title % 'eog', show=is_show)
report.add_figs_to_section(fig6, captions=['Scores of ICs related to EOG'],
section = 'ICA - EOG')
# check how many EoG ch we have
rs = np.shape(scores)
if len(rs)>1:
rr = rs[0]
show_picks = [np.abs(scores[i][:]).argsort()[::-1][:5] for i in range(rr)]
for i in range(rr):
fig7 = ica.plot_sources(raw, show_picks[i][:], exclude=eog_inds,
start = raw.times[0], stop = raw.times[-1],
title=ICA_title % 'eog',show=is_show)
fig8 = ica.plot_components(show_picks[i][:], title=ICA_title % 'eog', colorbar=True, show=is_show) # ICA nel tempo
fig = [fig7, fig8]
report.add_figs_to_section(fig, captions=['Scores of ICs related to EOG',
'Time Series plots of ICs (EOG)'],
section = 'ICA - EOG')
else:
show_picks = np.abs(scores).argsort()[::-1][:5]
fig7 = ica.plot_sources(raw, show_picks, exclude=eog_inds, title=ICA_title % 'eog', show=is_show)
fig8 = ica.plot_components(show_picks, title=ICA_title % 'eog', colorbar=True, show=is_show)
fig = [fig7, fig8]
report.add_figs_to_section(fig, captions=['Time Series plots of ICs (EOG)',
'TopoMap of ICs (EOG)',],
section = 'ICA - EOG')
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
if EoG_ch_name in raw.info['ch_names']:
eog_evoked = create_eog_epochs(raw, tmin=-.5, tmax=.5, picks=select_sensors,
ch_name=EoG_ch_name).average()
else:
eog_evoked = create_eog_epochs(raw, tmin=-.5, tmax=.5, picks=select_sensors).average()
fig9 = ica.plot_sources(eog_evoked, exclude=eog_inds, show=is_show) # plot EOG sources + selection
fig10 = ica.plot_overlay(eog_evoked, exclude=eog_inds, show=is_show) # plot EOG cleaning
fig = [fig9, fig10]
report.add_figs_to_section(fig, captions=['Time-locked EOG sources',
'EOG overlay'], section = 'ICA - EOG')
fig11 = ica.plot_overlay(raw, show=is_show)
report.add_figs_to_section(fig11, captions=['Signal'], section = 'Signal quality')
### plot all topographies and time seris of the ICA components
n_ica_components = ica.mixing_matrix_.shape[1]
n_topo = 10;
n_fig = n_ica_components/n_topo;
n_plot = n_ica_components%n_topo;
print '*************** n_fig = ' + str(n_fig) + ' n_plot = ' + str(n_plot) + '********************'
fig = []
t_start = 0
t_stop = None # 60 if we want to take the fist 60s
for n in range(0,n_fig):
fig_tmp = ica.plot_components(range(n_topo*n,n_topo*(n+1)),title='ICA components', show=is_show)
fig.append(fig_tmp)
fig_tmp = ica.plot_sources(raw, range(n_topo*n,n_topo*(n+1)),
start = t_start, stop = t_stop,
title='ICA components')
fig.append(fig_tmp)
# if n_plot > 0:
# fig_tmp = ica.plot_components(range(n_fig*n_topo,n_ica_components), title='ICA components', show=is_show)
# fig.append(fig_tmp)
# fig_tmp = ica.plot_sources(raw, range(n_fig*n_topo,n_ica_components),
# start = t_start, stop = t_stop,
# title='ICA components')
# fig.append(fig_tmp)
#
# for n in range(0, len(fig)):
# report.add_figs_to_section(fig[n], captions=['TOPO'], section = 'ICA Topo Maps')
if n_plot > 5:
n_fig_l = n_plot//5
print '*************************** ' + str(n_fig_l) + ' *********************************'
for n in range(0,n_fig_l):
print range(n_fig*n_topo+5*n, n_fig*n_topo+5*(n+1))
fig_tmp = ica.plot_components(range(n_fig*n_topo+5*n, n_fig*n_topo+5*(n+1)),title='ICA components')
fig.append(fig_tmp)
fig_tmp = ica.plot_sources(raw, range(n_fig*n_topo+5*n, n_fig*n_topo+5*(n+1)),
start = t_start, stop = t_stop,
title='ICA components')
fig.append(fig_tmp)
print range(n_fig*n_topo+5*(n+1),n_ica_components)
fig_tmp = ica.plot_components(range(n_fig*n_topo+5*(n+1),n_ica_components), title='ICA components')
fig.append(fig_tmp)
fig_tmp = ica.plot_sources(raw, range(n_fig*n_topo+5*(n+1),n_ica_components),
start = t_start, stop = t_stop,
title='ICA components')
fig.append(fig_tmp)
for n in range(0, len(fig)):
report.add_figs_to_section(fig[n], captions=['TOPO'], section = 'ICA Topo Maps')
report_filename = os.path.join(subj_path,basename + "-report.html")
print '******* ' + report_filename
report.save(report_filename, open_browser=False, overwrite=True)
# 3) apply ICA to raw data and save solution and report
# check the amplitudes do not change
# raw_ica_file = os.path.abspath(basename[:i_raw] + 'ica-raw.fif')
raw_ica_file = os.path.join(subj_path, basename + '-preproc-raw.fif')
raw_ica = ica.apply(raw)
raw_ica.resample(sfreq=down_sfreq, npad=0)
raw_ica.save(raw_ica_file, overwrite=True)
# save ICA solution
print ica_filename
if has_ICA is False:
ica.save(ica_filename)
# 4) save data
data_noIca, times = raw[select_sensors, :]
data, times = raw_ica[select_sensors, :]
print data.shape
print raw.info['sfreq']
ts_file = os.path.abspath(basename + "_ica.npy")
np.save(ts_file, data)
print '***** TS FILE ' + ts_file + '*****'
if is_sensor_space:
return ts_file, channel_coords_file, channel_names_file, raw.info['sfreq']
else:
# return raw_ica, channel_coords_file, channel_names_file, raw.info['sfreq']
return raw_ica_file, channel_coords_file, channel_names_file, raw.info['sfreq']
def generate_report(raw, ica, subj_name, basename,
ecg_evoked, ecg_scores, ecg_inds, ecg_ch_name,
eog_evoked, eog_scores, eog_inds, eog_ch_name):
"""Generate report for ica solution"""
from mne.report import Report
import numpy as np
import os
report = Report()
ica_title = 'Sources related to %s artifacts (red)'
is_show = False
# ------------------- Generate report for ECG ------------------------ #
fig_ecg_scores = ica.plot_scores(ecg_scores,
exclude=ecg_inds,
title=ica_title % 'ecg',
show=is_show)
# Pick the five largest ecg_scores and plot them
show_picks = np.abs(ecg_scores).argsort()[::-1][:5]
# Plot estimated latent sources given the unmixing matrix.
fig_ecg_ts = ica.plot_sources(raw, show_picks, exclude=ecg_inds,
title=ica_title % 'ecg' + ' in 30s',
start=0, stop=30, show=is_show)
# topoplot of unmixing matrix columns
fig_ecg_comp = ica.plot_components(show_picks,
title=ica_title % 'ecg',
colorbar=True, show=is_show)
# plot ECG sources + selection
fig_ecg_src = ica.plot_sources(ecg_evoked, exclude=ecg_inds, show=is_show)
fig = [fig_ecg_scores, fig_ecg_ts, fig_ecg_comp, fig_ecg_src]
report.add_figs_to_section(fig,
captions=['Scores of ICs related to ECG',
'Time Series plots of ICs (ECG)',
'TopoMap of ICs (ECG)',
'Time-locked ECG sources'],
section='ICA - ECG')
# -------------------- end generate report for ECG ---------------------- #
# -------------------------- Generate report for EoG -------------------- #
# check how many EoG ch we have
if set(eog_ch_name.split(',')).issubset(set(raw.info['ch_names'])):
fig_eog_scores = ica.plot_scores(eog_scores, exclude=eog_inds,
title=ica_title % 'eog', show=is_show)
report.add_figs_to_section(fig_eog_scores,
captions=['Scores of ICs related to EOG'],
section='ICA - EOG')
n_eogs = np.shape(eog_scores)
if len(n_eogs) > 1:
n_eog0 = n_eogs[0]
show_picks = [np.abs(eog_scores[i][:]).argsort()[::-1][:5]
for i in range(n_eog0)]
for i in range(n_eog0):
fig_eog_comp = ica.plot_components(show_picks[i][:],
title=ica_title % 'eog',
colorbar=True, show=is_show)
fig = [fig_eog_comp]
report.add_figs_to_section(fig,
captions=['Scores of EoG ICs'],
section='ICA - EOG')
else:
show_picks = np.abs(eog_scores).argsort()[::-1][:5]
fig_eog_comp = ica.plot_components(show_picks,
title=ica_title % 'eog',
colorbar=True, show=is_show)
fig = [fig_eog_comp]
report.add_figs_to_section(fig, captions=['TopoMap of ICs (EOG)'],
section='ICA - EOG')
fig_eog_src = ica.plot_sources(eog_evoked,
exclude=eog_inds,
show=is_show)
fig = [fig_eog_src]
report.add_figs_to_section(fig, captions=['Time-locked EOG sources'],
section='ICA - EOG')
# ----------------- end generate report for EoG ---------- #
ic_nums = list(range(ica.n_components_))
fig = ica.plot_components(picks=ic_nums, show=False)
report.add_figs_to_section(fig, captions=['All IC topographies'],
section='ICA - muscles')
fig = ica.plot_sources(raw, start=0, stop=None, title='All IC time series')
report.add_figs_to_section(fig, captions=['All IC time series'],
section='ICA - muscles')
psds = []
captions_psd = []
ica_src = ica.get_sources(raw)
for i_ic in ic_nums:
fig = ica_src.plot_psd(tmax=60, picks=[i_ic], fmax=140, show=False)
fig.set_figheight(3)
fig.set_figwidth(5)
psds.append(fig)
captions_psd.append('IC #' + str(i_ic))
report.add_figs_to_section(figs=psds, captions=captions_psd,
section='ICA - muscles')
report_filename = os.path.join(basename + "-report.html")
print(('******* ' + report_filename))
report.save(report_filename, open_browser=False, overwrite=True)
return report_filename
def generateReport(raw, ica, subj_name, subj_path, basename,
ecg_evoked, ecg_scores, ecg_inds, ECG_ch_name,
eog_evoked, eog_scores, eog_inds, EoG_ch_name):
from mne.report import Report
import numpy as np
import os
import HTML
report = Report()
ICA_title = 'Sources related to %s artifacts (red)'
is_show = False # visualization
File_length = str(round(raw.times[-1], 0))
# report.add_htmls_to_section(htmls=name_html, captions='File path', section='General')
name_html = '<h4 style="text-align:left;"> Path: ' + subj_path + '/' + basename + '.fif' + '</h4>'
ex_comps_table = [['', 'ICs to exclude'],['ECG', ecg_inds], ['EOG', eog_inds]]
ex_comps_html = '<h4>' + HTML.table(ex_comps_table) + '</h4>'
File_length_html = '<h4 style="text-align:left;">' + 'File length: ' + File_length + ' seconds' + '</h4>'
report.add_htmls_to_section(htmls=name_html + File_length_html + ex_comps_html, captions='General info', section='General info')
# --------------------- Generate report for ECG ---------------------------------------- #
fig1 = ica.plot_scores(
ecg_scores, exclude=ecg_inds, title=ICA_title % 'ecg', show=is_show)
# Pick the five largest ecg_scores and plot them
show_picks = np.abs(ecg_scores).argsort()[::-1][:5]
# Plot estimated latent sources given the unmixing matrix.
# topoplot of unmixing matrix columns
fig2 = ica.plot_components(
show_picks, title=ICA_title % 'ecg', colorbar=True, show=is_show)
# plot ECG sources + selection
fig3 = ica.plot_sources(ecg_evoked, exclude=ecg_inds, show=is_show)
fig = [fig1, fig2, fig3]
report.add_figs_to_section(fig, captions=['Scores of ICs related to ECG',
'TopoMap of ICs (ECG)',
'Time-locked ECG sources'], section='ICA - ECG')
# ----------------------------------- end generate report for ECG ------------------------------- #
# --------------------------------- Generate report for EoG --------------------------------------------- #
# check how many EoG ch we have
if set(EoG_ch_name.split(',')).issubset(set(raw.info['ch_names'])):
fig4 = ica.plot_scores(
eog_scores, exclude=eog_inds, title=ICA_title % 'eog', show=is_show)
report.add_figs_to_section(fig4, captions=['Scores of ICs related to EOG'],
section='ICA - EOG')
rs = np.shape(eog_scores)
if len(rs) > 1:
rr = rs[0]
show_picks = [np.abs(eog_scores[i][:]).argsort()[::-1][:5]
for i in range(rr)]
for i in range(rr):
fig5 = ica.plot_components(show_picks[i][:], title=ICA_title % 'eog',
colorbar=True, show=is_show) # ICA nel tempo
fig = [fig5]
report.add_figs_to_section(fig, captions=['Scores of ICs related to EOG'],
section='ICA - EOG')
else:
show_picks = np.abs(eog_scores).argsort()[::-1][:5]
fig5 = ica.plot_components(
show_picks, title=ICA_title % 'eog', colorbar=True, show=is_show)
fig = [fig5]
report.add_figs_to_section(fig, captions=['TopoMap of ICs (EOG)', ],
section='ICA - EOG')
fig9 = ica.plot_sources(eog_evoked, exclude=eog_inds, show=is_show) # plot EOG sources + selection
# fig10 = ica.plot_overlay(eog_evoked, exclude=eog_inds, show=is_show) # plot EOG cleaning
# fig = [fig9, fig10]
fig = [fig9]
report.add_figs_to_section(fig, captions=['Time-locked EOG sources'], section = 'ICA - EOG')
# -------------------- end generate report for EoG -------------------------------------------------------- #
# import ipdb; ipdb.set_trace()
IC_nums = range(ica.n_components_)
fig = ica.plot_components(picks=IC_nums, show=False)
report.add_figs_to_section(fig, captions=['All IC topographies'], section='ICA - muscles')
psds = []
captions_psd = []
ica_src = ica.get_sources(raw)
for iIC in IC_nums:
fig = ica_src.plot_psd(tmax=None, picks=[iIC], fmax=140, show=False)
fig.set_figheight(3)
fig.set_figwidth(5)
psds.append(fig)
captions_psd.append('IC #' + str(iIC))
# report.add_slider_to_section(figs=psds, captions=captions_psd, title='', section='ICA - muscles')
report.add_figs_to_section(figs=psds, captions=captions_psd, section='ICA - muscles')
report_filename = os.path.join(subj_path, basename + "-report.html")
print '******* ' + report_filename
report.save(report_filename, open_browser=False, overwrite=True)
def test_render_report():
"""Test rendering -*.fif files for mne report.
"""
report = Report(info_fname=raw_fname)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
report.parse_folder(data_path=base_dir)
assert_true(len(w) == 1)
# Check correct paths and filenames
assert_true(raw_fname in report.fnames)
assert_true(event_name in report.fnames)
assert_true(report.data_path == base_dir)
# Check if all files were rendered in the report
fnames = glob.glob(op.join(base_dir, '*.fif'))
bad_name = 'test_ctf_comp_raw-eve.fif'
decrement = any(fname.endswith(bad_name) for fname in fnames)
fnames = [fname for fname in fnames if
fname.endswith(('-eve.fif', '-ave.fif', '-cov.fif',
'-sol.fif', '-fwd.fif', '-inv.fif',
'-src.fif', '-trans.fif', 'raw.fif',
'sss.fif', '-epo.fif')) and
not fname.endswith(bad_name)]
# last file above gets created by another test, and it shouldn't be there
for fname in fnames:
assert_true(''.join(report.html).find(op.basename(fname)) != -1)
assert_equal(len(report.fnames), len(fnames))
assert_equal(len(report.html), len(report.fnames))
evoked1 = read_evokeds(evoked1_fname)
evoked2 = read_evokeds(evoked2_fname)
assert_equal(len(report.fnames) + len(evoked1) + len(evoked2) - 2,
report.initial_id - decrement)
# Check saving functionality
report.data_path = tempdir
report.save(fname=op.join(tempdir, 'report.html'), open_browser=False)
assert_true(op.isfile(op.join(tempdir, 'report.html')))
# Check add_figs_to_section functionality
fig = evoked1[0].plot(show=False)
report.add_figs_to_section(figs=fig, # test non-list input
captions=['evoked response'])
assert_equal(len(report.html), len(fnames) + 1)
assert_equal(len(report.html), len(report.fnames))
assert_raises(ValueError, report.add_figs_to_section, figs=[fig, fig],
captions='H')
# Check add_images_to_section
img_fname = op.join(tempdir, 'testimage.png')
fig.savefig(img_fname)
report.add_images_to_section(fnames=[img_fname],
captions=['evoked response'])
assert_raises(ValueError, report.add_images_to_section,
fnames=[img_fname, img_fname], captions='H')
# Check deprecation of add_section
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
report.add_section(figs=fig,
captions=['evoked response'])
assert_true(w[0].category == DeprecationWarning)
# Check saving same report to new filename
report.save(fname=op.join(tempdir, 'report2.html'), open_browser=False)
assert_true(op.isfile(op.join(tempdir, 'report2.html')))
# Check overwriting file
report.save(fname=op.join(tempdir, 'report.html'), open_browser=False,
overwrite=True)
assert_true(op.isfile(op.join(tempdir, 'report.html')))
brain.set_time(time)
fig_temp[hemi].append(brain.save_montage(filename=None,
orientation='h'))
fig_temp[hemi] = np.concatenate(fig_temp[hemi], axis=0)
brain.close()
# Correct for different lengths
min_len = min(len(fig_temp['lh']), len(fig_temp['rh']))
fig_temp['lh'] = fig_temp['lh'][0:min_len, :, :]
fig_temp['rh'] = fig_temp['rh'][0:min_len, :, :]
# Collapse hemispheric images into one
fig_array = np.concatenate([fig_temp['lh'], fig_temp['rh']], axis=1)
# Plot figure and save it to report
fig, ax = subplots(figsize=(20, len(times) * 2))
ax.imshow(fig_array)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
report.add_figs_to_section(fig, section=nip,
captions=bloc + ' - ' + stimulus)
# Save HTML report
report.save(os.getcwd() + '/4_sourceReconstruction.html', overwrite=True,
open_browser=False)
def apply_ica(subject, run, session):
print("Processing subject: %s" % subject)
# Construct the search path for the data file. `sub` is mandatory
subject_path = op.join('sub-{}'.format(subject))
# `session` is optional
if session is not None:
subject_path = op.join(subject_path, 'ses-{}'.format(session))
subject_path = op.join(subject_path, config.kind)
bids_basename = make_bids_basename(subject=subject,
session=session,
task=config.task,
acquisition=config.acq,
run=None,
processing=config.proc,
recording=config.rec,
space=config.space
)
fpath_deriv = op.join(config.bids_root, 'derivatives',
config.PIPELINE_NAME, subject_path)
fname_in = \
op.join(fpath_deriv, bids_basename + '-epo.fif')
fname_out = \
op.join(fpath_deriv, bids_basename + '_cleaned-epo.fif')
# load epochs to reject ICA components
epochs = mne.read_epochs(fname_in, preload=True)
print("Input: ", fname_in)
print("Output: ", fname_out)
# load first run of raw data for ecg /eog epochs
print(" Loading one run from raw data")
bids_basename = make_bids_basename(subject=subject,
session=session,
task=config.task,
acquisition=config.acq,
run=config.runs[0],
processing=config.proc,
recording=config.rec,
space=config.space
)
if config.use_maxwell_filter:
raw_fname_in = \
op.join(fpath_deriv, bids_basename + '_sss_raw.fif')
else:
raw_fname_in = \
op.join(fpath_deriv, bids_basename + '_filt_raw.fif')
raw = mne.io.read_raw_fif(raw_fname_in, preload=True)
# run ICA on MEG and EEG
picks_meg = mne.pick_types(raw.info, meg=True, eeg=False,
eog=False, stim=False, exclude='bads')
picks_eeg = mne.pick_types(raw.info, meg=False, eeg=True,
eog=False, stim=False, exclude='bads')
all_picks = {'meg': picks_meg, 'eeg': picks_eeg}
for ch_type in config.ch_types:
report = None
print(ch_type)
picks = all_picks[ch_type]
# Load ICA
fname_ica = \
op.join(fpath_deriv, bids_basename + '_%s-ica.fif' % ch_type)
print('Reading ICA: ' + fname_ica)
ica = read_ica(fname=fname_ica)
pick_ecg = mne.pick_types(raw.info, meg=False, eeg=False,
ecg=True, eog=False)
# ECG
# either needs an ecg channel, or avg of the mags (i.e. MEG data)
ecg_inds = list()
if pick_ecg or ch_type == 'meg':
picks_ecg = np.concatenate([picks, pick_ecg])
# Create ecg epochs
if ch_type == 'meg':
reject = {'mag': config.reject['mag'],
'grad': config.reject['grad']}
elif ch_type == 'eeg':
reject = {'eeg': config.reject['eeg']}
ecg_epochs = create_ecg_epochs(raw, picks=picks_ecg, reject=reject,
baseline=(None, 0), tmin=-0.5,
tmax=0.5)
ecg_average = ecg_epochs.average()
ecg_inds, scores = \
ica.find_bads_ecg(ecg_epochs, method='ctps',
threshold=config.ica_ctps_ecg_threshold)
del ecg_epochs
report_fname = \
op.join(fpath_deriv,
bids_basename + '_%s-reject_ica.html' % ch_type)
report = Report(report_fname, verbose=False)
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores,
exclude=ecg_inds,
show=config.plot),
captions=ch_type.upper() + ' - ECG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(ecg_average,
exclude=ecg_inds,
show=config.plot),
captions=ch_type.upper() + ' - ECG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(ecg_average,
exclude=ecg_inds,
show=config.plot),
captions=ch_type.upper() + ' - ECG - ' +
'Corrections')
else:
# XXX : to check when EEG only is processed
print('no ECG channel is present. Cannot automate ICAs component '
'detection for ECG!')
# EOG
pick_eog = mne.pick_types(raw.info, meg=False, eeg=False,
ecg=False, eog=True)
eog_inds = list()
if pick_eog.any():
print('using EOG channel')
picks_eog = np.concatenate([picks, pick_eog])
# Create eog epochs
eog_epochs = create_eog_epochs(raw, picks=picks_eog, reject=None,
baseline=(None, 0), tmin=-0.5,
tmax=0.5)
eog_average = eog_epochs.average()
eog_inds, scores = ica.find_bads_eog(eog_epochs, threshold=3.0)
del eog_epochs
params = dict(exclude=eog_inds, show=config.plot)
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores, **params),
captions=ch_type.upper() + ' - EOG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(eog_average, **params),
captions=ch_type.upper() + ' - EOG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(eog_average, **params),
captions=ch_type.upper() + ' - EOG - ' +
'Corrections')
report.save(report_fname, overwrite=True, open_browser=False)
else:
print('no EOG channel is present. Cannot automate ICAs component '
'detection for EOG!')
ica_reject = (list(ecg_inds) + list(eog_inds) +
list(config.rejcomps_man[subject][ch_type]))
# now reject the components
print('Rejecting from %s: %s' % (ch_type, ica_reject))
epochs = ica.apply(epochs, exclude=ica_reject)
print('Saving cleaned epochs')
epochs.save(fname_out)
if report is not None:
fig = ica.plot_overlay(raw, exclude=ica_reject, show=config.plot)
report.add_figs_to_section(fig, captions=ch_type.upper() +
' - ALL(epochs) - Corrections')
if config.plot:
epochs.plot_image(combine='gfp', group_by='type', sigma=2.,
cmap="YlGnBu_r", show=config.plot)
def compute_ica(raw, subject, n_components=0.99, picks=None, decim=None,
reject=None, ecg_tmin=-0.5, ecg_tmax=0.5, eog_tmin=-0.5,
eog_tmax=0.5, n_max_ecg=3, n_max_eog=1,
n_max_ecg_epochs=200, show=True, img_scale=1.0,
random_state=None, report=None, artifact_stats=None):
"""Run ICA in raw data
Parameters
----------,
raw : instance of Raw
Raw measurements to be decomposed.
subject : str
The name of the subject.
picks : array-like of int, shape(n_channels, ) | None
Channels to be included. This selection remains throughout the
initialized ICA solution. If None only good data channels are used.
Defaults to None.
n_components : int | float | None | 'rank'
The number of components used for ICA decomposition. If int, it must be
smaller then max_pca_components. If None, all PCA components will be
used. If float between 0 and 1 components can will be selected by the
cumulative percentage of explained variance.
If 'rank', the number of components equals the rank estimate.
Defaults to 0.99.
decim : int | None
Increment for selecting each nth time slice. If None, all samples
within ``start`` and ``stop`` are used. Defalts to None.
reject : dict | None
Rejection parameters based on peak to peak amplitude.
Valid keys are 'grad' | 'mag' | 'eeg' | 'eog' | 'ecg'.
If reject is None then no rejection is done. You should
use such parameters to reject big measurement artifacts
and not EOG for example. It only applies if `inst` is of type Raw.
Defaults to {'mag': 5e-12}
ecg_tmin : float
Start time before ECG event. Defaults to -0.5.
ecg_tmax : float
End time after ECG event. Defaults to 0.5.
eog_tmin : float
Start time before rog event. Defaults to -0.5.
eog_tmax : float
End time after rog event. Defaults to 0.5.
n_max_ecg : int | None
The maximum number of ECG components to exclude. Defaults to 3.
n_max_eog : int | None
The maximum number of EOG components to exclude. Defaults to 1.
n_max_ecg_epochs : int
The maximum number of ECG epochs to use for phase-consistency
estimation. Defaults to 200.
show : bool
Show figure if True
scale_img : float
The scaling factor for the report. Defaults to 1.0.
random_state : None | int | instance of np.random.RandomState
np.random.RandomState to initialize the FastICA estimation.
As the estimation is non-deterministic it can be useful to
fix the seed to have reproducible results. Defaults to None.
report : instance of Report | None
The report object. If None, a new report will be generated.
artifact_stats : None | dict
A dict that contains info on amplitude ranges of artifacts and
numbers of events, etc. by channel type.
Returns
-------
ica : instance of ICA
The ICA solution.
report : dict
A dict with an html report ('html') and artifact statistics ('stats').
"""
if report is None:
report = Report(subject=subject, title='ICA preprocessing')
if n_components == 'rank':
n_components = raw.estimate_rank(picks=picks)
ica = ICA(n_components=n_components, max_pca_components=None,
random_state=random_state, max_iter=256)
ica.fit(raw, picks=picks, decim=decim, reject=reject)
comment = []
for ch in ('mag', 'grad', 'eeg'):
if ch in ica:
comment += [ch.upper()]
if len(comment) > 0:
comment = '+'.join(comment) + ' '
else:
comment = ''
topo_ch_type = 'mag'
if 'GRAD' in comment and 'MAG' not in comment:
topo_ch_type = 'grad'
elif 'EEG' in comment:
topo_ch_type = 'eeg'
###########################################################################
# 2) identify bad components by analyzing latent sources.
title = '%s related to %s artifacts (red) ({})'.format(subject)
# generate ECG epochs use detection via phase statistics
reject_ = {'mag': 5e-12, 'grad': 5000e-13, 'eeg': 300e-6}
if reject is not None:
reject_.update(reject)
for ch_type in ['mag', 'grad', 'eeg']:
if ch_type not in ica:
reject_.pop(ch_type)
picks_ = np.array([raw.ch_names.index(k) for k in ica.ch_names])
if 'eeg' in ica:
if 'ecg' in raw:
picks_ = np.append(picks_,
pick_types(raw.info, meg=False, ecg=True)[0])
else:
logger.info('There is no ECG channel, trying to guess ECG from '
'magnetormeters')
if artifact_stats is None:
artifact_stats = dict()
ecg_epochs = create_ecg_epochs(raw, tmin=ecg_tmin, tmax=ecg_tmax,
keep_ecg=True, picks=picks_, reject=reject_)
n_ecg_epochs_found = len(ecg_epochs.events)
artifact_stats['ecg_n_events'] = n_ecg_epochs_found
n_max_ecg_epochs = min(n_max_ecg_epochs, n_ecg_epochs_found)
artifact_stats['ecg_n_used'] = n_max_ecg_epochs
sel_ecg_epochs = np.arange(n_ecg_epochs_found)
rng = np.random.RandomState(42)
rng.shuffle(sel_ecg_epochs)
ecg_ave = ecg_epochs.average()
report.add_figs_to_section(ecg_ave.plot(), 'ECG-full', 'artifacts')
ecg_epochs = ecg_epochs[sel_ecg_epochs[:n_max_ecg_epochs]]
ecg_ave = ecg_epochs.average()
report.add_figs_to_section(ecg_ave.plot(), 'ECG-used', 'artifacts')
_put_artifact_range(artifact_stats, ecg_ave, kind='ecg')
ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
if len(ecg_inds) > 0:
ecg_evoked = ecg_epochs.average()
del ecg_epochs
fig = ica.plot_scores(scores, exclude=ecg_inds, labels='ecg',
title='', show=show)
report.add_figs_to_section(fig, 'scores ({})'.format(subject),
section=comment + 'ECG',
scale=img_scale)
current_exclude = [e for e in ica.exclude] # issue #2608 MNE
fig = ica.plot_sources(raw, ecg_inds, exclude=ecg_inds,
title=title % ('components', 'ecg'), show=show)
report.add_figs_to_section(fig, 'sources ({})'.format(subject),
section=comment + 'ECG',
scale=img_scale)
ica.exclude = current_exclude
fig = ica.plot_components(ecg_inds, ch_type=topo_ch_type,
title='', colorbar=True, show=show)
report.add_figs_to_section(fig, title % ('sources', 'ecg'),
section=comment + 'ECG', scale=img_scale)
ica.exclude = current_exclude
ecg_inds = ecg_inds[:n_max_ecg]
ica.exclude += ecg_inds
fig = ica.plot_sources(ecg_evoked, exclude=ecg_inds, show=show)
report.add_figs_to_section(fig, 'evoked sources ({})'.format(subject),
section=comment + 'ECG',
scale=img_scale)
fig = ica.plot_overlay(ecg_evoked, exclude=ecg_inds, show=show)
report.add_figs_to_section(fig,
'rejection overlay ({})'.format(subject),
section=comment + 'ECG',
scale=img_scale)
# detect EOG by correlation
picks_eog = np.concatenate(
[picks_, pick_types(raw.info, meg=False, eeg=False, ecg=False,
eog=True)])
eog_epochs = create_eog_epochs(raw, tmin=eog_tmin, tmax=eog_tmax,
picks=picks_eog, reject=reject_)
artifact_stats['eog_n_events'] = len(eog_epochs.events)
artifact_stats['eog_n_used'] = artifact_stats['eog_n_events']
eog_ave = eog_epochs.average()
report.add_figs_to_section(eog_ave.plot(), 'EOG-used', 'artifacts')
_put_artifact_range(artifact_stats, eog_ave, kind='eog')
eog_inds = None
if len(eog_epochs.events) > 0:
eog_inds, scores = ica.find_bads_eog(eog_epochs)
if eog_inds is not None and len(eog_epochs.events) > 0:
fig = ica.plot_scores(scores, exclude=eog_inds, labels='eog',
show=show, title='')
report.add_figs_to_section(fig, 'scores ({})'.format(subject),
section=comment + 'EOG',
scale=img_scale)
current_exclude = [e for e in ica.exclude] # issue #2608 MNE
fig = ica.plot_sources(raw, eog_inds, exclude=ecg_inds,
title=title % ('sources', 'eog'), show=show)
report.add_figs_to_section(fig, 'sources', section=comment + 'EOG',
scale=img_scale)
ica.exclude = current_exclude
fig = ica.plot_components(eog_inds, ch_type=topo_ch_type,
title='', colorbar=True, show=show)
report.add_figs_to_section(fig, title % ('components', 'eog'),
section=comment + 'EOG', scale=img_scale)
ica.exclude = current_exclude
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
eog_evoked = eog_epochs.average()
fig = ica.plot_sources(eog_evoked, exclude=eog_inds, show=show)
report.add_figs_to_section(
fig, 'evoked sources ({})'.format(subject),
section=comment + 'EOG', scale=img_scale)
fig = ica.plot_overlay(eog_evoked, exclude=eog_inds, show=show)
report.add_figs_to_section(
fig, 'rejection overlay({})'.format(subject),
section=comment + 'EOG', scale=img_scale)
else:
del eog_epochs
# check the amplitudes do not change
if len(ica.exclude) > 0:
fig = ica.plot_overlay(raw, show=show) # EOG artifacts remain
html = _render_components_table(ica)
report.add_htmls_to_section(
html, captions='excluded components',
section='ICA rejection summary (%s)' % ch_type)
report.add_figs_to_section(
fig, 'rejection overlay({})'.format(subject),
section=comment + 'RAW', scale=img_scale)
return ica, dict(html=report, stats=artifact_stats)
def run_ica(subject, session=None):
"""Run ICA."""
deriv_path = config.get_subject_deriv_path(subject=subject,
session=session,
kind=config.get_kind())
bids_basename = BIDSPath(subject=subject,
session=session,
task=config.get_task(),
acquisition=config.acq,
recording=config.rec,
space=config.space,
prefix=deriv_path)
raw_list = list()
msg = 'Loading filtered raw data'
logger.info(
gen_log_message(message=msg, step=4, subject=subject, session=session))
for run in config.get_runs():
raw_fname_in = bids_basename.copy().update(run=run,
processing='filt',
kind=config.get_kind(),
extension='.fif')
raw = mne.io.read_raw_fif(raw_fname_in, preload=True)
raw_list.append(raw)
msg = 'Concatenating runs'
logger.info(
gen_log_message(message=msg, step=4, subject=subject, session=session))
raw = mne.concatenate_raws(raw_list)
events, event_id = mne.events_from_annotations(raw)
if config.get_kind() == 'eeg':
raw.set_eeg_reference(projection=True)
del raw_list
# don't reject based on EOG to keep blink artifacts
# in the ICA computation.
reject_ica = config.get_reject()
if reject_ica and 'eog' in reject_ica:
reject_ica = dict(reject_ica)
del reject_ica['eog']
# produce high-pass filtered version of the data for ICA
raw_ica = raw.copy().filter(l_freq=1., h_freq=None)
epochs_for_ica = mne.Epochs(raw_ica,
events,
event_id,
config.tmin,
config.tmax,
proj=True,
baseline=config.baseline,
preload=True,
decim=config.decim,
reject=reject_ica)
# get number of components for ICA
# compute_rank requires 0.18
# n_components_meg = (mne.compute_rank(epochs_for_ica.copy()
# .pick_types(meg=True)))['meg']
n_components_meg = 0.999
n_components = {'meg': n_components_meg, 'eeg': 0.999}
kind = config.get_kind()
msg = f'Running ICA for {kind}'
logger.info(
gen_log_message(message=msg, step=4, subject=subject, session=session))
if config.ica_algorithm == 'picard':
fit_params = dict(fastica_it=5)
elif config.ica_algorithm == 'extended_infomax':
fit_params = dict(extended=True)
elif config.ica_algorithm == 'fastica':
fit_params = None
ica = ICA(method=config.ica_algorithm,
random_state=config.random_state,
n_components=n_components[kind],
fit_params=fit_params,
max_iter=config.ica_max_iterations)
ica.fit(epochs_for_ica, decim=config.ica_decim)
msg = (f'Fit {ica.n_components_} components (explaining at least '
f'{100*n_components[kind]:.1f}% of the variance)')
logger.info(
gen_log_message(message=msg, step=4, subject=subject, session=session))
# Save ICA
ica_fname = bids_basename.copy().update(run=None,
kind=f'{kind}-ica',
extension='.fif')
ica.save(ica_fname)
if config.interactive:
# plot ICA components to html report
report_fname = bids_basename.copy().update(run=None,
kind=f'{kind}-ica',
extension='.html')
report = Report(report_fname, verbose=False)
for idx in range(0, ica.n_components_):
figure = ica.plot_properties(epochs_for_ica,
picks=idx,
psd_args={'fmax': 60},
show=False)
report.add_figs_to_section(figure,
section=subject,
captions=(kind.upper() +
' - ICA Components'))
report.save(report_fname, overwrite=True, open_browser=False)
def run_evoked(subject):
print("Processing subject: %s" % subject)
meg_subject_dir = op.join(config.meg_dir, subject)
# load epochs to reject ICA components
extension = '-epo'
fname_in = op.join(meg_subject_dir, config.base_fname.format(**locals()))
epochs = mne.read_epochs(fname_in, preload=True)
extension = 'cleaned-epo'
fname_out = op.join(meg_subject_dir, config.base_fname.format(**locals()))
print("Input: ", fname_in)
print("Output: ", fname_out)
# load first run of raw data for ecg /eog epochs
raw_list = list()
print(" Loading one run from raw data")
extension = config.runs[0] + '_sss_raw'
raw_fname_in = op.join(meg_subject_dir,
config.base_fname.format(**locals()))
raw = mne.io.read_raw_fif(raw_fname_in, preload=True)
# run ICA on MEG and EEG
picks_meg = mne.pick_types(raw.info,
meg=True,
eeg=False,
eog=False,
stim=False,
exclude='bads')
picks_eeg = mne.pick_types(raw.info,
meg=False,
eeg=True,
eog=False,
stim=False,
exclude='bads')
all_picks = {'meg': picks_meg, 'eeg': picks_eeg}
if config.eeg:
ch_types = ['meg', 'eeg']
else:
ch_types = ['meg']
for ch_type in ch_types:
print(ch_type)
picks = all_picks[ch_type]
# Load ICA
fname_ica = op.join(
meg_subject_dir,
'{0}_{1}_{2}-ica.fif'.format(subject, config.study_name, ch_type))
print('Reading ICA: ' + fname_ica)
ica = read_ica(fname=fname_ica)
pick_ecg = mne.pick_types(raw.info,
meg=False,
eeg=False,
ecg=True,
eog=False)
# ECG
# either needs an ecg channel, or avg of the mags (i.e. MEG data)
if pick_ecg or ch_type == 'meg':
picks_ecg = np.concatenate([picks, pick_ecg])
# Create ecg epochs
if ch_type == 'meg':
reject = {
'mag': config.reject['mag'],
'grad': config.reject['grad']
}
elif ch_type == 'eeg':
reject = {'eeg': config.reject['eeg']}
ecg_epochs = create_ecg_epochs(raw,
picks=picks_ecg,
reject=reject,
baseline=(None, 0),
tmin=-0.5,
tmax=0.5)
ecg_average = ecg_epochs.average()
# XXX I had to lower the threshold for ctps (default 0.25), otherwise it does not
# find any components
# check how this behaves on other data
ecg_inds, scores = ica.find_bads_ecg(ecg_epochs,
method='ctps',
threshold=0.1)
del ecg_epochs
report_name = op.join(
meg_subject_dir,
'{0}_{1}_{2}-reject_ica.html'.format(subject,
config.study_name,
ch_type))
report = Report(report_name, verbose=False)
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores,
exclude=ecg_inds),
captions=ch_type.upper() + ' - ECG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(ecg_average,
exclude=ecg_inds),
captions=ch_type.upper() + ' - ECG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(ecg_average,
exclude=ecg_inds),
captions=ch_type.upper() + ' - ECG - ' +
'Corrections')
else:
print('no ECG channel!')
# EOG
pick_eog = mne.pick_types(raw.info,
meg=False,
eeg=False,
ecg=False,
eog=True)
if pick_eog.any():
print('using EOG channel')
picks_eog = np.concatenate([picks, pick_eog])
# Create eog epochs
eog_epochs = create_eog_epochs(raw,
picks=picks_eog,
reject=None,
baseline=(None, 0),
tmin=-0.5,
tmax=0.5)
eog_average = eog_epochs.average()
eog_inds, scores = ica.find_bads_eog(eog_epochs)
del eog_epochs
# Plot r score
report.add_figs_to_section(ica.plot_scores(scores,
exclude=eog_inds),
captions=ch_type.upper() + ' - EOG - ' +
'R scores')
# Plot source time course
report.add_figs_to_section(ica.plot_sources(eog_average,
exclude=eog_inds),
captions=ch_type.upper() + ' - EOG - ' +
'Sources time course')
# Plot source time course
report.add_figs_to_section(ica.plot_overlay(eog_average,
exclude=eog_inds),
captions=ch_type.upper() + ' - EOG - ' +
'Corrections')
report.save(report_name, overwrite=True, open_browser=False)
else:
print('no EOG channel!')
ica_reject = (list(ecg_inds) + list(eog_inds) +
list(config.rejcomps_man[subject][ch_type]))
# now reject the components
print('Rejecting from ' + ch_type + ': ' + str(ica_reject))
epochs = ica.apply(epochs, exclude=ica_reject)
print('Saving epochs')
epochs.save(fname_out)
report.add_figs_to_section(ica.plot_overlay(raw.copy(),
exclude=ica_reject),
captions=ch_type.upper() +
' - ALL(epochs) - ' + 'Corrections')
if config.plot:
epochs.plot_image(combine='gfp',
group_by='type',
sigma=2.,
cmap="YlGnBu_r")
