def read_features(fname):
measures_classes = dict(inspect.getmembers(sys.modules['nice.measures']))
contents = h5_listdir(fname)
measures = list()
epochs = None
if 'nice/features/order' in contents:
measure_order = read_hdf5(fname,
title='nice/features/order',
slash='replace')
else:
measure_order = [k for k in contents if 'nice/measure/' in k]
if any('nice/data/epochs' in k for k in contents):
epochs = read_hdf5(fname, title='nice/data/epochs', slash='replace')
# MNE fix
if 'filename' in epochs['info']:
del (epochs['info']['filename'])
epochs = mne.EpochsArray(data=epochs.pop('_data'),
info=Info(epochs.pop('info')),
tmin=epochs.pop('tmin'),
event_id=epochs.pop('event_id'),
events=epochs.pop('events'),
reject=epochs.pop('reject'),
flat=epochs.pop('flat'))
# Read all PowerSpectralDensityEstimator estimators
estimators = [
k for k in contents
if 'nice/container/PowerSpectralDensityEstimator' in k
]
all_estimators = {}
for estimator_name in estimators:
estimator_comment = estimator_name.split('/')[-1]
this_estimator = read_psd_estimator(fname, comment=estimator_comment)
all_estimators[estimator_comment] = this_estimator
for content in measure_order:
_, _, my_class_name, comment = content.split('/')
my_class = measures_classes[my_class_name]
if issubclass(my_class, BaseTimeLocked):
if not epochs:
raise RuntimeError(
'Something weird has happened. You want to read a '
'measure that depends on epochs but '
'I could not find any epochs in the file you gave me.')
measures.append(my_class._read(fname, epochs, comment=comment))
elif issubclass(my_class, BasePowerSpectralDensity):
measures.append(
my_class._read(fname,
estimators=all_estimators,
comment=comment))
elif issubclass(my_class, BaseMeasure):
measures.append(my_class._read(fname, comment=comment))
else:
raise ValueError('Come on--this is not a Nice class!')
measures = Features(measures)
return measures
def _read_container(klass, fname, comment='default'):
data = read_hdf5(fname, _get_title(klass, comment), slash='replace')
init_params = {k: v for k, v in data.items() if not k.endswith('_')}
attrs = {k: v for k, v in data.items() if k.endswith('_')}
file_info = read_hdf5(fname, title='nice/data/ch_info', slash='replace')
if 'filename' in file_info:
del (file_info['filename'])
attrs['ch_info_'] = Info(file_info)
out = klass(**init_params)
for k, v in attrs.items():
if k.endswith('_'):
setattr(out, k, v)
return out
def _read_my_marker(klass, fname, comment='default'):
# MANDATORY: This method should work for any marker as it is now.
data = read_hdf5(fname, _get_title(klass, comment), slash='replace')
init_params = {k: v for k, v in data.items() if not k.endswith('_')}
attrs = {k: v for k, v in data.items() if k.endswith('_')}
file_info = read_hdf5(fname, title='nice/data/ch_info', slash='replace')
if 'filename' in file_info:
del (file_info['filename'])
attrs['ch_info_'] = Info(file_info)
out = klass(**init_params)
for k, v in attrs.items():
if k.endswith('_'):
setattr(out, k, v)
return out
def load_neighbours(paths, study=None, **kwargs):
chanlocs_dir = paths.get_path('chanpos', study=study)
if study in ['A', 'D', 'E']:
from mne.externals import h5io
full_fname = op.join(chanlocs_dir, 'neighbours.hdf5')
return h5io.read_hdf5(full_fname)
else:
from scipy.io import loadmat
full_fname = op.join(chanlocs_dir, 'neighbours.mat')
return loadmat(full_fname, squeeze_me=True)['neighbours']
def load_mapping(fname):
# create new instances
mapping = imwarp.DiffeomorphicMap(None, [])
affine = imaffine.AffineMap(None)
data = read_hdf5(fname + '.h5')
mapping.__dict__ = data.get(type(mapping).__name__)
affine.__dict__ = data.get(type(affine).__name__)
return mapping, affine
def read_auto_reject(fname):
"""Read AutoReject object.
Parameters
----------
fname : str
The filename where the AutoReject object is saved.
Returns
-------
ar : instance of autoreject.AutoReject
"""
state = read_hdf5(fname, title='autoreject')
ar = AutoReject()
ar.__setstate__(state)
return ar
def read_morph(fname):
if not fname.endswith('.h5'):
fname += '.h5'
morph_in = read_hdf5(fname)
morph = dict()
# create new instances
morph['mapping'] = imwarp.DiffeomorphicMap(None, [])
morph['mapping'].__dict__ = morph_in.get('mapping')
morph['affine'] = imaffine.AffineMap(None)
morph['affine'].__dict__ = morph_in.get('affine')
morph['affine_reg'] = morph_in.get('affine_reg')
morph['domain_shape'] = morph_in.get('domain_shape')
return morph
def read_auto_reject(fname):
"""Read AutoReject object.
Parameters
----------
fname : str
The filename where the AutoReject object is saved.
Returns
-------
ar : instance of autoreject.AutoReject
"""
state = read_hdf5(fname, title='autoreject')
init_kwargs = {param: state[param] for param in _INIT_PARAMS}
ar = AutoReject(**init_kwargs)
ar.__setstate__(state)
return ar
def test_chan_freq_clusters():
from mne import create_info
from mne.externals import h5io
import matplotlib.pyplot as plt
fname = op.join(data_dir, 'chan_alpha_range.hdf5')
data_dict = h5io.read_hdf5(fname)
try:
info = create_info(data_dict['dimcoords'][0],
sfreq=250.,
ch_types='eeg',
montage='easycap-M1')
except TypeError:
info = create_info(data_dict['dimcoords'][0],
sfreq=250.,
ch_types='eeg')
mntg = mne.channels.make_standard_montage('easycap-M1')
info.set_montage(mntg)
clst = Clusters(data_dict['stat'],
data_dict['clusters'],
data_dict['pvals'],
dimnames=data_dict['dimnames'],
dimcoords=data_dict['dimcoords'],
info=info,
description=data_dict['description'])
topo = clst.plot(cluster_idx=1, freq=(8, 8.5))
plt.close(topo.fig)
clst.clusters = None
topo = clst.plot(freq=(10, 11.5))
plt.close(topo.fig)
topo = clst.plot(freq=(10, 11.5), contours=4)
plt.close(topo.fig)
# multi axes:
topo = clst.plot(cluster_idx=1, freq=[8, 10])
assert len(topo.axes) == 2
plt.close(topo.fig)
marker_kwargs = dict(marker='+')
topo = clst.plot(cluster_idx=1, freq=[8, 10], mark_kwargs=marker_kwargs)
plt.close(topo.fig)
def read_connectivity(fname):
"""Read a Connectivity object from an HDF5 file.
Parameters
----------
fname : str
The name of the file to read the connectivity from. The extension '.h5'
will be appended if the given filename doesn't have it already.
Returns
-------
connectivity : instance of Connectivity
The Connectivity object that was stored in the file.
See Also
--------
Connectivity.save : For saving connectivity objects
"""
if not fname.endswith('.h5'):
fname += '.h5'
con_dict = read_hdf5(fname, title='conpy')
con_type = con_dict['type']
del con_dict['type']
if con_type == 'all-to-all':
return VertexConnectivity(
data=con_dict['data'],
pairs=con_dict['pairs'],
vertices=con_dict['vertices'],
vertex_degree=con_dict['source_degree'],
subject=con_dict['subject'],
)
elif con_type == 'label':
labels = [Label(**l) for l in con_dict['labels']]
return LabelConnectivity(
data=con_dict['data'],
pairs=con_dict['pairs'],
labels=labels,
label_degree=con_dict['source_degree'],
subject=con_dict['subject'],
)
def read_ged(fname):
'''Read GED object from hdf5 file.
Parameters
----------
fname : str
File name or full file path.
Returns
-------
ged : sarna.ged.GED
Read GED object.
'''
from mne.externals import h5io
data_dict = h5io.read_hdf5(fname)
ged = GED(eig=data_dict['eig'],
filters=data_dict['filters'],
patterns=data_dict['patterns'],
description=data_dict['description'])
return ged
def read_cluster(fname, subjects_dir=None, src=None, info=None):
'''
Read standard Clusters .hdf5 file and return Clusters object.
You need to pass correct subjects_dir and src to `read_cluster` if your
results are in source space or correct info if your results are in channel
space.
Parameters
----------
fname : str
File path for the file to read.
subjects_dir : str, optional
Path to Freesurfer subjects directory.
src : mne.SourceSpaces, optional
Source space that the results are reprseneted in.
info : mne.Info, optional
Channel space that the results are respresented in.
Returns
-------
clst : Clusters
Cluster results read from file.
'''
from mne.externals import h5io
# subjects_dir = mne.utils.get_subjects_dir(subjects_dir, raise_error=True)
data_dict = h5io.read_hdf5(fname)
clst = Clusters(data_dict['stat'],
data_dict['clusters'],
data_dict['pvals'],
dimnames=data_dict['dimnames'],
dimcoords=data_dict['dimcoords'],
subject=data_dict['subject'],
subjects_dir=subjects_dir,
info=info,
src=src,
description=data_dict['description'])
return clst
def _load_stat(fname):
from mne.externals import h5io
stat = h5io.read_hdf5(fname)
if 'clusters' in stat:
from borsar.cluster import Clusters
if 'src' in fname:
info = None
study = (stat['description']['study']
if 'study' in stat['description'] else 'C')
# FIXME: src should be different when 'asy'
src = pth.paths.get_data('fwd', study=study)['src']
selection = stat['description']['selection']
subject = 'fsaverage_sym' if 'asy' in selection else 'fsaverage'
subjects_dir = pth.paths.get_path('subjects_dir')
else:
import sarna
src = None
study = stat['description']['study']
info = pth.paths.get_data('info', study=study)
sarna.utils.fix_channel_pos(info)
subject, subjects_dir = None, None
clst = Clusters(stat['stat'],
stat['clusters'],
stat['pvals'],
dimnames=stat['dimnames'],
dimcoords=stat['dimcoords'],
info=info,
src=src,
description=stat['description'],
subject=subject,
subjects_dir=subjects_dir)
return clst
else:
return stat
def _test_raw_reader(reader,
test_preloading=True,
test_kwargs=True,
boundary_decimal=2,
**kwargs):
"""Test reading, writing and slicing of raw classes.
Parameters
----------
reader : function
Function to test.
test_preloading : bool
Whether not preloading is implemented for the reader. If True, both
cases and memory mapping to file are tested.
test_kwargs : dict
Test _init_kwargs support.
boundary_decimal : int
Number of decimals up to which the boundary should match.
**kwargs :
Arguments for the reader. Note: Do not use preload as kwarg.
Use ``test_preloading`` instead.
Returns
-------
raw : instance of Raw
A preloaded Raw object.
"""
tempdir = _TempDir()
rng = np.random.RandomState(0)
montage = None
if "montage" in kwargs:
montage = kwargs['montage']
del kwargs['montage']
if test_preloading:
raw = reader(preload=True, **kwargs)
rep = repr(raw)
assert rep.count('<') == 1
assert rep.count('>') == 1
if montage is not None:
raw.set_montage(montage)
# don't assume the first is preloaded
buffer_fname = op.join(tempdir, 'buffer')
picks = rng.permutation(np.arange(len(raw.ch_names) - 1))[:10]
picks = np.append(picks, len(raw.ch_names) - 1) # test trigger channel
bnd = min(int(round(raw.buffer_size_sec * raw.info['sfreq'])),
raw.n_times)
slices = [
slice(0, bnd),
slice(bnd - 1, bnd),
slice(3, bnd),
slice(3, 300),
slice(None),
slice(1, bnd)
]
if raw.n_times >= 2 * bnd: # at least two complete blocks
slices += [
slice(bnd, 2 * bnd),
slice(bnd, bnd + 1),
slice(0, bnd + 100)
]
other_raws = [
reader(preload=buffer_fname, **kwargs),
reader(preload=False, **kwargs)
]
for sl_time in slices:
data1, times1 = raw[picks, sl_time]
for other_raw in other_raws:
data2, times2 = other_raw[picks, sl_time]
assert_allclose(data1, data2)
assert_allclose(times1, times2)
else:
raw = reader(**kwargs)
full_data = raw._data
assert raw.__class__.__name__ in repr(raw) # to test repr
assert raw.info.__class__.__name__ in repr(raw.info)
assert isinstance(raw.info['dig'], (type(None), list))
data_max = full_data.max()
data_min = full_data.min()
# these limits could be relaxed if we actually find data with
# huge values (in SI units)
assert data_max < 1e5
assert data_min > -1e5
if isinstance(raw.info['dig'], list):
for di, d in enumerate(raw.info['dig']):
assert isinstance(d, DigPoint), (di, d)
# gh-5604
meas_date = raw.info['meas_date']
assert meas_date is None or meas_date >= _stamp_to_dt((0, 0))
# test resetting raw
if test_kwargs:
raw2 = reader(**raw._init_kwargs)
assert set(raw.info.keys()) == set(raw2.info.keys())
assert_array_equal(raw.times, raw2.times)
# Test saving and reading
out_fname = op.join(tempdir, 'test_raw.fif')
raw = concatenate_raws([raw])
raw.save(out_fname, tmax=raw.times[-1], overwrite=True, buffer_size_sec=1)
raw3 = read_raw_fif(out_fname)
assert set(raw.info.keys()) == set(raw3.info.keys())
assert_allclose(raw3[0:20][0], full_data[0:20], rtol=1e-6,
atol=1e-20) # atol is very small but > 0
assert_array_almost_equal(raw.times, raw3.times)
assert not math.isnan(raw3.info['highpass'])
assert not math.isnan(raw3.info['lowpass'])
assert not math.isnan(raw.info['highpass'])
assert not math.isnan(raw.info['lowpass'])
assert raw3.info['kit_system_id'] == raw.info['kit_system_id']
# Make sure concatenation works
first_samp = raw.first_samp
last_samp = raw.last_samp
concat_raw = concatenate_raws([raw.copy(), raw])
assert concat_raw.n_times == 2 * raw.n_times
assert concat_raw.first_samp == first_samp
assert concat_raw.last_samp - last_samp + first_samp == last_samp + 1
idx = np.where(concat_raw.annotations.description == 'BAD boundary')[0]
expected_bad_boundary_onset = raw._last_time
assert_array_almost_equal(concat_raw.annotations.onset[idx],
expected_bad_boundary_onset,
decimal=boundary_decimal)
if raw.info['meas_id'] is not None:
for key in ['secs', 'usecs', 'version']:
assert raw.info['meas_id'][key] == raw3.info['meas_id'][key]
assert_array_equal(raw.info['meas_id']['machid'],
raw3.info['meas_id']['machid'])
assert isinstance(raw.annotations, Annotations)
# Make a "soft" test on units: They have to be valid SI units as in
# mne.io.meas_info.valid_units, but we accept any lower/upper case for now.
valid_units = _get_valid_units()
valid_units_lower = [unit.lower() for unit in valid_units]
if raw._orig_units is not None:
assert isinstance(raw._orig_units, dict)
for ch_name, unit in raw._orig_units.items():
assert unit.lower() in valid_units_lower, ch_name
# Test picking with and without preload
if test_preloading:
preload_kwargs = (dict(preload=True), dict(preload=False))
else:
preload_kwargs = (dict(), )
n_ch = len(raw.ch_names)
picks = rng.permutation(n_ch)
for preload_kwarg in preload_kwargs:
these_kwargs = kwargs.copy()
these_kwargs.update(preload_kwarg)
# don't use the same filename or it could create problems
if isinstance(these_kwargs.get('preload', None), str) and \
op.isfile(these_kwargs['preload']):
these_kwargs['preload'] += '-1'
whole_raw = reader(**these_kwargs)
print(whole_raw) # __repr__
assert n_ch >= 2
picks_1 = picks[:n_ch // 2]
picks_2 = picks[n_ch // 2:]
raw_1 = whole_raw.copy().pick(picks_1)
raw_2 = whole_raw.copy().pick(picks_2)
data, times = whole_raw[:]
data_1, times_1 = raw_1[:]
data_2, times_2 = raw_2[:]
assert_array_equal(times, times_1)
assert_array_equal(data[picks_1], data_1)
assert_array_equal(
times,
times_2,
)
assert_array_equal(data[picks_2], data_2)
# Make sure that writing info to h5 format
# (all fields should be compatible)
if check_version('h5py'):
fname_h5 = op.join(tempdir, 'info.h5')
with _writing_info_hdf5(raw.info):
write_hdf5(fname_h5, raw.info)
new_info = Info(read_hdf5(fname_h5))
assert object_diff(new_info, raw.info) == ''
return raw
def gen_covariances(p, subjects, run_indices, decim):
"""Generate forward solutions
Can only complete successfully once preprocessing is performed.
Parameters
----------
p : instance of Parameters
Analysis parameters.
subjects : list of str
Subject names to analyze (e.g., ['Eric_SoP_001', ...]).
run_indices : array-like | None
Run indices to include.
decim : list of int
The subject decimations.
"""
for si, subj in enumerate(subjects):
print(' Subject %2d/%2d...' % (si + 1, len(subjects)), end='')
cov_dir = op.join(p.work_dir, subj, p.cov_dir)
if not op.isdir(cov_dir):
os.mkdir(cov_dir)
has_rank_arg = 'rank' in get_args(compute_covariance)
kwargs = dict()
kwargs_erm = dict()
if p.cov_rank == 'full': # backward compat
if has_rank_arg:
kwargs['rank'] = 'full'
else:
if not has_rank_arg:
raise RuntimeError(
'There is no "rank" argument of compute_covariance, '
'you need to update MNE-Python')
if p.cov_rank is None:
assert p.compute_rank # otherwise this is weird
kwargs['rank'] = _compute_rank(p, subj, run_indices[si])
else:
kwargs['rank'] = p.cov_rank
kwargs_erm['rank'] = kwargs['rank']
if p.force_erm_cov_rank_full and has_rank_arg:
kwargs_erm['rank'] = 'full'
# Use the same thresholds we used for primary Epochs
if p.autoreject_thresholds:
reject = get_epochs_evokeds_fnames(p, subj, [])[0][1]
reject = reject.replace('-epo.fif', '-reject.h5')
reject = read_hdf5(reject)
else:
reject = _handle_dict(p.reject, subj)
flat = _handle_dict(p.flat, subj)
# Make empty room cov
if p.runs_empty:
if len(p.runs_empty) > 1:
raise ValueError('Too many empty rooms; undefined output!')
new_run = safe_inserter(p.runs_empty[0], subj)
empty_cov_name = op.join(
cov_dir, new_run + p.pca_extra + p.inv_tag + '-cov.fif')
empty_fif = get_raw_fnames(p, subj, 'pca', 'only', False)[0]
raw = read_raw_fif(empty_fif, preload=True)
raw.pick_types(meg=True, eog=True, exclude='bads')
use_reject, use_flat = _restrict_reject_flat(reject, flat, raw)
if 'eeg' in use_reject:
del use_reject['eeg']
if 'eeg' in use_flat:
del use_flat['eeg']
cov = compute_raw_covariance(raw,
reject=use_reject,
flat=use_flat,
method=p.cov_method,
**kwargs_erm)
write_cov(empty_cov_name, cov)
# Make evoked covariances
for ii, (inv_name, inv_run) in enumerate(zip(p.inv_names, p.inv_runs)):
cov_name = op.join(
cov_dir,
safe_inserter(inv_name, subj) + ('-%d' % p.lp_cut) +
p.inv_tag + '-cov.fif')
if run_indices[si] is None:
ridx = inv_run
else:
ridx = np.intersect1d(run_indices[si], inv_run)
# read in raw files
raw_fnames = get_raw_fnames(p, subj, 'pca', False, False, ridx)
raws = []
first_samps = []
last_samps = []
for raw_fname in raw_fnames:
raws.append(read_raw_fif(raw_fname, preload=False))
first_samps.append(raws[-1]._first_samps[0])
last_samps.append(raws[-1]._last_samps[-1])
_fix_raw_eog_cals(raws) # safe b/c cov only needs MEEG
raw = concatenate_raws(raws)
# read in events
events = _read_events(p, subj, ridx, raw)
if p.pick_events_cov is not None:
old_count = sum(len(e) for e in events)
if callable(p.pick_events_cov):
picker = p.pick_events_cov
else:
picker = p.pick_events_cov[ii]
events = picker(events)
new_count = len(events)
print(' Using %s/%s events for %s' %
(new_count, old_count, op.basename(cov_name)))
# create epochs
use_reject, use_flat = _restrict_reject_flat(reject, flat, raw)
baseline = _get_baseline(p)
epochs = Epochs(
raw,
events,
event_id=None,
tmin=baseline[0],
tmax=baseline[1],
baseline=(None, None),
proj=False,
reject=use_reject,
flat=use_flat,
preload=True,
decim=decim[si],
verbose='error', # ignore decim-related warnings
on_missing=p.on_missing,
reject_by_annotation=p.reject_epochs_by_annot)
epochs.pick_types(meg=True, eeg=True, exclude=[])
cov = compute_covariance(epochs, method=p.cov_method, **kwargs)
if kwargs.get('rank', None) not in (None, 'full'):
want_rank = sum(kwargs['rank'].values())
out_rank = compute_whitener(cov,
epochs.info,
return_rank=True,
verbose='error')[2]
if want_rank != out_rank:
# Hopefully we never hit this code path, but let's keep
# some debugging stuff around just in case
plot_cov(cov, epochs.info)
epochs_fnames, _ = get_epochs_evokeds_fnames(
p, subj, p.analyses)
epochs2 = read_epochs(epochs_fnames[1], preload=True)
idx = np.searchsorted(epochs.events[:, 0],
epochs2.events[:, 0])
assert len(np.unique(idx)) == len(idx)
epochs = epochs[idx]
assert np.array_equal(epochs.events[:, 0],
epochs2.events[:, 0])
epochs2.pick_types(meg=True, eeg=True, exclude=[])
import matplotlib.pyplot as plt
plt.figure()
for eps in (epochs, epochs2):
eps = eps.get_data().transpose([1, 0, 2])
eps = eps.reshape(len(eps), -1)
plt.plot(
np.log10(np.maximum(linalg.svdvals(eps), 1e-50)))
epochs.plot()
baseline = _get_baseline(p)
epochs2.copy().crop(*baseline).plot()
raise RuntimeError('Error computing rank')
write_cov(cov_name, cov)
print()
- MAE of one modality versus MAE of another modality.
"""
# Author: Denis A. Engemann <*****@*****.**>
#
# License: BSD (3-clause)
from itertools import combinations
import numpy as np
import pandas as pd
from mne.externals import h5io
IN_DEPENDENCE = './data/age_stacked_dependence_model-full.h5'
OUT_DEPENDENCE_1D = './data/age_stacked_dependence_model-full-1d.csv'
OUT_DEPENDENCE_2D = './data/age_stacked_dependence_model-full-2d.csv'
dependence = h5io.read_hdf5(IN_DEPENDENCE.format('model-full'))
# check source code for contour potting:
# https://github.com/scikit-learn/scikit-learn/blob/1495f6924/sklearn/inspection/partial_dependence.py#L619
if False:
preds = dependence[0]['2d'][
'MEG envelope diag--MEG mne_envelope_cross alpha'][0]
values = dependence[0]['2d'][
'MEG envelope diag--MEG mne_envelope_cross alpha'][1]
XX, YY = np.meshgrid(values[0], values[1])
Z = preds.T
min_pd = Z.min()
def _test_raw_reader(reader, test_preloading=True, test_kwargs=True,
boundary_decimal=2, test_scaling=True, test_rank=True,
**kwargs):
"""Test reading, writing and slicing of raw classes.
Parameters
----------
reader : function
Function to test.
test_preloading : bool
Whether not preloading is implemented for the reader. If True, both
cases and memory mapping to file are tested.
test_kwargs : dict
Test _init_kwargs support.
boundary_decimal : int
Number of decimals up to which the boundary should match.
**kwargs :
Arguments for the reader. Note: Do not use preload as kwarg.
Use ``test_preloading`` instead.
Returns
-------
raw : instance of Raw
A preloaded Raw object.
"""
tempdir = _TempDir()
rng = np.random.RandomState(0)
montage = None
if "montage" in kwargs:
montage = kwargs['montage']
del kwargs['montage']
if test_preloading:
raw = reader(preload=True, **kwargs)
rep = repr(raw)
assert rep.count('<') == 1
assert rep.count('>') == 1
if montage is not None:
raw.set_montage(montage)
# don't assume the first is preloaded
buffer_fname = op.join(tempdir, 'buffer')
picks = rng.permutation(np.arange(len(raw.ch_names) - 1))[:10]
picks = np.append(picks, len(raw.ch_names) - 1) # test trigger channel
bnd = min(int(round(raw.buffer_size_sec *
raw.info['sfreq'])), raw.n_times)
slices = [slice(0, bnd), slice(bnd - 1, bnd), slice(3, bnd),
slice(3, 300), slice(None), slice(1, bnd)]
if raw.n_times >= 2 * bnd: # at least two complete blocks
slices += [slice(bnd, 2 * bnd), slice(bnd, bnd + 1),
slice(0, bnd + 100)]
other_raws = [reader(preload=buffer_fname, **kwargs),
reader(preload=False, **kwargs)]
for sl_time in slices:
data1, times1 = raw[picks, sl_time]
for other_raw in other_raws:
data2, times2 = other_raw[picks, sl_time]
assert_allclose(data1, data2)
assert_allclose(times1, times2)
# test projection vs cals and data units
other_raw = reader(preload=False, **kwargs)
other_raw.del_proj()
eeg = meg = fnirs = False
if 'eeg' in raw:
eeg, atol = True, 1e-18
elif 'grad' in raw:
meg, atol = 'grad', 1e-24
elif 'mag' in raw:
meg, atol = 'mag', 1e-24
else:
assert 'fnirs_cw_amplitude' in raw, 'New channel type necessary?'
fnirs, atol = 'fnirs_cw_amplitude', 1e-10
picks = pick_types(
other_raw.info, meg=meg, eeg=eeg, fnirs=fnirs)
col_names = [other_raw.ch_names[pick] for pick in picks]
proj = np.ones((1, len(picks)))
proj /= proj.shape[1]
proj = Projection(
data=dict(data=proj, nrow=1, row_names=None,
col_names=col_names, ncol=len(picks)),
active=False)
assert len(other_raw.info['projs']) == 0
other_raw.add_proj(proj)
assert len(other_raw.info['projs']) == 1
# Orders of projector application, data loading, and reordering
# equivalent:
# 1. load->apply->get
data_load_apply_get = \
other_raw.copy().load_data().apply_proj().get_data(picks)
# 2. apply->get (and don't allow apply->pick)
apply = other_raw.copy().apply_proj()
data_apply_get = apply.get_data(picks)
data_apply_get_0 = apply.get_data(picks[0])[0]
with pytest.raises(RuntimeError, match='loaded'):
apply.copy().pick(picks[0]).get_data()
# 3. apply->load->get
data_apply_load_get = apply.copy().load_data().get_data(picks)
data_apply_load_get_0, data_apply_load_get_1 = \
apply.copy().load_data().pick(picks[:2]).get_data()
# 4. reorder->apply->load->get
all_picks = np.arange(len(other_raw.ch_names))
reord = np.concatenate((
picks[1::2],
picks[0::2],
np.setdiff1d(all_picks, picks)))
rev = np.argsort(reord)
assert_array_equal(reord[rev], all_picks)
assert_array_equal(rev[reord], all_picks)
reorder = other_raw.copy().pick(reord)
assert reorder.ch_names == [other_raw.ch_names[r] for r in reord]
assert reorder.ch_names[0] == other_raw.ch_names[picks[1]]
assert_allclose(reorder.get_data([0]), other_raw.get_data(picks[1]))
reorder_apply = reorder.copy().apply_proj()
assert reorder_apply.ch_names == reorder.ch_names
assert reorder_apply.ch_names[0] == apply.ch_names[picks[1]]
assert_allclose(reorder_apply.get_data([0]), apply.get_data(picks[1]),
atol=1e-18)
data_reorder_apply_load_get = \
reorder_apply.load_data().get_data(rev[:len(picks)])
data_reorder_apply_load_get_1 = \
reorder_apply.copy().load_data().pick([0]).get_data()[0]
assert reorder_apply.ch_names[0] == apply.ch_names[picks[1]]
assert (data_load_apply_get.shape ==
data_apply_get.shape ==
data_apply_load_get.shape ==
data_reorder_apply_load_get.shape)
del apply
# first check that our data are (probably) in the right units
data = data_load_apply_get.copy()
data = data - np.mean(data, axis=1, keepdims=True) # can be offsets
np.abs(data, out=data)
if test_scaling:
maxval = atol * 1e16
assert_array_less(data, maxval)
minval = atol * 1e6
assert_array_less(minval, np.median(data))
else:
atol = 1e-7 * np.median(data) # 1e-7 * MAD
# ranks should all be reduced by 1
if test_rank == 'less':
cmp = np.less
elif test_rank is False:
cmp = None
else: # anything else is like True or 'equal'
assert test_rank is True or test_rank == 'equal', test_rank
cmp = np.equal
rank_load_apply_get = np.linalg.matrix_rank(data_load_apply_get)
rank_apply_get = np.linalg.matrix_rank(data_apply_get)
rank_apply_load_get = np.linalg.matrix_rank(data_apply_load_get)
if cmp is not None:
assert cmp(rank_load_apply_get, len(col_names) - 1)
assert cmp(rank_apply_get, len(col_names) - 1)
assert cmp(rank_apply_load_get, len(col_names) - 1)
# and they should all match
t_kw = dict(
atol=atol, err_msg='before != after, likely _mult_cal_one prob')
assert_allclose(data_apply_get[0], data_apply_get_0, **t_kw)
assert_allclose(data_apply_load_get_1,
data_reorder_apply_load_get_1, **t_kw)
assert_allclose(data_load_apply_get[0], data_apply_load_get_0, **t_kw)
assert_allclose(data_load_apply_get, data_apply_get, **t_kw)
assert_allclose(data_load_apply_get, data_apply_load_get, **t_kw)
if 'eeg' in raw:
other_raw.del_proj()
direct = \
other_raw.copy().load_data().set_eeg_reference().get_data()
other_raw.set_eeg_reference(projection=True)
assert len(other_raw.info['projs']) == 1
this_proj = other_raw.info['projs'][0]['data']
assert this_proj['col_names'] == col_names
assert this_proj['data'].shape == proj['data']['data'].shape
assert_allclose(this_proj['data'], proj['data']['data'])
proj = other_raw.apply_proj().get_data()
assert_allclose(proj[picks], data_load_apply_get, atol=1e-10)
assert_allclose(proj, direct, atol=1e-10, err_msg=t_kw['err_msg'])
else:
raw = reader(**kwargs)
assert_named_constants(raw.info)
# smoke test for gh #9743
ids = [id(ch['loc']) for ch in raw.info['chs']]
assert len(set(ids)) == len(ids)
full_data = raw._data
assert raw.__class__.__name__ in repr(raw) # to test repr
assert raw.info.__class__.__name__ in repr(raw.info)
assert isinstance(raw.info['dig'], (type(None), list))
data_max = full_data.max()
data_min = full_data.min()
# these limits could be relaxed if we actually find data with
# huge values (in SI units)
assert data_max < 1e5
assert data_min > -1e5
if isinstance(raw.info['dig'], list):
for di, d in enumerate(raw.info['dig']):
assert isinstance(d, DigPoint), (di, d)
# gh-5604
meas_date = raw.info['meas_date']
assert meas_date is None or meas_date >= _stamp_to_dt((0, 0))
# test repr_html
assert 'Good channels' in raw.info._repr_html_()
# test resetting raw
if test_kwargs:
raw2 = reader(**raw._init_kwargs)
assert set(raw.info.keys()) == set(raw2.info.keys())
assert_array_equal(raw.times, raw2.times)
# Test saving and reading
out_fname = op.join(tempdir, 'test_raw.fif')
raw = concatenate_raws([raw])
raw.save(out_fname, tmax=raw.times[-1], overwrite=True, buffer_size_sec=1)
# Test saving with not correct extension
out_fname_h5 = op.join(tempdir, 'test_raw.h5')
with pytest.raises(IOError, match='raw must end with .fif or .fif.gz'):
raw.save(out_fname_h5)
raw3 = read_raw_fif(out_fname)
assert_named_constants(raw3.info)
assert set(raw.info.keys()) == set(raw3.info.keys())
assert_allclose(raw3[0:20][0], full_data[0:20], rtol=1e-6,
atol=1e-20) # atol is very small but > 0
assert_allclose(raw.times, raw3.times, atol=1e-6, rtol=1e-6)
assert not math.isnan(raw3.info['highpass'])
assert not math.isnan(raw3.info['lowpass'])
assert not math.isnan(raw.info['highpass'])
assert not math.isnan(raw.info['lowpass'])
assert raw3.info['kit_system_id'] == raw.info['kit_system_id']
# Make sure concatenation works
first_samp = raw.first_samp
last_samp = raw.last_samp
concat_raw = concatenate_raws([raw.copy(), raw])
assert concat_raw.n_times == 2 * raw.n_times
assert concat_raw.first_samp == first_samp
assert concat_raw.last_samp - last_samp + first_samp == last_samp + 1
idx = np.where(concat_raw.annotations.description == 'BAD boundary')[0]
expected_bad_boundary_onset = raw._last_time
assert_array_almost_equal(concat_raw.annotations.onset[idx],
expected_bad_boundary_onset,
decimal=boundary_decimal)
if raw.info['meas_id'] is not None:
for key in ['secs', 'usecs', 'version']:
assert raw.info['meas_id'][key] == raw3.info['meas_id'][key]
assert_array_equal(raw.info['meas_id']['machid'],
raw3.info['meas_id']['machid'])
assert isinstance(raw.annotations, Annotations)
# Make a "soft" test on units: They have to be valid SI units as in
# mne.io.meas_info.valid_units, but we accept any lower/upper case for now.
valid_units = _get_valid_units()
valid_units_lower = [unit.lower() for unit in valid_units]
if raw._orig_units is not None:
assert isinstance(raw._orig_units, dict)
for ch_name, unit in raw._orig_units.items():
assert unit.lower() in valid_units_lower, ch_name
# Test picking with and without preload
if test_preloading:
preload_kwargs = (dict(preload=True), dict(preload=False))
else:
preload_kwargs = (dict(),)
n_ch = len(raw.ch_names)
picks = rng.permutation(n_ch)
for preload_kwarg in preload_kwargs:
these_kwargs = kwargs.copy()
these_kwargs.update(preload_kwarg)
# don't use the same filename or it could create problems
if isinstance(these_kwargs.get('preload', None), str) and \
op.isfile(these_kwargs['preload']):
these_kwargs['preload'] += '-1'
whole_raw = reader(**these_kwargs)
print(whole_raw) # __repr__
assert n_ch >= 2
picks_1 = picks[:n_ch // 2]
picks_2 = picks[n_ch // 2:]
raw_1 = whole_raw.copy().pick(picks_1)
raw_2 = whole_raw.copy().pick(picks_2)
data, times = whole_raw[:]
data_1, times_1 = raw_1[:]
data_2, times_2 = raw_2[:]
assert_array_equal(times, times_1)
assert_array_equal(data[picks_1], data_1)
assert_array_equal(times, times_2,)
assert_array_equal(data[picks_2], data_2)
# Make sure that writing info to h5 format
# (all fields should be compatible)
if check_version('h5py'):
fname_h5 = op.join(tempdir, 'info.h5')
with _writing_info_hdf5(raw.info):
write_hdf5(fname_h5, raw.info)
new_info = Info(read_hdf5(fname_h5))
assert object_diff(new_info, raw.info) == ''
# Make sure that changing directory does not break anything
if test_preloading:
these_kwargs = kwargs.copy()
key = None
for key in ('fname',
'input_fname', # artemis123
'vhdr_fname', # BV
'pdf_fname', # BTi
'directory', # CTF
'filename', # nedf
):
try:
fname = kwargs[key]
except KeyError:
key = None
else:
break
# len(kwargs) == 0 for the fake arange reader
if len(kwargs):
assert key is not None, sorted(kwargs.keys())
dirname = op.dirname(fname)
these_kwargs[key] = op.basename(fname)
these_kwargs['preload'] = False
orig_dir = os.getcwd()
try:
os.chdir(dirname)
raw_chdir = reader(**these_kwargs)
finally:
os.chdir(orig_dir)
raw_chdir.load_data()
return raw
def load_psd(path,
study='C',
eyes='closed',
space='avg',
winlen=2.,
step=0.5,
reg='.+',
weight_norm='.+',
task=None):
'''
Load power spectrum density for given analysis.
'''
from scipy.io import loadmat
prefix = 'psd_study-{}_eyes-{}_space-{}'.format(study, eyes, space)
if space in ['avg', 'csd']:
prefix = prefix + '_winlen-{}_step-{}'.format(winlen, step)
elif space == 'src':
reg_pattern = '_reg-{:.2f}' if not isinstance(reg, str) else '_reg-{}'
prefix = prefix + (reg_pattern + 'weightnorm-{}')
prefix = prefix.format(reg, weight_norm)
# all psds are in C directory for convenience
study_dir = path.get_path('main', study='C')
psd_dir = op.join(study_dir, 'analysis', 'psd')
files_with_prefix = [
f for f in os.listdir(psd_dir)
if (f.endswith('.mat') or f.endswith('.hdf5'))
and len(re.findall(prefix, f)) > 0
]
num_good_files = len(files_with_prefix)
if num_good_files == 0:
msg = 'Could not find file matching prefix: {}'.format(prefix)
raise FileNotFoundError(msg)
if num_good_files > 1:
from warnings import warn
fls = ', '.join(files_with_prefix)
warn('More than one psd file matching specified criteria: {}.'
'Loading the first of the matching files.'.format(fls))
fname = files_with_prefix[0]
if fname.endswith('.mat'):
psds_mat = loadmat(op.join(psd_dir, fname))
# cleanup data
if space == 'src':
keys = ['psd', 'freq', 'subject_id']
psds, *rest = [psds_mat[k] for k in keys]
freq, subj_id = [x.ravel() for x in rest]
return psds, freq, None, subj_id
else:
keys = ['psd', 'freq', 'ch_names', 'subj_id']
psds, *rest = [psds_mat[k] for k in keys]
freq, ch_names, subj_id = [x.ravel() for x in rest]
ch_names = [ch.replace(' ', '') for ch in ch_names]
return psds, freq, ch_names, subj_id
elif fname.endswith('.hdf5') and space == 'src':
from mne.externals import h5io
temp = h5io.read_hdf5(op.join(psd_dir, fname))
if isinstance(temp['subject_id'], list):
temp['subject_id'] = np.array(temp['subject_id'])
return temp['psd'], temp['freq'], None, temp['subject_id']
def summarize_stats_clusters(reduce_columns=True, stat_dir='stats'):
'''Summarize multiple analyses (saved in analysis dir) in a dataframe.
Parameters
----------
reduce_columns : bool
Whether to remove columns with no variability from the output. Defaults
to ``True``.
stat_dir : str
Subdirectory to use (``'stats'``, ``'add1'`` or ``'add2'``, unless
additional subdirectories were created).
Returns
-------
df : pandas.DataFrame
Dataframe with summarized DiamSar analyses.
'''
from mne.externals import h5io
stat_dir = op.join(pth.paths.get_path('main', 'C'), 'analysis', stat_dir)
stat_files = [f for f in os.listdir(stat_dir) if f.endswith('.hdf5')]
n_stat = len(stat_files)
polarity_to_str = {True: 'pos', False: 'neg'}
# first, create an empty dataframe
stat_params = [
'study', 'contrast', 'space', 'N_low', 'N_high', 'N_all', 'eyes',
'selection', 'freq_range', 'avg_freq', 'transform', 'div_by_sum'
]
stat_summary = [
'min t', 'max t', 'n signif points', 'n clusters',
'largest cluster size', 'min cluster p', 'eff dir', 'n signif clusters'
]
df = pd.DataFrame(index=np.arange(1, n_stat + 1),
columns=stat_params + stat_summary)
row_idx = 0
for fname in stat_files:
stat = h5io.read_hdf5(op.join(stat_dir, fname))
if 'description' not in stat:
continue
row_idx += 1
# fill the basic columns
for col in stat_params:
value = (stat['description'][col]
if col in stat['description'] else np.nan)
if isinstance(value, (list, tuple, np.ndarray)):
value = str(value)
df.loc[row_idx, col] = value
# summarize clusters
n_clst = len(stat['clusters']) if stat['clusters'] is not None else 0
min_cluster_p = stat['pvals'].min() if n_clst > 0 else np.nan
n_below_thresh = (stat['pvals'] < 0.05).sum() if n_clst > 0 else 0
n_signif_points = stat['clusters'].sum() if n_clst > 0 else 0
if n_clst > 0:
largest_cluster_size = max([c.sum() for c in stat['clusters']])
polarity = stat['stat'][stat['clusters'][0]].mean() > 0
polarity_str = polarity_to_str[polarity]
largest_cluster_direction = polarity_str
else:
largest_cluster_size, largest_cluster_direction = np.nan, np.nan
df.loc[row_idx, 'min t'] = stat['stat'].min()
df.loc[row_idx, 'max t'] = stat['stat'].max()
df.loc[row_idx, 'n signif points'] = n_signif_points
df.loc[row_idx, 'n clusters'] = n_clst
df.loc[row_idx, 'min cluster p'] = min_cluster_p
df.loc[row_idx, 'n signif clusters'] = n_below_thresh
df.loc[row_idx, 'largest cluster size'] = largest_cluster_size
df.loc[row_idx, 'eff dir'] = largest_cluster_direction
# reduce columns
df = df.loc[:row_idx, :]
if reduce_columns:
df = remove_columns_with_no_variability(df)
return utils.reformat_stat_table(df)
def summarize_stats_pairs(reduce_columns=True,
stat_dir='stats',
confounds=False,
progressbar='text'):
'''Summarize multiple channel pair analyses (saved in analysis dir) in a
dataframe.
This takes longer than ``summarize_stats`` because it adds effect sizes and
bootstrap confidence intervals for effect sizes.
Parameters
----------
reduce_columns : bool
Whether to remove columns with no variability from the output. Defaults
to ``True``.
Returns
-------
df : pandas.DataFrame
Dataframe with summarized results.
'''
from mne.externals import h5io
from DiamSar.utils import progressbar as pbarobj
stat_dir = op.join(pth.paths.get_path('main', 'C'), 'analysis', stat_dir)
stat_files = [
f for f in os.listdir(stat_dir)
if f.endswith('.hdf5') and 'asy_pairs' in f
]
n_stat = len(stat_files)
# first, create an empty dataframe
stat_params = [
'study', 'contrast', 'space', 'ch_pair', 'N_low', 'N_high', 'N_all',
'eyes', 'selection', 'freq_range', 'avg_freq', 'transform',
'div_by_sum'
]
stats = ['t', 'p', 'es', 'ci']
stat_summary = [x + ' 1' for x in stats] + [x + ' 2' for x in stats]
df = pd.DataFrame(index=np.arange(1, n_stat + 1),
columns=stat_params + stat_summary)
pbar = pbarobj(progressbar, n_stat * 2)
for idx, fname in enumerate(stat_files, start=1):
stat = h5io.read_hdf5(op.join(stat_dir, fname))
for col in stat_params:
df.loc[idx, col] = stat[col] if col in stat else np.nan
# for ES and CI we need the original data:
sel_keys = [
'study', 'contrast', 'eyes', 'space', 'freq_range', 'avg_freq',
'selection', 'div_by_sum', 'transform'
]
kwargs = {key: stat[key] for key in sel_keys}
data = io.prepare_data(pth.paths,
confounds=confounds,
verbose=False,
**kwargs)
# add stats
for ch_idx in range(2):
stat_col = ch_idx * 4
t, p = stat['stat'][ch_idx], stat['pval'][ch_idx]
df.loc[idx, stat_summary[stat_col]] = t
df.loc[idx, stat_summary[stat_col + 1]] = p
if 'vs' in contrast and not confounds:
psd_high, psd_low, ch_names = (data['hi'], data['lo'],
data['ch_names'])
esci = esci_indep_cohens_d(psd_high[:, ch_idx],
psd_low[:, ch_idx])
else:
psd_sel, bdi_sel, ch_names = (data['hilo'], data['bdi'],
data['ch_names'])
if 'vs' in contrast:
onechan = psd_sel[:, [ch_idx]]
beh = data['bdi']
grp1 = beh[:, -1] == beh[0, -1]
data1 = np.concatenate([onechan[grp1, :], beh[grp1, :]],
axis=1)
data2 = np.concatenate([onechan[~grp1, :], beh[~grp1, :]],
axis=1)
esci = esci_indep_cohens_d(data1,
data2,
n_boot=5000,
has_preds=True)
else:
if confounds:
beh = data['bdi']
esci = esci_regression_r(beh, psd_sel[:, ch_idx])
else:
esci = esci_regression_r(bdi_sel, psd_sel[:, ch_idx])
df.loc[idx, stat_summary[stat_col + 2]] = esci['es']
df.loc[idx, stat_summary[stat_col + 3]] = esci['ci']
pbar.update(1)
pbar.close()
# split into two dfs
pair_rows = df['selection'].str.contains('pairs').values
df = df.loc[pair_rows, :].reset_index(drop=True)
df = remove_columns_with_no_variability(df)
df = utils.reformat_stat_table(df)
return df
p.lp_cut = 80
# Subject loop
for si, ss in enumerate(df.id.values):
subject = f"genz{ss}"
del ss
subj_dir = os.path.join(defaults.megdata, subject)
inv_fname = os.path.join(subj_dir, "inverse", f"{subject}-meg-erm-inv.fif")
# Load raw
print("Loading data for %s ..." % subject, end="")
src_dir = os.path.join(subj_dir, "source")
if not os.path.exists(src_dir):
os.mkdir(src_dir)
out_fname = os.path.join(src_dir, f"{state}_envcorr.h5")
if op.isfile(out_fname):
data = h5io.read_hdf5(out_fname)
degree[si] = data["degree"]
laplacian[si] = data["laplacian"]
print(" previous run data used")
continue
if state == "task":
p.pca_dir = "sss_pca_fif"
raws = get_raw_fnames(p, subject, which="pca")
if len(raws) == 0:
print(" No raws found")
continue
raw = _concat_resamp_raws(p, subject, raws)[0]
else:
raw_fname = os.path.join(
subj_dir,
# NOTE: change the name to the intended cluster result
clu_fname = op.join(rst_dir, 'aud_left_eq_vs_vis_left_eq_0_to_200.h5')
# prepare spatial adjency
fsaverage_src = mne.read_source_spaces(
op.join(subjects_dir, 'fsaverage', 'bem', f'fsaverage-{spacing}-src.fif'))
fsaverage_vertices = [s['vertno'] for s in fsaverage_src]
# get info
info_data = mne.read_source_estimate(
op.join(meg_dir, 'sample', f'sample_audvis-dSPM-{spacing}-inverse-morph-filt-sss-aud_left_eq'))
tstep = info_data.tstep
cluster_result = read_hdf5(clu_fname, title='mnepython')
clu = (cluster_result['t_obs'], cluster_result['clusters'],
cluster_result['cluster_pv'], cluster_result['H0'])
os.environ["SUBJECTS_DIR"] = subjects_dir
# In[3]:
stc_cluster = mne.stats.summarize_clusters_stc(
clu,
tstep=tstep * 1000,
vertices=fsaverage_vertices,
subject='fsaverage')
