def _interpolate_bads_eeg_epochs(epochs, bad_channels_by_epoch=None):
"""Interpolate bad channels per epoch
Parameters
----------
inst : mne.io.Raw, mne.Epochs or mne.Evoked
The data to interpolate. Must be preloaded.
bad_channels_by_epoch : list of list of str
Bad channel names specified for each epoch. For example, for an Epochs
instance containing 3 epochs: ``[['F1'], [], ['F3', 'FZ']]``
"""
if len(bad_channels_by_epoch) != len(epochs):
raise ValueError("Unequal length of epochs (%i) and "
"bad_channels_by_epoch (%i)" %
(len(epochs), len(bad_channels_by_epoch)))
interp_cache = {}
for i, bad_channels in enumerate(bad_channels_by_epoch):
if not bad_channels:
continue
# find interpolation matrix
key = tuple(sorted(bad_channels))
if key in interp_cache:
goods_idx, bads_idx, interpolation = interp_cache[key]
else:
goods_idx, bads_idx, interpolation = interp_cache[key] \
= _make_interpolator(epochs, key)
# apply interpolation
logger.info('Interpolating %i sensors on epoch %i', bads_idx.sum(), i)
epochs._data[i, bads_idx, :] = np.dot(interpolation,
epochs._data[i, goods_idx, :])
def _raw_to_epochs_array(x, sfreq, events, tmin, tmax):
"""Aux function to create epochs from a 2D array"""
if events.ndim != 1:
raise ValueError('events must be 1D')
if events.dtype != int:
raise ValueError('events must be of dtype int')
# Check that events won't be cut off
n_times = x.shape[-1]
min_ix = 0 - sfreq * tmin
max_ix = n_times - sfreq * tmax
msk_keep = np.logical_and(events > min_ix, events < max_ix)
if not all(msk_keep):
logger.info('Some event windows extend beyond data limits,'
' and will be cut off...')
events = events[msk_keep]
# Pull events from the raw data
epochs = []
for ix in events:
ix_min, ix_max = [ix + int(i_tlim * sfreq)
for i_tlim in [tmin, tmax]]
epochs.append(x[np.newaxis, :, ix_min:ix_max])
epochs = np.concatenate(epochs, axis=0)
times = np.arange(epochs.shape[-1]) / float(sfreq) + tmin
return epochs, times, msk_keep
def _save_annotations(*, annotations, bids_path, verbose):
# Attach the new Annotations to our raw data so we can easily convert them
# to events, which will be stored in the *_events.tsv sidecar.
extra_params = dict()
if bids_path.extension == '.fif':
extra_params['allow_maxshield'] = True
raw = read_raw_bids(bids_path=bids_path,
extra_params=extra_params,
verbose='warning')
raw.set_annotations(annotations)
events, durs, descrs = _read_events(events_data=None,
event_id=None,
raw=raw,
verbose=False)
# Write sidecar – or remove it if no events are left.
events_tsv_fname = (bids_path.copy().update(suffix='events',
extension='.tsv').fpath)
if len(events) > 0:
_events_tsv(events=events,
durations=durs,
raw=raw,
fname=events_tsv_fname,
trial_type=descrs,
overwrite=True,
verbose=verbose)
elif events_tsv_fname.exists():
logger.info(f'No events remaining after interactive inspection, '
f'removing {events_tsv_fname.name}')
events_tsv_fname.unlink()
def _handle_coordsystem_reading(coordsystem_fpath, datatype):
"""Read associated coordsystem.json.
Handle reading the coordinate frame and coordinate unit
of each electrode.
"""
# open coordinate system sidecar json
with open(coordsystem_fpath, 'r', encoding='utf-8-sig') as fin:
coordsystem_json = json.load(fin)
if datatype == 'meg':
coord_frame = coordsystem_json['MEGCoordinateSystem']
coord_unit = coordsystem_json['MEGCoordinateUnits']
coord_frame_desc = coordsystem_json.get('MEGCoordinateDescription',
None)
elif datatype == 'eeg':
coord_frame = coordsystem_json['EEGCoordinateSystem']
coord_unit = coordsystem_json['EEGCoordinateUnits']
coord_frame_desc = coordsystem_json.get('EEGCoordinateDescription',
None)
elif datatype == 'ieeg':
coord_frame = coordsystem_json['iEEGCoordinateSystem']
coord_unit = coordsystem_json['iEEGCoordinateUnits']
coord_frame_desc = coordsystem_json.get('iEEGCoordinateDescription',
None)
logger.info(f"Reading in coordinate system frame {coord_frame}: "
f"{coord_frame_desc}.")
return coord_frame, coord_unit
def _create_folds(X, y, n_folds=None):
"""Split the observations in X into stratified folds."""
if y is None:
# No folding
return X[np.newaxis, ...]
y = np.asarray(y)
if len(y) != len(X):
raise ValueError(f'The length of y ({len(y)}) does not match the '
f'number of items ({len(X)}).')
y_one_hot = _convert_to_one_hot(y)
n_items = y_one_hot.shape[1]
if n_folds is None:
# Set n_folds to maximum value
n_folds = len(X) // n_items
logger.info(f'Automatic dermination of folds: {n_folds}' +
' (no cross-validation)' if n_folds == 1 else '')
if n_folds == 1:
# Making one fold is easy
folds = [_compute_item_means(X, y_one_hot)]
else:
folds = []
for _, fold in StratifiedKFold(n_folds).split(X, y):
folds.append(_compute_item_means(X, y_one_hot, fold))
return np.array(folds)
def chop_raw_data(raw, start_time=60.0, stop_time=360.0, save=True):
'''
This function extracts specified duration of raw data
and writes it into a fif file.
Five mins of data will be extracted by default.
Parameters
----------
raw: Raw object or raw file name as a string.
start_time: Time to extract data from in seconds. Default is 60.0 seconds.
stop_time: Time up to which data is to be extracted. Default is 360.0 seconds.
save: bool, If True the raw file is written to disk.
'''
if isinstance(raw, str):
print 'Raw file name provided, loading raw object...'
raw = mne.io.Raw(raw, preload=True)
# Check if data is longer than required chop duration.
if (raw.n_times / (raw.info['sfreq'])) < (stop_time + start_time):
logger.info("The data is not long enough for file %s.") % (raw.info['filename'])
return
# Obtain indexes for start and stop times.
assert start_time < stop_time, "Start time is greater than stop time."
start_idx = raw.time_as_index(start_time)
stop_idx = raw.time_as_index(stop_time)
data, times = raw[:, start_idx:stop_idx]
raw._data,raw._times = data, times
dur = int((stop_time - start_time) / 60)
if save:
#raw.save(raw.info['filename'].split('/')[-1].split('.')[0] + '_' + str(dur) + 'm-raw.fif')
raw.save(raw.info['filename'].split('-raw.fif')[0] + ',' + str(dur) + 'm-raw.fif')
raw.close()
return
def chop_raw_data(raw, start_time=60.0, stop_time=360.0):
'''
This function extracts specified duration of raw data
and write it into a fif file.
Five mins of data will be extracted by default.
Parameters
----------
raw: Raw object.
start_time: Time to extract data from in seconds. Default is 60.0 seconds.
stop_time: Time up to which data is to be extracted. Default is 360.0 seconds.
'''
# Check if data is longer than required chop duration.
if (raw.n_times / (raw.info['sfreq'])) < (stop_time + 60.0):
logger.info("The data is not long enough.")
return
# Obtain indexes for start and stop times.
assert start_time < stop_time, "Start time is greater than stop time."
start_idx = raw.time_as_index(start_time)
stop_idx = raw.time_as_index(stop_time)
data, times = raw[:, start_idx:stop_idx]
raw._data, raw._times = data, times
dur = int((stop_time - start_time) / 60)
raw.save(raw.info['filename'].split('/')[-1].split('.')[0] + '_' +
str(dur) + 'm.fif')
# For the moment, simply warn.
logger.warning('The file name is not saved in standard form.')
return
def save(self, fname, overwrite=False):
if not isinstance(fname, Path):
fname = Path(fname)
self._save_info(fname, overwrite=overwrite)
save_vars = self._get_save_vars(
exclude=['data_', 'parent', 'tmin', 'tmax'])
parent_name = self.parent._get_title()
save_vars['parent_name_'] = parent_name
has_parent = False
with h5py.File(fname) as h5fid:
if parent_name in h5fid:
has_parent = True
logger.info('Parent already present in HDF5 file, '
'will not be overwritten')
if not has_parent:
logger.info('Writing numerator to HDF5 file')
self.parent.save(fname, overwrite=overwrite)
write_hdf5(
fname, save_vars, overwrite=overwrite,
title=self._get_title(), slash='replace')
def chop_raw_data(raw, start_time=60.0, stop_time=360.0, save=True, return_chop=False):
'''
This function extracts specified duration of raw data
and writes it into a fif file.
Five mins of data will be extracted by default.
Parameters
----------
raw: Raw object or raw file name as a string.
start_time: Time to extract data from in seconds. Default is 60.0 seconds.
stop_time: Time up to which data is to be extracted. Default is 360.0 seconds.
save: bool, If True the raw file is written to disk. (default: True)
return_chop: bool, Return the chopped raw object. (default: False)
'''
if isinstance(raw, str):
print 'Raw file name provided, loading raw object...'
raw = mne.io.Raw(raw, preload=True)
# Check if data is longer than required chop duration.
if (raw.n_times / (raw.info['sfreq'])) < (stop_time + start_time):
logger.info("The data is not long enough for file %s.") % (raw.filenames[0])
return
# Obtain indexes for start and stop times.
assert start_time < stop_time, "Start time is greater than stop time."
crop = raw.copy().crop(tmin=start_time, tmax=stop_time)
dur = int((stop_time - start_time) / 60)
if save:
crop.save(crop.filenames[0].split('-raw.fif')[0] + ',' + str(dur) + 'm-raw.fif')
raw.close()
if return_chop:
return crop
else:
crop.close()
return
def create_epochs(raw: Raw) -> Epochs:
"""
Create non-overlapping segments from Raw data with a fixed duration.
Note that temporal filtering should be done before creating the epochs.
The duration of epochs is defined in the configuration file (config.py).
Parameters
----------
raw: the continuous data to be segmented into non-overlapping epochs
Returns
-------
Epochs instance
"""
epoch_duration_in_seconds = settings["epochs"]["duration"]
logger.info("Creating epochs from continuous data ...")
events = make_fixed_length_events(raw,
id=1,
first_samp=True,
duration=epoch_duration_in_seconds)
epochs = Epochs(
raw=raw,
events=events,
picks="all",
event_id=list(np.unique(events[..., 2])),
baseline=None,
tmin=0.0,
tmax=epoch_duration_in_seconds, # - (1 / raw.info['sfreq']
preload=False,
)
return epochs
def _rectify_resolution_matrix(resmat):
"""
Ensure resolution matrix is square matrix.
If resmat is not a square matrix, it is assumed that the inverse operator
had free or loose orientation constraint, i.e. multiple values per source
location. The Euclidean length for values at each location is computed to
make resmat a square matrix.
"""
shape = resmat.shape
if not shape[0] == shape[1]:
if shape[0] < shape[1]:
raise ValueError('Number of target sources (%d) cannot be lower '
'than number of input sources (%d)' % shape[0],
shape[1])
if np.mod(shape[0], shape[1]): # if ratio not integer
raise ValueError('Number of target sources (%d) must be a '
'multiple of the number of input sources (%d)'
% shape[0], shape[1])
ns = shape[0] // shape[1] # number of source components per vertex
# Combine rows of resolution matrix
resmatl = [np.sqrt((resmat[ns * i:ns * (i + 1), :]**2).sum(axis=0))
for i in np.arange(0, shape[1], dtype=int)]
resmat = np.array(resmatl)
logger.info('Rectified resolution matrix from (%d, %d) to (%d, %d).' %
(shape[0], shape[1], resmat.shape[0], resmat.shape[1]))
return resmat
def chop_raw_data(raw, start_time=60.0, stop_time=360.0):
'''
This function extracts specified duration of raw data
and write it into a fif file.
Five mins of data will be extracted by default.
Parameters
----------
raw: Raw object.
start_time: Time to extract data from in seconds. Default is 60.0 seconds.
stop_time: Time up to which data is to be extracted. Default is 360.0 seconds.
'''
# Check if data is longer than required chop duration.
if (raw.n_times / (raw.info['sfreq'])) < (stop_time + 60.0):
logger.info("The data is not long enough.")
return
# Obtain indexes for start and stop times.
assert start_time < stop_time, "Start time is greater than stop time."
start_idx = raw.time_as_index(start_time)
stop_idx = raw.time_as_index(stop_time)
data, times = raw[:, start_idx:stop_idx]
raw._data,raw._times = data, times
dur = int((stop_time - start_time) / 60)
raw.save(raw.info['filename'].split('/')[-1].split('.')[0]+'_'+str(dur)+'m.fif')
# For the moment, simply warn.
logger.warning('The file name is not saved in standard form.')
return
def _handle_electrodes_reading(electrodes_fname, coord_frame, coord_unit):
"""Read associated electrodes.tsv and populate raw.
Handle xyz coordinates and coordinate frame of each channel.
"""
logger.info('Reading electrode '
'coords from {}.'.format(electrodes_fname))
electrodes_dict = _from_tsv(electrodes_fname)
ch_names_tsv = electrodes_dict['name']
def _float_or_nan(val):
if val == "n/a":
return np.nan
else:
return float(val)
# convert coordinates to float and create list of tuples
electrodes_dict['x'] = [_float_or_nan(x) for x in electrodes_dict['x']]
electrodes_dict['y'] = [_float_or_nan(x) for x in electrodes_dict['y']]
electrodes_dict['z'] = [_float_or_nan(x) for x in electrodes_dict['z']]
ch_names_raw = [
x for i, x in enumerate(ch_names_tsv)
if electrodes_dict['x'][i] != "n/a"
]
ch_locs = np.c_[electrodes_dict['x'], electrodes_dict['y'],
electrodes_dict['z']]
# convert coordinates to meters
ch_locs = _scale_coord_to_meters(ch_locs, coord_unit)
# create mne.DigMontage
ch_pos = dict(zip(ch_names_raw, ch_locs))
montage = mne.channels.make_dig_montage(ch_pos=ch_pos,
coord_frame=coord_frame)
return montage
def __init__(self,
tmin=None,
tmax=None,
fmin=None,
fmax=None,
method_params=None,
n_jobs='auto',
comment='default'):
BaseMarkerSandbox.__init__(self, tmin=None, tmax=None, comment=comment)
if method_params is None:
method_params = {}
if fmax is None:
fmax = np.inf
self.fmin = fmin
self.fmax = fmax
self.method_params = method_params
if n_jobs == 'auto':
try:
import multiprocessing as mp
import mkl
n_jobs = int(mp.cpu_count() / mkl.get_max_threads())
logger.info('Autodetected number of jobs {}'.format(n_jobs))
except Exception:
logger.info('Cannot autodetect number of jobs')
n_jobs = 1
self.n_jobs = n_jobs
def faster_bad_components(ica, epochs, thres=3, use_metrics=None):
"""Implements the third step of the FASTER algorithm.
This function attempts to automatically mark bad ICA components by
performing outlier detection.
Parameters
----------
ica : Instance of ICA
The ICA operator, already fitted to the supplied Epochs object.
epochs : Instance of Epochs
The untransformed epochs to analyze.
thres : float
The threshold value, in standard deviations, to apply. A component
crossing this threshold value is marked as bad. Defaults to 3.
use_metrics : list of str
List of metrics to use. Can be any combination of:
'eog_correlation', 'kurtosis', 'power_gradient', 'hurst',
'median_gradient'
Defaults to all of them.
Returns
-------
bads : list of int
The indices of the bad components.
See also
--------
ICA.find_bads_ecg
ICA.find_bads_eog
"""
source_data = ica.get_sources(epochs).get_data().transpose(1, 0, 2)
source_data = source_data.reshape(source_data.shape[0], -1)
metrics = {
'eog_correlation':
lambda x: x.find_bads_eog(epochs)[1],
'kurtosis':
lambda x: kurtosis(np.dot(x.mixing_matrix_.T, x.
pca_components_[:x.n_components_]),
axis=1),
'power_gradient':
lambda x: _power_gradient(x, source_data),
'hurst':
lambda x: hurst(source_data),
'median_gradient':
lambda x: np.median(np.abs(np.diff(source_data)), axis=1),
'line_noise':
lambda x: _freqs_power(source_data, epochs.info['sfreq'], [50, 60]),
}
if use_metrics is None:
use_metrics = metrics.keys()
bads = []
for m in use_metrics:
scores = np.atleast_2d(metrics[m](ica))
for s in scores:
b = find_outliers(s, thres)
logger.info('Bad by %s:\n\t%s' % (m, b))
bads.append(b)
return np.unique(np.concatenate(bads)).tolist()
def _raw_to_epochs_array(x, sfreq, events, tmin, tmax):
"""Aux function to create epochs from a 2D array"""
if events.ndim != 1:
raise ValueError('events must be 1D')
if events.dtype != int:
raise ValueError('events must be of dtype int')
# Check that events won't be cut off
n_times = x.shape[-1]
min_ix = 0 - sfreq * tmin
max_ix = n_times - sfreq * tmax
msk_keep = np.logical_and(events > min_ix, events < max_ix)
if not all(msk_keep):
logger.info('Some event windows extend beyond data limits,'
' and will be cut off...')
events = events[msk_keep]
# Pull events from the raw data
epochs = []
for ix in events:
ix_min, ix_max = [ix + int(i_tlim * sfreq) for i_tlim in [tmin, tmax]]
epochs.append(x[np.newaxis, :, ix_min:ix_max])
epochs = np.concatenate(epochs, axis=0)
times = np.arange(epochs.shape[-1]) / float(sfreq) + tmin
return epochs, times, msk_keep
def cv_decode_sliding(X,
y,
clf=None,
cv=None,
class_weight=None,
scoring='roc_auc',
random_state=None,
picks=None,
n_jobs=-1):
all_scores = []
if not isinstance(random_state, list):
random_state = [random_state]
for t_random in random_state:
logger.info('Using random state {}'.format(t_random))
clf, cv = _check_clf(clf, cv, class_weight, t_random)
se = mne.decoding.SlidingEstimator(clf, scoring=scoring)
if picks is not None:
logger.info('Picking channels')
X = X[:, picks, :]
scores = mne.decoding.cross_val_multiscore(se,
X,
y,
cv=cv,
n_jobs=n_jobs)
all_scores.append(scores)
return np.concatenate(all_scores, axis=0)
def decode_generalization(X_train,
y_train,
X_test,
y_test,
clf=None,
class_weight=None,
scoring='roc_auc',
random_state=None,
picks=None,
n_jobs=-1):
clf, _ = _check_clf(clf, None, class_weight, random_state)
ge = mne.decoding.GeneralizingEstimator(clf,
scoring=scoring,
n_jobs=n_jobs)
if picks is not None:
logger.info('Picking channels')
X_train = X_train[:, picks, :]
X_test = X_test[:, picks, :]
ge.fit(X_train, y_train)
scores = ge.score(X_test, y_test)
return scores[None, :]
def _interpolate_bads_eeg_epochs(epochs, bad_channels_by_epoch=None):
"""Interpolate bad channels per epoch
Parameters
----------
inst : mne.io.Raw, mne.Epochs or mne.Evoked
The data to interpolate. Must be preloaded.
bad_channels_by_epoch : list of list of str
Bad channel names specified for each epoch. For example, for an Epochs
instance containing 3 epochs: ``[['F1'], [], ['F3', 'FZ']]``
"""
if len(bad_channels_by_epoch) != len(epochs):
raise ValueError("Unequal length of epochs (%i) and "
"bad_channels_by_epoch (%i)"
% (len(epochs), len(bad_channels_by_epoch)))
interp_cache = {}
for i, bad_channels in enumerate(bad_channels_by_epoch):
if not bad_channels:
continue
# find interpolation matrix
key = tuple(sorted(bad_channels))
if key in interp_cache:
goods_idx, bads_idx, interpolation = interp_cache[key]
else:
goods_idx, bads_idx, interpolation = interp_cache[key] \
= _make_interpolator(epochs, key)
# apply interpolation
logger.info('Interpolating %i sensors on epoch %i', bads_idx.sum(), i)
epochs._data[i, bads_idx, :] = np.dot(interpolation,
epochs._data[i, goods_idx, :])
def cluster_threshold(con,
src,
min_size=20,
max_spread=0.013,
method='single',
verbose=None):
"""Threshold connectivity using clustering.
First, connections are grouped into "bundles". A bundle is a group of
connections which start and end points are close together. Then, only
bundles with a sufficient amount of connections are retained.
Parameters
----------
con : instance of Connectivity
Connectivity to threshold.
src : instance of SourceSpace
The source space for which the connectivity is defined.
min_size : int
Minimum amount of connections that a bundle must contain in order to be
accepted.
max_spread : float
Maximum amount the position (in metres) of the start and end points
of the connections may vary in order for them to be considered part of
the same "bundle". Defaults to 0.013.
method : str
Linkage method for fclusterdata. Defaults to 'single'. See
documentation for ``scipy.cluster.hierarchy.fclusterdata`` for for more
information.
verbose : bool | str | int | None
If not ``None``, override default verbose level
(see :func:`mne.verbose` and :ref:`Logging documentation <tut_logging>`
for more).
Returns
-------
thresholded_connectivity : instance of Connectivity
Instance of connectivity with the thresholded data.
"""
grid_points = np.vstack([s['rr'][v] for s, v in zip(src, con.vertices)])
X = np.hstack([grid_points[inds] for inds in con.pairs])
clust_no = fclusterdata(X, max_spread, criterion='distance', method=method)
# Remove clusters that do not pass the threshold
clusters, counts = np.unique(clust_no, return_counts=True)
big_clusters = clusters[counts >= min_size]
logger.info('Found %d bundles, of which %d are of sufficient size.' %
(len(clusters), len(big_clusters)))
# Restrict the connections to only those found in the big bundles
mask = np.in1d(clust_no, big_clusters)
data = con.data[mask]
pairs = [p[mask] for p in con.pairs]
return VertexConnectivity(data=data,
pairs=pairs,
vertices=con.vertices,
vertex_degree=con.source_degree,
subject=con.subject)
def faster_bad_channels_in_epochs(epochs,
picks=None,
thres=3,
use_metrics=None):
"""Implements the fourth step of the FASTER algorithm.
This function attempts to automatically mark bad channels in each epochs by
performing outlier detection.
Parameters
----------
epochs : Instance of Epochs
The epochs to analyze.
picks : list of int | None
Channels to operate on. Defaults to EEG channels.
thres : float
The threshold value, in standard deviations, to apply. An epoch
crossing this threshold value is marked as bad. Defaults to 3.
use_metrics : list of str
List of metrics to use. Can be any combination of:
'amplitude', 'variance', 'deviation', 'median_gradient'
Defaults to all of them.
Returns
-------
bads : list of lists of int
For each epoch, the indices of the bad channels.
"""
metrics = {
'amplitude': lambda x: np.ptp(x, axis=2),
'deviation': lambda x: _deviation(x),
'variance': lambda x: np.var(x, axis=2),
'median_gradient': lambda x: np.median(np.abs(np.diff(x)), axis=2),
'line_noise':
lambda x: _freqs_power(x, epochs.info['sfreq'], [50, 60]),
}
if picks is None:
picks = mne.pick_types(epochs.info,
meg=False,
eeg=True,
exclude='bads')
if use_metrics is None:
use_metrics = metrics.keys()
data = epochs.get_data()[:, picks, :]
bads = [[] for i in range(len(epochs))]
for m in use_metrics:
s_epochs = metrics[m](data)
for i, s in enumerate(s_epochs):
b = [epochs.ch_names[picks[j]] for j in find_outliers(s, thres)]
logger.info('Epoch %d, Bad by %s:\n\t%s' % (i, m, b))
bads[i].append(b)
for i, b in enumerate(bads):
if len(b) > 0:
bads[i] = np.unique(np.concatenate(b)).tolist()
return bads
def _find_bad_channels(epochs, picks, use_metrics, thresh, max_iter):
"""Implements the first step of the FASTER algorithm.
This function attempts to automatically mark bad EEG channels by performing
outlier detection. It operated on epoched data, to make sure only relevant
data is analyzed.
Additional Parameters
---------------------
use_metrics : list of str
List of metrics to use. Can be any combination of:
'variance', 'correlation', 'hurst', 'kurtosis', 'line_noise'
Defaults to all of them.
thresh : float
The threshold value, in standard deviations, to apply. A channel
crossing this threshold value is marked as bad. Defaults to 3.
max_iter : int
The maximum number of iterations performed during outlier detection
(defaults to 1, as in the original FASTER paper).
"""
from scipy.stats import kurtosis
metrics = {
'variance':
lambda x: np.var(x, axis=1),
'correlation':
lambda x: np.mean(np.ma.masked_array(np.corrcoef(x),
np.identity(len(x), dtype=bool)),
axis=0),
'hurst':
lambda x: _hurst(x),
'kurtosis':
lambda x: kurtosis(x, axis=1),
'line_noise':
lambda x: _freqs_power(x, epochs.info['sfreq'], [50, 60]),
}
if use_metrics is None:
use_metrics = metrics.keys()
# Concatenate epochs in time
data = epochs.get_data()[:, picks]
data = data.transpose(1, 0, 2).reshape(data.shape[1], -1)
# Find bad channels
bads = defaultdict(list)
info = pick_info(epochs.info, picks, copy=True)
for ch_type, chs in _picks_by_type(info):
logger.info('Bad channel detection on %s channels:' % ch_type.upper())
for metric in use_metrics:
scores = metrics[metric](data[chs])
bad_channels = [
epochs.ch_names[picks[chs[i]]]
for i in find_outliers(scores, thresh, max_iter)
]
logger.info('\tBad by %s: %s' % (metric, bad_channels))
bads[metric].append(bad_channels)
bads = dict((k, np.concatenate(v).tolist()) for k, v in bads.items())
return bads
def _iter_spatial(self):
"""Generate spatial searchlight patches only."""
logger.info('Creating spatial searchlight patches')
for series in self.sel_series:
patch = list(self.patch_template) # Copy the template
series_i = np.flatnonzero(self.dist[series] < self.spatial_radius)
patch[self.series_dim] = series_i
yield tuple(patch)
def _iter_temporal(self):
"""Generate temporal searchlight patches only."""
logger.info('Creating temporal searchlight patches')
for sample in self.time_centers:
patch = list(self.patch_template) # Copy the template
patch[self.samples_dim] = slice(sample - self.temporal_radius,
sample + self.temporal_radius + 1)
yield tuple(patch)
def _write_tsv(fname, dictionary, overwrite=False, verbose=None):
"""Write an ordered dictionary to a .tsv file."""
if op.exists(fname) and not overwrite:
raise FileExistsError(f'"{fname}" already exists. '
'Please set overwrite to True.')
_to_tsv(dictionary, fname)
logger.info(f"Writing '{fname}'...")
def _iter_spatial(self):
"""Generate spatial searchlight patches only."""
logger.info('Creating spatial searchlight patches')
patch = list(self.patch_template) # Copy the template
for series in self.sel_series:
spat_ind = _get_in_radius(self.dist, series, self.spatial_radius)
patch[self.series_dim] = spat_ind
yield tuple(patch)
def faster_bad_channels(epochs, picks=None, thres=3, use_metrics=None):
"""Implements the first step of the FASTER algorithm.
This function attempts to automatically mark bad EEG channels by performing
outlier detection. It operated on epoched data, to make sure only relevant
data is analyzed.
Parameters
----------
epochs : Instance of Epochs
The epochs for which bad channels need to be marked
picks : list of int | None
Channels to operate on. Defaults to EEG channels.
thres : float
The threshold value, in standard deviations, to apply. A channel
crossing this threshold value is marked as bad. Defaults to 3.
use_metrics : list of str
List of metrics to use. Can be any combination of:
'variance', 'correlation', 'hurst', 'kurtosis', 'line_noise'
Defaults to all of them.
Returns
-------
bads : list of str
The names of the bad EEG channels.
"""
metrics = {
'variance':
lambda x: np.var(x, axis=1),
'correlation':
lambda x: nanmean(np.ma.masked_array(np.corrcoef(x),
np.identity(len(x), dtype=bool)),
axis=0),
'hurst':
lambda x: hurst(x),
'kurtosis':
lambda x: kurtosis(x, axis=1),
'line_noise':
lambda x: _freqs_power(x, epochs.info['sfreq'], [50, 60]),
}
if picks is None:
picks = mne.pick_types(epochs.info, meg=False, eeg=True, exclude=[])
if use_metrics is None:
use_metrics = metrics.keys()
# Concatenate epochs in time
data = epochs.get_data()
data = data.transpose(1, 0, 2).reshape(data.shape[1], -1)
data = data[picks]
# Find bad channels
bads = []
for m in use_metrics:
s = metrics[m](data)
b = [epochs.ch_names[picks[i]] for i in find_outliers(s, thres)]
logger.info('Bad by %s:\n\t%s' % (m, b))
bads.append(b)
return np.unique(np.concatenate(bads)).tolist()
def _read_antcnt_events(eeg, event_id=None, event_id_func='strip_to_integer'):
"""Create events array from ANT cnt structure
An event array is constructed by looking up events in the
event_id, trying to reduce them to their integer part otherwise, and
entirely dropping them (with a warning) if this is impossible.
Returns a 1x3 array of zeros if no events are found."""
if event_id_func is 'strip_to_integer':
event_id_func = _strip_to_integer
if event_id is None:
event_id = dict()
types = [eeg.get_trigger(i)[0] for i in range(eeg.get_trigger_count())]
latencies= [eeg.get_trigger(i)[1] for i in range(eeg.get_trigger_count())]
if len(types) < 1: # if there are 0 events, we can exit here
logger.info('No events found, returning empty stim channel ...')
return np.zeros((0, 3))
not_in_event_id = set(x for x in types if x not in event_id)
not_purely_numeric = set(x for x in not_in_event_id if not x.isdigit())
no_numbers = set([x for x in not_purely_numeric
if not any([d.isdigit() for d in x])])
have_integers = set([x for x in not_purely_numeric
if x not in no_numbers])
if len(not_purely_numeric) > 0:
basewarn = "Events like the following will be dropped"
n_no_numbers, n_have_integers = len(no_numbers), len(have_integers)
if n_no_numbers > 0:
no_num_warm = " entirely: {0}, {1} in total"
warn(basewarn + no_num_warm.format(list(no_numbers)[:5],
n_no_numbers))
if n_have_integers > 0 and event_id_func is None:
intwarn = (", but could be reduced to their integer part "
"instead with the default `event_id_func`: "
"{0}, {1} in total")
warn(basewarn + intwarn.format(list(have_integers)[:5],
n_have_integers))
events = list()
for tt, latency in zip(types, latencies):
try: # look up the event in event_id and if not, try event_id_func
event_code = event_id[tt] if tt in event_id else event_id_func(tt)
events.append([int(latency), 1, event_code])
except (ValueError, TypeError): # if event_id_func fails
pass # We're already raising warnings above, so we just drop
if len(events) < len(types):
missings = len(types) - len(events)
msg = ("{0}/{1} event codes could not be mapped to integers. Use "
"the 'event_id' parameter to map such events manually.")
warn(msg.format(missings, len(types)))
if len(events) < 1:
warn("As is, the trigger channel will consist entirely of zeros.")
return np.zeros((0, 3))
return np.asarray(events)
def _compute_maps(data,
n_states=4,
max_iter=1000,
thresh=1e-6,
random_state=None,
verbose=None):
"""The modified K-means clustering algorithm.
See :func:`segment` for the meaning of the parameters and return
values.
"""
if not isinstance(random_state, np.random.RandomState):
random_state = np.random.RandomState(random_state)
n_channels, n_samples = data.shape
# Cache this value for later
data_sum_sq = np.sum(data**2)
# Select random timepoints for our initial topographic maps
init_times = random_state.choice(n_samples, size=n_states, replace=False)
maps = data[:, init_times].T
maps /= np.linalg.norm(maps, axis=1, keepdims=True) # Normalize the maps
prev_residual = np.inf
for _ in range(max_iter):
# Assign each sample to the best matching microstate
activation = maps.dot(data)
segmentation = np.argmax(np.abs(activation), axis=0)
# Recompute the topographic maps of the microstates, based on the
# samples that were assigned to each state.
for state in range(n_states):
idx = (segmentation == state)
if np.sum(idx) == 0:
logger.info('Some microstates are never activated')
maps[state] = 0
continue
# Find largest eigenvector
# cov = data[:, idx].dot(data[:, idx].T)
# _, vec = eigh(cov, eigvals=(n_channels - 1, n_channels - 1))
# maps[state] = vec.ravel()
maps[state] = data[:, idx].dot(activation[state, idx])
maps[state] /= np.linalg.norm(maps[state])
# Estimate residual noise
act_sum_sq = np.sum(np.sum(maps[segmentation].T * data, axis=0)**2)
residual = abs(data_sum_sq - act_sum_sq)
residual /= float(n_samples * (n_channels - 1))
# Have we converged?
if (prev_residual - residual) < (thresh * residual):
# logger.info('Converged at %d iterations.' % iteration)
break
prev_residual = residual
else:
logger.info('Modified K-means algorithm failed to converge.')
return maps
def __init__(self,
input_fname,
montage=None,
eog=None,
misc=(-4, -3, -2, -1),
stim_channel=None,
scale=1e-6,
sfreq=250,
missing_tol=1,
preload=True,
verbose=None):
bci_info = {'missing_tol': missing_tol, 'stim_channel': stim_channel}
if not eog:
eog = list()
if not misc:
misc = list()
nsamps, nchan = self._get_data_dims(input_fname)
last_samps = [nsamps - 1]
ch_names = ['EEG %03d' % num for num in range(1, nchan + 1)]
ch_types = ['eeg'] * nchan
if misc:
misc_names = ['MISC %03d' % ii for ii in range(1, len(misc) + 1)]
misc_types = ['misc'] * len(misc)
for ii, mi in enumerate(misc):
ch_names[mi] = misc_names[ii]
ch_types[mi] = misc_types[ii]
if eog:
eog_names = ['EOG %03d' % ii for ii in range(len(eog))]
eog_types = ['eog'] * len(eog)
for ii, ei in enumerate(eog):
ch_names[ei] = eog_names[ii]
ch_types[ei] = eog_types[ii]
if stim_channel:
ch_names[stim_channel] = 'STI 014'
ch_types[stim_channel] = 'stim'
# mark last channel as the timestamp channel
ch_names[-1] = "Timestamps"
ch_types[-1] = "misc"
# fix it for eog and misc marking
info = create_info(ch_names, sfreq, ch_types, montage)
info["buffer_size_sec"] = 1.
super(RawOpenBCI, self).__init__(info,
last_samps=last_samps,
raw_extras=[bci_info],
filenames=[input_fname],
preload=False,
verbose=verbose)
# load data
if preload:
self.preload = preload
logger.info('Reading raw data from %s...' % input_fname)
self._data = self._read_segment()
def reduce_to_epochs(self, marker_params):
"""Reduce each marker of the collection to a single value per epoch.
Parameters
----------
marker_params : dict with reduction parameters
Each key of the dict should be of the form MarkerClass or
MarkerClass/comment. Each value should be a dictionary with two
keys: 'reduction_func' and 'picks'.
reduction_func: list of dictionaries. Each dictionary should have
two keys: 'axis' and 'function'. The marker is going to be
reduced following the order of the list. Selecting the
corresponding axis and applying the corresponding function.
picks: dictionary of axis to array. Before appling the reduction
function, the corresponding axis will be subselected by picks.
A value of None indicates all the elements.
Example:
marker_params = dict()
reduction_params['PowerSpectralDensity'] = {
'reduction_func':
[{'axis': 'frequency', 'function': np.sum},
{'axis': 'channels', 'function': np.mean},
{'axis': 'epochs', 'function': np.mean}],
'picks': {
'epochs': None,
'channels': np.arange(224)}}
Returns
-------
out : dict
Each marker of the collection will be a key, with a value
representing the marker value for each epoch (
np.ndarray of float, shape(n_epochs,))
"""
logger.info('Reducing to epochs')
self._check_marker_params_keys(marker_params)
ch_picks = mne.pick_types(self.ch_info_, eeg=True, meg=True)
if ch_picks is not None: # XXX think if info is needed down-stream
info = mne.io.pick.pick_info(self.ch_info_, ch_picks, copy=True)
else:
info = self.ch_info_
markers_to_epochs = [
meas for meas in self.values()
if isin_info(info_source=info, info_target=meas.ch_info_)
and 'epochs' in meas._axis_map
]
n_markers = len(markers_to_epochs)
n_epochs = markers_to_epochs[0].data_.shape[
markers_to_epochs[0]._axis_map['epochs']]
out = OrderedDict()
for ii, meas in enumerate(markers_to_epochs):
logger.info('Reducing {}'.format(meas._get_title()))
this_params = _get_reduction_params(marker_params, meas)
out[meas._get_title()] = meas.reduce_to_epochs(**this_params)
return out
def epochs_compute_pe(epochs,
kernel,
tau,
tmin=None,
tmax=None,
backend='python',
method_params=None):
"""Compute Permutation Entropy (PE)
Parameters
----------
epochs : instance of mne.Epochs
The epochs on which to compute the PE.
kernel : int
The number of samples to use to transform to a symbol
tau : int
The number of samples left between the ones that defines a symbol.
backend : {'python', 'c'}
The backend to be used. Defaults to 'python'.
"""
if method_params is None:
method_params = {}
freq = epochs.info['sfreq']
picks = mne.io.pick.pick_types(epochs.info, meg=True, eeg=True)
data = epochs.get_data()[:, picks, ...]
n_epochs = len(data)
data = np.hstack(data)
if 'filter_freq' in method_params:
filter_freq = method_params['filter_freq']
else:
filter_freq = np.double(freq) / kernel / tau
logger.info('Filtering at %.2f Hz' % filter_freq)
b, a = butter(6, 2.0 * filter_freq / np.double(freq), 'lowpass')
fdata = np.transpose(
np.array(np.split(filtfilt(b, a, data), n_epochs, axis=1)), [1, 2, 0])
time_mask = _time_mask(epochs.times, tmin, tmax)
fdata = fdata[:, time_mask, :]
if backend == 'python':
logger.info("Performing symbolic transformation")
sym, count = _symb_python(fdata, kernel, tau)
pe = np.nan_to_num(-np.nansum(count * np.log(count), axis=1))
elif backend == 'c':
from ..optimizations.jivaro import pe as jpe
pe, sym = jpe(fdata, kernel, tau)
else:
raise ValueError('backend %s not supported for PE' % backend)
nsym = math.factorial(kernel)
pe = pe / np.log(nsym)
return pe, sym
def _handle_electrodes_reading(electrodes_fname, coord_frame, raw, verbose):
"""Read associated electrodes.tsv and populate raw.
Handle xyz coordinates and coordinate frame of each channel.
Assumes units of coordinates are in 'm'.
"""
logger.info('Reading electrode '
'coords from {}.'.format(electrodes_fname))
electrodes_dict = _from_tsv(electrodes_fname)
# First, make sure that ordering of names in channels.tsv matches the
# ordering of names in the raw data. The "name" column is mandatory in BIDS
ch_names_raw = list(raw.ch_names)
ch_names_tsv = electrodes_dict['name']
if ch_names_raw != ch_names_tsv:
msg = ('Channels do not correspond between raw data and the '
'channels.tsv file. For MNE-BIDS, the channel names in the '
'tsv MUST be equal and in the same order as the channels in '
'the raw data.\n\n'
'{} channels in tsv file: "{}"\n\n --> {}\n\n'
'{} channels in raw file: "{}"\n\n --> {}\n\n'
.format(len(ch_names_tsv), electrodes_fname, ch_names_tsv,
len(ch_names_raw), raw.filenames, ch_names_raw)
)
# XXX: this could be due to MNE inserting a 'STI 014' channel as the
# last channel: In that case, we can work. --> Can be removed soon,
# because MNE will stop the synthesis of stim channels in the near
# future
if not (ch_names_raw[-1] == 'STI 014' and
ch_names_raw[:-1] == ch_names_tsv):
raise RuntimeError(msg)
if verbose:
print("The read in electrodes file is: \n", electrodes_dict)
# convert coordinates to float and create list of tuples
ch_names_raw = [x for i, x in enumerate(ch_names_raw)
if electrodes_dict['x'][i] != "n/a"]
electrodes_dict['x'] = [float(x) for x in electrodes_dict['x']
if x != "n/a"]
electrodes_dict['y'] = [float(x) for x in electrodes_dict['y']
if x != "n/a"]
electrodes_dict['z'] = [float(x) for x in electrodes_dict['z']
if x != "n/a"]
ch_locs = list(zip(electrodes_dict['x'],
electrodes_dict['y'],
electrodes_dict['z']))
ch_pos = dict(zip(ch_names_raw, ch_locs))
# create mne.DigMontage
montage = mne.channels.make_dig_montage(ch_pos=ch_pos,
coord_frame=coord_frame)
raw.set_montage(montage)
return raw
def fit(self, epochs):
for meas in self.values():
if isinstance(meas, BasePowerSpectralDensity):
meas._check_freq_time_range(epochs)
for meas in self.values():
logger.info('Fitting {}'.format(meas._get_title()))
meas.fit(epochs)
self.ch_info_ = list(self.values())[0].ch_info_
def label_svd(sub_leadfield, n_svd_comp, ch_names):
""" Computes SVD of subleadfield for sensor types separately
Parameters:
-----------
sub_leadfield: numpy array (n_sens x n_vert) with part of the
leadfield matrix
n_svd_comp: scalar, number of SVD components required
ch_names: list of channel names
Returns:
--------
this_label_lfd_summary: numpy array, n_svd_comp scaled SVD components
of subleadfield
OH Aug 2015
"""
logger.info("\nComputing SVD within labels, using %d component(s)"
% n_svd_comp)
EEG_idx = [cc for cc in range(len(ch_names)) if ch_names[cc][:3]=='EEG']
MAG_idx = [cc for cc in range(len(ch_names)) if (ch_names[cc][:3]=='MEG'
and ch_names[cc][-1:]=='1')]
GRA_idx = [cc for cc in range(len(ch_names)) if (ch_names[cc][:3]=='MEG'
and (ch_names[cc][-1:]=='2' or ch_names[cc][-1:]=='3'))]
list_idx = []
u_idx = -1 # keep track of which element of u_svd belongs t which sensor type
if MAG_idx:
list_idx.append(MAG_idx)
u_idx += 1
u_mag = u_idx
if GRA_idx:
list_idx.append(GRA_idx)
u_idx += 1
u_gra = u_idx
if EEG_idx:
list_idx.append(EEG_idx)
u_idx += 1
u_eeg = u_idx
# # compute SVD of sub-leadfield for individual sensor types
u_svd = [get_svd_comps(sub_leadfield[ch_idx,:], n_svd_comp) for ch_idx
in list_idx]
# put sensor types back together
this_label_lfd_summary = np.zeros([len(ch_names),u_svd[0].shape[1]])
if MAG_idx:
this_label_lfd_summary[MAG_idx] = u_svd[u_mag]
if GRA_idx:
this_label_lfd_summary[GRA_idx] = u_svd[u_gra]
if EEG_idx:
this_label_lfd_summary[EEG_idx] = u_svd[u_eeg]
return this_label_lfd_summary
def _interpolate_bads_eeg(inst, picks=None, verbose=None):
""" Interpolate bad EEG channels.
Operates in place.
Parameters
----------
inst : mne.io.Raw, mne.Epochs or mne.Evoked
The data to interpolate. Must be preloaded.
picks: np.ndarray, shape(n_channels, ) | list | None
The channel indices to be used for interpolation.
"""
from mne.bem import _fit_sphere
from mne.utils import logger, warn
from mne.channels.interpolation import _do_interp_dots
from mne.channels.interpolation import _make_interpolation_matrix
import numpy as np
if picks is None:
picks = pick_types(inst.info, meg=False, eeg=True, exclude=[])
bads_idx = np.zeros(len(inst.ch_names), dtype=np.bool)
goods_idx = np.zeros(len(inst.ch_names), dtype=np.bool)
inst.info._check_consistency()
bads_idx[picks] = [inst.ch_names[ch] in inst.info['bads'] for ch in picks]
if len(picks) == 0 or bads_idx.sum() == 0:
return
goods_idx[picks] = True
goods_idx[bads_idx] = False
pos = inst._get_channel_positions(picks)
# Make sure only good EEG are used
bads_idx_pos = bads_idx[picks]
goods_idx_pos = goods_idx[picks]
pos_good = pos[goods_idx_pos]
pos_bad = pos[bads_idx_pos]
# test spherical fit
radius, center = _fit_sphere(pos_good)
distance = np.sqrt(np.sum((pos_good - center) ** 2, 1))
distance = np.mean(distance / radius)
if np.abs(1. - distance) > 0.1:
warn('Your spherical fit is poor, interpolation results are '
'likely to be inaccurate.')
logger.info('Computing interpolation matrix from {0} sensor '
'positions'.format(len(pos_good)))
interpolation = _make_interpolation_matrix(pos_good, pos_bad)
logger.info('Interpolating {0} sensors'.format(len(pos_bad)))
_do_interp_dots(inst, interpolation, goods_idx, bads_idx)
def _check_fname(fname, overwrite=False, must_exist=False):
"""Check for file existence."""
_validate_type(fname, 'str', 'fname')
from mne.utils import logger
if must_exist and not op.isfile(fname):
raise IOError('File "%s" does not exist' % fname)
if op.isfile(fname):
if not overwrite:
raise IOError('Destination file exists. Please use option '
'"overwrite=True" to force overwriting.')
elif overwrite != 'read':
logger.info('Overwriting existing file.')
def _find_bad_channels(epochs, picks, use_metrics, thresh, max_iter):
"""Implements the first step of the FASTER algorithm.
This function attempts to automatically mark bad EEG channels by performing
outlier detection. It operated on epoched data, to make sure only relevant
data is analyzed.
Additional Parameters
---------------------
use_metrics : list of str
List of metrics to use. Can be any combination of:
'variance', 'correlation', 'hurst', 'kurtosis', 'line_noise'
Defaults to all of them.
thresh : float
The threshold value, in standard deviations, to apply. A channel
crossing this threshold value is marked as bad. Defaults to 3.
max_iter : int
The maximum number of iterations performed during outlier detection
(defaults to 1, as in the original FASTER paper).
"""
from scipy.stats import kurtosis
metrics = {
'variance': lambda x: np.var(x, axis=1),
'correlation': lambda x: np.mean(
np.ma.masked_array(np.corrcoef(x),
np.identity(len(x), dtype=bool)), axis=0),
'hurst': lambda x: _hurst(x),
'kurtosis': lambda x: kurtosis(x, axis=1),
'line_noise': lambda x: _freqs_power(x, epochs.info['sfreq'],
[50, 60]),
}
if use_metrics is None:
use_metrics = metrics.keys()
# Concatenate epochs in time
data = epochs.get_data()[:, picks]
data = data.transpose(1, 0, 2).reshape(data.shape[1], -1)
# Find bad channels
bads = defaultdict(list)
info = pick_info(epochs.info, picks, copy=True)
for ch_type, chs in _picks_by_type(info):
logger.info('Bad channel detection on %s channels:' % ch_type.upper())
for metric in use_metrics:
scores = metrics[metric](data[chs])
bad_channels = [epochs.ch_names[picks[chs[i]]]
for i in find_outliers(scores, thresh, max_iter)]
logger.info('\tBad by %s: %s' % (metric, bad_channels))
bads[metric].append(bad_channels)
bads = dict((k, np.concatenate(v).tolist()) for k, v in bads.items())
return bads
def _find_bad_channels_in_epochs(epochs, picks, use_metrics, thresh, max_iter):
"""Implements the fourth step of the FASTER algorithm.
This function attempts to automatically mark bad channels in each epochs by
performing outlier detection.
Additional Parameters
---------------------
use_metrics : list of str
List of metrics to use. Can be any combination of:
'amplitude', 'variance', 'deviation', 'median_gradient'
Defaults to all of them.
thresh : float
The threshold value, in standard deviations, to apply. A channel
crossing this threshold value is marked as bad. Defaults to 3.
max_iter : int
The maximum number of iterations performed during outlier detection
(defaults to 1, as in the original FASTER paper).
"""
metrics = {
'amplitude': lambda x: np.ptp(x, axis=2),
'deviation': lambda x: _deviation(x),
'variance': lambda x: np.var(x, axis=2),
'median_gradient': lambda x: np.median(np.abs(np.diff(x)), axis=2),
'line_noise': lambda x: _freqs_power(x, epochs.info['sfreq'],
[50, 60]),
}
if use_metrics is None:
use_metrics = metrics.keys()
info = pick_info(epochs.info, picks, copy=True)
data = epochs.get_data()[:, picks]
bads = dict((m, np.zeros((len(data), len(picks)), dtype=bool)) for
m in metrics)
for ch_type, chs in _picks_by_type(info):
ch_names = [info['ch_names'][k] for k in chs]
chs = np.array(chs)
for metric in use_metrics:
logger.info('Bad channel-in-epoch detection on %s channels:'
% ch_type.upper())
s_epochs = metrics[metric](data[:, chs])
for i_epochs, epoch in enumerate(s_epochs):
outliers = find_outliers(epoch, thresh, max_iter)
if len(outliers) > 0:
bad_segment = [ch_names[k] for k in outliers]
logger.info('Epoch %d, Bad by %s:\n\t%s' % (
i_epochs, metric, bad_segment))
bads[metric][i_epochs, chs[outliers]] = True
return bads
def __init__(self, input_fname, montage=None, eog=None,
misc=(-4, -3, -2, -1), stim_channel=None, scale=1e-6, sfreq=250,
missing_tol=1, preload=True, verbose=None):
bci_info = {'missing_tol': missing_tol, 'stim_channel': stim_channel}
openbci_channames = ["FP1", "FP2", "C3", "C4", "P7", "P8", "O1", "O2", "F7", "F8", "F3", "F4", "T7", "T8", "P3", "P4"]
if not eog:
eog = list()
if not misc:
misc = list()
nsamps, nchan = self._get_data_dims(input_fname)
last_samps = [nsamps - 1]
ch_names = ['EEG %03d' % num for num in range(1, nchan + 1)]
ch_names[:nchan-4] = openbci_channames[:nchan-4]
ch_types = ['eeg'] * nchan
if misc:
misc_names = ['MISC %03d' % ii for ii in range(1, len(misc) + 1)]
misc_types = ['misc'] * len(misc)
for ii, mi in enumerate(misc):
ch_names[mi] = misc_names[ii]
ch_types[mi] = misc_types[ii]
if eog:
eog_names = ['EOG %03d' % ii for ii in range(len(eog))]
eog_types = ['eog'] * len(eog)
for ii, ei in enumerate(eog):
ch_names[ei] = eog_names[ii]
ch_types[ei] = eog_types[ii]
if stim_channel:
ch_names[stim_channel] = 'STI 014'
ch_types[stim_channel] = 'stim'
# mark last channel as the timestamp channel
ch_names[-1] = "Timestamps"
ch_types[-1] = "misc"
# fix it for eog and misc marking
info = create_info(ch_names, sfreq, ch_types, montage)
info["buffer_size_sec"] = 1.
super(RawOpenBCI, self).__init__(info, last_samps=last_samps,
raw_extras=[bci_info],
filenames=[input_fname],
preload=False, verbose=verbose)
# load data
if preload:
self.preload = preload
logger.info('Reading raw data from %s...' % input_fname)
self._data = self._read_segment()
def read_info(subject, data_type, run_index=0, hcp_path=op.curdir):
"""Read info from unprocessed data
Parameters
----------
subject : str, file_map
The subject
data_type : str
The kind of data to read. The following options are supported:
'rest'
'task_motor'
'task_story_math'
'task_working_memory'
'noise_empty_room'
'noise_subject'
run_index : int
The run index. For the first run, use 0, for the second, use 1.
Also see HCP documentation for the number of runs for a given data
type.
hcp_path : str
The HCP directory, defaults to op.curdir.
Returns
-------
info : instance of mne.io.meas_info.Info
The MNE channel info object.
.. note::
HCP MEG does not deliver only 3 of the 5 task packages from MRI HCP.
"""
raw, config = get_file_paths(
subject=subject, data_type=data_type, output='raw',
run_index=run_index, hcp_path=hcp_path)
if not op.exists(raw):
raw = None
meg_info = _read_bti_info(raw, config)
if raw is None:
logger.info('Did not find Raw data. Guessing EMG, ECG and EOG '
'channels')
rename_channels(meg_info, dict(_label_mapping))
return meg_info
def _make_interpolator(inst, bad_channels):
"""Find indexes and interpolation matrix to interpolate bad channels
Parameters
----------
inst : mne.io.Raw, mne.Epochs or mne.Evoked
The data to interpolate. Must be preloaded.
"""
bads_idx = np.zeros(len(inst.ch_names), dtype=np.bool)
goods_idx = np.zeros(len(inst.ch_names), dtype=np.bool)
picks = pick_types(inst.info, meg=False, eeg=True, exclude=[])
bads_idx[picks] = [inst.ch_names[ch] in bad_channels for ch in picks]
goods_idx[picks] = True
goods_idx[bads_idx] = False
if bads_idx.sum() != len(bad_channels):
logger.warning('Channel interpolation is currently only implemented '
'for EEG. The MEG channels marked as bad will remain '
'untouched.')
pos = get_channel_positions(inst, picks)
# Make sure only EEG are used
bads_idx_pos = bads_idx[picks]
goods_idx_pos = goods_idx[picks]
pos_good = pos[goods_idx_pos]
pos_bad = pos[bads_idx_pos]
# test spherical fit
radius, center = _fit_sphere(pos_good)
distance = np.sqrt(np.sum((pos_good - center) ** 2, 1))
distance = np.mean(distance / radius)
if np.abs(1. - distance) > 0.1:
logger.warning('Your spherical fit is poor, interpolation results are '
'likely to be inaccurate.')
logger.info('Computing interpolation matrix from {0} sensor '
'positions'.format(len(pos_good)))
interpolation = _make_interpolation_matrix(pos_good, pos_bad)
return goods_idx, bads_idx, interpolation
def _find_bad_epochs(epochs, picks, use_metrics, thresh, max_iter):
"""Implements the second step of the FASTER algorithm.
This function attempts to automatically mark bad epochs by performing
outlier detection.
Additional Parameters
---------------------
use_metrics : list of str
List of metrics to use. Can be any combination of:
'amplitude', 'variance', 'deviation'. Defaults to all of them.
thresh : float
The threshold value, in standard deviations, to apply. A channel
crossing this threshold value is marked as bad. Defaults to 3.
max_iter : int
The maximum number of iterations performed during outlier detection
(defaults to 1, as in the original FASTER paper).
"""
metrics = {
'amplitude': lambda x: np.mean(np.ptp(x, axis=2), axis=1),
'deviation': lambda x: np.mean(_deviation(x), axis=1),
'variance': lambda x: np.mean(np.var(x, axis=2), axis=1),
}
if use_metrics is None:
use_metrics = metrics.keys()
info = pick_info(epochs.info, picks, copy=True)
data = epochs.get_data()[:, picks]
bads = defaultdict(list)
for ch_type, chs in _picks_by_type(info):
logger.info('Bad epoch detection on %s channels:' % ch_type.upper())
for metric in use_metrics:
scores = metrics[metric](data[:, chs])
bad_epochs = find_outliers(scores, thresh, max_iter)
logger.info('\tBad by %s: %s' % (metric, bad_epochs))
bads[metric].append(bad_epochs)
bads = dict((k, np.concatenate(v).tolist()) for k, v in bads.items())
return bads
def __init__(self, input_fname, montage=None, eog=None,
misc=(-3, -2, -1), stim_channel=None, scale=1e-6, sfreq=250,
missing_tol=1, preload=True, verbose=None):
bci_info = {'missing_tol': missing_tol, 'stim_channel': stim_channel}
if not eog:
eog = list()
if not misc:
misc = list()
nsamps, nchan = self._get_data_dims(input_fname)
last_samps = [nsamps - 1]
ch_names = ['EEG %03d' % num for num in range(1, nchan + 1)]
ch_types = ['eeg'] * nchan
if misc:
misc_names = ['MISC %03d' % ii for ii in range(1, len(misc) + 1)]
misc_types = ['misc'] * len(misc)
for ii, mi in enumerate(misc):
ch_names[mi] = misc_names[ii]
ch_types[mi] = misc_types[ii]
if eog:
eog_names = ['EOG %03d' % ii for ii in range(len(eog))]
eog_types = ['eog'] * len(eog)
for ii, ei in enumerate(eog):
ch_names[ei] = eog_names[ii]
ch_types[ei] = eog_types[ii]
if stim_channel:
ch_names[stim_channel] = 'STI 014'
ch_types[stim_channel] = 'stim'
# fix it for eog and misc marking
info = create_info(ch_names, sfreq, ch_types, montage)
super(RawOpenBCI, self).__init__(info, last_samps=last_samps,
raw_extras=[bci_info],
filenames=[input_fname],
preload=False, verbose=verbose)
# load data
if preload:
self.preload = preload
logger.info('Reading raw data from %s...' % input_fname)
self._data, _ = self._read_segment()
def get_svd_comps(sub_leadfield, n_svd_comp):
""" Compute SVD components of sub-leadfield for selected channels
(all channels in one SVD)
Parameters:
-----------
sub_leadfield: numpy array (n_sens x n_vert) with part of the leadfield matrix
n_svd_comp: scalar, number of SVD components required
Returns:
--------
u_svd: numpy array, n_svd_comp scaled SVD components of subleadfield for
selected channels
s_svd: corresponding singular values
"""
u_svd, s_svd, _ = np.linalg.svd(sub_leadfield,
full_matrices=False,
compute_uv=True)
# get desired first vectors of u_svd
u_svd = u_svd[:, :n_svd_comp]
# project SVD components on sub-leadfield, take sum over vertices
u_svd_proj = u_svd.T.dot(sub_leadfield).sum(axis=1)
# make sure overall projection has positive sign
u_svd = u_svd.dot(np.sign(np.diag(u_svd_proj)))
u_svd = u_svd * s_svd[:n_svd_comp][np.newaxis, :]
logger.info("\nFirst 5 singular values (n=%d): %s" % (u_svd.shape[0], \
s_svd[0:5]))
# explained variance by chosen components within sub-leadfield
my_comps = s_svd[0:n_svd_comp]
comp_var = (100. * np.sum(my_comps * my_comps) /
np.sum(s_svd * s_svd))
logger.info("Your %d component(s) explain(s) %.1f%% "
"variance." % (n_svd_comp, comp_var))
return u_svd
def grid_search(epochs, n_interpolates, consensus_percs, prefix, n_folds=3):
"""Grid search to find optimal values of n_interpolate and consensus_perc.
Parameters
----------
epochs : instance of mne.Epochs
The epochs object for which bad epochs must be found.
n_interpolates : array
The number of sensors to interpolate.
consensus_percs : array
The percentage of channels to be interpolated.
n_folds : int
Number of folds for cross-validation.
prefix : str
Prefix to the log
"""
cv = KFold(len(epochs), n_folds=n_folds, random_state=42)
err_cons = np.zeros((len(consensus_percs), len(n_interpolates),
n_folds))
auto_reject = ConsensusAutoReject()
# The thresholds must be learnt from the entire data
auto_reject.fit(epochs)
for fold, (train, test) in enumerate(cv):
for jdx, n_interp in enumerate(n_interpolates):
for idx, consensus_perc in enumerate(consensus_percs):
logger.info('%s[Val fold %d] Trying consensus '
'perc %0.2f, n_interp %d' % (
prefix, fold + 1, consensus_perc, n_interp))
# set the params
auto_reject.consensus_perc = consensus_perc
auto_reject.n_interpolate = n_interp
# not do the transform
auto_reject.transform(epochs[train])
# score using this param
X = epochs[test].get_data()
err_cons[idx, jdx, fold] = -auto_reject.score(X)
return err_cons
def _interpolate_bads_meg(epochs, bad_channels_by_epoch, mode='fast'):
"""Interpolate bad MEG channels per epoch
Parameters
----------
inst : mne.io.Raw, mne.Epochs or mne.Evoked
The data to interpolate. Must be preloaded.
bad_channels_by_epoch : list of list of str
Bad channel names specified for each epoch. For example, for an Epochs
instance containing 3 epochs: ``[['F1'], [], ['F3', 'FZ']]``
Notes
-----
Based on mne 0.9.0 MEG channel interpolation.
"""
if len(bad_channels_by_epoch) != len(epochs):
raise ValueError("Unequal length of epochs (%i) and "
"bad_channels_by_epoch (%i)"
% (len(epochs), len(bad_channels_by_epoch)))
interp_cache = {}
for i, bad_channels in enumerate(bad_channels_by_epoch):
if not bad_channels:
continue
# find interpolation matrix
key = tuple(sorted(bad_channels))
if key in interp_cache:
picks_good, picks_bad, interpolation = interp_cache[key]
else:
picks_good = pick_types(epochs.info, ref_meg=False, exclude=key)
picks_bad = pick_channels(epochs.ch_names, key)
interpolation = _map_meg_channels(epochs, picks_good, picks_bad, mode)
interp_cache[key] = picks_good, picks_bad, interpolation
# apply interpolation
logger.info('Interpolating sensors %s on epoch %s', picks_bad, i)
epochs._data[i, picks_bad, :] = interpolation.dot(epochs._data[i, picks_good, :])
def _run(subjects_dir, subject, layers, ico, overwrite):
this_env = copy.copy(os.environ)
this_env['SUBJECTS_DIR'] = subjects_dir
this_env['SUBJECT'] = subject
if 'SUBJECTS_DIR' not in this_env:
raise RuntimeError('The environment variable SUBJECTS_DIR should '
'be set')
if not op.isdir(subjects_dir):
raise RuntimeError('subjects directory %s not found, specify using '
'the environment variable SUBJECTS_DIR or '
'the command line option --subjects-dir')
if 'FREESURFER_HOME' not in this_env:
raise RuntimeError('The FreeSurfer environment needs to be set up '
'for this script')
subj_path = op.join(subjects_dir, subject)
if not op.exists(subj_path):
raise RuntimeError('%s does not exits. Please check your subject '
'directory path.' % subj_path)
logger.info('1. Setting up MRI files...')
if overwrite:
run_subprocess(['mne_setup_mri', '--mri', 'T1', '--subject', subject, '--overwrite'], env=this_env)
else:
run_subprocess(['mne_setup_mri', '--mri', 'T1', '--subject', subject], env=this_env)
logger.info('2. Setting up %d layer BEM...' % layers)
if layers == 3:
flash05 = op.join(subjects_dir, subject, 'nii/FLASH5.nii')
flash30 = op.join(subjects_dir, subject, 'nii/FLASH30.nii')
run_subprocess(['mne', 'flash_bem_model', '-s', subject, '-d', subjects_dir,
'--flash05', flash05, '--flash30', flash30, '-v'], env=this_env)
for srf in ('inner_skull', 'outer_skull', 'outer_skin'):
shutil.copy(op.join(subjects_dir, subject, 'bem/flash/%s.surf' % srf),
op.join(subjects_dir, subject, 'bem/%s.surf' % srf))
else:
if overwrite:
run_subprocess(['mne', 'watershed_bem', '-s', subject, '-d', subjects_dir, '--overwrite'], env=this_env)
else:
run_subprocess(['mne', 'watershed_bem', '-s', subject, '-d', subjects_dir], env=this_env)
# Create dense head surface and symbolic link to head.fif file
logger.info('3. Creating high resolution skin surface for coregisteration...')
run_subprocess(['mne', 'make_scalp_surfaces', '--overwrite', '--subject', subject])
if op.isfile(op.join(subjects_dir, subject, 'bem/%s-head.fif' % subject)):
os.rename(op.join(subjects_dir, subject, 'bem/%s-head.fif' % subject),
op.join(subjects_dir, subject, 'bem/%s-head-sparse.fif' % subject))
os.symlink((op.join(subjects_dir, subject, 'bem/%s-head-dense.fif' % subject)),
(op.join(subjects_dir, subject, 'bem/%s-head.fif' % subject)))
# Create source space
run_subprocess(['mne_setup_source_space', '--subject', subject, '--spacing', '%.0f' % 5, '--cps'],
env=this_env)
def test_noise_reducer():
data_path = os.environ['SUBJECTS_DIR']
subject = os.environ['SUBJECT']
dname = data_path + '/' + 'empty_room_files' + '/109925_empty_room_file-raw.fif'
subjects_dir = data_path + '/subjects'
#
checkresults = True
exclart = False
use_reffilter = True
refflt_lpfreq = 52.
refflt_hpfreq = 48.
print "########## before of noisereducer call ##########"
sigchanlist = ['MEG ..1', 'MEG ..3', 'MEG ..5', 'MEG ..7', 'MEG ..9']
sigchanlist = None
refchanlist = ['RFM 001', 'RFM 003', 'RFM 005', 'RFG ...']
tmin = 15.
noise_reducer(dname, signals=sigchanlist, noiseref=refchanlist, tmin=tmin,
reflp=refflt_lpfreq, refhp=refflt_hpfreq,
exclude_artifacts=exclart, complementary_signal=True)
print "########## behind of noisereducer call ##########"
print "########## Read raw data:"
tc0 = time.clock()
tw0 = time.time()
raw = mne.io.Raw(dname, preload=True)
tc1 = time.clock()
tw1 = time.time()
print "loading raw data took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tc0), (tw1 - tw0))
# Time window selection
# weights are calc'd based on [tmin,tmax], but applied to the entire data set.
# tstep is used in artifact detection
tmax = raw.times[raw.last_samp]
tstep = 0.2
itmin = int(floor(tmin * raw.info['sfreq']))
itmax = int(ceil(tmax * raw.info['sfreq']))
itstep = int(ceil(tstep * raw.info['sfreq']))
print ">>> Set time-range to [%7.3f,%7.3f]" % (tmin, tmax)
if sigchanlist is None:
sigpick = mne.pick_types(raw.info, meg='mag', eeg=False, stim=False, eog=False, exclude='bads')
else:
sigpick = channel_indices_from_list(raw.info['ch_names'][:], sigchanlist)
nsig = len(sigpick)
print "sigpick: %3d chans" % nsig
if nsig == 0:
raise ValueError("No channel selected for noise compensation")
if refchanlist is None:
# References are not limited to 4D ref-chans, but can be anything,
# incl. ECG or powerline monitor.
print ">>> Using all refchans."
refexclude = "bads"
refpick = mne.pick_types(raw.info, ref_meg=True, meg=False, eeg=False,
stim=False, eog=False, exclude=refexclude)
else:
refpick = channel_indices_from_list(raw.info['ch_names'][:], refchanlist)
print "refpick = '%s'" % refpick
nref = len(refpick)
print "refpick: %3d chans" % nref
if nref == 0:
raise ValueError("No channel selected as noise reference")
print "########## Refchan geo data:"
# This is just for info to locate special 4D-refs.
for iref in refpick:
print raw.info['chs'][iref]['ch_name'], raw.info['chs'][iref]['loc'][0:3]
print ""
if use_reffilter:
print "########## Filter reference channels:"
if refflt_lpfreq is not None:
print " low-pass with cutoff-freq %.1f" % refflt_lpfreq
if refflt_hpfreq is not None:
print "high-pass with cutoff-freq %.1f" % refflt_hpfreq
# Adapt followg drop-chans cmd to use 'all-but-refpick'
droplist = [raw.info['ch_names'][k] for k in xrange(raw.info['nchan']) if not k in refpick]
fltref = raw.drop_channels(droplist, copy=True)
tct = time.clock()
twt = time.time()
fltref.filter(refflt_hpfreq, refflt_lpfreq, picks=np.array(xrange(nref)), method='iir')
tc1 = time.clock()
tw1 = time.time()
print "filtering ref-chans took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
print "########## Calculating sig-ref/ref-ref-channel covariances:"
# Calculate sig-ref/ref-ref-channel covariance:
# (there is no need to calc inter-signal-chan cov,
# but there seems to be no appropriat fct available)
# Here we copy the idea from compute_raw_data_covariance()
# and truncate it as appropriate.
tct = time.clock()
twt = time.time()
# The following reject and info{sig,ref} entries are only
# used in _is_good-calls.
# _is_good() from mne-0.9.git-py2.7.egg/mne/epochs.py seems to
# ignore ref-channels (not covered by dict) and checks individual
# data segments - artifacts across a buffer boundary are not found.
reject = dict(grad=4000e-13, # T / m (gradiometers)
mag=4e-12, # T (magnetometers)
eeg=40e-6, # uV (EEG channels)
eog=250e-6) # uV (EOG channels)
infosig = copy.copy(raw.info)
infosig['chs'] = [raw.info['chs'][k] for k in sigpick]
infosig['ch_names'] = [raw.info['ch_names'][k] for k in sigpick]
infosig['nchan'] = len(sigpick)
idx_by_typesig = channel_indices_by_type(infosig)
# inforef not good w/ filtering, but anyway useless
inforef = copy.copy(raw.info)
inforef['chs'] = [raw.info['chs'][k] for k in refpick]
inforef['ch_names'] = [raw.info['ch_names'][k] for k in refpick]
inforef['nchan'] = len(refpick)
idx_by_typeref = channel_indices_by_type(inforef)
# Read data in chunks:
sigmean = 0
refmean = 0
sscovdata = 0
srcovdata = 0
rrcovdata = 0
n_samples = 0
for first in range(itmin, itmax, itstep):
last = first + itstep
if last >= itmax:
last = itmax
raw_segmentsig, times = raw[sigpick, first:last]
if use_reffilter:
raw_segmentref, times = fltref[:, first:last]
else:
raw_segmentref, times = raw[refpick, first:last]
# if True:
# if _is_good(raw_segmentsig, infosig['ch_names'], idx_by_typesig, reject, flat=None,
# ignore_chs=raw.info['bads']) and _is_good(raw_segmentref,
# inforef['ch_names'], idx_by_typeref, reject, flat=None,
# ignore_chs=raw.info['bads']):
if not exclart or \
_is_good(raw_segmentsig, infosig['ch_names'], idx_by_typesig, reject,
flat=None, ignore_chs=raw.info['bads']):
sigmean += raw_segmentsig.sum(axis=1)
refmean += raw_segmentref.sum(axis=1)
sscovdata += (raw_segmentsig * raw_segmentsig).sum(axis=1)
srcovdata += np.dot(raw_segmentsig, raw_segmentref.T)
rrcovdata += np.dot(raw_segmentref, raw_segmentref.T)
n_samples += raw_segmentsig.shape[1]
else:
logger.info("Artefact detected in [%d, %d]" % (first, last))
#_check_n_samples(n_samples, len(picks))
sigmean /= n_samples
refmean /= n_samples
sscovdata -= n_samples * sigmean[:] * sigmean[:]
sscovdata /= (n_samples - 1)
srcovdata -= n_samples * sigmean[:, None] * refmean[None, :]
srcovdata /= (n_samples - 1)
rrcovdata -= n_samples * refmean[:, None] * refmean[None, :]
rrcovdata /= (n_samples - 1)
sscovinit = sscovdata
print "Normalize srcov..."
rrslopedata = copy.copy(rrcovdata)
for iref in xrange(nref):
dtmp = rrcovdata[iref][iref]
if dtmp > TINY:
for isig in xrange(nsig):
srcovdata[isig][iref] /= dtmp
for jref in xrange(nref):
rrslopedata[jref][iref] /= dtmp
else:
for isig in xrange(nsig):
srcovdata[isig][iref] = 0.
for jref in xrange(nref):
rrslopedata[jref][iref] = 0.
logger.info("Number of samples used : %d" % n_samples)
tc1 = time.clock()
tw1 = time.time()
print "sigrefchn covar-calc took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
print "########## Calculating sig-ref/ref-ref-channel covariances (robust):"
# Calculate sig-ref/ref-ref-channel covariance:
# (usg B.P.Welford, "Note on a method for calculating corrected sums
# of squares and products", Technometrics4 (1962) 419-420)
# (there is no need to calc inter-signal-chan cov,
# but there seems to be no appropriat fct available)
# Here we copy the idea from compute_raw_data_covariance()
# and truncate it as appropriate.
tct = time.clock()
twt = time.time()
# The following reject and info{sig,ref} entries are only
# used in _is_good-calls.
# _is_good() from mne-0.9.git-py2.7.egg/mne/epochs.py seems to
# ignore ref-channels (not covered by dict) and checks individual
# data segments - artifacts across a buffer boundary are not found.
reject = dict(grad=4000e-13, # T / m (gradiometers)
mag=4e-12, # T (magnetometers)
eeg=40e-6, # uV (EEG channels)
eog=250e-6) # uV (EOG channels)
infosig = copy.copy(raw.info)
infosig['chs'] = [raw.info['chs'][k] for k in sigpick]
infosig['ch_names'] = [raw.info['ch_names'][k] for k in sigpick]
infosig['nchan'] = len(sigpick)
idx_by_typesig = channel_indices_by_type(infosig)
# inforef not good w/ filtering, but anyway useless
inforef = copy.copy(raw.info)
inforef['chs'] = [raw.info['chs'][k] for k in refpick]
inforef['ch_names'] = [raw.info['ch_names'][k] for k in refpick]
inforef['nchan'] = len(refpick)
idx_by_typeref = channel_indices_by_type(inforef)
# Read data in chunks:
smean = np.zeros(nsig)
smold = np.zeros(nsig)
rmean = np.zeros(nref)
rmold = np.zeros(nref)
sscov = 0
srcov = 0
rrcov = np.zeros((nref, nref))
srcov = np.zeros((nsig, nref))
n_samples = 0
for first in range(itmin, itmax, itstep):
last = first + itstep
if last >= itmax:
last = itmax
raw_segmentsig, times = raw[sigpick, first:last]
if use_reffilter:
raw_segmentref, times = fltref[:, first:last]
else:
raw_segmentref, times = raw[refpick, first:last]
# if True:
# if _is_good(raw_segmentsig, infosig['ch_names'], idx_by_typesig, reject, flat=None,
# ignore_chs=raw.info['bads']) and _is_good(raw_segmentref,
# inforef['ch_names'], idx_by_typeref, reject, flat=None,
# ignore_chs=raw.info['bads']):
if not exclart or \
_is_good(raw_segmentsig, infosig['ch_names'], idx_by_typesig, reject,
flat=None, ignore_chs=raw.info['bads']):
for isl in xrange(raw_segmentsig.shape[1]):
nsl = isl + n_samples + 1
cnslm1dnsl = float((nsl - 1)) / float(nsl)
sslsubmean = (raw_segmentsig[:, isl] - smold)
rslsubmean = (raw_segmentref[:, isl] - rmold)
smean = smold + sslsubmean / nsl
rmean = rmold + rslsubmean / nsl
sscov += sslsubmean * (raw_segmentsig[:, isl] - smean)
srcov += cnslm1dnsl * np.dot(sslsubmean.reshape((nsig, 1)), rslsubmean.reshape((1, nref)))
rrcov += cnslm1dnsl * np.dot(rslsubmean.reshape((nref, 1)), rslsubmean.reshape((1, nref)))
smold = smean
rmold = rmean
n_samples += raw_segmentsig.shape[1]
else:
logger.info("Artefact detected in [%d, %d]" % (first, last))
#_check_n_samples(n_samples, len(picks))
sscov /= (n_samples - 1)
srcov /= (n_samples - 1)
rrcov /= (n_samples - 1)
print "Normalize srcov..."
rrslope = copy.copy(rrcov)
for iref in xrange(nref):
dtmp = rrcov[iref][iref]
if dtmp > TINY:
srcov[:, iref] /= dtmp
rrslope[:, iref] /= dtmp
else:
srcov[:, iref] = 0.
rrslope[:, iref] = 0.
logger.info("Number of samples used : %d" % n_samples)
print "Compare results with 'standard' values:"
print "cmp(sigmean,smean):", np.allclose(smean, sigmean, atol=0.)
print "cmp(refmean,rmean):", np.allclose(rmean, refmean, atol=0.)
print "cmp(sscovdata,sscov):", np.allclose(sscov, sscovdata, atol=0.)
print "cmp(srcovdata,srcov):", np.allclose(srcov, srcovdata, atol=0.)
print "cmp(rrcovdata,rrcov):", np.allclose(rrcov, rrcovdata, atol=0.)
tc1 = time.clock()
tw1 = time.time()
print "sigrefchn covar-calc took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
if checkresults:
print "########## Calculated initial signal channel covariance:"
# Calculate initial signal channel covariance:
# (only used as quality measure)
print "initl rt(avg sig pwr) = %12.5e" % np.sqrt(np.mean(sscov))
for i in xrange(5):
print "initl signal-rms[%3d] = %12.5e" % (i, np.sqrt(sscov.flatten()[i]))
print " "
if nref < 6:
print "rrslope-entries:"
for i in xrange(nref):
print rrslope[i][:]
U, s, V = np.linalg.svd(rrslope, full_matrices=True)
print s
print "Applying cutoff for smallest SVs:"
dtmp = s.max() * SVD_RELCUTOFF
sinv = np.zeros(nref)
for i in xrange(nref):
if abs(s[i]) >= dtmp:
sinv[i] = 1. / s[i]
else:
s[i] = 0.
# s *= (abs(s)>=dtmp)
# sinv = ???
print s
stat = np.allclose(rrslope, np.dot(U, np.dot(np.diag(s), V)))
print ">>> Testing svd-result: %s" % stat
if not stat:
print " (Maybe due to SV-cutoff?)"
# Solve for inverse coefficients:
print ">>> Setting RRinvtr=U diag(sinv) V"
RRinvtr = np.zeros((nref, nref))
RRinvtr = np.dot(U, np.dot(np.diag(sinv), V))
if checkresults:
# print ">>> RRinvtr-result:"
# print RRinvtr
stat = np.allclose(np.identity(nref), np.dot(rrslope.transpose(), RRinvtr))
if stat:
print ">>> Testing RRinvtr-result (shld be unit-matrix): ok"
else:
print ">>> Testing RRinvtr-result (shld be unit-matrix): failed"
print np.dot(rrslope.transpose(), RRinvtr)
# np.less_equal(np.abs(np.dot(rrslope.transpose(),RRinvtr)-np.identity(nref)),0.01*np.ones((nref,nref)))
print ""
print "########## Calc weight matrix..."
# weights-matrix will be somewhat larger than necessary,
# (to simplify indexing in compensation loop):
weights = np.zeros((raw._data.shape[0], nref))
for isig in xrange(nsig):
for iref in xrange(nref):
weights[sigpick[isig]][iref] = np.dot(srcov[isig][:], RRinvtr[iref][:])
if np.allclose(np.zeros(weights.shape), np.abs(weights), atol=1.e-8):
print ">>> all weights are small (<=1.e-8)."
else:
print ">>> largest weight %12.5e" % np.max(np.abs(weights))
wlrg = np.where(np.abs(weights) >= 0.99 * np.max(np.abs(weights)))
for iwlrg in xrange(len(wlrg[0])):
print ">>> weights[%3d,%2d] = %12.5e" % \
(wlrg[0][iwlrg], wlrg[1][iwlrg], weights[wlrg[0][iwlrg], wlrg[1][iwlrg]])
if nref < 5:
print "weights-entries for first sigchans:"
for i in xrange(5):
print 'weights[sp(%2d)][r]=[' % i + ' '.join([' %+10.7f' %
val for val in weights[sigpick[i]][:]]) + ']'
print "########## Compensating signal channels:"
tct = time.clock()
twt = time.time()
# data,times = raw[:,raw.time_as_index(tmin)[0]:raw.time_as_index(tmax)[0]:]
# Work on entire data stream:
for isl in xrange(raw._data.shape[1]):
slice = np.take(raw._data, [isl], axis=1)
if use_reffilter:
refslice = np.take(fltref._data, [isl], axis=1)
refarr = refslice[:].flatten() - rmean
# refarr = fltres[:,isl]-rmean
else:
refarr = slice[refpick].flatten() - rmean
subrefarr = np.dot(weights[:], refarr)
# data[:,isl] -= subrefarr will not modify raw._data?
raw._data[:, isl] -= subrefarr
if isl%10000 == 0:
print "\rProcessed slice %6d" % isl
print "\nDone."
tc1 = time.clock()
tw1 = time.time()
print "compensation loop took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
if checkresults:
print "########## Calculating final signal channel covariance:"
# Calculate final signal channel covariance:
# (only used as quality measure)
tct = time.clock()
twt = time.time()
sigmean = 0
sscovdata = 0
n_samples = 0
for first in range(itmin, itmax, itstep):
last = first + itstep
if last >= itmax:
last = itmax
raw_segmentsig, times = raw[sigpick, first:last]
# Artifacts found here will probably differ from pre-noisered artifacts!
if not exclart or \
_is_good(raw_segmentsig, infosig['ch_names'], idx_by_typesig, reject,
flat=None, ignore_chs=raw.info['bads']):
sigmean += raw_segmentsig.sum(axis=1)
sscovdata += (raw_segmentsig * raw_segmentsig).sum(axis=1)
n_samples += raw_segmentsig.shape[1]
sigmean /= n_samples
sscovdata -= n_samples * sigmean[:] * sigmean[:]
sscovdata /= (n_samples - 1)
print ">>> no channel got worse: ", np.all(np.less_equal(sscovdata, sscovinit))
print "final rt(avg sig pwr) = %12.5e" % np.sqrt(np.mean(sscovdata))
for i in xrange(5):
print "final signal-rms[%3d] = %12.5e" % (i, np.sqrt(sscovdata.flatten()[i]))
tc1 = time.clock()
tw1 = time.time()
print "signal covar-calc took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
print " "
nrname = dname[:dname.rfind('-raw.fif')] + ',nold-raw.fif'
print "Saving '%s'..." % nrname
raw.save(nrname, overwrite=True)
tc1 = time.clock()
tw1 = time.time()
print "Total run took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tc0), (tw1 - tw0))
def log_elapsed(t, verbose=None):
"""Log elapsed time."""
logger.info('Report complete in %s seconds' % round(t, 1))
def _phase_amplitude_coupling(data, sfreq, f_phase, f_amp, ixs,
pac_func='ozkurt', events=None,
tmin=None, tmax=None, n_cycles_ph=3,
n_cycles_am=3, scale_amp_func=None,
return_data=False, concat_epochs=False,
n_jobs=1, verbose=None):
""" Compute phase-amplitude coupling using pacpy.
Parameters
----------
data : array, shape ([n_epochs], n_channels, n_times)
The data used to calculate PAC
sfreq : float
The sampling frequency of the data.
f_phase : array, dtype float, shape (n_bands_phase, 2,)
The frequency ranges to use for the phase carrier. PAC will be
calculated between n_bands_phase * n_bands_amp frequencies.
f_amp : array, dtype float, shape (n_bands_amp, 2,)
The frequency ranges to use for the phase-modulated amplitude.
PAC will be calculated between n_bands_phase * n_bands_amp frequencies.
ixs : array-like, shape (n_ch_pairs x 2)
The indices for low/high frequency channels. PAC will be estimated
between n_ch_pairs of channels. Indices correspond to rows of `data`.
pac_func : {'plv', 'glm', 'mi_canolty', 'mi_tort', 'ozkurt'} |
list of strings
The function for estimating PAC. Corresponds to functions in
`pacpy.pac`. Defaults to 'ozkurt'. If multiple frequency bands are used
then `plv` cannot be calculated.
events : array, shape (n_events, 3) | array, shape (n_events,) | None
MNE events array. To be supplied if data is 2D and output should be
split by events. In this case, `tmin` and `tmax` must be provided. If
`ndim == 1`, it is assumed to be event indices, and all events will be
grouped together.
tmin : float | list of floats, shape (n_pac_windows,) | None
If `events` is not provided, it is the start time to use in `inst`.
If `events` is provided, it is the time (in seconds) to include before
each event index. If a list of floats is given, then PAC is calculated
for each pair of `tmin` and `tmax`. Defaults to `min(inst.times)`.
tmax : float | list of floats, shape (n_pac_windows,) | None
If `events` is not provided, it is the stop time to use in `inst`.
If `events` is provided, it is the time (in seconds) to include after
each event index. If a list of floats is given, then PAC is calculated
for each pair of `tmin` and `tmax`. Defaults to `max(inst.n_times)`.
n_cycles_ph : float, int | array of floats, shape (n_bands_phase,)
The number of cycles to be included in the window for each band-pass
filter for phase. Defaults to 3.
n_cycles_am : float, int | array of floats, shape (n_bands_amp,)
The number of cycles to be included in the window for each band-pass
filter for amplitude. Defaults to 3.
scale_amp_func : None | function
If not None, will be called on each amplitude signal in order to scale
the values. Function must accept an N-D input and will operate on the
last dimension. E.g., `sklearn.preprocessing.scale`.
Defaults to no scaling.
return_data : bool
If False, output will be `[pac_out]`. If True, output will be,
`[pac_out, phase_signal, amp_signal]`.
concat_epochs : bool
If True, epochs will be concatenated before calculating PAC values. If
epochs are relatively short, this is a good idea in order to improve
stability of the PAC metric.
n_jobs : int
Number of jobs to run in parallel. Defaults to 1.
verbose : bool, str, int, or None
If not None, override default verbose level (see `mne.verbose`).
Returns
-------
pac_out : array, list of arrays, dtype float,
shape([n_pac_funcs], n_epochs, n_channel_pairs,
n_freq_pairs, n_pac_windows).
The computed phase-amplitude coupling between each pair of data sources
given in ixs. If multiple pac metrics are specified, there will be one
array per metric in the output list. If n_pac_funcs is 1, then the
first dimension will be dropped.
[phase_signal] : array, shape (n_phase_signals, n_times,)
Only returned if `return_data` is True. The phase timeseries of the
phase signals (first column of `ixs`).
[amp_signal] : array, shape (n_amp_signals, n_times,)
Only returned if `return_data` is True. The amplitude timeseries of the
amplitude signals (second column of `ixs`).
"""
from ..externals.pacpy import pac as ppac
pac_func = np.atleast_1d(pac_func)
for i_func in pac_func:
if i_func not in _pac_funcs:
raise ValueError("PAC function %s is not supported" % i_func)
n_pac_funcs = pac_func.shape[0]
ixs = np.array(ixs, ndmin=2)
n_ch_pairs = ixs.shape[0]
tmin = 0 if tmin is None else tmin
tmin = np.atleast_1d(tmin)
n_pac_windows = len(tmin)
tmax = (data.shape[-1] - 1) / float(sfreq) if tmax is None else tmax
tmax = np.atleast_1d(tmax)
f_phase = np.atleast_2d(f_phase)
f_amp = np.atleast_2d(f_amp)
n_cycles_ph = np.atleast_1d(n_cycles_ph)
n_cycles_am = np.atleast_1d(n_cycles_am)
if n_cycles_ph.shape[0] == 1:
n_cycles_ph = np.repeat(n_cycles_ph, f_phase.shape[0])
if n_cycles_am.shape[0] == 1:
n_cycles_am = np.repeat(n_cycles_am, f_amp.shape[0])
if data.ndim != 2:
raise ValueError('Data must be shape (n_channels, n_times)')
if ixs.shape[1] != 2:
raise ValueError('Indices must have have a 2nd dimension of length 2')
if f_phase.shape[-1] != 2 or f_amp.shape[-1] != 2:
raise ValueError('Frequencies must be specified w/ a low/hi tuple')
if len(tmin) != len(tmax):
raise ValueError('tmin and tmax have differing lengths')
if any(i_f.shape[0] > 1 and 'plv' in pac_func for i_f in (f_amp, f_phase)):
raise ValueError('If calculating PLV, must use a single pair of freqs')
for icyc, i_f in zip([n_cycles_ph, n_cycles_am], [f_phase, f_amp]):
if icyc.shape[0] != i_f.shape[0]:
raise ValueError("n_cycles must match n_freq_bands")
if icyc.ndim > 1:
raise ValueError("n_cycles must be 1-d, not {}d".format(icyc.ndim))
logger.info('Pre-filtering data and extracting phase/amplitude...')
hi_phase = np.unique([i_func in _hi_phase_funcs for i_func in pac_func])
if len(hi_phase) != 1:
raise ValueError("Can't mix pac funcs that use both hi-freq phase/amp")
hi_phase = bool(hi_phase[0])
data_ph, data_am, ix_map_ph, ix_map_am = _pre_filter_ph_am(
data, sfreq, ixs, f_phase, f_amp, hi_phase=hi_phase,
scale_amp_func=scale_amp_func, n_cycles_ph=n_cycles_ph,
n_cycles_am=n_cycles_am)
# So we know how big the PAC output will be
if events is None:
n_epochs = 1
elif concat_epochs is True:
if events.ndim == 1:
n_epochs = 1
else:
n_epochs = np.unique(events[:, -1]).shape[0]
else:
n_epochs = events.shape[0]
# Iterate through each pair of frequencies
ixs_freqs = product(range(data_ph.shape[1]), range(data_am.shape[1]))
ixs_freqs = np.atleast_2d(list(ixs_freqs))
freq_pac = np.array([[f_phase[ii], f_amp[jj]] for ii, jj in ixs_freqs])
n_f_pairs = len(ixs_freqs)
pac = np.zeros([n_pac_funcs, n_epochs, n_ch_pairs,
n_f_pairs, n_pac_windows])
for i_f_pair, (ix_f_ph, ix_f_am) in enumerate(ixs_freqs):
# Second dimension is frequency
i_f_data_ph = data_ph[:, ix_f_ph, ...]
i_f_data_am = data_am[:, ix_f_am, ...]
# Redefine indices to match the new data arrays
ixs_new = [(ix_map_ph[i], ix_map_am[j]) for i, j in ixs]
i_f_data_ph = mne.io.RawArray(
i_f_data_ph, mne.create_info(i_f_data_ph.shape[0], sfreq))
i_f_data_am = mne.io.RawArray(
i_f_data_am, mne.create_info(i_f_data_am.shape[0], sfreq))
# Turn into Epochs if we have defined events
if events is not None:
i_f_data_ph = _raw_to_epochs_mne(i_f_data_ph, events, tmin, tmax)
i_f_data_am = _raw_to_epochs_mne(i_f_data_am, events, tmin, tmax)
# Data is either Raw or Epochs
pbar = ProgressBar(n_epochs)
for itime, (i_tmin, i_tmax) in enumerate(zip(tmin, tmax)):
# Pull times of interest
with warnings.catch_warnings(): # To suppress a depracation
warnings.simplefilter("ignore")
# Not sure how to do this w/o copying
i_t_data_am = i_f_data_am.copy().crop(i_tmin, i_tmax)
i_t_data_ph = i_f_data_ph.copy().crop(i_tmin, i_tmax)
if concat_epochs is True:
# Iterate through each event type and hstack
con_data_ph = []
con_data_am = []
for i_ev in i_t_data_am.event_id.keys():
con_data_ph.append(np.hstack(i_t_data_ph[i_ev]._data))
con_data_am.append(np.hstack(i_t_data_am[i_ev]._data))
i_t_data_ph = np.vstack(con_data_ph)
i_t_data_am = np.vstack(con_data_am)
else:
# Just pull all epochs separately
i_t_data_ph = i_t_data_ph._data
i_t_data_am = i_t_data_am._data
# Now make sure that inputs to the loop are ep x chan x time
if i_t_data_am.ndim == 2:
i_t_data_ph = i_t_data_ph[np.newaxis, ...]
i_t_data_am = i_t_data_am[np.newaxis, ...]
# Loop through epochs (or epoch grps), each index pair, and funcs
data_iter = zip(i_t_data_ph, i_t_data_am)
for iep, (ep_ph, ep_am) in enumerate(data_iter):
for iix, (i_ix_ph, i_ix_am) in enumerate(ixs_new):
for ix_func, i_pac_func in enumerate(pac_func):
func = getattr(ppac, i_pac_func)
pac[ix_func, iep, iix, i_f_pair, itime] = func(
ep_ph[i_ix_ph], ep_am[i_ix_am],
f_phase, f_amp, filterfn=False)
pbar.update_with_increment_value(1)
if pac.shape[0] == 1:
pac = pac[0]
if return_data:
return pac, freq_pac, data_ph, data_am
else:
return pac, freq_pac
def _run(subjects_dir, subject, force, overwrite, no_decimate, verbose=None):
this_env = copy.copy(os.environ)
subjects_dir = get_subjects_dir(subjects_dir, raise_error=True)
this_env['SUBJECTS_DIR'] = subjects_dir
this_env['SUBJECT'] = subject
if 'FREESURFER_HOME' not in this_env:
raise RuntimeError('The FreeSurfer environment needs to be set up '
'for this script')
incomplete = 'warn' if force else 'raise'
subj_path = op.join(subjects_dir, subject)
if not op.exists(subj_path):
raise RuntimeError('%s does not exist. Please check your subject '
'directory path.' % subj_path)
mri = 'T1.mgz' if op.exists(op.join(subj_path, 'mri', 'T1.mgz')) else 'T1'
logger.info('1. Creating a dense scalp tessellation with mkheadsurf...')
def check_seghead(surf_path=op.join(subj_path, 'surf')):
surf = None
for k in ['lh.seghead', 'lh.smseghead']:
this_surf = op.join(surf_path, k)
if op.exists(this_surf):
surf = this_surf
break
return surf
my_seghead = check_seghead()
if my_seghead is None:
run_subprocess(['mkheadsurf', '-subjid', subject, '-srcvol', mri],
env=this_env)
surf = check_seghead()
if surf is None:
raise RuntimeError('mkheadsurf did not produce the standard output '
'file.')
bem_dir = op.join(subjects_dir, subject, 'bem')
if not op.isdir(bem_dir):
os.mkdir(bem_dir)
dense_fname = op.join(bem_dir, '%s-head-dense.fif' % subject)
logger.info('2. Creating %s ...' % dense_fname)
_check_file(dense_fname, overwrite)
surf = mne.bem._surfaces_to_bem(
[surf], [mne.io.constants.FIFF.FIFFV_BEM_SURF_ID_HEAD], [1],
incomplete=incomplete)[0]
mne.write_bem_surfaces(dense_fname, surf)
levels = 'medium', 'sparse'
tris = [] if no_decimate else [30000, 2500]
if os.getenv('_MNE_TESTING_SCALP', 'false') == 'true':
tris = [len(surf['tris'])] # don't actually decimate
for ii, (n_tri, level) in enumerate(zip(tris, levels), 3):
logger.info('%i. Creating %s tessellation...' % (ii, level))
logger.info('%i.1 Decimating the dense tessellation...' % ii)
with ETSContext():
points, tris = mne.decimate_surface(points=surf['rr'],
triangles=surf['tris'],
n_triangles=n_tri)
dec_fname = dense_fname.replace('dense', level)
logger.info('%i.2 Creating %s' % (ii, dec_fname))
_check_file(dec_fname, overwrite)
dec_surf = mne.bem._surfaces_to_bem(
[dict(rr=points, tris=tris)],
[mne.io.constants.FIFF.FIFFV_BEM_SURF_ID_HEAD], [1], rescale=False,
incomplete=incomplete)
mne.write_bem_surfaces(dec_fname, dec_surf)
def setup_provenance(script, results_dir, config=None, use_agg=True,
run_id=None):
"""Setup provenance tracking
Parameters
----------
script : str
The script that was executed.
results_dir : str
The results directory.
config : None | str
The name of the config file. By default, the function expects the
config to be under `__script__/' named `config.py`. It can also
be another kind of textfile, e.g. .json.
use_agg : bool
Whether to use the 'Agg' backend for matplotlib or not.
Returns
-------
report : mne.report.Report
The mne report.
Side-effects
------------
- make results dir if it does not exists
- sets log file for sterr output
- writes log file with runtime information
"""
if use_agg is True:
import matplotlib
matplotlib.use('Agg')
if not callable(script):
if not op.isfile(script):
raise ValueError('sorry, this is not a script!')
if not op.isdir(results_dir) and not callable(script):
results_dir = op.join(op.dirname(op.dirname(script)), results_dir)
else:
results_dir = op.join(op.curdir, results_dir)
if not callable(script):
step = op.splitext(op.split(script)[1])[0]
else:
step = script.__name__
if not op.isabs(results_dir):
results_dir = op.abspath(results_dir)
start_path = op.dirname(results_dir)
results_dir = op.join(results_dir, step)
if not op.exists(results_dir):
logger.info('generating results dir')
_forec_create_dir(results_dir, start=start_path)
if run_id is None:
run_id = create_run_id()
logger.info('generated run id: %s' % run_id)
else:
logger.info('using existing run id: %s' % run_id)
logger.info('preparing logging:')
logging_dir = op.join(results_dir, run_id)
if not op.exists(logging_dir):
logger.info('... making logging directory: %s' % logging_dir)
os.mkdir(logging_dir)
else:
logger.info('... using logging directory: %s' % logging_dir)
modules = get_versions(sys)
runtime_log = op.join(logging_dir, 'run_time.json')
with open(runtime_log, 'w') as fid:
json.dump(modules, fid)
logger.info('... writing runtime info to: %s' % runtime_log)
script_code_out = op.join(logging_dir, 'script.py')
if callable(script):
script_code_in = inspect.getsourcefile(script)
else:
script_code_in = script
if not op.isfile(script_code_out):
with open(script_code_out, 'w') as fid:
with open(script_code_in) as script_fid:
source_code = script_fid.read()
fid.write(source_code)
logger.info('... logging source code of calling script')
if config is None:
config = 'config.py'
if op.isabs(config):
config_fname = config
else:
config_fname = ''
config_code = op.join( # weird behavior of join if last arg is path
results_dir, run_id, op.split(config_fname)[-1])
if not op.isfile(config_fname):
logger.info('... No config found. Logging nothing.')
elif op.isfile(config_code):
logger.info('... Config already written. I assume that you are using'
' the same run_id for different runs of your script.')
else:
with open(config_code, 'w') as fid:
with open(config_fname) as config_fid:
source_code = config_fid.read()
fid.write(source_code)
logger.info('... logging source code of "%s".' % config_fname)
logger.info('... preparing Report')
report = Report(title=step)
report.data_path = logging_dir
std_logfile = op.join(logging_dir, 'run_output.log')
logger.info('... setting logfile: %s' % std_logfile)
set_log_file(std_logfile)
return report, run_id, results_dir, logger
def printer(x):
logger.info('exec')
return x
def _run(subjects_dir, subject, force, overwrite, verbose=None):
this_env = copy.copy(os.environ)
this_env['SUBJECTS_DIR'] = subjects_dir
this_env['SUBJECT'] = subject
if 'SUBJECTS_DIR' not in this_env:
raise RuntimeError('The environment variable SUBJECTS_DIR should '
'be set')
if not op.isdir(subjects_dir):
raise RuntimeError('subjects directory %s not found, specify using '
'the environment variable SUBJECTS_DIR or '
'the command line option --subjects-dir')
if 'MNE_ROOT' not in this_env:
raise RuntimeError('MNE_ROOT environment variable is not set')
if 'FREESURFER_HOME' not in this_env:
raise RuntimeError('The FreeSurfer environment needs to be set up '
'for this script')
force = '--force' if force else '--check'
subj_path = op.join(subjects_dir, subject)
if not op.exists(subj_path):
raise RuntimeError('%s does not exits. Please check your subject '
'directory path.' % subj_path)
if op.exists(op.join(subj_path, 'mri', 'T1.mgz')):
mri = 'T1.mgz'
else:
mri = 'T1'
logger.info('1. Creating a dense scalp tessellation with mkheadsurf...')
def check_seghead(surf_path=op.join(subj_path, 'surf')):
for k in ['/lh.seghead', '/lh.smseghead']:
surf = surf_path + k if op.exists(surf_path + k) else None
if surf is not None:
break
return surf
my_seghead = check_seghead()
if my_seghead is None:
run_subprocess(['mkheadsurf', '-subjid', subject, '-srcvol', mri],
env=this_env)
surf = check_seghead()
if surf is None:
raise RuntimeError('mkheadsurf did not produce the standard output '
'file.')
dense_fname = '{0}/{1}/bem/{1}-head-dense.fif'.format(subjects_dir,
subject)
logger.info('2. Creating %s ...' % dense_fname)
_check_file(dense_fname, overwrite)
run_subprocess(['mne_surf2bem', '--surf', surf, '--id', '4', force,
'--fif', dense_fname], env=this_env)
levels = 'medium', 'sparse'
my_surf = mne.read_bem_surfaces(dense_fname)[0]
tris = [30000, 2500]
if os.getenv('_MNE_TESTING_SCALP', 'false') == 'true':
tris = [len(my_surf['tris'])] # don't actually decimate
for ii, (n_tri, level) in enumerate(zip(tris, levels), 3):
logger.info('%i. Creating %s tessellation...' % (ii, level))
logger.info('%i.1 Decimating the dense tessellation...' % ii)
points, tris = mne.decimate_surface(points=my_surf['rr'],
triangles=my_surf['tris'],
n_triangles=n_tri)
other_fname = dense_fname.replace('dense', level)
logger.info('%i.2 Creating %s' % (ii, other_fname))
_check_file(other_fname, overwrite)
tempdir = _TempDir()
surf_fname = tempdir + '/tmp-surf.surf'
# convert points to meters, make mne_analyze happy
mne.write_surface(surf_fname, points * 1e3, tris)
# XXX for some reason --check does not work here.
try:
run_subprocess(['mne_surf2bem', '--surf', surf_fname, '--id', '4',
'--force', '--fif', other_fname], env=this_env)
finally:
del tempdir
def compute_ica(raw, subject, n_components=0.99, picks=None, decim=None,
reject=None, ecg_tmin=-0.5, ecg_tmax=0.5, eog_tmin=-0.5,
eog_tmax=0.5, n_max_ecg=3, n_max_eog=1,
n_max_ecg_epochs=200, show=True, img_scale=1.0,
random_state=None, report=None, artifact_stats=None):
"""Run ICA in raw data
Parameters
----------,
raw : instance of Raw
Raw measurements to be decomposed.
subject : str
The name of the subject.
picks : array-like of int, shape(n_channels, ) | None
Channels to be included. This selection remains throughout the
initialized ICA solution. If None only good data channels are used.
Defaults to None.
n_components : int | float | None | 'rank'
The number of components used for ICA decomposition. If int, it must be
smaller then max_pca_components. If None, all PCA components will be
used. If float between 0 and 1 components can will be selected by the
cumulative percentage of explained variance.
If 'rank', the number of components equals the rank estimate.
Defaults to 0.99.
decim : int | None
Increment for selecting each nth time slice. If None, all samples
within ``start`` and ``stop`` are used. Defalts to None.
reject : dict | None
Rejection parameters based on peak to peak amplitude.
Valid keys are 'grad' | 'mag' | 'eeg' | 'eog' | 'ecg'.
If reject is None then no rejection is done. You should
use such parameters to reject big measurement artifacts
and not EOG for example. It only applies if `inst` is of type Raw.
Defaults to {'mag': 5e-12}
ecg_tmin : float
Start time before ECG event. Defaults to -0.5.
ecg_tmax : float
End time after ECG event. Defaults to 0.5.
eog_tmin : float
Start time before rog event. Defaults to -0.5.
eog_tmax : float
End time after rog event. Defaults to 0.5.
n_max_ecg : int | None
The maximum number of ECG components to exclude. Defaults to 3.
n_max_eog : int | None
The maximum number of EOG components to exclude. Defaults to 1.
n_max_ecg_epochs : int
The maximum number of ECG epochs to use for phase-consistency
estimation. Defaults to 200.
show : bool
Show figure if True
scale_img : float
The scaling factor for the report. Defaults to 1.0.
random_state : None | int | instance of np.random.RandomState
np.random.RandomState to initialize the FastICA estimation.
As the estimation is non-deterministic it can be useful to
fix the seed to have reproducible results. Defaults to None.
report : instance of Report | None
The report object. If None, a new report will be generated.
artifact_stats : None | dict
A dict that contains info on amplitude ranges of artifacts and
numbers of events, etc. by channel type.
Returns
-------
ica : instance of ICA
The ICA solution.
report : dict
A dict with an html report ('html') and artifact statistics ('stats').
"""
if report is None:
report = Report(subject=subject, title='ICA preprocessing')
if n_components == 'rank':
n_components = raw.estimate_rank(picks=picks)
ica = ICA(n_components=n_components, max_pca_components=None,
random_state=random_state, max_iter=256)
ica.fit(raw, picks=picks, decim=decim, reject=reject)
comment = []
for ch in ('mag', 'grad', 'eeg'):
if ch in ica:
comment += [ch.upper()]
if len(comment) > 0:
comment = '+'.join(comment) + ' '
else:
comment = ''
topo_ch_type = 'mag'
if 'GRAD' in comment and 'MAG' not in comment:
topo_ch_type = 'grad'
elif 'EEG' in comment:
topo_ch_type = 'eeg'
###########################################################################
# 2) identify bad components by analyzing latent sources.
title = '%s related to %s artifacts (red) ({})'.format(subject)
# generate ECG epochs use detection via phase statistics
reject_ = {'mag': 5e-12, 'grad': 5000e-13, 'eeg': 300e-6}
if reject is not None:
reject_.update(reject)
for ch_type in ['mag', 'grad', 'eeg']:
if ch_type not in ica:
reject_.pop(ch_type)
picks_ = np.array([raw.ch_names.index(k) for k in ica.ch_names])
if 'eeg' in ica:
if 'ecg' in raw:
picks_ = np.append(picks_,
pick_types(raw.info, meg=False, ecg=True)[0])
else:
logger.info('There is no ECG channel, trying to guess ECG from '
'magnetormeters')
if artifact_stats is None:
artifact_stats = dict()
ecg_epochs = create_ecg_epochs(raw, tmin=ecg_tmin, tmax=ecg_tmax,
keep_ecg=True, picks=picks_, reject=reject_)
n_ecg_epochs_found = len(ecg_epochs.events)
artifact_stats['ecg_n_events'] = n_ecg_epochs_found
n_max_ecg_epochs = min(n_max_ecg_epochs, n_ecg_epochs_found)
artifact_stats['ecg_n_used'] = n_max_ecg_epochs
sel_ecg_epochs = np.arange(n_ecg_epochs_found)
rng = np.random.RandomState(42)
rng.shuffle(sel_ecg_epochs)
ecg_ave = ecg_epochs.average()
report.add_figs_to_section(ecg_ave.plot(), 'ECG-full', 'artifacts')
ecg_epochs = ecg_epochs[sel_ecg_epochs[:n_max_ecg_epochs]]
ecg_ave = ecg_epochs.average()
report.add_figs_to_section(ecg_ave.plot(), 'ECG-used', 'artifacts')
_put_artifact_range(artifact_stats, ecg_ave, kind='ecg')
ecg_inds, scores = ica.find_bads_ecg(ecg_epochs, method='ctps')
if len(ecg_inds) > 0:
ecg_evoked = ecg_epochs.average()
del ecg_epochs
fig = ica.plot_scores(scores, exclude=ecg_inds, labels='ecg',
title='', show=show)
report.add_figs_to_section(fig, 'scores ({})'.format(subject),
section=comment + 'ECG',
scale=img_scale)
current_exclude = [e for e in ica.exclude] # issue #2608 MNE
fig = ica.plot_sources(raw, ecg_inds, exclude=ecg_inds,
title=title % ('components', 'ecg'), show=show)
report.add_figs_to_section(fig, 'sources ({})'.format(subject),
section=comment + 'ECG',
scale=img_scale)
ica.exclude = current_exclude
fig = ica.plot_components(ecg_inds, ch_type=topo_ch_type,
title='', colorbar=True, show=show)
report.add_figs_to_section(fig, title % ('sources', 'ecg'),
section=comment + 'ECG', scale=img_scale)
ica.exclude = current_exclude
ecg_inds = ecg_inds[:n_max_ecg]
ica.exclude += ecg_inds
fig = ica.plot_sources(ecg_evoked, exclude=ecg_inds, show=show)
report.add_figs_to_section(fig, 'evoked sources ({})'.format(subject),
section=comment + 'ECG',
scale=img_scale)
fig = ica.plot_overlay(ecg_evoked, exclude=ecg_inds, show=show)
report.add_figs_to_section(fig,
'rejection overlay ({})'.format(subject),
section=comment + 'ECG',
scale=img_scale)
# detect EOG by correlation
picks_eog = np.concatenate(
[picks_, pick_types(raw.info, meg=False, eeg=False, ecg=False,
eog=True)])
eog_epochs = create_eog_epochs(raw, tmin=eog_tmin, tmax=eog_tmax,
picks=picks_eog, reject=reject_)
artifact_stats['eog_n_events'] = len(eog_epochs.events)
artifact_stats['eog_n_used'] = artifact_stats['eog_n_events']
eog_ave = eog_epochs.average()
report.add_figs_to_section(eog_ave.plot(), 'EOG-used', 'artifacts')
_put_artifact_range(artifact_stats, eog_ave, kind='eog')
eog_inds = None
if len(eog_epochs.events) > 0:
eog_inds, scores = ica.find_bads_eog(eog_epochs)
if eog_inds is not None and len(eog_epochs.events) > 0:
fig = ica.plot_scores(scores, exclude=eog_inds, labels='eog',
show=show, title='')
report.add_figs_to_section(fig, 'scores ({})'.format(subject),
section=comment + 'EOG',
scale=img_scale)
current_exclude = [e for e in ica.exclude] # issue #2608 MNE
fig = ica.plot_sources(raw, eog_inds, exclude=ecg_inds,
title=title % ('sources', 'eog'), show=show)
report.add_figs_to_section(fig, 'sources', section=comment + 'EOG',
scale=img_scale)
ica.exclude = current_exclude
fig = ica.plot_components(eog_inds, ch_type=topo_ch_type,
title='', colorbar=True, show=show)
report.add_figs_to_section(fig, title % ('components', 'eog'),
section=comment + 'EOG', scale=img_scale)
ica.exclude = current_exclude
eog_inds = eog_inds[:n_max_eog]
ica.exclude += eog_inds
eog_evoked = eog_epochs.average()
fig = ica.plot_sources(eog_evoked, exclude=eog_inds, show=show)
report.add_figs_to_section(
fig, 'evoked sources ({})'.format(subject),
section=comment + 'EOG', scale=img_scale)
fig = ica.plot_overlay(eog_evoked, exclude=eog_inds, show=show)
report.add_figs_to_section(
fig, 'rejection overlay({})'.format(subject),
section=comment + 'EOG', scale=img_scale)
else:
del eog_epochs
# check the amplitudes do not change
if len(ica.exclude) > 0:
fig = ica.plot_overlay(raw, show=show) # EOG artifacts remain
html = _render_components_table(ica)
report.add_htmls_to_section(
html, captions='excluded components',
section='ICA rejection summary (%s)' % ch_type)
report.add_figs_to_section(
fig, 'rejection overlay({})'.format(subject),
section=comment + 'RAW', scale=img_scale)
return ica, dict(html=report, stats=artifact_stats)
def _run(subjects_dir, subject, raw_dir, force, mp, volume):
this_env = copy.copy(os.environ)
this_env['SUBJECTS_DIR'] = subjects_dir
this_env['SUBJECT'] = subject
parrec_dir = op.join(subjects_dir, raw_dir, subject)
if 'SUBJECTS_DIR' not in this_env:
raise RuntimeError('The environment variable SUBJECTS_DIR should '
'be set')
if not op.isdir(subjects_dir):
raise RuntimeError('subjects directory %s not found, specify using '
'the environment variable SUBJECTS_DIR or '
'the command line option --subjects-dir')
if not op.isdir(parrec_dir):
raise RuntimeError('%s directory not found, specify using '
'the command line option --raw-dir' % parrec_dir)
if 'FREESURFER_HOME' not in this_env:
raise RuntimeError('The FreeSurfer environment needs to be set up '
'for this script')
if op.isdir(op.join(subjects_dir, subject)) and not force:
raise RuntimeError('%s FreeSurfer directory exists. '
'Use command line option --force to overwrite '
'previous reconstruction results.' % subject)
if force:
shutil.rmtree(op.join(subjects_dir, subject))
os.mkdir(op.join(subjects_dir, subject))
os.makedirs(op.join(subjects_dir, subject, 'mri/orig/'))
os.mkdir(op.join(subjects_dir, subject, 'mri/nii'))
fs_nii_dir = op.join(subjects_dir, subject, 'mri/nii')
logger.info('1. Processing raw MRI data...')
for root, _, filenames in os.walk(parrec_dir):
for filename in fnmatch.filter(filenames, '*Quiet_Survey*'):
os.remove(op.join(root, filename))
parfiles = []
for root, dirnames, filenames in os.walk(parrec_dir):
for filename in fnmatch.filter(filenames, '*.PAR'):
parfiles.append(op.join(root, filename))
parfiles.sort()
for pf in parfiles:
if (volume in pf) or ('FLASH' in pf):
print('Converting {0}'.format(pf))
pimg = nibabel.load(pf)
pr_hdr = pimg.header
raw_data = pimg.dataobj.get_unscaled()
affine = pr_hdr.get_affine(origin='fov')
nimg = nibabel.Nifti1Image(raw_data, affine, pr_hdr)
nimg.to_filename(op.join(parrec_dir, op.basename(pf)[:-4]))
shutil.copy(nimg.get_filename(), fs_nii_dir)
for ff in glob.glob(op.join(fs_nii_dir, '*.nii')):
if volume in op.basename(ff):
os.symlink(ff, op.join(fs_nii_dir, 'MPRAGE.nii'))
elif 'FLASH5' in op.basename(ff):
os.symlink(ff, op.join(fs_nii_dir, 'FLASH5.nii'))
elif 'FLASH30' in op.basename(ff):
os.symlink(ff, op.join(fs_nii_dir, 'FLASH30.nii'))
logger.info('2. Starting FreeSurfer reconstruction process...')
mri = op.join(fs_nii_dir, 'MPRAGE.nii')
run_subprocess(['mri_concat', '--rms', '--i', mri,
'--o', op.join(subjects_dir, subject, 'mri/orig/001.mgz')],
env=this_env)
run_subprocess(['recon-all', '-openmp', mp, '-subject', subject, '-all'], env=this_env)
for morph_to in ['fsaverage', subject]:
run_subprocess(['mne_make_morph_maps', '--to', morph_to, '--from', subject], env=this_env)
def _make_forward_solutions(info, mri, src, bem, bem_eog, dev_head_ts, mindist,
chpi_rrs, eog_rrs, ecg_rrs, n_jobs):
"""Calculate a forward solution for a subject
Parameters
----------
info : instance of mne.io.meas_info.Info | str
If str, then it should be a filename to a Raw, Epochs, or Evoked
file with measurement information. If dict, should be an info
dict (such as one from Raw, Epochs, or Evoked).
mri : dict | str
Either a transformation filename (usually made using mne_analyze)
or an info dict (usually opened using read_trans()).
If string, an ending of `.fif` or `.fif.gz` will be assumed to
be in FIF format, any other ending will be assumed to be a text
file with a 4x4 transformation matrix (like the `--trans` MNE-C
option).
src : str | instance of SourceSpaces
If string, should be a source space filename. Can also be an
instance of loaded or generated SourceSpaces.
bem : str
Filename of the BEM (e.g., "sample-5120-5120-5120-bem-sol.fif") to
use.
bem_eog : dict
Spherical BEM to use for EOG (and ECG) simulation.
dev_head_ts : list
List of device<->head transforms.
mindist : float
Minimum distance of sources from inner skull surface (in mm).
chpi_rrs : ndarray
CHPI dipoles to simulate (magnetic dipoles).
eog_rrs : ndarray
EOG dipoles to simulate.
ecg_rrs : ndarray
ECG dipoles to simulate.
n_jobs : int
Number of jobs to run in parallel.
Returns
-------
fwd : generator
A generator for each forward solution in dev_head_ts.
Notes
-----
Some of the forward solution calculation options from the C code
(e.g., `--grad`, `--fixed`) are not implemented here. For those,
consider using the C command line tools or the Python wrapper
`do_forward_solution`.
"""
mri_head_t, mri = _get_mri_head_t(mri)
assert mri_head_t['from'] == FIFF.FIFFV_COORD_MRI
if not isinstance(src, string_types):
if not isinstance(src, SourceSpaces):
raise TypeError('src must be a string or SourceSpaces')
else:
if not op.isfile(src):
raise IOError('Source space file "%s" not found' % src)
if isinstance(bem, dict):
bem_extra = 'dict'
else:
bem_extra = bem
if not op.isfile(bem):
raise IOError('BEM file "%s" not found' % bem)
if not isinstance(info, (dict, string_types)):
raise TypeError('info should be a dict or string')
if isinstance(info, string_types):
info = read_info(info, verbose=False)
# set default forward solution coordinate frame to HEAD
# this could, in principle, be an option
coord_frame = FIFF.FIFFV_COORD_HEAD
# Report the setup
logger.info('Setting up forward solutions')
# Read the source locations
if isinstance(src, string_types):
src = read_source_spaces(src, verbose=False)
else:
# let's make a copy in case we modify something
src = src.copy()
nsource = sum(s['nuse'] for s in src)
if nsource == 0:
raise RuntimeError('No sources are active in these source spaces. '
'"do_all" option should be used.')
logger.info('Read %d source spaces a total of %d active source locations'
% (len(src), nsource))
# make a new dict with the relevant information
mri_id = dict(machid=np.zeros(2, np.int32), version=0, secs=0, usecs=0)
info = dict(nchan=info['nchan'], chs=info['chs'], comps=info['comps'],
ch_names=info['ch_names'],
mri_file='', mri_id=mri_id, meas_file='',
meas_id=None, working_dir=os.getcwd(),
command_line='', bads=info['bads'])
# Only get the EEG channels here b/c we can do MEG later
_, _, eegels, _, eegnames, _ = \
_prep_channels(info, False, True, True, verbose=False)
# Transform the source spaces into the appropriate coordinates
# (will either be HEAD or MRI)
for s in src:
transform_surface_to(s, coord_frame, mri_head_t)
# Prepare the BEM model
bem = _setup_bem(bem, bem_extra, len(eegnames), mri_head_t, verbose=False)
# Circumvent numerical problems by excluding points too close to the skull
if not bem['is_sphere']:
inner_skull = _bem_find_surface(bem, 'inner_skull')
_filter_source_spaces(inner_skull, mindist, mri_head_t, src, n_jobs,
verbose=False)
# Time to do the heavy lifting: EEG first, then MEG
rr = np.concatenate([s['rr'][s['vertno']] for s in src])
eegfwd = _compute_forwards(rr, bem, [eegels], [None],
[None], ['eeg'], n_jobs, verbose=False)[0]
eegfwd = _to_forward_dict(eegfwd, None, eegnames, coord_frame,
FIFF.FIFFV_MNE_FREE_ORI)
eegeog = _compute_forwards(eog_rrs, bem_eog, [eegels], [None],
[None], ['eeg'], n_jobs, verbose=False)[0]
eegeog = _to_forward_dict(eegeog, None, eegnames, coord_frame,
FIFF.FIFFV_MNE_FREE_ORI)
for ti, dev_head_t in enumerate(dev_head_ts):
# could be *slightly* more efficient not to do this N times,
# but the cost here is tiny compared to actual fwd calculation
logger.info('Computing gain matrix for transform #%s/%s'
% (ti + 1, len(dev_head_ts)))
info = deepcopy(info)
info['dev_head_t'] = dev_head_t
megcoils, compcoils, _, megnames, _, meg_info = \
_prep_channels(info, True, False, False, verbose=False)
# make sure our sensors are all outside our BEM
coil_rr = [coil['r0'] for coil in megcoils]
if not bem['is_sphere']:
idx = np.where(np.array([s['id'] for s in bem['surfs']]) ==
FIFF.FIFFV_BEM_SURF_ID_BRAIN)[0]
assert len(idx) == 1
bem_surf = transform_surface_to(bem['surfs'][idx[0]], coord_frame,
mri_head_t)
outside = _points_outside_surface(coil_rr, bem_surf, n_jobs,
verbose=False)
else:
rad = bem['layers'][-1]['rad']
outside = np.sqrt(np.sum((coil_rr - bem['r0']) ** 2)) >= rad
if not np.all(outside):
raise RuntimeError('MEG sensors collided with inner skull '
'surface for transform %s' % ti)
# compute forward
megfwd = _compute_forwards(rr, bem, [megcoils], [compcoils],
[meg_info], ['meg'], n_jobs,
verbose=False)[0]
megfwd = _to_forward_dict(megfwd, None, megnames, coord_frame,
FIFF.FIFFV_MNE_FREE_ORI)
fwd = _merge_meg_eeg_fwds(megfwd, eegfwd, verbose=False)
# pick out final dict info
nsource = fwd['sol']['data'].shape[1] // 3
source_nn = np.tile(np.eye(3), (nsource, 1))
fwd.update(dict(nchan=fwd['sol']['data'].shape[0], nsource=nsource,
info=info, src=src, source_nn=source_nn,
source_rr=rr, surf_ori=False, mri_head_t=mri_head_t))
fwd['info']['mri_head_t'] = mri_head_t
fwd['info']['dev_head_t'] = dev_head_t
megeog = _compute_forwards(eog_rrs, bem_eog, [megcoils], [compcoils],
[meg_info], ['meg'], n_jobs,
verbose=False)[0]
megeog = _to_forward_dict(megeog, None, megnames, coord_frame,
FIFF.FIFFV_MNE_FREE_ORI)
fwd_eog = _merge_meg_eeg_fwds(megeog, eegeog, verbose=False)
megecg = _compute_forwards(ecg_rrs, bem_eog, [megcoils], [compcoils],
[meg_info], ['meg'], n_jobs,
verbose=False)[0]
fwd_ecg = _to_forward_dict(megecg, None, megnames, coord_frame,
FIFF.FIFFV_MNE_FREE_ORI)
fwd_chpi = _magnetic_dipole_field_vec(chpi_rrs, megcoils).T
yield fwd, fwd_eog, fwd_ecg, fwd_chpi
def noise_reducer(fname_raw, raw=None, signals=[], noiseref=[], detrending=None,
tmin=None, tmax=None, reflp=None, refhp=None, refnotch=None,
exclude_artifacts=True, checkresults=True, return_raw=False,
complementary_signal=False, fnout=None, verbose=False):
"""Apply noise reduction to signal channels using reference channels.
Parameters
----------
fname_raw : (list of) rawfile names
raw : mne Raw objects
Allows passing of raw object as well.
signals : list of string
List of channels to compensate using noiseref.
If empty use the meg signal channels.
noiseref : list of string | str
List of channels to use as noise reference.
If empty use the magnetic reference channsls (default).
signals and noiseref may contain regexp, which are resolved
using mne.pick_channels_regexp(). All other channels are copied.
tmin : lower latency bound for weight-calc [start of trace]
tmax : upper latency bound for weight-calc [ end of trace]
Weights are calc'd for (tmin,tmax), but applied to entire data set
refhp : high-pass frequency for reference signal filter [None]
reflp : low-pass frequency for reference signal filter [None]
reflp < refhp: band-stop filter
reflp > refhp: band-pass filter
reflp is not None, refhp is None: low-pass filter
reflp is None, refhp is not None: high-pass filter
refnotch : (base) notch frequency for reference signal filter [None]
use raw(ref)-notched(ref) as reference signal
exclude_artifacts: filter signal-channels thru _is_good() [True]
(parameters are at present hard-coded!)
return_raw : bool
If return_raw is true, the raw object is returned and raw file
is not written to disk. It is suggested that this option be used in cases
where the noise_reducer is applied multiple times. [False]
complementary_signal : replaced signal by traces that would be subtracted [False]
(can be useful for debugging)
detrending: boolean to ctrl subtraction of linear trend from all magn. chans [False]
checkresults : boolean to control internal checks and overall success [True]
Outputfile
----------
<wawa>,nr-raw.fif for input <wawa>-raw.fif
Returns
-------
If return_raw is True, then mne.io.Raw instance is returned.
Bugs
----
- artifact checking is incomplete (and with arb. window of tstep=0.2s)
- no accounting of channels used as signal/reference
- non existing input file handled ungracefully
"""
if type(complementary_signal) != bool:
raise ValueError("Argument complementary_signal must be of type bool")
# handle error if Raw object passed with file list
if raw and isinstance(fname_raw, list):
raise ValueError('List of file names cannot be combined with one Raw object')
# handle error if return_raw is requested with file list
if return_raw and isinstance(fname_raw, list):
raise ValueError('List of file names cannot be combined return_raw.'
'Please pass one file at a time.')
# handle error if Raw object is passed with detrending option
#TODO include perform_detrending for Raw objects
if raw and detrending:
raise ValueError('Please perform detrending on the raw file directly. Cannot perform'
'detrending on the raw object')
fnraw = get_files_from_list(fname_raw)
# loop across all filenames
for fname in fnraw:
if verbose:
print "########## Read raw data:"
tc0 = time.clock()
tw0 = time.time()
if raw is None:
if detrending:
raw = perform_detrending(fname, save=False)
else:
raw = mne.io.Raw(fname, preload=True)
else:
# perform sanity check to make sure Raw object and file are same
if os.path.basename(fname) != os.path.basename(raw.info['filename']):
warnings.warn('The file name within the Raw object and provided'
'fname are not the same. Please check again.')
tc1 = time.clock()
tw1 = time.time()
if verbose:
print ">>> loading raw data took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tc0), (tw1 - tw0))
# Time window selection
# weights are calc'd based on [tmin,tmax], but applied to the entire data set.
# tstep is used in artifact detection
# tmin,tmax variables must not be changed here!
if tmin is None:
itmin = 0
else:
itmin = int(floor(tmin * raw.info['sfreq']))
if tmax is None:
itmax = raw.last_samp
else:
itmax = int(ceil(tmax * raw.info['sfreq']))
if itmax - itmin < 2:
raise ValueError("Time-window for noise compensation empty or too short")
if verbose:
print ">>> Set time-range to [%7.3f,%7.3f]" % \
(raw.times[itmin], raw.times[itmax])
if signals is None or len(signals) == 0:
sigpick = mne.pick_types(raw.info, meg='mag', eeg=False, stim=False,
eog=False, exclude='bads')
else:
sigpick = channel_indices_from_list(raw.info['ch_names'][:], signals,
raw.info.get('bads'))
nsig = len(sigpick)
if nsig == 0:
raise ValueError("No channel selected for noise compensation")
if noiseref is None or len(noiseref) == 0:
# References are not limited to 4D ref-chans, but can be anything,
# incl. ECG or powerline monitor.
if verbose:
print ">>> Using all refchans."
refexclude = "bads"
refpick = mne.pick_types(raw.info, ref_meg=True, meg=False, eeg=False,
stim=False, eog=False, exclude='bads')
else:
refpick = channel_indices_from_list(raw.info['ch_names'][:], noiseref,
raw.info.get('bads'))
nref = len(refpick)
if nref == 0:
raise ValueError("No channel selected as noise reference")
if verbose:
print ">>> sigpick: %3d chans, refpick: %3d chans" % (nsig, nref)
if reflp is None and refhp is None and refnotch is None:
use_reffilter = False
use_refantinotch = False
else:
use_reffilter = True
if verbose:
print "########## Filter reference channels:"
use_refantinotch = False
if refnotch is not None:
if reflp is None and reflp is None:
use_refantinotch = True
freqlast = np.min([5.01 * refnotch, 0.5 * raw.info['sfreq']])
if verbose:
print ">>> notches at freq %.1f and harmonics below %.1f" % (refnotch, freqlast)
else:
raise ValueError("Cannot specify notch- and high-/low-pass"
"reference filter together")
else:
if verbose:
if reflp is not None:
print ">>> low-pass with cutoff-freq %.1f" % reflp
if refhp is not None:
print ">>> high-pass with cutoff-freq %.1f" % refhp
# Adapt followg drop-chans cmd to use 'all-but-refpick'
droplist = [raw.info['ch_names'][k] for k in xrange(raw.info['nchan']) if not k in refpick]
tct = time.clock()
twt = time.time()
fltref = raw.copy().drop_channels(droplist)
if use_refantinotch:
rawref = raw.copy().drop_channels(droplist)
freqlast = np.min([5.01 * refnotch, 0.5 * raw.info['sfreq']])
fltref.notch_filter(np.arange(refnotch, freqlast, refnotch),
picks=np.array(xrange(nref)), method='iir')
fltref._data = (rawref._data - fltref._data)
else:
fltref.filter(refhp, reflp, picks=np.array(xrange(nref)), method='iir')
tc1 = time.clock()
tw1 = time.time()
if verbose:
print ">>> filtering ref-chans took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
if verbose:
print "########## Calculating sig-ref/ref-ref-channel covariances:"
# Calculate sig-ref/ref-ref-channel covariance:
# (there is no need to calc inter-signal-chan cov,
# but there seems to be no appropriat fct available)
# Here we copy the idea from compute_raw_data_covariance()
# and truncate it as appropriate.
tct = time.clock()
twt = time.time()
# The following reject and infosig entries are only
# used in _is_good-calls.
# _is_good() from mne-0.9.git-py2.7.egg/mne/epochs.py seems to
# ignore ref-channels (not covered by dict) and checks individual
# data segments - artifacts across a buffer boundary are not found.
reject = dict(grad=4000e-13, # T / m (gradiometers)
mag=4e-12, # T (magnetometers)
eeg=40e-6, # uV (EEG channels)
eog=250e-6) # uV (EOG channels)
infosig = copy.copy(raw.info)
infosig['chs'] = [raw.info['chs'][k] for k in sigpick]
infosig['ch_names'] = [raw.info['ch_names'][k] for k in sigpick]
infosig['nchan'] = len(sigpick)
idx_by_typesig = channel_indices_by_type(infosig)
# Read data in chunks:
tstep = 0.2
itstep = int(ceil(tstep * raw.info['sfreq']))
sigmean = 0
refmean = 0
sscovdata = 0
srcovdata = 0
rrcovdata = 0
n_samples = 0
for first in range(itmin, itmax, itstep):
last = first + itstep
if last >= itmax:
last = itmax
raw_segmentsig, times = raw[sigpick, first:last]
if use_reffilter:
raw_segmentref, times = fltref[:, first:last]
else:
raw_segmentref, times = raw[refpick, first:last]
if not exclude_artifacts or \
_is_good(raw_segmentsig, infosig['ch_names'], idx_by_typesig, reject, flat=None,
ignore_chs=raw.info['bads']):
sigmean += raw_segmentsig.sum(axis=1)
refmean += raw_segmentref.sum(axis=1)
sscovdata += (raw_segmentsig * raw_segmentsig).sum(axis=1)
srcovdata += np.dot(raw_segmentsig, raw_segmentref.T)
rrcovdata += np.dot(raw_segmentref, raw_segmentref.T)
n_samples += raw_segmentsig.shape[1]
else:
logger.info("Artefact detected in [%d, %d]" % (first, last))
if n_samples <= 1:
raise ValueError('Too few samples to calculate weights')
sigmean /= n_samples
refmean /= n_samples
sscovdata -= n_samples * sigmean[:] * sigmean[:]
sscovdata /= (n_samples - 1)
srcovdata -= n_samples * sigmean[:, None] * refmean[None, :]
srcovdata /= (n_samples - 1)
rrcovdata -= n_samples * refmean[:, None] * refmean[None, :]
rrcovdata /= (n_samples - 1)
sscovinit = np.copy(sscovdata)
if verbose:
print ">>> Normalize srcov..."
rrslope = copy.copy(rrcovdata)
for iref in xrange(nref):
dtmp = rrcovdata[iref, iref]
if dtmp > TINY:
srcovdata[:, iref] /= dtmp
rrslope[:, iref] /= dtmp
else:
srcovdata[:, iref] = 0.
rrslope[:, iref] = 0.
if verbose:
print ">>> Number of samples used : %d" % n_samples
tc1 = time.clock()
tw1 = time.time()
print ">>> sigrefchn covar-calc took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
if checkresults:
if verbose:
print "########## Calculated initial signal channel covariance:"
# Calculate initial signal channel covariance:
# (only used as quality measure)
print ">>> initl rt(avg sig pwr) = %12.5e" % np.sqrt(np.mean(sscovdata))
for i in xrange(5):
print ">>> initl signal-rms[%3d] = %12.5e" % (i, np.sqrt(sscovdata.flatten()[i]))
print ">>>"
U, s, V = np.linalg.svd(rrslope, full_matrices=True)
if verbose:
print ">>> singular values:"
print s
print ">>> Applying cutoff for smallest SVs:"
dtmp = s.max() * SVD_RELCUTOFF
s *= (abs(s) >= dtmp)
sinv = [1. / s[k] if s[k] != 0. else 0. for k in xrange(nref)]
if verbose:
print ">>> singular values (after cutoff):"
print s
stat = np.allclose(rrslope, np.dot(U, np.dot(np.diag(s), V)))
if verbose:
print ">>> Testing svd-result: %s" % stat
if not stat:
print " (Maybe due to SV-cutoff?)"
# Solve for inverse coefficients:
# Set RRinv.tr=U diag(sinv) V
RRinv = np.transpose(np.dot(U, np.dot(np.diag(sinv), V)))
if checkresults:
stat = np.allclose(np.identity(nref), np.dot(RRinv, rrslope))
if stat:
if verbose:
print ">>> Testing RRinv-result (should be unit-matrix): ok"
else:
print ">>> Testing RRinv-result (should be unit-matrix): failed"
print np.transpose(np.dot(RRinv, rrslope))
print ">>>"
if verbose:
print "########## Calc weight matrix..."
# weights-matrix will be somewhat larger than necessary,
# (to simplify indexing in compensation loop):
weights = np.zeros((raw._data.shape[0], nref))
for isig in xrange(nsig):
for iref in xrange(nref):
weights[sigpick[isig],iref] = np.dot(srcovdata[isig,:], RRinv[:,iref])
if verbose:
print "########## Compensating signal channels:"
if complementary_signal:
print ">>> Caveat: REPLACING signal by compensation signal"
tct = time.clock()
twt = time.time()
# Work on entire data stream:
for isl in xrange(raw._data.shape[1]):
slice = np.take(raw._data, [isl], axis=1)
if use_reffilter:
refslice = np.take(fltref._data, [isl], axis=1)
refarr = refslice[:].flatten() - refmean
# refarr = fltres[:,isl]-refmean
else:
refarr = slice[refpick].flatten() - refmean
subrefarr = np.dot(weights[:], refarr)
if not complementary_signal:
raw._data[:, isl] -= subrefarr
else:
raw._data[:, isl] = subrefarr
if (isl % 10000 == 0) and verbose:
print "\rProcessed slice %6d" % isl
if verbose:
print "\nDone."
tc1 = time.clock()
tw1 = time.time()
print ">>> compensation loop took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
if checkresults:
if verbose:
print "########## Calculating final signal channel covariance:"
# Calculate final signal channel covariance:
# (only used as quality measure)
tct = time.clock()
twt = time.time()
sigmean = 0
sscovdata = 0
n_samples = 0
for first in range(itmin, itmax, itstep):
last = first + itstep
if last >= itmax:
last = itmax
raw_segmentsig, times = raw[sigpick, first:last]
# Artifacts found here will probably differ from pre-noisered artifacts!
if not exclude_artifacts or \
_is_good(raw_segmentsig, infosig['ch_names'], idx_by_typesig, reject,
flat=None, ignore_chs=raw.info['bads']):
sigmean += raw_segmentsig.sum(axis=1)
sscovdata += (raw_segmentsig * raw_segmentsig).sum(axis=1)
n_samples += raw_segmentsig.shape[1]
sigmean /= n_samples
sscovdata -= n_samples * sigmean[:] * sigmean[:]
sscovdata /= (n_samples - 1)
if verbose:
print ">>> no channel got worse: ", np.all(np.less_equal(sscovdata, sscovinit))
print ">>> final rt(avg sig pwr) = %12.5e" % np.sqrt(np.mean(sscovdata))
for i in xrange(5):
print ">>> final signal-rms[%3d] = %12.5e" % (i, np.sqrt(sscovdata.flatten()[i]))
tc1 = time.clock()
tw1 = time.time()
print ">>> signal covar-calc took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tct), (tw1 - twt))
print ">>>"
if fnout is not None:
fnoutloc = fnout
else:
fnoutloc = fname[:fname.rfind('-raw.fif')] + ',nr-raw.fif'
if verbose:
print ">>> Saving '%s'..." % fnoutloc
if return_raw:
return raw
else:
raw.save(fnoutloc, overwrite=True)
tc1 = time.clock()
tw1 = time.time()
if verbose:
print ">>> Total run took %.1f ms (%.2f s walltime)" % (1000. * (tc1 - tc0), (tw1 - tw0))
def combine_meeg(raw_fname, eeg_fname, flow=0.6, fhigh=200,
filter_order=2, njobs=-1):
'''
Functions combines meg data with eeg data. This is done by: -
1. Adjust MEG and EEG data length.
2. Resampling EEG data channels to match sampling
frequency of MEG signals.
3. Write EEG channels into MEG fif file and write to disk.
Parameters
----------
raw_fname: FIF file containing MEG data.
eeg_fname: FIF file containing EEG data.
flow, fhigh: Low and high frequency limits for filtering.
(default 0.6-200 Hz)
filter_order: Order of the Butterworth filter used for filtering.
njobs : Number of jobs.
Warning: Please make sure that the filter settings provided
are stable for both MEG and EEG data.
Only channels ECG 001, EOG 001, EOG 002 and STI 014 are written.
'''
import numpy as np
import mne
from mne.utils import logger
if not raw_fname.endswith('-meg.fif') and \
not eeg_fname.endswith('-eeg.fif'):
logger.warning('Files names are not standard. \
Please use standard file name extensions.')
raw = mne.io.Raw(raw_fname, preload=True)
eeg = mne.io.Raw(eeg_fname, preload=True)
# Filter both signals
filter_type = 'butter'
logger.info('The MEG and EEG signals will be filtered from %s to %s' \
% (flow, fhigh))
picks_fil = mne.pick_types(raw.info, meg=True, eog=True, \
ecg=True, exclude='bads')
raw.filter(flow, fhigh, picks=picks_fil, n_jobs=njobs, method='iir', \
iir_params={'ftype': filter_type, 'order': filter_order})
picks_fil = mne.pick_types(eeg.info, meg=False, eeg=True, exclude='bads')
eeg.filter(flow, fhigh, picks=picks_fil, n_jobs=njobs, method='iir', \
iir_params={'ftype': filter_type, 'order': filter_order})
# Find sync pulse S128 in stim channel of EEG signal.
start_idx_eeg = mne.find_events(eeg, stim_channel='STI 014', \
output='onset')[0, 0]
# Find sync pulse S128 in stim channel of MEG signal.
start_idx_raw = mne.find_events(raw, stim_channel='STI 014', \
output='onset')[0, 0]
# Start times for both eeg and meg channels
start_time_eeg = eeg.times[start_idx_eeg]
start_time_raw = raw.times[start_idx_raw]
# Stop times for both eeg and meg channels
stop_time_eeg = eeg.times[eeg.last_samp]
stop_time_raw = raw.times[raw.last_samp]
# Choose channel with shortest duration (usually MEG)
meg_duration = stop_time_eeg - start_time_eeg
eeg_duration = stop_time_raw - start_time_raw
diff_time = min(meg_duration, eeg_duration)
# Reset both the channel times based on shortest duration
end_time_eeg = diff_time + start_time_eeg
end_time_raw = diff_time + start_time_raw
# Calculate the index of the last time points
stop_idx_eeg = eeg.time_as_index(round(end_time_eeg, 3))[0]
stop_idx_raw = raw.time_as_index(round(end_time_raw, 3))[0]
events = mne.find_events(eeg, stim_channel='STI 014', output='onset',
consecutive=True)
events = events[np.where(events[:, 0] < stop_idx_eeg)[0], :]
events = events[np.where(events[:, 0] > start_idx_eeg)[0], :]
events[:, 0] -= start_idx_eeg
eeg_data, eeg_times = eeg[:, start_idx_eeg:stop_idx_eeg]
_, raw_times = raw[:, start_idx_raw:stop_idx_raw]
# Resample eeg signal
resamp_list = jumeg_resample(raw.info['sfreq'], eeg.info['sfreq'], \
raw_times.shape[0], events=events)
# Update eeg signal
eeg._data, eeg._times = eeg_data[:, resamp_list], eeg_times[resamp_list]
# Update meg signal
raw._data, raw._times = raw[:, start_idx_raw:stop_idx_raw]
raw._first_samps[0] = 0
raw._last_samps[0] = raw._data.shape[1] - 1
# Identify raw channels for ECG, EOG and STI and replace it with relevant data.
logger.info('Only ECG 001, EOG 001, EOG002 and STI 014 will be updated.')
raw._data[raw.ch_names.index('ECG 001')] = eeg._data[0]
raw._data[raw.ch_names.index('EOG 001')] = eeg._data[1]
raw._data[raw.ch_names.index('EOG 002')] = eeg._data[2]
raw._data[raw.ch_names.index('STI 014')] = eeg._data[3]
# Write the combined FIF file to disk.
raw.save(raw_fname.split('-')[0] + '-raw.fif', overwrite=True)
