def test_progressbar_parallel_more(capsys):
"""Test ProgressBar with parallel computing, advanced version."""
assert capsys.readouterr().out == ''
# This must be "1" because "capsys" won't get stdout properly otherwise
parallel, p_fun, _ = parallel_func(_identity_block_wide,
n_jobs=1,
verbose=False)
arr = np.arange(10)
with use_log_level(True):
with ProgressBar(len(arr) * 2) as pb:
out = parallel(
p_fun(x, pb.subset(pb_idx))
for pb_idx, x in array_split_idx(arr, 2, n_per_split=2))
idxs = np.concatenate([o[1] for o in out])
assert_array_equal(idxs, np.arange(len(arr) * 2))
out = np.concatenate([o[0] for o in out])
assert op.isfile(pb._mmap_fname)
sum_ = np.memmap(pb._mmap_fname,
dtype='bool',
mode='r',
shape=len(arr) * 2).sum()
assert sum_ == len(arr) * 2
assert not op.isfile(pb._mmap_fname), '__exit__ not called?'
cap = capsys.readouterr()
out = cap.err
assert '100%' in out
def main():
"""Run sliding estimator."""
if not config.contrasts:
msg = 'No contrasts specified; not performing decoding.'
logger.info(**gen_log_kwargs(message=msg))
return
if not config.decode:
msg = 'No decoding requested by user.'
logger.info(**gen_log_kwargs(message=msg))
return
# Here we go parallel inside the :class:`mne.decoding.SlidingEstimator`
# so we don't dispatch manually to multiple jobs.
parallel, run_func, _ = parallel_func(run_time_decoding, n_jobs=1)
logs = parallel(
run_func(cfg=get_config(),
subject=subject,
condition1=cond_1,
condition2=cond_2,
session=session)
for subject, session, (cond_1, cond_2) in itertools.product(
config.get_subjects(), config.get_sessions(), config.contrasts))
config.save_logs(logs)
def predict(self, X):
"""Predict all features.
Parameters
----------
X : array, shape (n_sample, n_feature)
The data.
Returns
-------
X_pred : array, shape(n_sample, n_feature)
"""
n_sample, n_feature = X.shape
if n_feature != self.n_feature_:
raise ValueError('X must have same dims in fit and predict.')
n_splits = n_jobs = np.min([self.n_jobs, self.n_feature_])
parallel, p_func, n_jobs = parallel_func(_predict_loop, n_jobs,
verbose=None,
max_nbytes='auto')
splits = np.array_split(np.arange(n_feature), n_splits)
y_pred = parallel(p_func(self.estimators_[split], X, split)
for split in splits)
self.y_pred_ = np.hstack(y_pred)
return self.y_pred_
def main():
"""Run epochs."""
# Here we use fewer N_JOBS to prevent potential memory problems
parallel, run_func, _ = parallel_func(run_epochs, n_jobs=N_JOBS)
parallel(
run_func(subject, session) for subject, session in itertools.product(
config.subjects_list, config.sessions))
def predict(self, X):
"""Predict all features.
Parameters
----------
X : array, shape (n_sample, n_feature)
The data.
Returns
-------
X_pred : array, shape(n_sample, n_feature)
"""
n_sample, n_feature = X.shape
if n_feature != self.n_feature_:
raise ValueError('X must have same dims in fit and predict.')
n_splits = n_jobs = np.min([self.n_jobs, self.n_feature_])
parallel, p_func, n_jobs = parallel_func(_predict_loop,
n_jobs,
verbose=None,
max_nbytes='auto')
splits = np.array_split(np.arange(n_feature), n_splits)
y_pred = parallel(
p_func(self.estimators_[split], X, split) for split in splits)
self.y_pred_ = np.hstack(y_pred)
return self.y_pred_
def main():
"""Apply ssp."""
if not config.use_ssp:
return
parallel, run_func, _ = parallel_func(apply_ssp, n_jobs=config.N_JOBS)
parallel(run_func(subject, session) for subject, session in
itertools.product(config.subjects_list, config.sessions))
def run_group_average_source(*, cfg, subject='average'):
"""Run group average in source space"""
if not config.run_source_estimation:
msg = ' … skipping: run_source_estimation is set to False.'
logger.info(**gen_log_kwargs(message=msg))
return
mne.datasets.fetch_fsaverage(subjects_dir=config.get_fs_subjects_dir())
parallel, run_func, _ = parallel_func(morph_stc,
n_jobs=config.get_n_jobs())
all_morphed_stcs = parallel(
run_func(cfg=cfg,
subject=subject,
fs_subject=config.get_fs_subject(subject),
session=session) for subject, session in itertools.product(
config.get_subjects(), config.get_sessions()))
mean_morphed_stcs = np.array(all_morphed_stcs).mean(axis=0)
# XXX to fix
sessions = config.get_sessions()
if sessions:
session = sessions[0]
else:
session = None
run_average(cfg=cfg, session=session, mean_morphed_stcs=mean_morphed_stcs)
def main():
"""Run tf."""
parallel, run_func, _ = parallel_func(run_time_frequency,
n_jobs=config.N_JOBS)
parallel(
run_func(subject, session) for subject, session in itertools.product(
config.subjects_list, config.sessions))
def main():
"""Make reports."""
parallel, run_func, _ = parallel_func(run_report,
n_jobs=config.get_n_jobs())
logs = parallel(
run_func(
cfg=get_config(subject=subject), subject=subject, session=session)
for subject, session in itertools.product(config.get_subjects(),
config.get_sessions()))
config.save_logs(logs)
sessions = config.get_sessions()
if not sessions:
sessions = [None]
if (config.get_task() is not None and config.get_task().lower() == 'rest'):
msg = ' … skipping "average" report for "rest" task.'
logger.info(**gen_log_kwargs(message=msg))
return
for session in sessions:
run_report_average(cfg=get_config(subject='average'),
subject='average',
session=session)
def main():
"""Run group average in source space"""
msg = 'Running Step 13: Grand-average source estimates'
logger.info(gen_log_message(step=13, message=msg))
if not config.run_source_estimation:
msg = ' … skipping: run_source_estimation is set to False.'
logger.info(gen_log_message(step=13, message=msg))
return
mne.datasets.fetch_fsaverage(subjects_dir=config.get_fs_subjects_dir())
parallel, run_func, _ = parallel_func(morph_stc, n_jobs=config.N_JOBS)
all_morphed_stcs = parallel(run_func(subject, session)
for subject, session in
itertools.product(config.get_subjects(),
config.get_sessions()))
all_morphed_stcs = [morphed_stcs for morphed_stcs, subject in
zip(all_morphed_stcs, config.get_subjects())]
mean_morphed_stcs = map(sum, zip(*all_morphed_stcs))
subject = 'average'
# XXX to fix
if config.get_sessions():
session = config.get_sessions()[0]
else:
session = None
bids_path = BIDSPath(subject=subject,
session=session,
task=config.get_task(),
acquisition=config.acq,
run=None,
processing=config.proc,
recording=config.rec,
space=config.space,
datatype=config.get_datatype(),
root=config.deriv_root,
check=False)
if isinstance(config.conditions, dict):
conditions = list(config.conditions.keys())
else:
conditions = config.conditions
for condition, this_stc in zip(conditions, mean_morphed_stcs):
this_stc /= len(all_morphed_stcs)
method = config.inverse_method
cond_str = config.sanitize_cond_name(condition)
inverse_str = method
hemi_str = 'hemi' # MNE will auto-append '-lh' and '-rh'.
morph_str = 'morph2fsaverage'
fname_stc_avg = bids_path.copy().update(
suffix=f'{cond_str}+{inverse_str}+{morph_str}+{hemi_str}')
this_stc.save(fname_stc_avg)
msg = 'Completed Step 13: Grand-average source estimates'
logger.info(gen_log_message(step=13, message=msg))
def parallel_stats(X, function=_my_wilcoxon, correction='FDR', n_jobs=2):
# check if correction method was provided
if correction not in [False, None, 'FDR']:
raise ValueError('Unknown correction')
# reshape to 2D
X = np.array(X)
dims = X.shape
X.resize([dims[0], np.prod(dims[1:])])
# prepare parallel
n_cols = X.shape[1]
parallel, pfunc, n_jobs = parallel_func(_loop, n_jobs)
n_chunks = min(n_cols, n_jobs)
chunks = np.array_split(range(n_cols), n_chunks)
p_values = parallel(pfunc(X[:, chunk], function) for chunk in chunks)
p_values = np.reshape(np.hstack(p_values), dims[1:])
X.resize(dims)
# apply correction
if correction == 'FDR':
dims = p_values.shape
_, p_values = fdr_correction(p_values)
p_values = np.reshape(p_values, dims)
return p_values
def fit(self,
raw: mne.io.RawArray,
start: float = None,
stop: float = None,
reject_by_annotation: bool = True,
gfp: bool = False,
n_jobs: int = 1,
verbose=None) -> mod_Kmeans:
"""[summary]
Args:
raw (mne.io.RawArray): [description]
start (float, optional): [description]. Defaults to None.
stop (float, optional): [description]. Defaults to None.
reject_by_annotation (bool, optional): [description]. Defaults to True.
gfp (bool, optional): [description]. Defaults to False.
n_jobs (int, optional): [description]. Defaults to 1.
verbose ([type], optional): [description]. Defaults to None.
Returns:
mod_Kmeans: [description]
"""
_validate_type(raw, (BaseRaw), 'raw', 'Raw')
reject_by_annotation = 'omit' if reject_by_annotation else None
start, stop = _check_start_stop(raw, start, stop)
n_jobs = check_n_jobs(n_jobs)
if len(raw.info['bads']) is not 0:
warn('Bad channels are present in the recording. '
'They will still be used to compute microstate topographies. '
'Consider using Raw.pick() or Raw.interpolate_bads()'
' before fitting.')
data = raw.get_data(start,
stop,
reject_by_annotation=reject_by_annotation)
if gfp is True:
data = _extract_gfps(data)
best_gev = 0
if n_jobs == 1:
for _ in range(self.n_init):
gev, maps, segmentation = self._run_mod_kmeans(data)
if gev > best_gev:
best_gev, best_maps, best_segmentation = gev, maps, segmentation
else:
parallel, p_fun, _ = parallel_func(self._run_mod_kmeans,
total=self.n_init,
n_jobs=n_jobs)
runs = parallel(p_fun(data) for i in range(self.n_init))
runs = np.array(runs)
best_run = np.argmax(runs[:, 0])
best_gev, best_maps, best_segmentation = runs[best_run]
self.cluster_centers = best_maps
self.GEV = best_gev
self.labels = best_segmentation
self.current_fit = True
return (self)
def transform(self, X):
parallel, p_func, n_jobs = parallel_func(_predict_gat, self.n_jobs)
y_pred = parallel(
p_func(self.estimators_, x_split, self.method)
for x_split in np.array_split(X, n_jobs, axis=2))
y_pred = np.concatenate(y_pred, axis=2)
return y_pred
def fit(self, X, y):
self.estimators_ = list()
parallel, p_func, n_jobs = parallel_func(_fit, self.n_jobs)
estimators = parallel(
p_func(self.estimator, split, y)
for split in np.array_split(X, n_jobs, axis=2))
self.estimators_ = np.concatenate(estimators, 0)
return self
def main():
"""Run ICA."""
if not config.use_ica:
return
parallel, run_func, _ = parallel_func(run_ica, n_jobs=config.N_JOBS)
parallel(
run_func(subject, session) for subject, session in itertools.product(
config.subjects_list, config.sessions))
def test_progressbar_parallel_basic(capsys):
"""Test ProgressBar with parallel computing, basic version."""
assert capsys.readouterr().out == ''
parallel, p_fun, _ = parallel_func(_identity, total=10, n_jobs=1,
verbose=True)
out = parallel(p_fun(x) for x in range(10))
assert out == list(range(10))
assert '100.00%' in capsys.readouterr().out
def make_surrogates_ctps(phase_array, nrepeat=1000, mode='shuffle', n_jobs=4,
verbose=None):
''' calculate surrogates from an array of (phase) trials
by means of shuffling the phase
Parameters
----------
phase_trial : 4d ndarray of dimension [nfreqs x ntrials x nchan x nsamples]
Optional:
nrepeat:
mode: 2 different modi are allowed.
'mode=shuffle' whill randomly shuffle the phase values. This is the default
'mode=shift' whill randomly shift the phase values
n_jobs: number of cpu nodes to use
verbose: verbose level (does not work yet)
Returns
-------
pt : shuffled phase trials
'''
from joblib import Parallel, delayed
from mne.parallel import parallel_func
from mne.preprocessing.ctps_ import kuiper
nfreq, ntrials, nsources, nsamples = phase_array.shape
pk = np.zeros((nfreq, nrepeat, nsources, nsamples), dtype='float32')
# create surrogates: parallised over nrepeats
parallel, my_kuiper, _ = parallel_func(kuiper, n_jobs, verbose=verbose)
for ifreq in range(nfreq):
for isource in range(nsources):
# print ">>> working on frequency: ",bp[ifreq,:]," source: ",isource+1
print ">>> working on frequency range: ",ifreq + 1," source: ",isource + 1
pt = phase_array[ifreq, :, isource, :] # extract [ntrials, nsamp]
if(mode=='shuffle'):
# shuffle phase values for all repetitions
pt_s = Parallel(n_jobs=n_jobs, verbose=0)(delayed(shuffle_data)
(pt) for i in range(nrepeat))
else:
# shift all phase values for all repetitions
pt_s = Parallel(n_jobs=n_jobs, verbose=0)(delayed(shift_data)
(pt) for i in range(nrepeat))
# calculate Kuiper's statistics for each phase array
out = parallel(my_kuiper(i) for i in pt_s)
# store stat and pk in different arrays
out = np.array(out, dtype='float32')
# ks[ifreq,:,isource,:] = out[:,0,:] # is actually not needed
pk[ifreq, :, isource, :] = out[:, 1, :] # [nrepeat, pk_idx, nsamp]
return pk
def make_surrogates_ctps(phase_array, nrepeat=1000, mode='shuffle', n_jobs=4,
verbose=None):
''' calculate surrogates from an array of (phase) trials
by means of shuffling the phase
Parameters
----------
phase_trial : 4d ndarray of dimension [nfreqs x ntrials x nchan x nsamples]
Optional:
nrepeat:
mode: 2 different modi are allowed.
'mode=shuffle' whill randomly shuffle the phase values. This is the default
'mode=shift' whill randomly shift the phase values
n_jobs: number of cpu nodes to use
verbose: verbose level (does not work yet)
Returns
-------
pt : shuffled phase trials
'''
from joblib import Parallel, delayed
from mne.parallel import parallel_func
from mne.preprocessing.ctps_ import kuiper
nfreq, ntrials, nsources, nsamples = phase_array.shape
pk = np.zeros((nfreq, nrepeat, nsources, nsamples), dtype='float32')
# create surrogates: parallised over nrepeats
parallel, my_kuiper, _ = parallel_func(kuiper, n_jobs, verbose=verbose)
for ifreq in range(nfreq):
for isource in range(nsources):
# print ">>> working on frequency: ",bp[ifreq,:]," source: ",isource+1
print ">>> working on frequency range: ",ifreq + 1," source: ",isource + 1
pt = phase_array[ifreq, :, isource, :] # extract [ntrials, nsamp]
if(mode=='shuffle'):
# shuffle phase values for all repetitions
pt_s = Parallel(n_jobs=n_jobs, verbose=0)(delayed(shuffle_data)
(pt) for i in range(nrepeat))
else:
# shift all phase values for all repetitions
pt_s = Parallel(n_jobs=n_jobs, verbose=0)(delayed(shift_data)
(pt) for i in range(nrepeat))
# calculate Kuiper's statistics for each phase array
out = parallel(my_kuiper(i) for i in pt_s)
# store stat and pk in different arrays
out = np.array(out, dtype='float32')
# ks[ifreq,:,isource,:] = out[:,0,:] # is actually not needed
pk[ifreq, :, isource, :] = out[:, 1, :] # [nrepeat, pk_idx, nsamp]
return pk
def main():
"""Run epochs."""
parallel, run_func, _ = parallel_func(drop_ptp, n_jobs=config.get_n_jobs())
logs = parallel(
run_func(cfg=get_config(), subject=subject, session=session)
for subject, session in itertools.product(config.get_subjects(),
config.get_sessions()))
config.save_logs(logs)
def main():
# Ensure we're also processing fsaverage if present
subjects = config.get_subjects()
if (Path(config.get_fs_subjects_dir()) / 'fsaverage').exists():
subjects.append('fsaverage')
parallel, run_func, _ = parallel_func(make_coreg_surfaces,
n_jobs=config.get_n_jobs())
parallel(run_func(get_config(), subject) for subject in subjects)
def main():
"""Run BEM surface extraction."""
msg = 'Running Step 10: Create BEM surfaces'
logger.info(gen_log_message(step=10, message=msg))
parallel, run_func, _ = parallel_func(make_bem, n_jobs=config.N_JOBS)
parallel(run_func(subject) for subject in config.get_subjects())
msg = 'Completed Step 10: Create BEM surfaces'
logger.info(gen_log_message(step=10, message=msg))
def main():
"""Run forward."""
msg = 'Running Step 10: Create forward solution'
logger.info(gen_log_message(step=10, message=msg))
parallel, run_func, _ = parallel_func(run_forward, n_jobs=config.N_JOBS)
parallel(run_func(subject, session) for subject, session in
itertools.product(config.get_subjects(), config.get_sessions()))
msg = 'Completed Step 10: Create forward solution'
logger.info(gen_log_message(step=10, message=msg))
def main():
"""Run inv."""
msg = 'Running Step 12: Compute and apply inverse solution'
logger.info(gen_log_message(step=12, message=msg))
parallel, run_func, _ = parallel_func(run_inverse, n_jobs=config.N_JOBS)
parallel(run_func(subject, session) for subject, session in
itertools.product(config.get_subjects(), config.get_sessions()))
msg = 'Completed Step 12: Compute and apply inverse solution'
logger.info(gen_log_message(step=12, message=msg))
def main():
"""Run evoked."""
msg = 'Running Step 6: Create evoked data'
logger.info(gen_log_message(step=6, message=msg))
parallel, run_func, _ = parallel_func(run_evoked, n_jobs=config.N_JOBS)
parallel(run_func(subject, session) for subject, session in
itertools.product(config.get_subjects(), config.get_sessions()))
msg = 'Completed Step 6: Create evoked data'
logger.info(gen_log_message(step=6, message=msg))
def main():
"""Run tf."""
msg = 'Running Step 8: Time-frequency decomposition'
logger.info(gen_log_message(message=msg, step=8))
parallel, run_func, _ = parallel_func(run_time_frequency,
n_jobs=config.N_JOBS)
parallel(run_func(subject, session) for subject, session in
itertools.product(config.get_subjects(), config.get_sessions()))
msg = 'Completed Step 8: Time-frequency decomposition'
logger.info(gen_log_message(message=msg, step=8))
def main():
"""Run cov."""
msg = 'Running Step 11: Estimate noise covariance'
logger.info(gen_log_message(step=11, message=msg))
parallel, run_func, _ = parallel_func(run_covariance, n_jobs=config.N_JOBS)
parallel(
run_func(subject, session) for subject, session in itertools.product(
config.get_subjects(), config.get_sessions()))
msg = 'Completed Step 11: Estimate noise covariance'
logger.info(gen_log_message(step=11, message=msg))
def main():
"""Run grp ave."""
msg = 'Running Step 13: Grand-average source estimates'
logger.info(gen_log_message(step=13, message=msg))
mne.datasets.fetch_fsaverage(subjects_dir=config.get_fs_subjects_dir())
parallel, run_func, _ = parallel_func(morph_stc, n_jobs=config.N_JOBS)
all_morphed_stcs = parallel(run_func(subject, session)
for subject, session in
itertools.product(config.get_subjects(),
config.get_sessions()))
all_morphed_stcs = [morphed_stcs for morphed_stcs, subject in
zip(all_morphed_stcs, config.get_subjects())]
mean_morphed_stcs = map(sum, zip(*all_morphed_stcs))
subject = 'average'
# XXX to fix
if config.get_sessions():
session = config.get_sessions()[0]
else:
session = None
deriv_path = config.get_subject_deriv_path(subject=subject,
session=session,
kind=config.get_kind())
bids_basename = BIDSPath(subject=subject,
session=session,
task=config.get_task(),
acquisition=config.acq,
run=None,
processing=config.proc,
recording=config.rec,
space=config.space,
prefix=deriv_path,
check=False)
for condition, this_stc in zip(config.conditions, mean_morphed_stcs):
this_stc /= len(all_morphed_stcs)
method = config.inverse_method
cond_str = condition.replace(op.sep, '').replace('_', '')
inverse_str = method
hemi_str = 'hemi' # MNE will auto-append '-lh' and '-rh'.
morph_str = 'morph2fsaverage'
fname_stc_avg = bids_basename.copy().update(
kind=f'{cond_str}+{inverse_str}+{morph_str}+{hemi_str}')
this_stc.save(fname_stc_avg)
msg = 'Completed Step 13: Grand-average source estimates'
logger.info(gen_log_message(step=13, message=msg))
def main():
"""Run filter."""
msg = 'Running Step 2: Frequency filtering'
logger.info(gen_log_message(step=2, message=msg))
parallel, run_func, _ = parallel_func(run_filter, n_jobs=config.N_JOBS)
parallel(run_func(subject, run, session) for subject, run, session in
itertools.product(config.get_subjects(), config.get_runs(),
config.get_sessions()))
msg = 'Completed 2: Frequency filtering'
logger.info(gen_log_message(step=2, message=msg))
def main():
"""Run maxwell_filter."""
msg = 'Running Step 1: Data import and Maxwell filtering'
logger.info(gen_log_message(step=1, message=msg))
parallel, run_func, _ = parallel_func(run_maxwell_filter,
n_jobs=config.N_JOBS)
parallel(
run_func(subject, session) for subject, session in itertools.product(
config.get_subjects(), config.get_sessions()))
msg = 'Completed Step 1: Data import and Maxwell filtering'
logger.info(gen_log_message(step=1, message=msg))
def transform(self, X):
parallel, p_func, n_jobs = parallel_func(_predict_decod, self.n_jobs)
X_splits = np.array_split(X, n_jobs, axis=2)
est_splits = np.array_split(self.estimators_, n_jobs)
y_pred = parallel(
p_func(est_split, x_split, self.method)
for (est_split, x_split) in zip(est_splits, X_splits))
if n_jobs > 1:
y_pred = np.concatenate(y_pred, axis=1)
else:
y_pred = y_pred[0]
return y_pred
def main():
"""Run epochs."""
msg = 'Running Step 3: Epoching'
logger.info(gen_log_message(step=3, message=msg))
# Here we use fewer N_JOBS to prevent potential memory problems
parallel, run_func, _ = parallel_func(run_epochs,
n_jobs=max(config.N_JOBS // 4, 1))
parallel(
run_func(subject, session) for subject, session in itertools.product(
config.get_subjects(), config.get_sessions()))
msg = 'Completed Step 3: Epoching'
logger.info(gen_log_message(step=3, message=msg))
def main():
"""Run ICA."""
msg = 'Running Step 4: Compute ICA'
logger.info(gen_log_message(step=4, message=msg))
if config.use_ica:
parallel, run_func, _ = parallel_func(run_ica, n_jobs=config.N_JOBS)
parallel(
run_func(subject, session)
for subject, session in itertools.product(config.get_subjects(),
config.get_sessions()))
msg = 'Completed Step 4: Compute ICA'
logger.info(gen_log_message(step=4, message=msg))
def pairwise(X, y, func, n_jobs=-1):
"""Applies pairwise operations on two matrices using multicore:
function(X[:, jj, kk, ...], y[:, jj, kk, ...])
Parameters
----------
X : np.ndarray, shape(n, ...)
y : np.array, shape(n, ...) | shape(n,)
If shape == X.shape:
parallel(X[:, chunk], y[:, chunk ] for chunk in n_chunks)
If shape == X.shape[0]:
parallel(X[:, chunk], y for chunk in n_chunks)
func : function
n_jobs : int, optional
Number of parallel cpu.
Returns
-------
out : np.array, shape(func(X, y))
"""
import numpy as np
from mne.parallel import parallel_func
dims = X.shape
if y.shape[0] != dims[0]:
raise ValueError('X and y must have identical shapes')
X.resize([dims[0], np.prod(dims[1:])])
if y.ndim > 1:
Y = np.reshape(y, [dims[0], np.prod(dims[1:])])
parallel, pfunc, n_jobs = parallel_func(func, n_jobs)
n_cols = X.shape[1]
n_chunks = min(n_cols, n_jobs)
chunks = np.array_split(range(n_cols), n_chunks)
if y.ndim == 1:
out = parallel(pfunc(X[:, chunk], y) for chunk in chunks)
else:
out = parallel(pfunc(X[:, chunk], Y[:, chunk]) for chunk in chunks)
# size back in case higher dependencies
X.resize(dims)
# unpack
if isinstance(out[0], tuple):
return [np.reshape(out_, dims[1:]) for out_ in zip(*out)]
else:
return np.reshape(np.hstack(out), dims[1:])
def fast_wilcoxon(X, y=None, zero_method='wilcox', correction=False,
n_jobs=-1):
from mne.parallel import parallel_func
if y is not None:
X -= y
dims = X.shape
X = X.reshape(len(X), -1)
parallel, p_time_gen, n_jobs = parallel_func(_loop_wilcoxon, n_jobs)
n_chunks = np.min([n_jobs, X.shape[1]])
out = parallel(p_time_gen(X[..., chunk],
zero_method=zero_method, correction=correction)
for chunk in np.array_split(range(X.shape[1]), n_chunks))
stats, p_val = map(list, zip(*out))
stats = np.hstack(stats).reshape(dims[1:])
p_val = np.hstack(p_val).reshape(dims[1:])
return stats, p_val
def test_progressbar_parallel_advanced(capsys):
"""Test ProgressBar with parallel computing, advanced version."""
assert capsys.readouterr().out == ''
# This must be "1" because "capsys" won't get stdout properly otherwise
parallel, p_fun, _ = parallel_func(_identity_block, n_jobs=1,
verbose=False)
arr = np.arange(10)
with ProgressBar(len(arr), verbose_bool=True) as pb:
out = parallel(p_fun(x, pb.subset(pb_idx))
for pb_idx, x in array_split_idx(arr, 2))
assert op.isfile(pb._mmap_fname)
sum_ = np.memmap(pb._mmap_fname, dtype='bool', mode='r',
shape=10).sum()
assert sum_ == len(arr)
assert not op.isfile(pb._mmap_fname), '__exit__ not called?'
out = np.concatenate(out)
assert_array_equal(out, arr)
assert '100.00%' in capsys.readouterr().out
def fit(self, X):
"""Fits a regressor for each feature.
Parameters
----------
X : array, shape (n_sample, n_feature)
The data.
"""
from sklearn.base import clone
n_sample, self.n_feature_ = X.shape
# Setup parallel
n_splits = n_jobs = np.min([self.n_jobs, self.n_feature_])
parallel, p_func, n_jobs = parallel_func(_fit_loop, n_jobs,
verbose=None,
max_nbytes='auto')
# Split chunks of features to avoid overheads
splits = np.array_split(np.arange(self.n_feature_), n_splits)
out = parallel(p_func([clone(self.estimator) for f in split], X, split)
for split in splits)
self.estimators_ = np.concatenate(out, axis=0)
def compute_psd(signals, Fs, NFFT=2048, fmin=0, fmax=np.inf, n_jobs=1):
"""Based on the code in compute_raw_psd"""
NFFT = int(NFFT)
print "Effective window size : %0.3f (s)" % (NFFT / float(Fs))
if n_jobs > 1:
parallel, my_psd, n_jobs = parallel_func(mlab.psd, n_jobs)
out = parallel(my_psd(d, Fs=Fs, NFFT=NFFT) for d in signals)
freqs = out[0][1]
psd = np.array(zip(*out)[0])
else:
psd = []
for d in signals:
p, freqs = mlab.psd(d, Fs=Fs, NFFT=NFFT)
psd.append(p)
psd = np.array(psd)
mask = (freqs >= fmin) & (freqs <= fmax)
freqs = freqs[mask]
psd = np.squeeze(psd[:, mask])
return psd, freqs
def fast_mannwhitneyu(X, Y, use_continuity=True, n_jobs=-1):
from mne.parallel import parallel_func
X = np.array(X)
Y = np.array(Y)
nx, ny = len(X), len(Y)
dims = X.shape
X = np.reshape(X, [nx, -1])
Y = np.reshape(Y, [ny, -1])
parallel, p_time_gen, n_jobs = parallel_func(_loop_mannwhitneyu, n_jobs)
n_chunks = np.min([n_jobs, X.shape[1]])
chunks = np.array_split(range(X.shape[1]), n_chunks)
out = parallel(p_time_gen(X[..., chunk],
Y[..., chunk], use_continuity=use_continuity)
for chunk in chunks)
# Unpack estimators into time slices X folds list of lists.
U, p_value = map(list, zip(*out))
U = np.concatenate(U, axis=1).reshape(dims[1:])
p_value = np.concatenate(p_value, axis=1).reshape(dims[1:])
AUC = U / (nx * ny)
# correct directionality of U stats imposed by mannwhitneyu
if nx > ny:
AUC = 1 - AUC
return U, p_value, AUC
def parallel_stats(X, function=_my_wilcoxon, correction="FDR", n_jobs=-1):
from mne.parallel import parallel_func
if correction not in [False, None, "FDR"]:
raise ValueError("Unknown correction")
# reshape to 2D
X = np.array(X)
dims = X.shape
X.resize([dims[0], np.prod(dims[1:])])
# prepare parallel
n_cols = X.shape[1]
parallel, pfunc, n_jobs = parallel_func(_loop, n_jobs)
n_chunks = min(n_cols, n_jobs)
chunks = np.array_split(range(n_cols), n_chunks)
p_values = parallel(pfunc(X[:, chunk], function) for chunk in chunks)
p_values = np.reshape(np.hstack(p_values), dims[1:])
X.resize(dims)
# apply correction
if correction == "FDR":
dims = p_values.shape
_, p_values = fdr_correction(p_values)
p_values = np.reshape(p_values, dims)
return p_values
from mne.preprocessing import ICA
from mne.parallel import parallel_func
from library.config import meg_dir, N_JOBS
def run_ica(subject_id):
subject = "sub%03d" % subject_id
print("processing subject: %s" % subject)
data_path = op.join(meg_dir, subject)
for run in range(1, 7):
print("Run: %s" % run)
run_fname = op.join(data_path, 'run_%02d_filt_sss_raw.fif' % run)
if not os.path.exists(run_fname):
warn('Could not find file %s. '
'Skipping run %s for subject %s.' % (run_fname, run, subject))
continue
raw = mne.io.read_raw_fif(run_fname, add_eeg_ref=False)
ica_name = op.join(meg_dir, subject, 'run_%02d-ica.fif' % run)
ica = ICA(method='fastica', random_state=42, n_components=0.98)
picks = mne.pick_types(raw.info, meg=True, eeg=False, eog=False,
stim=False, exclude='bads')
ica.fit(raw, picks=picks, reject=dict(grad=4000e-13, mag=4e-12),
decim=8)
ica.save(ica_name)
parallel, run_func, _ = parallel_func(run_ica, n_jobs=N_JOBS)
parallel(run_func(subject_id) for subject_id in range(1, 20))
picks = mne.pick_types(raw.info, meg=True, eeg=True, stim=True,
eog=True)
# Read epochs
epochs = mne.Epochs(raw, events, events_id, tmin, tmax, proj=True,
picks=picks, baseline=baseline, preload=True,
decim=2, reject=reject, add_eeg_ref=False)
# ICA
ica_name = op.join(meg_dir, subject, 'run_%02d-ica.fif' % run)
ica = read_ica(ica_name)
n_max_ecg = 3 # use max 3 components
ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5)
ecg_inds, scores_ecg = ica.find_bads_ecg(ecg_epochs, method='ctps',
threshold=0.8)
ica.exclude += ecg_inds[:n_max_ecg]
ica.apply(epochs)
all_epochs.append(epochs)
epochs = mne.epochs.concatenate_epochs(all_epochs)
epochs.save(op.join(data_path, '%s-epo.fif' % subject))
###############################################################################
# Let us make the script parallel across subjects
parallel, run_func, _ = parallel_func(run_epochs, n_jobs=N_JOBS)
parallel(run_func(subject_id) for subject_id in range(1, 20))
data_path = op.join(meg_dir, subject)
epochs = mne.read_epochs(op.join(data_path, '%s-epo.fif' % subject),
preload=False)
evoked_famous = epochs['face/famous'].average()
evoked_scrambled = epochs['scrambled'].average()
evoked_unfamiliar = epochs['face/unfamiliar'].average()
# Simplify comment
evoked_famous.comment = 'famous'
evoked_scrambled.comment = 'scrambled'
evoked_unfamiliar.comment = 'unfamiliar'
contrast = mne.combine_evoked([evoked_famous, evoked_unfamiliar,
evoked_scrambled], weights=[0.5, 0.5, -1.])
contrast.comment = 'contrast'
faces = mne.combine_evoked([evoked_famous, evoked_unfamiliar], 'nave')
faces.comment = 'faces'
mne.evoked.write_evokeds(op.join(data_path, '%s-ave.fif' % subject),
[evoked_famous, evoked_scrambled,
evoked_unfamiliar, contrast, faces])
# take care of noise cov
cov = mne.compute_covariance(epochs, tmax=0, method='shrunk')
cov.save(op.join(data_path, '%s-cov.fif' % subject))
parallel, run_func, _ = parallel_func(run_evoked, n_jobs=N_JOBS)
parallel(run_func(subject_id) for subject_id in range(1, 20))
# List storing a dict of results for each .fif file
sf_list = [None] * len(erm_filenames)
# Print relevant processing info
print '\nMaxwell filter fine calibrations choosen:'
pp.pprint(params['cal_keys'])
if '1D' in params['cal_keys']:
print '1D calibration file: ' + fineCal_1d_fname
if '3D' in params['cal_keys']:
print '3D calibration file: ' + fineCal_3d_fname
print ('\nProcessing {num} files (started @ '.format(num=len(erm_filenames)) +
strftime('%D %H:%M:%S') + '):')
pp.pprint(erm_filenames)
# Parallelized functions
parallel, my_maxwell, n_jobs = parallel_func(_maxwell_fun, n_jobs=-1,
verbose=False)
sf_list = parallel(my_maxwell(f_name) for f_name in erm_filenames)
####################################################
# Save data
####################################################
if pkl_data:
print '\nSaving data',
with open(op.join(save_dir, 'erm_shielding_factors.pkl'), 'wb') as pkl_file:
pkl.dump(sf_list, pkl_file)
with open(op.join(save_dir, 'shielding_factor_params.pkl'), 'wb') as pkl_file:
params['finish_time'] = strftime('%D %H:%M:%S')
pkl.dump(params, pkl_file)
print ' ... Done'
print 'Finished @ ' + strftime('%D %H:%M:%S')
fname_ave = op.join(data_path, '%s-ave.fif' % subject)
fname_cov = op.join(data_path, '%s-cov.fif' % subject)
fname_fwd = op.join(data_path, '%s-meg-%s-fwd.fif' % (subject, spacing))
fname_inv = op.join(data_path, '%s-meg-%s-inv.fif' % (subject, spacing))
evokeds = mne.read_evokeds(fname_ave, condition=[0, 1, 2, 3, 4])
cov = mne.read_cov(fname_cov)
forward = mne.read_forward_solution(fname_fwd, surf_ori=True)
# make an M/EEG, MEG-only, and EEG-only inverse operators
info = evokeds[0].info
inverse_operator = make_inverse_operator(info, forward, cov, loose=0.2,
depth=0.8)
write_inverse_operator(fname_inv, inverse_operator)
# Compute inverse solution
snr = 3.0
lambda2 = 1.0 / snr ** 2
for evoked in evokeds:
stc = apply_inverse(evoked, inverse_operator, lambda2, "dSPM",
pick_ori=None)
stc.save(op.join(data_path, 'mne_dSPM_inverse-%s' % evoked.comment))
parallel, run_func, _ = parallel_func(run_inverse, n_jobs=N_JOBS)
parallel(run_func(subject_id) for subject_id in range(1, 20))
except AttributeError:
# Some files on openfmri are corrupted and cannot be read.
warn('Could not read file %s. '
'Skipping run %s from subject %s.' % (raw_in, run, subject))
continue
if run_tsss:
# Hackish way of reading bad channels.
with open(op.join(study_path, 'ds117', subject, 'MEG',
'run_%02d_sss_log.txt' % run)) as fid:
for line in fid:
if line.startswith('Static bad channels'):
chs = line.split(':')[-1].split()
bads = ['MEG%04d' % int(ch) for ch in chs]
break
raw.info['bads'] += bads
raw = mne.preprocessing.maxwell_filter(raw, calibration=cal,
cross_talk=ctc,
st_duration=10.)
raw_out = raw_fname_out % run
if not op.exists(op.join(meg_dir, subject)):
os.mkdir(op.join(meg_dir, subject))
raw.filter(1, 40, l_trans_bandwidth=0.5, h_trans_bandwidth='auto',
filter_length='auto', phase='zero', fir_window='hann')
raw.save(raw_out, overwrite=True)
parallel, run_func, _ = parallel_func(run_filter, n_jobs=N_JOBS)
parallel(run_func(subject_id) for subject_id in range(1, 20))
import mne
from mne.parallel import parallel_func
from library.config import meg_dir, N_JOBS
def run_events(subject_id):
subject = "sub%03d" % subject_id
print("processing subject: %s" % subject)
data_path = op.join(meg_dir, subject)
for run in range(1, 7):
run_fname = op.join(data_path, 'run_%02d_filt_sss_raw.fif' % run)
if not os.path.exists(run_fname):
continue
raw = mne.io.Raw(run_fname, add_eeg_ref=False)
mask = 4096 + 256 # mask for excluding high order bits
events = mne.find_events(raw, stim_channel='STI101',
consecutive='increasing', mask=mask,
mask_type='not_and', min_duration=0.003,
verbose=True)
print("S %s - R %s" % (subject, run))
fname_events = op.join(data_path, 'run_%02d_filt_sss-eve.fif' % run)
mne.write_events(fname_events, events)
parallel, run_func, _ = parallel_func(run_events, n_jobs=N_JOBS)
parallel(run_func(subject_id) for subject_id in range(1, 20))
n_cycles = freqs / 2.
def run_time_frequency(subject_id):
print("processing subject: %s" % subject_id)
subject = "sub%03d" % subject_id
data_path = op.join(meg_dir, subject)
epochs = mne.read_epochs(op.join(data_path, '%s-epo.fif' % subject))
faces = epochs['face']
idx = [faces.ch_names.index('EEG070')]
power_faces, itc_faces = mne.time_frequency.tfr_morlet(
faces, freqs=freqs, return_itc=True, n_cycles=n_cycles, picks=idx)
power_scrambled, itc_scrambled = mne.time_frequency.tfr_morlet(
epochs['scrambled'], freqs=freqs, return_itc=True, n_cycles=n_cycles,
picks=idx)
power_faces.save(op.join(data_path, '%s-faces-tfr.h5' % subject),
overwrite=True)
itc_faces.save(op.join(data_path, '%s-itc_faces-tfr.h5' % subject),
overwrite=True)
power_scrambled.save(op.join(data_path, '%s-scrambled-tfr.h5' % subject),
overwrite=True)
itc_scrambled.save(op.join(data_path, '%s-itc_scrambled-tfr.h5' % subject),
overwrite=True)
parallel, run_func, _ = parallel_func(run_time_frequency, n_jobs=N_JOBS)
parallel(run_func(subject_id) for subject_id in range(1, 20))
from mne.decoding import TimeDecoding
times = dict(step=0.005) # fit a classifier only every 5 ms
# Use AUC because chance level is same regardless of the class balance
td = TimeDecoding(predict_mode='cross-validation',
times=times, scorer='roc_auc')
td.fit(epochs, y)
# let's save the scores now
a_vs_b = '%s_vs_%s' % (os.path.basename(condition1),
os.path.basename(condition2))
fname_td = os.path.join(data_path, '%s-td-auc-%s.mat'
% (subject, a_vs_b))
from scipy.io import savemat
savemat(fname_td, {'scores': td.score(epochs),
'times': td.times_['times']})
###############################################################################
# Finally we make this script parallel across subjects and write the results
#
# .. warning::
# This may take a large amount of memory because the epochs will be
# replicated for each parallel job
parallel, run_func, _ = parallel_func(run_time_decoding, n_jobs=N_JOBS)
parallel(run_func(subject_id, 'face', 'scrambled')
for subject_id in range(1, 20))
parallel(run_func(subject_id, 'face/famous', 'face/unfamiliar')
for subject_id in range(1, 20))
def run_forward(subject_id):
subject = "sub%03d" % subject_id
print("processing subject: %s" % subject)
data_path = op.join(meg_dir, subject)
fname_ave = op.join(data_path, '%s-ave.fif' % subject)
fname_fwd = op.join(data_path, '%s-meg-%s-fwd.fif' % (subject, spacing))
fname_trans = op.join(study_path, 'ds117', subject, 'MEG',
'%s-trans.fif' % subject)
src = mne.setup_source_space(subject, spacing=spacing,
subjects_dir=subjects_dir, overwrite=True,
n_jobs=1, add_dist=False)
src_fname = op.join(subjects_dir, subject, '%s-src.fif' % spacing)
mne.write_source_spaces(src_fname, src)
bem_model = mne.make_bem_model(subject, ico=4, subjects_dir=subjects_dir,
conductivity=(0.3,))
bem = mne.make_bem_solution(bem_model)
info = mne.read_evokeds(fname_ave, condition=0).info
fwd = mne.make_forward_solution(info, trans=fname_trans, src=src, bem=bem,
fname=None, meg=True, eeg=False,
mindist=mindist, n_jobs=1, overwrite=True)
fwd = mne.convert_forward_solution(fwd, surf_ori=True)
mne.write_forward_solution(fname_fwd, fwd, overwrite=True)
parallel, run_func, _ = parallel_func(run_forward, n_jobs=N_JOBS)
parallel(run_func(subject_id) for subject_id in range(1, 20))
