def test_progressbar_parallel_more(capsys): """Test ProgressBar with parallel computing, advanced version.""" assert capsys.readouterr().out == '' # This must be "1" because "capsys" won't get stdout properly otherwise parallel, p_fun, _ = parallel_func(_identity_block_wide, n_jobs=1, verbose=False) arr = np.arange(10) with use_log_level(True): with ProgressBar(len(arr) * 2) as pb: out = parallel( p_fun(x, pb.subset(pb_idx)) for pb_idx, x in array_split_idx(arr, 2, n_per_split=2)) idxs = np.concatenate([o[1] for o in out]) assert_array_equal(idxs, np.arange(len(arr) * 2)) out = np.concatenate([o[0] for o in out]) assert op.isfile(pb._mmap_fname) sum_ = np.memmap(pb._mmap_fname, dtype='bool', mode='r', shape=len(arr) * 2).sum() assert sum_ == len(arr) * 2 assert not op.isfile(pb._mmap_fname), '__exit__ not called?' cap = capsys.readouterr() out = cap.err assert '100%' in out
def main(): """Run sliding estimator.""" if not config.contrasts: msg = 'No contrasts specified; not performing decoding.' logger.info(**gen_log_kwargs(message=msg)) return if not config.decode: msg = 'No decoding requested by user.' logger.info(**gen_log_kwargs(message=msg)) return # Here we go parallel inside the :class:`mne.decoding.SlidingEstimator` # so we don't dispatch manually to multiple jobs. parallel, run_func, _ = parallel_func(run_time_decoding, n_jobs=1) logs = parallel( run_func(cfg=get_config(), subject=subject, condition1=cond_1, condition2=cond_2, session=session) for subject, session, (cond_1, cond_2) in itertools.product( config.get_subjects(), config.get_sessions(), config.contrasts)) config.save_logs(logs)
def predict(self, X): """Predict all features. Parameters ---------- X : array, shape (n_sample, n_feature) The data. Returns ------- X_pred : array, shape(n_sample, n_feature) """ n_sample, n_feature = X.shape if n_feature != self.n_feature_: raise ValueError('X must have same dims in fit and predict.') n_splits = n_jobs = np.min([self.n_jobs, self.n_feature_]) parallel, p_func, n_jobs = parallel_func(_predict_loop, n_jobs, verbose=None, max_nbytes='auto') splits = np.array_split(np.arange(n_feature), n_splits) y_pred = parallel(p_func(self.estimators_[split], X, split) for split in splits) self.y_pred_ = np.hstack(y_pred) return self.y_pred_
def main(): """Run epochs.""" # Here we use fewer N_JOBS to prevent potential memory problems parallel, run_func, _ = parallel_func(run_epochs, n_jobs=N_JOBS) parallel( run_func(subject, session) for subject, session in itertools.product( config.subjects_list, config.sessions))
def predict(self, X): """Predict all features. Parameters ---------- X : array, shape (n_sample, n_feature) The data. Returns ------- X_pred : array, shape(n_sample, n_feature) """ n_sample, n_feature = X.shape if n_feature != self.n_feature_: raise ValueError('X must have same dims in fit and predict.') n_splits = n_jobs = np.min([self.n_jobs, self.n_feature_]) parallel, p_func, n_jobs = parallel_func(_predict_loop, n_jobs, verbose=None, max_nbytes='auto') splits = np.array_split(np.arange(n_feature), n_splits) y_pred = parallel( p_func(self.estimators_[split], X, split) for split in splits) self.y_pred_ = np.hstack(y_pred) return self.y_pred_
def main(): """Apply ssp.""" if not config.use_ssp: return parallel, run_func, _ = parallel_func(apply_ssp, n_jobs=config.N_JOBS) parallel(run_func(subject, session) for subject, session in itertools.product(config.subjects_list, config.sessions))
def run_group_average_source(*, cfg, subject='average'): """Run group average in source space""" if not config.run_source_estimation: msg = ' … skipping: run_source_estimation is set to False.' logger.info(**gen_log_kwargs(message=msg)) return mne.datasets.fetch_fsaverage(subjects_dir=config.get_fs_subjects_dir()) parallel, run_func, _ = parallel_func(morph_stc, n_jobs=config.get_n_jobs()) all_morphed_stcs = parallel( run_func(cfg=cfg, subject=subject, fs_subject=config.get_fs_subject(subject), session=session) for subject, session in itertools.product( config.get_subjects(), config.get_sessions())) mean_morphed_stcs = np.array(all_morphed_stcs).mean(axis=0) # XXX to fix sessions = config.get_sessions() if sessions: session = sessions[0] else: session = None run_average(cfg=cfg, session=session, mean_morphed_stcs=mean_morphed_stcs)
def main(): """Run tf.""" parallel, run_func, _ = parallel_func(run_time_frequency, n_jobs=config.N_JOBS) parallel( run_func(subject, session) for subject, session in itertools.product( config.subjects_list, config.sessions))
def main(): """Make reports.""" parallel, run_func, _ = parallel_func(run_report, n_jobs=config.get_n_jobs()) logs = parallel( run_func( cfg=get_config(subject=subject), subject=subject, session=session) for subject, session in itertools.product(config.get_subjects(), config.get_sessions())) config.save_logs(logs) sessions = config.get_sessions() if not sessions: sessions = [None] if (config.get_task() is not None and config.get_task().lower() == 'rest'): msg = ' … skipping "average" report for "rest" task.' logger.info(**gen_log_kwargs(message=msg)) return for session in sessions: run_report_average(cfg=get_config(subject='average'), subject='average', session=session)
def main(): """Run group average in source space""" msg = 'Running Step 13: Grand-average source estimates' logger.info(gen_log_message(step=13, message=msg)) if not config.run_source_estimation: msg = ' … skipping: run_source_estimation is set to False.' logger.info(gen_log_message(step=13, message=msg)) return mne.datasets.fetch_fsaverage(subjects_dir=config.get_fs_subjects_dir()) parallel, run_func, _ = parallel_func(morph_stc, n_jobs=config.N_JOBS) all_morphed_stcs = parallel(run_func(subject, session) for subject, session in itertools.product(config.get_subjects(), config.get_sessions())) all_morphed_stcs = [morphed_stcs for morphed_stcs, subject in zip(all_morphed_stcs, config.get_subjects())] mean_morphed_stcs = map(sum, zip(*all_morphed_stcs)) subject = 'average' # XXX to fix if config.get_sessions(): session = config.get_sessions()[0] else: session = None bids_path = BIDSPath(subject=subject, session=session, task=config.get_task(), acquisition=config.acq, run=None, processing=config.proc, recording=config.rec, space=config.space, datatype=config.get_datatype(), root=config.deriv_root, check=False) if isinstance(config.conditions, dict): conditions = list(config.conditions.keys()) else: conditions = config.conditions for condition, this_stc in zip(conditions, mean_morphed_stcs): this_stc /= len(all_morphed_stcs) method = config.inverse_method cond_str = config.sanitize_cond_name(condition) inverse_str = method hemi_str = 'hemi' # MNE will auto-append '-lh' and '-rh'. morph_str = 'morph2fsaverage' fname_stc_avg = bids_path.copy().update( suffix=f'{cond_str}+{inverse_str}+{morph_str}+{hemi_str}') this_stc.save(fname_stc_avg) msg = 'Completed Step 13: Grand-average source estimates' logger.info(gen_log_message(step=13, message=msg))
def parallel_stats(X, function=_my_wilcoxon, correction='FDR', n_jobs=2): # check if correction method was provided if correction not in [False, None, 'FDR']: raise ValueError('Unknown correction') # reshape to 2D X = np.array(X) dims = X.shape X.resize([dims[0], np.prod(dims[1:])]) # prepare parallel n_cols = X.shape[1] parallel, pfunc, n_jobs = parallel_func(_loop, n_jobs) n_chunks = min(n_cols, n_jobs) chunks = np.array_split(range(n_cols), n_chunks) p_values = parallel(pfunc(X[:, chunk], function) for chunk in chunks) p_values = np.reshape(np.hstack(p_values), dims[1:]) X.resize(dims) # apply correction if correction == 'FDR': dims = p_values.shape _, p_values = fdr_correction(p_values) p_values = np.reshape(p_values, dims) return p_values
def fit(self, raw: mne.io.RawArray, start: float = None, stop: float = None, reject_by_annotation: bool = True, gfp: bool = False, n_jobs: int = 1, verbose=None) -> mod_Kmeans: """[summary] Args: raw (mne.io.RawArray): [description] start (float, optional): [description]. Defaults to None. stop (float, optional): [description]. Defaults to None. reject_by_annotation (bool, optional): [description]. Defaults to True. gfp (bool, optional): [description]. Defaults to False. n_jobs (int, optional): [description]. Defaults to 1. verbose ([type], optional): [description]. Defaults to None. Returns: mod_Kmeans: [description] """ _validate_type(raw, (BaseRaw), 'raw', 'Raw') reject_by_annotation = 'omit' if reject_by_annotation else None start, stop = _check_start_stop(raw, start, stop) n_jobs = check_n_jobs(n_jobs) if len(raw.info['bads']) is not 0: warn('Bad channels are present in the recording. ' 'They will still be used to compute microstate topographies. ' 'Consider using Raw.pick() or Raw.interpolate_bads()' ' before fitting.') data = raw.get_data(start, stop, reject_by_annotation=reject_by_annotation) if gfp is True: data = _extract_gfps(data) best_gev = 0 if n_jobs == 1: for _ in range(self.n_init): gev, maps, segmentation = self._run_mod_kmeans(data) if gev > best_gev: best_gev, best_maps, best_segmentation = gev, maps, segmentation else: parallel, p_fun, _ = parallel_func(self._run_mod_kmeans, total=self.n_init, n_jobs=n_jobs) runs = parallel(p_fun(data) for i in range(self.n_init)) runs = np.array(runs) best_run = np.argmax(runs[:, 0]) best_gev, best_maps, best_segmentation = runs[best_run] self.cluster_centers = best_maps self.GEV = best_gev self.labels = best_segmentation self.current_fit = True return (self)
def transform(self, X): parallel, p_func, n_jobs = parallel_func(_predict_gat, self.n_jobs) y_pred = parallel( p_func(self.estimators_, x_split, self.method) for x_split in np.array_split(X, n_jobs, axis=2)) y_pred = np.concatenate(y_pred, axis=2) return y_pred
def fit(self, X, y): self.estimators_ = list() parallel, p_func, n_jobs = parallel_func(_fit, self.n_jobs) estimators = parallel( p_func(self.estimator, split, y) for split in np.array_split(X, n_jobs, axis=2)) self.estimators_ = np.concatenate(estimators, 0) return self
def main(): """Run ICA.""" if not config.use_ica: return parallel, run_func, _ = parallel_func(run_ica, n_jobs=config.N_JOBS) parallel( run_func(subject, session) for subject, session in itertools.product( config.subjects_list, config.sessions))
def test_progressbar_parallel_basic(capsys): """Test ProgressBar with parallel computing, basic version.""" assert capsys.readouterr().out == '' parallel, p_fun, _ = parallel_func(_identity, total=10, n_jobs=1, verbose=True) out = parallel(p_fun(x) for x in range(10)) assert out == list(range(10)) assert '100.00%' in capsys.readouterr().out
def make_surrogates_ctps(phase_array, nrepeat=1000, mode='shuffle', n_jobs=4, verbose=None): ''' calculate surrogates from an array of (phase) trials by means of shuffling the phase Parameters ---------- phase_trial : 4d ndarray of dimension [nfreqs x ntrials x nchan x nsamples] Optional: nrepeat: mode: 2 different modi are allowed. 'mode=shuffle' whill randomly shuffle the phase values. This is the default 'mode=shift' whill randomly shift the phase values n_jobs: number of cpu nodes to use verbose: verbose level (does not work yet) Returns ------- pt : shuffled phase trials ''' from joblib import Parallel, delayed from mne.parallel import parallel_func from mne.preprocessing.ctps_ import kuiper nfreq, ntrials, nsources, nsamples = phase_array.shape pk = np.zeros((nfreq, nrepeat, nsources, nsamples), dtype='float32') # create surrogates: parallised over nrepeats parallel, my_kuiper, _ = parallel_func(kuiper, n_jobs, verbose=verbose) for ifreq in range(nfreq): for isource in range(nsources): # print ">>> working on frequency: ",bp[ifreq,:]," source: ",isource+1 print ">>> working on frequency range: ",ifreq + 1," source: ",isource + 1 pt = phase_array[ifreq, :, isource, :] # extract [ntrials, nsamp] if(mode=='shuffle'): # shuffle phase values for all repetitions pt_s = Parallel(n_jobs=n_jobs, verbose=0)(delayed(shuffle_data) (pt) for i in range(nrepeat)) else: # shift all phase values for all repetitions pt_s = Parallel(n_jobs=n_jobs, verbose=0)(delayed(shift_data) (pt) for i in range(nrepeat)) # calculate Kuiper's statistics for each phase array out = parallel(my_kuiper(i) for i in pt_s) # store stat and pk in different arrays out = np.array(out, dtype='float32') # ks[ifreq,:,isource,:] = out[:,0,:] # is actually not needed pk[ifreq, :, isource, :] = out[:, 1, :] # [nrepeat, pk_idx, nsamp] return pk
def main(): """Run epochs.""" parallel, run_func, _ = parallel_func(drop_ptp, n_jobs=config.get_n_jobs()) logs = parallel( run_func(cfg=get_config(), subject=subject, session=session) for subject, session in itertools.product(config.get_subjects(), config.get_sessions())) config.save_logs(logs)
def main(): # Ensure we're also processing fsaverage if present subjects = config.get_subjects() if (Path(config.get_fs_subjects_dir()) / 'fsaverage').exists(): subjects.append('fsaverage') parallel, run_func, _ = parallel_func(make_coreg_surfaces, n_jobs=config.get_n_jobs()) parallel(run_func(get_config(), subject) for subject in subjects)
def main(): """Run BEM surface extraction.""" msg = 'Running Step 10: Create BEM surfaces' logger.info(gen_log_message(step=10, message=msg)) parallel, run_func, _ = parallel_func(make_bem, n_jobs=config.N_JOBS) parallel(run_func(subject) for subject in config.get_subjects()) msg = 'Completed Step 10: Create BEM surfaces' logger.info(gen_log_message(step=10, message=msg))
def main(): """Run forward.""" msg = 'Running Step 10: Create forward solution' logger.info(gen_log_message(step=10, message=msg)) parallel, run_func, _ = parallel_func(run_forward, n_jobs=config.N_JOBS) parallel(run_func(subject, session) for subject, session in itertools.product(config.get_subjects(), config.get_sessions())) msg = 'Completed Step 10: Create forward solution' logger.info(gen_log_message(step=10, message=msg))
def main(): """Run inv.""" msg = 'Running Step 12: Compute and apply inverse solution' logger.info(gen_log_message(step=12, message=msg)) parallel, run_func, _ = parallel_func(run_inverse, n_jobs=config.N_JOBS) parallel(run_func(subject, session) for subject, session in itertools.product(config.get_subjects(), config.get_sessions())) msg = 'Completed Step 12: Compute and apply inverse solution' logger.info(gen_log_message(step=12, message=msg))
def main(): """Run evoked.""" msg = 'Running Step 6: Create evoked data' logger.info(gen_log_message(step=6, message=msg)) parallel, run_func, _ = parallel_func(run_evoked, n_jobs=config.N_JOBS) parallel(run_func(subject, session) for subject, session in itertools.product(config.get_subjects(), config.get_sessions())) msg = 'Completed Step 6: Create evoked data' logger.info(gen_log_message(step=6, message=msg))
def main(): """Run tf.""" msg = 'Running Step 8: Time-frequency decomposition' logger.info(gen_log_message(message=msg, step=8)) parallel, run_func, _ = parallel_func(run_time_frequency, n_jobs=config.N_JOBS) parallel(run_func(subject, session) for subject, session in itertools.product(config.get_subjects(), config.get_sessions())) msg = 'Completed Step 8: Time-frequency decomposition' logger.info(gen_log_message(message=msg, step=8))
def main(): """Run cov.""" msg = 'Running Step 11: Estimate noise covariance' logger.info(gen_log_message(step=11, message=msg)) parallel, run_func, _ = parallel_func(run_covariance, n_jobs=config.N_JOBS) parallel( run_func(subject, session) for subject, session in itertools.product( config.get_subjects(), config.get_sessions())) msg = 'Completed Step 11: Estimate noise covariance' logger.info(gen_log_message(step=11, message=msg))
def main(): """Run grp ave.""" msg = 'Running Step 13: Grand-average source estimates' logger.info(gen_log_message(step=13, message=msg)) mne.datasets.fetch_fsaverage(subjects_dir=config.get_fs_subjects_dir()) parallel, run_func, _ = parallel_func(morph_stc, n_jobs=config.N_JOBS) all_morphed_stcs = parallel(run_func(subject, session) for subject, session in itertools.product(config.get_subjects(), config.get_sessions())) all_morphed_stcs = [morphed_stcs for morphed_stcs, subject in zip(all_morphed_stcs, config.get_subjects())] mean_morphed_stcs = map(sum, zip(*all_morphed_stcs)) subject = 'average' # XXX to fix if config.get_sessions(): session = config.get_sessions()[0] else: session = None deriv_path = config.get_subject_deriv_path(subject=subject, session=session, kind=config.get_kind()) bids_basename = BIDSPath(subject=subject, session=session, task=config.get_task(), acquisition=config.acq, run=None, processing=config.proc, recording=config.rec, space=config.space, prefix=deriv_path, check=False) for condition, this_stc in zip(config.conditions, mean_morphed_stcs): this_stc /= len(all_morphed_stcs) method = config.inverse_method cond_str = condition.replace(op.sep, '').replace('_', '') inverse_str = method hemi_str = 'hemi' # MNE will auto-append '-lh' and '-rh'. morph_str = 'morph2fsaverage' fname_stc_avg = bids_basename.copy().update( kind=f'{cond_str}+{inverse_str}+{morph_str}+{hemi_str}') this_stc.save(fname_stc_avg) msg = 'Completed Step 13: Grand-average source estimates' logger.info(gen_log_message(step=13, message=msg))
def main(): """Run filter.""" msg = 'Running Step 2: Frequency filtering' logger.info(gen_log_message(step=2, message=msg)) parallel, run_func, _ = parallel_func(run_filter, n_jobs=config.N_JOBS) parallel(run_func(subject, run, session) for subject, run, session in itertools.product(config.get_subjects(), config.get_runs(), config.get_sessions())) msg = 'Completed 2: Frequency filtering' logger.info(gen_log_message(step=2, message=msg))
def main(): """Run maxwell_filter.""" msg = 'Running Step 1: Data import and Maxwell filtering' logger.info(gen_log_message(step=1, message=msg)) parallel, run_func, _ = parallel_func(run_maxwell_filter, n_jobs=config.N_JOBS) parallel( run_func(subject, session) for subject, session in itertools.product( config.get_subjects(), config.get_sessions())) msg = 'Completed Step 1: Data import and Maxwell filtering' logger.info(gen_log_message(step=1, message=msg))
def transform(self, X): parallel, p_func, n_jobs = parallel_func(_predict_decod, self.n_jobs) X_splits = np.array_split(X, n_jobs, axis=2) est_splits = np.array_split(self.estimators_, n_jobs) y_pred = parallel( p_func(est_split, x_split, self.method) for (est_split, x_split) in zip(est_splits, X_splits)) if n_jobs > 1: y_pred = np.concatenate(y_pred, axis=1) else: y_pred = y_pred[0] return y_pred
def main(): """Run epochs.""" msg = 'Running Step 3: Epoching' logger.info(gen_log_message(step=3, message=msg)) # Here we use fewer N_JOBS to prevent potential memory problems parallel, run_func, _ = parallel_func(run_epochs, n_jobs=max(config.N_JOBS // 4, 1)) parallel( run_func(subject, session) for subject, session in itertools.product( config.get_subjects(), config.get_sessions())) msg = 'Completed Step 3: Epoching' logger.info(gen_log_message(step=3, message=msg))
def main(): """Run ICA.""" msg = 'Running Step 4: Compute ICA' logger.info(gen_log_message(step=4, message=msg)) if config.use_ica: parallel, run_func, _ = parallel_func(run_ica, n_jobs=config.N_JOBS) parallel( run_func(subject, session) for subject, session in itertools.product(config.get_subjects(), config.get_sessions())) msg = 'Completed Step 4: Compute ICA' logger.info(gen_log_message(step=4, message=msg))
def pairwise(X, y, func, n_jobs=-1): """Applies pairwise operations on two matrices using multicore: function(X[:, jj, kk, ...], y[:, jj, kk, ...]) Parameters ---------- X : np.ndarray, shape(n, ...) y : np.array, shape(n, ...) | shape(n,) If shape == X.shape: parallel(X[:, chunk], y[:, chunk ] for chunk in n_chunks) If shape == X.shape[0]: parallel(X[:, chunk], y for chunk in n_chunks) func : function n_jobs : int, optional Number of parallel cpu. Returns ------- out : np.array, shape(func(X, y)) """ import numpy as np from mne.parallel import parallel_func dims = X.shape if y.shape[0] != dims[0]: raise ValueError('X and y must have identical shapes') X.resize([dims[0], np.prod(dims[1:])]) if y.ndim > 1: Y = np.reshape(y, [dims[0], np.prod(dims[1:])]) parallel, pfunc, n_jobs = parallel_func(func, n_jobs) n_cols = X.shape[1] n_chunks = min(n_cols, n_jobs) chunks = np.array_split(range(n_cols), n_chunks) if y.ndim == 1: out = parallel(pfunc(X[:, chunk], y) for chunk in chunks) else: out = parallel(pfunc(X[:, chunk], Y[:, chunk]) for chunk in chunks) # size back in case higher dependencies X.resize(dims) # unpack if isinstance(out[0], tuple): return [np.reshape(out_, dims[1:]) for out_ in zip(*out)] else: return np.reshape(np.hstack(out), dims[1:])
def fast_wilcoxon(X, y=None, zero_method='wilcox', correction=False, n_jobs=-1): from mne.parallel import parallel_func if y is not None: X -= y dims = X.shape X = X.reshape(len(X), -1) parallel, p_time_gen, n_jobs = parallel_func(_loop_wilcoxon, n_jobs) n_chunks = np.min([n_jobs, X.shape[1]]) out = parallel(p_time_gen(X[..., chunk], zero_method=zero_method, correction=correction) for chunk in np.array_split(range(X.shape[1]), n_chunks)) stats, p_val = map(list, zip(*out)) stats = np.hstack(stats).reshape(dims[1:]) p_val = np.hstack(p_val).reshape(dims[1:]) return stats, p_val
def test_progressbar_parallel_advanced(capsys): """Test ProgressBar with parallel computing, advanced version.""" assert capsys.readouterr().out == '' # This must be "1" because "capsys" won't get stdout properly otherwise parallel, p_fun, _ = parallel_func(_identity_block, n_jobs=1, verbose=False) arr = np.arange(10) with ProgressBar(len(arr), verbose_bool=True) as pb: out = parallel(p_fun(x, pb.subset(pb_idx)) for pb_idx, x in array_split_idx(arr, 2)) assert op.isfile(pb._mmap_fname) sum_ = np.memmap(pb._mmap_fname, dtype='bool', mode='r', shape=10).sum() assert sum_ == len(arr) assert not op.isfile(pb._mmap_fname), '__exit__ not called?' out = np.concatenate(out) assert_array_equal(out, arr) assert '100.00%' in capsys.readouterr().out
def fit(self, X): """Fits a regressor for each feature. Parameters ---------- X : array, shape (n_sample, n_feature) The data. """ from sklearn.base import clone n_sample, self.n_feature_ = X.shape # Setup parallel n_splits = n_jobs = np.min([self.n_jobs, self.n_feature_]) parallel, p_func, n_jobs = parallel_func(_fit_loop, n_jobs, verbose=None, max_nbytes='auto') # Split chunks of features to avoid overheads splits = np.array_split(np.arange(self.n_feature_), n_splits) out = parallel(p_func([clone(self.estimator) for f in split], X, split) for split in splits) self.estimators_ = np.concatenate(out, axis=0)
def compute_psd(signals, Fs, NFFT=2048, fmin=0, fmax=np.inf, n_jobs=1): """Based on the code in compute_raw_psd""" NFFT = int(NFFT) print "Effective window size : %0.3f (s)" % (NFFT / float(Fs)) if n_jobs > 1: parallel, my_psd, n_jobs = parallel_func(mlab.psd, n_jobs) out = parallel(my_psd(d, Fs=Fs, NFFT=NFFT) for d in signals) freqs = out[0][1] psd = np.array(zip(*out)[0]) else: psd = [] for d in signals: p, freqs = mlab.psd(d, Fs=Fs, NFFT=NFFT) psd.append(p) psd = np.array(psd) mask = (freqs >= fmin) & (freqs <= fmax) freqs = freqs[mask] psd = np.squeeze(psd[:, mask]) return psd, freqs
def fast_mannwhitneyu(X, Y, use_continuity=True, n_jobs=-1): from mne.parallel import parallel_func X = np.array(X) Y = np.array(Y) nx, ny = len(X), len(Y) dims = X.shape X = np.reshape(X, [nx, -1]) Y = np.reshape(Y, [ny, -1]) parallel, p_time_gen, n_jobs = parallel_func(_loop_mannwhitneyu, n_jobs) n_chunks = np.min([n_jobs, X.shape[1]]) chunks = np.array_split(range(X.shape[1]), n_chunks) out = parallel(p_time_gen(X[..., chunk], Y[..., chunk], use_continuity=use_continuity) for chunk in chunks) # Unpack estimators into time slices X folds list of lists. U, p_value = map(list, zip(*out)) U = np.concatenate(U, axis=1).reshape(dims[1:]) p_value = np.concatenate(p_value, axis=1).reshape(dims[1:]) AUC = U / (nx * ny) # correct directionality of U stats imposed by mannwhitneyu if nx > ny: AUC = 1 - AUC return U, p_value, AUC
def parallel_stats(X, function=_my_wilcoxon, correction="FDR", n_jobs=-1): from mne.parallel import parallel_func if correction not in [False, None, "FDR"]: raise ValueError("Unknown correction") # reshape to 2D X = np.array(X) dims = X.shape X.resize([dims[0], np.prod(dims[1:])]) # prepare parallel n_cols = X.shape[1] parallel, pfunc, n_jobs = parallel_func(_loop, n_jobs) n_chunks = min(n_cols, n_jobs) chunks = np.array_split(range(n_cols), n_chunks) p_values = parallel(pfunc(X[:, chunk], function) for chunk in chunks) p_values = np.reshape(np.hstack(p_values), dims[1:]) X.resize(dims) # apply correction if correction == "FDR": dims = p_values.shape _, p_values = fdr_correction(p_values) p_values = np.reshape(p_values, dims) return p_values
from mne.preprocessing import ICA from mne.parallel import parallel_func from library.config import meg_dir, N_JOBS def run_ica(subject_id): subject = "sub%03d" % subject_id print("processing subject: %s" % subject) data_path = op.join(meg_dir, subject) for run in range(1, 7): print("Run: %s" % run) run_fname = op.join(data_path, 'run_%02d_filt_sss_raw.fif' % run) if not os.path.exists(run_fname): warn('Could not find file %s. ' 'Skipping run %s for subject %s.' % (run_fname, run, subject)) continue raw = mne.io.read_raw_fif(run_fname, add_eeg_ref=False) ica_name = op.join(meg_dir, subject, 'run_%02d-ica.fif' % run) ica = ICA(method='fastica', random_state=42, n_components=0.98) picks = mne.pick_types(raw.info, meg=True, eeg=False, eog=False, stim=False, exclude='bads') ica.fit(raw, picks=picks, reject=dict(grad=4000e-13, mag=4e-12), decim=8) ica.save(ica_name) parallel, run_func, _ = parallel_func(run_ica, n_jobs=N_JOBS) parallel(run_func(subject_id) for subject_id in range(1, 20))
picks = mne.pick_types(raw.info, meg=True, eeg=True, stim=True, eog=True) # Read epochs epochs = mne.Epochs(raw, events, events_id, tmin, tmax, proj=True, picks=picks, baseline=baseline, preload=True, decim=2, reject=reject, add_eeg_ref=False) # ICA ica_name = op.join(meg_dir, subject, 'run_%02d-ica.fif' % run) ica = read_ica(ica_name) n_max_ecg = 3 # use max 3 components ecg_epochs = create_ecg_epochs(raw, tmin=-.5, tmax=.5) ecg_inds, scores_ecg = ica.find_bads_ecg(ecg_epochs, method='ctps', threshold=0.8) ica.exclude += ecg_inds[:n_max_ecg] ica.apply(epochs) all_epochs.append(epochs) epochs = mne.epochs.concatenate_epochs(all_epochs) epochs.save(op.join(data_path, '%s-epo.fif' % subject)) ############################################################################### # Let us make the script parallel across subjects parallel, run_func, _ = parallel_func(run_epochs, n_jobs=N_JOBS) parallel(run_func(subject_id) for subject_id in range(1, 20))
data_path = op.join(meg_dir, subject) epochs = mne.read_epochs(op.join(data_path, '%s-epo.fif' % subject), preload=False) evoked_famous = epochs['face/famous'].average() evoked_scrambled = epochs['scrambled'].average() evoked_unfamiliar = epochs['face/unfamiliar'].average() # Simplify comment evoked_famous.comment = 'famous' evoked_scrambled.comment = 'scrambled' evoked_unfamiliar.comment = 'unfamiliar' contrast = mne.combine_evoked([evoked_famous, evoked_unfamiliar, evoked_scrambled], weights=[0.5, 0.5, -1.]) contrast.comment = 'contrast' faces = mne.combine_evoked([evoked_famous, evoked_unfamiliar], 'nave') faces.comment = 'faces' mne.evoked.write_evokeds(op.join(data_path, '%s-ave.fif' % subject), [evoked_famous, evoked_scrambled, evoked_unfamiliar, contrast, faces]) # take care of noise cov cov = mne.compute_covariance(epochs, tmax=0, method='shrunk') cov.save(op.join(data_path, '%s-cov.fif' % subject)) parallel, run_func, _ = parallel_func(run_evoked, n_jobs=N_JOBS) parallel(run_func(subject_id) for subject_id in range(1, 20))
# List storing a dict of results for each .fif file sf_list = [None] * len(erm_filenames) # Print relevant processing info print '\nMaxwell filter fine calibrations choosen:' pp.pprint(params['cal_keys']) if '1D' in params['cal_keys']: print '1D calibration file: ' + fineCal_1d_fname if '3D' in params['cal_keys']: print '3D calibration file: ' + fineCal_3d_fname print ('\nProcessing {num} files (started @ '.format(num=len(erm_filenames)) + strftime('%D %H:%M:%S') + '):') pp.pprint(erm_filenames) # Parallelized functions parallel, my_maxwell, n_jobs = parallel_func(_maxwell_fun, n_jobs=-1, verbose=False) sf_list = parallel(my_maxwell(f_name) for f_name in erm_filenames) #################################################### # Save data #################################################### if pkl_data: print '\nSaving data', with open(op.join(save_dir, 'erm_shielding_factors.pkl'), 'wb') as pkl_file: pkl.dump(sf_list, pkl_file) with open(op.join(save_dir, 'shielding_factor_params.pkl'), 'wb') as pkl_file: params['finish_time'] = strftime('%D %H:%M:%S') pkl.dump(params, pkl_file) print ' ... Done' print 'Finished @ ' + strftime('%D %H:%M:%S')
fname_ave = op.join(data_path, '%s-ave.fif' % subject) fname_cov = op.join(data_path, '%s-cov.fif' % subject) fname_fwd = op.join(data_path, '%s-meg-%s-fwd.fif' % (subject, spacing)) fname_inv = op.join(data_path, '%s-meg-%s-inv.fif' % (subject, spacing)) evokeds = mne.read_evokeds(fname_ave, condition=[0, 1, 2, 3, 4]) cov = mne.read_cov(fname_cov) forward = mne.read_forward_solution(fname_fwd, surf_ori=True) # make an M/EEG, MEG-only, and EEG-only inverse operators info = evokeds[0].info inverse_operator = make_inverse_operator(info, forward, cov, loose=0.2, depth=0.8) write_inverse_operator(fname_inv, inverse_operator) # Compute inverse solution snr = 3.0 lambda2 = 1.0 / snr ** 2 for evoked in evokeds: stc = apply_inverse(evoked, inverse_operator, lambda2, "dSPM", pick_ori=None) stc.save(op.join(data_path, 'mne_dSPM_inverse-%s' % evoked.comment)) parallel, run_func, _ = parallel_func(run_inverse, n_jobs=N_JOBS) parallel(run_func(subject_id) for subject_id in range(1, 20))
except AttributeError: # Some files on openfmri are corrupted and cannot be read. warn('Could not read file %s. ' 'Skipping run %s from subject %s.' % (raw_in, run, subject)) continue if run_tsss: # Hackish way of reading bad channels. with open(op.join(study_path, 'ds117', subject, 'MEG', 'run_%02d_sss_log.txt' % run)) as fid: for line in fid: if line.startswith('Static bad channels'): chs = line.split(':')[-1].split() bads = ['MEG%04d' % int(ch) for ch in chs] break raw.info['bads'] += bads raw = mne.preprocessing.maxwell_filter(raw, calibration=cal, cross_talk=ctc, st_duration=10.) raw_out = raw_fname_out % run if not op.exists(op.join(meg_dir, subject)): os.mkdir(op.join(meg_dir, subject)) raw.filter(1, 40, l_trans_bandwidth=0.5, h_trans_bandwidth='auto', filter_length='auto', phase='zero', fir_window='hann') raw.save(raw_out, overwrite=True) parallel, run_func, _ = parallel_func(run_filter, n_jobs=N_JOBS) parallel(run_func(subject_id) for subject_id in range(1, 20))
import mne from mne.parallel import parallel_func from library.config import meg_dir, N_JOBS def run_events(subject_id): subject = "sub%03d" % subject_id print("processing subject: %s" % subject) data_path = op.join(meg_dir, subject) for run in range(1, 7): run_fname = op.join(data_path, 'run_%02d_filt_sss_raw.fif' % run) if not os.path.exists(run_fname): continue raw = mne.io.Raw(run_fname, add_eeg_ref=False) mask = 4096 + 256 # mask for excluding high order bits events = mne.find_events(raw, stim_channel='STI101', consecutive='increasing', mask=mask, mask_type='not_and', min_duration=0.003, verbose=True) print("S %s - R %s" % (subject, run)) fname_events = op.join(data_path, 'run_%02d_filt_sss-eve.fif' % run) mne.write_events(fname_events, events) parallel, run_func, _ = parallel_func(run_events, n_jobs=N_JOBS) parallel(run_func(subject_id) for subject_id in range(1, 20))
n_cycles = freqs / 2. def run_time_frequency(subject_id): print("processing subject: %s" % subject_id) subject = "sub%03d" % subject_id data_path = op.join(meg_dir, subject) epochs = mne.read_epochs(op.join(data_path, '%s-epo.fif' % subject)) faces = epochs['face'] idx = [faces.ch_names.index('EEG070')] power_faces, itc_faces = mne.time_frequency.tfr_morlet( faces, freqs=freqs, return_itc=True, n_cycles=n_cycles, picks=idx) power_scrambled, itc_scrambled = mne.time_frequency.tfr_morlet( epochs['scrambled'], freqs=freqs, return_itc=True, n_cycles=n_cycles, picks=idx) power_faces.save(op.join(data_path, '%s-faces-tfr.h5' % subject), overwrite=True) itc_faces.save(op.join(data_path, '%s-itc_faces-tfr.h5' % subject), overwrite=True) power_scrambled.save(op.join(data_path, '%s-scrambled-tfr.h5' % subject), overwrite=True) itc_scrambled.save(op.join(data_path, '%s-itc_scrambled-tfr.h5' % subject), overwrite=True) parallel, run_func, _ = parallel_func(run_time_frequency, n_jobs=N_JOBS) parallel(run_func(subject_id) for subject_id in range(1, 20))
from mne.decoding import TimeDecoding times = dict(step=0.005) # fit a classifier only every 5 ms # Use AUC because chance level is same regardless of the class balance td = TimeDecoding(predict_mode='cross-validation', times=times, scorer='roc_auc') td.fit(epochs, y) # let's save the scores now a_vs_b = '%s_vs_%s' % (os.path.basename(condition1), os.path.basename(condition2)) fname_td = os.path.join(data_path, '%s-td-auc-%s.mat' % (subject, a_vs_b)) from scipy.io import savemat savemat(fname_td, {'scores': td.score(epochs), 'times': td.times_['times']}) ############################################################################### # Finally we make this script parallel across subjects and write the results # # .. warning:: # This may take a large amount of memory because the epochs will be # replicated for each parallel job parallel, run_func, _ = parallel_func(run_time_decoding, n_jobs=N_JOBS) parallel(run_func(subject_id, 'face', 'scrambled') for subject_id in range(1, 20)) parallel(run_func(subject_id, 'face/famous', 'face/unfamiliar') for subject_id in range(1, 20))
def run_forward(subject_id): subject = "sub%03d" % subject_id print("processing subject: %s" % subject) data_path = op.join(meg_dir, subject) fname_ave = op.join(data_path, '%s-ave.fif' % subject) fname_fwd = op.join(data_path, '%s-meg-%s-fwd.fif' % (subject, spacing)) fname_trans = op.join(study_path, 'ds117', subject, 'MEG', '%s-trans.fif' % subject) src = mne.setup_source_space(subject, spacing=spacing, subjects_dir=subjects_dir, overwrite=True, n_jobs=1, add_dist=False) src_fname = op.join(subjects_dir, subject, '%s-src.fif' % spacing) mne.write_source_spaces(src_fname, src) bem_model = mne.make_bem_model(subject, ico=4, subjects_dir=subjects_dir, conductivity=(0.3,)) bem = mne.make_bem_solution(bem_model) info = mne.read_evokeds(fname_ave, condition=0).info fwd = mne.make_forward_solution(info, trans=fname_trans, src=src, bem=bem, fname=None, meg=True, eeg=False, mindist=mindist, n_jobs=1, overwrite=True) fwd = mne.convert_forward_solution(fwd, surf_ori=True) mne.write_forward_solution(fname_fwd, fwd, overwrite=True) parallel, run_func, _ = parallel_func(run_forward, n_jobs=N_JOBS) parallel(run_func(subject_id) for subject_id in range(1, 20))