# -------- # # Restrict further analysis to specified channels, here C3, C4, and Cz. # Also, use a specific resampling. In this example, all datasets are # set to 200 Hz. paradigm = LeftRightImagery(channels=['C3', 'C4', 'Cz'], resample=200.) ############################################################################## # Evaluation # ---------- # # The evaluation is conducted on with CSP+LDA, only on the 3 electrodes, with # a sampling rate of 200 Hz. evaluation = WithinSessionEvaluation(paradigm=paradigm, datasets=datasets) csp_lda = make_pipeline(CSP(n_components=2), LDA()) ts_lr = make_pipeline(Covariances(estimator='oas'), TangentSpace(metric='riemann'), LR(C=1.0)) results = evaluation.process({'csp+lda': csp_lda, 'ts+lr': ts_lr}) print(results.head()) ############################################################################## # Electrode selection # ------------------- # # It is possible to select the electrodes that are shared by all datasets # using the `find_intersecting_channels` function. Datasets that have 0 # overlap with others are discarded. It returns the set of common channels, # as well as the list of datasets with valid channels.
from sklearn.externals import joblib from sklearn.model_selection import StratifiedKFold from sklearn.metrics import roc_auc_score from sklearn.pipeline import make_pipeline from pyriemann.estimation import Covariances from riemann_lab import power_means import moabb moabb.set_log_level('info') from moabb.datasets import BNCI2014001 from moabb.paradigms import LeftRightImagery from moabb.evaluations import WithinSessionEvaluation # define the dataset, the paradigm, and the evaluation procedure datasets = [BNCI2014001()] paradigm = LeftRightImagery() evaluation = WithinSessionEvaluation(paradigm, datasets) # define the pipelines for classification -- MDM and MeansField classifier pipelines = {} pipelines['MDM'] = make_pipeline(Covariances('oas'), MDM()) plist = [1.00, 0.75, 0.50, 0.25, 0.10, 0.01, -0.01, -0.10, -0.25, -0.50, -0.75, -1.00] pipelines['MeansField'] = make_pipeline(Covariances('oas'), power_means.MeanFieldClassifier(plist=plist)) # loop through the subjects and datasets results = evaluation.process(pipelines)
paradigm = P300(resample=processing_sampling_rate) dataset = BNCI2014009() # Remove the slicing of the subject list to evaluate multiple subjects dataset.subject_list = dataset.subject_list[0:1] datasets = [dataset] overwrite = True # set to True if we want to overwrite cached results data_size = dict(policy="ratio", value=np.geomspace(0.02, 1, 6)) # When the training data is sparse, peform more permutations than when we have a lot of data n_perms = np.floor(np.geomspace(20, 2, len(data_size["value"]))).astype(int) print(n_perms) # Guarantee reproducibility np.random.seed(7536298) evaluation = WithinSessionEvaluation( paradigm=paradigm, datasets=datasets, data_size=data_size, n_perms=n_perms, suffix="examples_lr", overwrite=overwrite, ) results = evaluation.process(pipelines) ############################################################################## # Plot Results # ------------ # # Here we plot the results. fig, ax = plt.subplots(facecolor="white", figsize=[8, 4]) n_subs = len(dataset.subject_list)
# # To evaluate the score of this pipeline, we use the `evaluation` class. When # instantiating it, we say which paradigm we want to consider, a list with the # datasets to analyze, and whether the scores should be recalculated each time # we run the evaluation or if MOABB should create a cache file. # # Note that there are different ways of evaluating a classifier; in this # example, we choose `WithinSessionEvaluation`, which consists of doing a # cross-validation procedure where the training and testing partitions are from # the same recording session of the dataset. We could have used # `CrossSessionEvaluation`, which takes all but one session as training # partition and the remaining one as testing partition. evaluation = WithinSessionEvaluation( paradigm=paradigm, datasets=[dataset], overwrite=True, hdf5_path=None, ) ############################################################################## # We obtain the results in the form of a pandas dataframe results = evaluation.process({"csp+lda": pipeline}) ############################################################################## # The results are stored in locally, to avoid recomputing the results each time. # It is saved in `hdf5_path` if defined or in ~/mne_data/results otherwise. # To export the results in CSV: results.to_csv("./results_part2-1.csv")
if isinstance(config['pipeline'], BaseEstimator): pipeline = deepcopy(config['pipeline']) else: log.error(config['pipeline']) raise ( ValueError('pipeline must be a list or a sklearn estimator')) # append the pipeline in the paradigm list if paradigm not in paradigms.keys(): paradigms[paradigm] = {} # FIXME name are not unique log.debug('Pipeline: \n\n {} \n'.format(get_string_rep(pipeline))) paradigms[paradigm][config['name']] = pipeline all_results = [] for paradigm in paradigms: # get the context if len(context_params) == 0: context_params[paradigm] = {} log.debug('{}: {}'.format(paradigm, context_params[paradigm])) p = getattr(moabb_paradigms, paradigm)(**context_params[paradigm]) context = WithinSessionEvaluation(paradigm=p, random_state=42, n_jobs=options.threads) results = context.process(pipelines=paradigms[paradigm]) all_results.append(results) analyze(pd.concat(all_results, ignore_index=True), options.output, plot=options.plot)
# # We define the paradigm (P300) and use all three datasets available for it. # The evaluation will return a dataframe containing a single AUC score for # each subject / session of the dataset, and for each pipeline. # # Results are saved into the database, so that if you add a new pipeline, it # will not run again the evaluation unless a parameter has changed. Results can # be overwritten if necessary. paradigm = P300(resample=128) dataset = EPFLP300() dataset.subject_list = dataset.subject_list[:2] datasets = [dataset] overwrite = True # set to True if we want to overwrite cached results evaluation = WithinSessionEvaluation(paradigm=paradigm, datasets=datasets, suffix="examples", overwrite=overwrite) results = evaluation.process(pipelines) ############################################################################## # Plot Results # ---------------- # # Here we plot the results. fig, ax = plt.subplots(facecolor="white", figsize=[8, 4]) sns.stripplot( data=results, y="score", x="pipeline",
if paradigm not in context_params.keys(): log.debug(context_params) log.warning("Paradigm {} not in context file {}".format( paradigm, context_params.keys())) if isinstance(config['pipeline'], list): pipeline = create_pipeline_from_config(config['pipeline']) elif isinstance(config['pipeline'], BaseEstimator): pipeline = deepcopy(config['pipeline']) else: log.error(config['pipeline']) raise ( ValueError('pipeline must be a list or a sklearn estimator')) # append the pipeline in the paradigm list if paradigm not in paradigms.keys(): paradigms[paradigm] = {} # FIXME name are not unique log.debug('Pipeline: \n\n {} \n'.format(repr(pipeline.get_params()))) paradigms[paradigm][config['name']] = pipeline for paradigm in paradigms: # get the context if len(context_params) == 0: context_params[paradigm] = {} log.debug('{}: {}'.format(paradigm, context_params[paradigm])) p = getattr(moabb_paradigms, paradigm)(**context_params[paradigm]) context = WithinSessionEvaluation(paradigm=p, random_state=42) results = context.process(pipelines=paradigms[paradigm])
raise ValueError(f'Invalid benchmark extent.') # IF YOU WANT TO DOWNLOAD ALL DATA FIRST, UNCOMMENT THE NEXT LINE #pipelines = dict(test=pipelines['jm_few_lda_p_cov']) ############################################################################## # Evaluation ############################################################################## identifier = f'{dataset_name}_subj_{subjects if subjects is not None else "all"}' \ f'_sess_{sessions if sessions is not None else "all"}_{start_timestamp_as_str}'.replace(' ', '') unique_suffix = f'{identifier}_{uuid.uuid4()}' debug_path = RESULTS_FOLDER / f'{identifier}_DEBUG.txt' with open(debug_path, 'w') as debug_f: debug_f.writelines([f'{l}: {os.environ[l]}\n' for l in sorted(os.environ)]) overwrite = True error_score = np.nan evaluation = WithinSessionEvaluation(paradigm=bench_cfg['paradigm'], datasets=bench_cfg['dataset'], suffix=unique_suffix, overwrite=overwrite, random_state=8, error_score=error_score) results = evaluation.process(pipelines) result_path = RESULTS_FOLDER / f'{identifier}_results.csv' results.to_csv(result_path, encoding='utf-8', index=False) t1 = time.time() print(f'Benchmark run completed. Elapsed time: {(t1-t0)/3600} hours.')
# # To evaluate the score of this pipeline, we use the `evaluation` class. When # instantiating it, we say which paradigm we want to consider, a list with the # datasets to analyze, and whether the scores should be recalculated each time # we run the evaluation or if MOABB should create a cache file. # # Note that there are different ways of evaluating a classifier; in this # example, we choose `WithinSessionEvaluation`, which consists of doing a # cross-validation procedure where the training and testing partitions are from # the same recording session of the dataset. We could have used # `BetweenSessionEvaluation`, which takes all but one session as training # partition and the remaining one as testing partition. evaluation = WithinSessionEvaluation( paradigm=paradigm, datasets=[dataset], hdf5_path=None, ) # We obtain the results in the form of a pandas dataframe results = evaluation.process({"csp+lda": pipeline}) # The results are stored in locally, to avoid recomputing the results each time. # It is saved in `hdf5_path` if defined or in ~/mne_data/results otherwise. # To export the results in CSV: results.to_csv("./results_part2-1.csv") # To load previously obtained results saved in CSV results = pd.read_csv("./results_part2-1.csv") ##############################################################################
############################################################################## # Using the ExampleDataset # ------------------------ # # Now that the `ExampleDataset` is defined, it could be instanciated directly. # The rest of the code follows the steps described in the previous tutorials. dataset = ExampleDataset() paradigm = LeftRightImagery() X, labels, meta = paradigm.get_data(dataset=dataset, subjects=[1]) evaluation = WithinSessionEvaluation(paradigm=paradigm, datasets=dataset, overwrite=False, suffix="newdataset") pipelines = {} pipelines["MDM"] = make_pipeline(Covariances("oas"), MDM(metric="riemann")) scores = evaluation.process(pipelines) print(scores) ############################################################################## # Pushing on MOABB Github # ----------------------- # # If you want to make your dataset available to everyone, you could upload # your data on public server (like Zenodo or Figshare) and signal that you # want to add your dataset to MOABB in the `dedicated issue <https://github.com/NeuroTechX/moabb/issues/1>`_. # noqa: E501 # You could then follow the instructions on `how to contribute <https://github.com/NeuroTechX/moabb/blob/master/CONTRIBUTING.md>`_ # noqa: E501
import logging import coloredlogs logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger() coloredlogs.install(level=logging.DEBUG) datasets = utils.dataset_search('imagery', events=['supination', 'hand_close'], has_all_events=False, min_subjects=2, multi_session=False) for d in datasets: d.subject_list = d.subject_list[:10] paradigm = ImageryNClass(2) context = WithinSessionEvaluation(paradigm=paradigm, datasets=datasets, random_state=42) pipelines = OrderedDict() pipelines['av+TS'] = make_pipeline(Covariances(estimator='oas'), TSclassifier()) pipelines['av+CSP+LDA'] = make_pipeline(Covariances(estimator='oas'), CSP(8), LDA()) results = context.process(pipelines, overwrite=True) analyze(results, './')
# -------- # # Restrict further analysis to specified channels, here C3, C4, and Cz. # Also, use a specific resampling. In this example, all datasets are # set to 200 Hz. paradigm = LeftRightImagery(channels=["C3", "C4", "Cz"], resample=200.0) ############################################################################## # Evaluation # ---------- # # The evaluation is conducted on with CSP+LDA, only on the 3 electrodes, with # a sampling rate of 200 Hz. evaluation = WithinSessionEvaluation(paradigm=paradigm, datasets=datasets) csp_lda = make_pipeline(CSP(n_components=2), LDA()) ts_lr = make_pipeline(Covariances(estimator="oas"), TangentSpace(metric="riemann"), LR(C=1.0)) results = evaluation.process({"csp+lda": csp_lda, "ts+lr": ts_lr}) print(results.head()) ############################################################################## # Electrode selection # ------------------- # # It is possible to select the electrodes that are shared by all datasets # using the `find_intersecting_channels` function. Datasets that have 0 # overlap with others are discarded. It returns the set of common channels, # as well as the list of datasets with valid channels.