# --------
#
# Restrict further analysis to specified channels, here C3, C4, and Cz.
# Also, use a specific resampling. In this example, all datasets are
# set to 200 Hz.
paradigm = LeftRightImagery(channels=['C3', 'C4', 'Cz'], resample=200.)
##############################################################################
# Evaluation
# ----------
#
# The evaluation is conducted on with CSP+LDA, only on the 3 electrodes, with
# a sampling rate of 200 Hz.
evaluation = WithinSessionEvaluation(paradigm=paradigm, datasets=datasets)
csp_lda = make_pipeline(CSP(n_components=2), LDA())
ts_lr = make_pipeline(Covariances(estimator='oas'),
TangentSpace(metric='riemann'), LR(C=1.0))
results = evaluation.process({'csp+lda': csp_lda, 'ts+lr': ts_lr})
print(results.head())
##############################################################################
# Electrode selection
# -------------------
#
# It is possible to select the electrodes that are shared by all datasets
# using the `find_intersecting_channels` function. Datasets that have 0
# overlap with others are discarded. It returns the set of common channels,
# as well as the list of datasets with valid channels.
paradigm = P300(resample=processing_sampling_rate)
dataset = BNCI2014009()
# Remove the slicing of the subject list to evaluate multiple subjects
dataset.subject_list = dataset.subject_list[0:1]
datasets = [dataset]
overwrite = True # set to True if we want to overwrite cached results
data_size = dict(policy="ratio", value=np.geomspace(0.02, 1, 6))
# When the training data is sparse, peform more permutations than when we have a lot of data
n_perms = np.floor(np.geomspace(20, 2, len(data_size["value"]))).astype(int)
print(n_perms)
# Guarantee reproducibility
np.random.seed(7536298)
evaluation = WithinSessionEvaluation(
paradigm=paradigm,
datasets=datasets,
data_size=data_size,
n_perms=n_perms,
suffix="examples_lr",
overwrite=overwrite,
)
results = evaluation.process(pipelines)
##############################################################################
# Plot Results
# ------------
#
# Here we plot the results.
fig, ax = plt.subplots(facecolor="white", figsize=[8, 4])
n_subs = len(dataset.subject_list)
# We instantiate the two different datasets that follow the MI paradigm
# (with left-hand/right-hand classes) but were recorded with different number
# of electrodes, different number of trials, etc.
datasets = [Zhou2016(), BNCI2014001()]
subj = [1, 2, 3]
for d in datasets:
d.subject_list = subj
# The following lines go exactly as in the previous example, where we end up
# obtaining a pandas dataframe containing the results of the evaluation. We
# could set `overwrite` to False to cache the results, avoiding to restart all
# the evaluation from scratch if a problem occurs.
paradigm = LeftRightImagery()
evaluation = WithinSessionEvaluation(paradigm=paradigm,
datasets=datasets,
overwrite=False)
pipeline = make_pipeline(CSP(n_components=8), LDA())
results = evaluation.process({"csp+lda": pipeline})
##############################################################################
# Plotting Results
# ----------------
#
# We plot the results using the seaborn library. Note how easy it
# is to plot the results from the three datasets with just one line.
results["subj"] = [str(resi).zfill(2) for resi in results["subject"]]
g = sns.catplot(
kind="bar",
x="score",
# other two pipelines rely on Riemannian geometry, using an SVM classification
# in the tangent space of the covariance matrices estimated from the EEG or a
# MDM classifier that works directly on covariance matrices.
pipelines = {}
pipelines["csp+lda"] = make_pipeline(CSP(n_components=8), LDA())
pipelines["tgsp+svm"] = make_pipeline(Covariances('oas'),
TangentSpace(metric='riemann'),
SVC(kernel='linear'))
pipelines["MDM"] = make_pipeline(Covariances('oas'), MDM(metric='riemann'))
# The following lines go exactly as in the previous example, where we end up
# obtaining a pandas dataframe containing the results of the evaluation.
datasets = [BNCI2014001(), Weibo2014(), Zhou2016()]
paradigm = LeftRightImagery()
evaluation = WithinSessionEvaluation(paradigm=paradigm,
datasets=datasets,
overwrite=True)
results = evaluation.process(pipelines)
if not os.path.exists("./results"):
os.mkdir("./results")
results.to_csv("./results/results_part2-3.csv")
results = pd.read_csv('./results/results_part2-3.csv')
##############################################################################
# Plotting Results
# ----------------
#
# The following plot shows a comparison of the three classification pipelines
# for each subject of each dataset.
results["subj"] = [str(resi).zfill(2) for resi in results["subject"]]
from sklearn.externals import joblib
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score
from sklearn.pipeline import make_pipeline
from pyriemann.estimation import Covariances
from riemann_lab import power_means
import moabb
moabb.set_log_level('info')
from moabb.datasets import BNCI2014001
from moabb.paradigms import LeftRightImagery
from moabb.evaluations import WithinSessionEvaluation
# define the dataset, the paradigm, and the evaluation procedure
datasets = [BNCI2014001()]
paradigm = LeftRightImagery()
evaluation = WithinSessionEvaluation(paradigm, datasets)
# define the pipelines for classification -- MDM and MeansField classifier
pipelines = {}
pipelines['MDM'] = make_pipeline(Covariances('oas'), MDM())
plist = [1.00, 0.75, 0.50, 0.25, 0.10, 0.01, -0.01, -0.10, -0.25, -0.50, -0.75, -1.00]
pipelines['MeansField'] = make_pipeline(Covariances('oas'), power_means.MeanFieldClassifier(plist=plist))
# loop through the subjects and datasets
results = evaluation.process(pipelines)
if isinstance(config['pipeline'], BaseEstimator):
pipeline = deepcopy(config['pipeline'])
else:
log.error(config['pipeline'])
raise (
ValueError('pipeline must be a list or a sklearn estimator'))
# append the pipeline in the paradigm list
if paradigm not in paradigms.keys():
paradigms[paradigm] = {}
# FIXME name are not unique
log.debug('Pipeline: \n\n {} \n'.format(get_string_rep(pipeline)))
paradigms[paradigm][config['name']] = pipeline
all_results = []
for paradigm in paradigms:
# get the context
if len(context_params) == 0:
context_params[paradigm] = {}
log.debug('{}: {}'.format(paradigm, context_params[paradigm]))
p = getattr(moabb_paradigms, paradigm)(**context_params[paradigm])
context = WithinSessionEvaluation(paradigm=p,
random_state=42,
n_jobs=options.threads)
results = context.process(pipelines=paradigms[paradigm])
all_results.append(results)
analyze(pd.concat(all_results, ignore_index=True),
options.output,
plot=options.plot)
# ----------
#
# To evaluate the score of this pipeline, we use the `evaluation` class. When
# instantiating it, we say which paradigm we want to consider, a list with the
# datasets to analyze, and whether the scores should be recalculated each time
# we run the evaluation or if MOABB should create a cache file.
#
# Note that there are different ways of evaluating a classifier; in this
# example, we choose `WithinSessionEvaluation`, which consists of doing a
# cross-validation procedure where the training and testing partitions are from
# the same recording session of the dataset. We could have used
# `BetweenSessionEvaluation`, which takes all but one session as training
# partition and the remaining one as testing partition.
evaluation = WithinSessionEvaluation(paradigm=paradigm,
datasets=[dataset],
overwrite=True)
# We obtain the results in the form of a pandas dataframe
results = evaluation.process({'csp+lda': pipeline})
# To export the results in CSV within a directory:
if not os.path.exists('./results'):
os.mkdir('./results')
results.to_csv('./results/results_part2-1.csv')
# To load previously obtained results saved in CSV
results = pd.read_csv('./results/results_part2-1.csv')
##############################################################################
# Plotting Results
if paradigm not in context_params.keys():
log.debug(context_params)
log.warning("Paradigm {} not in context file {}".format(
paradigm, context_params.keys()))
if isinstance(config['pipeline'], list):
pipeline = create_pipeline_from_config(config['pipeline'])
elif isinstance(config['pipeline'], BaseEstimator):
pipeline = deepcopy(config['pipeline'])
else:
log.error(config['pipeline'])
raise (
ValueError('pipeline must be a list or a sklearn estimator'))
# append the pipeline in the paradigm list
if paradigm not in paradigms.keys():
paradigms[paradigm] = {}
# FIXME name are not unique
log.debug('Pipeline: \n\n {} \n'.format(repr(pipeline.get_params())))
paradigms[paradigm][config['name']] = pipeline
for paradigm in paradigms:
# get the context
if len(context_params) == 0:
context_params[paradigm] = {}
log.debug('{}: {}'.format(paradigm, context_params[paradigm]))
p = getattr(moabb_paradigms, paradigm)(**context_params[paradigm])
context = WithinSessionEvaluation(paradigm=p, random_state=42)
results = context.process(pipelines=paradigms[paradigm])
#
# To evaluate the score of this pipeline, we use the `evaluation` class. When
# instantiating it, we say which paradigm we want to consider, a list with the
# datasets to analyze, and whether the scores should be recalculated each time
# we run the evaluation or if MOABB should create a cache file.
#
# Note that there are different ways of evaluating a classifier; in this
# example, we choose `WithinSessionEvaluation`, which consists of doing a
# cross-validation procedure where the training and testing partitions are from
# the same recording session of the dataset. We could have used
# `CrossSessionEvaluation`, which takes all but one session as training
# partition and the remaining one as testing partition.
evaluation = WithinSessionEvaluation(
paradigm=paradigm,
datasets=[dataset],
overwrite=True,
hdf5_path=None,
)
##############################################################################
# We obtain the results in the form of a pandas dataframe
results = evaluation.process({"csp+lda": pipeline})
##############################################################################
# The results are stored in locally, to avoid recomputing the results each time.
# It is saved in `hdf5_path` if defined or in ~/mne_data/results otherwise.
# To export the results in CSV:
results.to_csv("./results_part2-1.csv")
#
# We define the paradigm (P300) and use all three datasets available for it.
# The evaluation will return a dataframe containing a single AUC score for
# each subject / session of the dataset, and for each pipeline.
#
# Results are saved into the database, so that if you add a new pipeline, it
# will not run again the evaluation unless a parameter has changed. Results can
# be overwritten if necessary.
paradigm = P300(resample=128)
dataset = EPFLP300()
dataset.subject_list = dataset.subject_list[:2]
datasets = [dataset]
overwrite = True # set to True if we want to overwrite cached results
evaluation = WithinSessionEvaluation(paradigm=paradigm,
datasets=datasets,
suffix="examples",
overwrite=overwrite)
results = evaluation.process(pipelines)
##############################################################################
# Plot Results
# ----------------
#
# Here we plot the results.
fig, ax = plt.subplots(facecolor="white", figsize=[8, 4])
sns.stripplot(
data=results,
y="score",
x="pipeline",
raise ValueError(f'Invalid benchmark extent.')
# IF YOU WANT TO DOWNLOAD ALL DATA FIRST, UNCOMMENT THE NEXT LINE
#pipelines = dict(test=pipelines['jm_few_lda_p_cov'])
##############################################################################
# Evaluation
##############################################################################
identifier = f'{dataset_name}_subj_{subjects if subjects is not None else "all"}' \
f'_sess_{sessions if sessions is not None else "all"}_{start_timestamp_as_str}'.replace(' ', '')
unique_suffix = f'{identifier}_{uuid.uuid4()}'
debug_path = RESULTS_FOLDER / f'{identifier}_DEBUG.txt'
with open(debug_path, 'w') as debug_f:
debug_f.writelines([f'{l}: {os.environ[l]}\n' for l in sorted(os.environ)])
overwrite = True
error_score = np.nan
evaluation = WithinSessionEvaluation(paradigm=bench_cfg['paradigm'],
datasets=bench_cfg['dataset'],
suffix=unique_suffix,
overwrite=overwrite,
random_state=8,
error_score=error_score)
results = evaluation.process(pipelines)
result_path = RESULTS_FOLDER / f'{identifier}_results.csv'
results.to_csv(result_path, encoding='utf-8', index=False)
t1 = time.time()
print(f'Benchmark run completed. Elapsed time: {(t1-t0)/3600} hours.')
##############################################################################
# Using the ExampleDataset
# ------------------------
#
# Now that the `ExampleDataset` is defined, it could be instanciated directly.
# The rest of the code follows the steps described in the previous tutorials.
dataset = ExampleDataset()
paradigm = LeftRightImagery()
X, labels, meta = paradigm.get_data(dataset=dataset, subjects=[1])
evaluation = WithinSessionEvaluation(paradigm=paradigm,
datasets=dataset,
overwrite=False,
suffix="newdataset")
pipelines = {}
pipelines["MDM"] = make_pipeline(Covariances("oas"), MDM(metric="riemann"))
scores = evaluation.process(pipelines)
print(scores)
##############################################################################
# Pushing on MOABB Github
# -----------------------
#
# If you want to make your dataset available to everyone, you could upload
# your data on public server (like Zenodo or Figshare) and signal that you
# want to add your dataset to MOABB in the `dedicated issue <https://github.com/NeuroTechX/moabb/issues/1>`_. # noqa: E501
# You could then follow the instructions on `how to contribute <https://github.com/NeuroTechX/moabb/blob/master/CONTRIBUTING.md>`_ # noqa: E501
import logging
import coloredlogs
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger()
coloredlogs.install(level=logging.DEBUG)
datasets = utils.dataset_search('imagery',
events=['supination', 'hand_close'],
has_all_events=False,
min_subjects=2,
multi_session=False)
for d in datasets:
d.subject_list = d.subject_list[:10]
paradigm = ImageryNClass(2)
context = WithinSessionEvaluation(paradigm=paradigm,
datasets=datasets,
random_state=42)
pipelines = OrderedDict()
pipelines['av+TS'] = make_pipeline(Covariances(estimator='oas'),
TSclassifier())
pipelines['av+CSP+LDA'] = make_pipeline(Covariances(estimator='oas'), CSP(8),
LDA())
results = context.process(pipelines, overwrite=True)
analyze(results, './')
# --------
#
# Restrict further analysis to specified channels, here C3, C4, and Cz.
# Also, use a specific resampling. In this example, all datasets are
# set to 200 Hz.
paradigm = LeftRightImagery(channels=["C3", "C4", "Cz"], resample=200.0)
##############################################################################
# Evaluation
# ----------
#
# The evaluation is conducted on with CSP+LDA, only on the 3 electrodes, with
# a sampling rate of 200 Hz.
evaluation = WithinSessionEvaluation(paradigm=paradigm, datasets=datasets)
csp_lda = make_pipeline(CSP(n_components=2), LDA())
ts_lr = make_pipeline(Covariances(estimator="oas"),
TangentSpace(metric="riemann"), LR(C=1.0))
results = evaluation.process({"csp+lda": csp_lda, "ts+lr": ts_lr})
print(results.head())
##############################################################################
# Electrode selection
# -------------------
#
# It is possible to select the electrodes that are shared by all datasets
# using the `find_intersecting_channels` function. Datasets that have 0
# overlap with others are discarded. It returns the set of common channels,
# as well as the list of datasets with valid channels.
