def test_all_policies_work(self):
kwargs = dict(paradigm=FakeImageryParadigm(),
datasets=[dataset],
n_perms=[2, 2])
# The next two should work without issue
ev.WithinSessionEvaluation(data_size={
"policy": "per_class",
"value": [5, 10]
},
**kwargs)
ev.WithinSessionEvaluation(data_size={
"policy": "ratio",
"value": [0.2, 0.5]
},
**kwargs)
self.assertRaises(
ValueError,
ev.WithinSessionEvaluation,
**dict(data_size={
"policy": "does_not_exist",
"value": [0.2, 0.5]
},
**kwargs),
)
def test_correct_results_integrity(self):
learning_curve_eval = ev.WithinSessionEvaluation(
paradigm=FakeImageryParadigm(),
datasets=[dataset],
data_size={
"policy": "ratio",
"value": np.array([0.2, 0.5])
},
n_perms=np.array([2, 2]),
)
results = [r for r in learning_curve_eval.evaluate(dataset, pipelines)]
keys = results[0].keys()
self.assertEqual(len(keys), 10) # 8 + 2 new for learning curve
self.assertTrue("permutation" in keys)
self.assertTrue("data_size" in keys)
def test_data_sanity(self):
# need this helper to iterate over the generator
def run_evaluation(eval, dataset, pipelines):
list(eval.evaluate(dataset, pipelines))
# E.g. if number of samples too high -> expect error
kwargs = dict(paradigm=FakeImageryParadigm(),
datasets=[dataset],
n_perms=[2, 2])
should_work = ev.WithinSessionEvaluation(data_size={
"policy": "per_class",
"value": [5, 10]
},
**kwargs)
too_many_samples = ev.WithinSessionEvaluation(data_size={
"policy": "per_class",
"value": [5, 100000]
},
**kwargs)
# This one should run
run_evaluation(should_work, dataset, pipelines)
self.assertRaises(ValueError, run_evaluation, too_many_samples,
dataset, pipelines)
pass
def setUp(self):
self.eval = ev.WithinSessionEvaluation(paradigm=FakeImageryParadigm(),
datasets=[dataset])
def setUp(self):
self.eval = ev.WithinSessionEvaluation(
paradigm=FakeImageryParadigm(),
datasets=[dataset],
additional_columns=["one", "two"],
)
def return_eval(self):
return ev.WithinSessionEvaluation(paradigm=FakeImageryParadigm(),
datasets=[dataset])