def test_can_run(self) -> None:
X, y = load_dataset("iris")
# Should be able to run with the most basic configuration
plumber = SKPlumber("classification", 1)
plumber.fit(X, y)
# Should be able to run using a non-default metric
plumber = SKPlumber("classification", 1, metric="f1macro")
plumber.fit(X, y)
def test_args_are_validated(self) -> None:
# metric should be valid
with self.assertRaises(ValueError):
SKPlumber("classification", 1, metric="foobar")
# metric should be valid for problem type
with self.assertRaises(ValueError):
SKPlumber("classification", 1, metric="rmse")
# problem type should be valid
with self.assertRaises(Exception):
SKPlumber("foobar", 1)
def test_default_works(self):
"""
`SKPlumber.fit`'s default evaluator should work
out of the box.
"""
plumber = SKPlumber("classification", 1)
X, y = load_dataset("iris")
plumber.fit(X, y)
def test_can_do_train_test(self):
"""
The evaluator returned by `make_train_test_evaluator`
should work, and should cupport custom test size.
"""
plumber = SKPlumber("classification",
1,
evaluator=make_train_test_evaluator(0.2))
X, y = load_dataset("iris")
plumber.fit(X, y)
def test_can_do_down_sample_evaluation(self):
"""
The evaluator returned by `make_down_sample_evaluator`
should work, and should cupport custom test size.
"""
plumber = SKPlumber("classification",
1,
evaluator=make_down_sample_evaluator(0.8, 0.2))
X, y = load_dataset("iris")
# Should be able to do down-sampled train/test validation.
plumber.fit(X, y)
def test_can_do_k_fold_cv(self):
"""
The evaluator returned by `make_kfold_evaluator`
should work, and should cupport custom number of folds.
"""
plumber = SKPlumber("classification",
1,
evaluator=make_kfold_evaluator(3))
X, y = load_dataset("iris")
# Should be able to do k-fold cross validation.
plumber.fit(X, y)
def test_can_take_callback(self) -> None:
self.n_iters = 0
X, y = load_dataset("iris")
def cb(state) -> bool:
self.n_iters = state.n_iters
return True if state.n_iters == 2 else False
plumber = SKPlumber("classification", 100, callback=cb)
plumber.fit(X, y)
assert self.n_iters < 3 and self.n_iters > 0
def test_can_sample_for_classification(self) -> None:
sampler = OneStackPipelineSampler()
plumber = SKPlumber("classification", 1, sampler=sampler)
X, y = load_dataset("titanic")
plumber.fit(X, y)
def test_can_sample_for_regression(self) -> None:
sampler = OneStackPipelineSampler()
plumber = SKPlumber("regression", 1, sampler=sampler)
X, y = load_dataset("boston")
plumber.fit(X, y)
def test_can_sample_multiple_preprocessors(self) -> None:
sampler = StraightPipelineSampler(preprocessors=2)
X, y = load_dataset("boston")
plumber = SKPlumber("regression", 1, sampler=sampler)
plumber.fit(X, y)
self.assertEqual(len(plumber.best_pipeline.steps), 5)