def test_LinearSVM(self):
# This warning is irrelevant here
warnings.filterwarnings("ignore", ".*", ConvergenceWarning)
learn = LinearSVMLearner()
res = CrossValidation(self.data, [learn], k=2)
self.assertGreater(CA(res)[0], 0.8)
self.assertLess(CA(res)[0], 0.9)
def test_NaiveBayes(self):
results = CrossValidation(self.table, [self.learner], k=10)
ca = CA(results)
self.assertGreater(ca, 0.7)
self.assertLess(ca, 0.9)
results = CrossValidation(Table("iris"), [self.learner], k=10)
ca = CA(results)
self.assertGreater(ca, 0.7)
def test_init(self):
res = Results(nmethods=2, nrows=100)
res.actual[:50] = 0
res.actual[50:] = 1
res.predicted = np.vstack((res.actual, res.actual))
np.testing.assert_almost_equal(CA(res), [1, 1])
res.predicted[0][0] = 1
np.testing.assert_almost_equal(CA(res), [0.99, 1])
res.predicted[1] = 1 - res.predicted[1]
np.testing.assert_almost_equal(CA(res), [0.99, 0])
def test_bayes(self):
x = np.random.randint(2, size=(100, 5))
col = np.random.randint(5)
y = x[:, col].copy().reshape(100, 1)
t = Table(x, y)
t = Discretize(method=discretize.EqualWidth(n=3))(t)
nb = NaiveBayesLearner()
res = TestOnTrainingData()(t, [nb])
np.testing.assert_almost_equal(CA(res), [1])
t.Y[-20:] = 1 - t.Y[-20:]
res = TestOnTrainingData()(t, [nb])
self.assertGreaterEqual(CA(res)[0], 0.75)
self.assertLess(CA(res)[0], 1)
def test_bayes(self):
x = np.random.random_integers(0, 1, (100, 5))
col = np.random.randint(5)
y = x[:, col].copy().reshape(100, 1)
t = Orange.data.Table(x, y)
t = Orange.preprocess.Discretize(method=discretize.EqualWidth(n=3))(t)
nb = Orange.classification.NaiveBayesLearner()
res = Orange.evaluation.TestOnTrainingData(t, [nb])
np.testing.assert_almost_equal(CA(res), [1])
t.Y[-20:] = 1 - t.Y[-20:]
res = Orange.evaluation.TestOnTrainingData(t, [nb])
self.assertGreaterEqual(CA(res)[0], 0.75)
self.assertLess(CA(res)[0], 1)
def test_missing_values(self):
data = self.heart
model = RandomForestLearner(random_state=0)(data)
res = permutation_feature_importance(model, data, CA(), self.n_repeats)
shape = len(data.domain.attributes), self.n_repeats
self.assertEqual(res[0].shape, shape)
self.assertEqual(res[1], [a.name for a in data.domain.attributes])
def test_retain_data(self):
data = self.heart
orig_X = data.X.copy()
model = RandomForestLearner(random_state=0)(data)
permutation_feature_importance(model, data, CA(), self.n_repeats)
np.testing.assert_array_equal(data.X, orig_X)
def test_compare_to_skl(self):
data = self.iris
model = LogisticRegressionLearner()(data)
res1 = _permutation_feature_importance_skl(model, data, self.n_repeats)
res2 = permutation_feature_importance(model, data, CA(),
self.n_repeats)
np.testing.assert_array_equal(res1, res2[0])
def test_GBTrees(self):
booster = CatGBClassifier()
cv = CrossValidation(k=3)
results = cv(self.iris, [booster])
ca = CA(results)
self.assertGreater(ca, 0.9)
self.assertLess(ca, 0.99)
def test_SoftmaxRegression(self):
learner = SoftmaxRegressionLearner()
cv = CrossValidation(k=3)
results = cv(self.iris, [learner])
ca = CA(results)
self.assertGreater(ca, 0.9)
self.assertLess(ca, 1.0)
def test_XGB(self, learner_class: XGBBase):
booster = learner_class()
cv = CrossValidation(k=10)
results = cv(self.iris, [booster])
ca = CA(results)
self.assertGreater(ca, 0.9)
self.assertLess(ca, 0.99)
def test_discrete_variables(self):
data = Table("zoo")
booster = CatGBClassifier()
cv = CrossValidation(k=3)
results = cv(data, [booster])
ca = CA(results)
self.assertGreater(ca, 0.9)
self.assertLess(ca, 0.99)
data = Table("titanic")
booster = CatGBClassifier()
cv = CrossValidation(k=3)
results = cv(data, [booster])
ca = CA(results)
self.assertGreater(ca, 0.75)
self.assertLess(ca, 0.99)
def test_adaboost(self):
learn = SklAdaBoostClassificationLearner()
cv = CrossValidation(k=3)
results = cv(self.iris, [learn])
ca = CA(results)
self.assertGreater(ca, 0.9)
self.assertLess(ca, 0.99)
def test_RandomForest(self):
table = Table('iris')
forest = RandomForestLearner()
results = CrossValidation(table, [forest], k=10)
ca = CA(results)
self.assertGreater(ca, 0.9)
self.assertLess(ca, 0.99)
def test_RandomForest(self):
forest = RandomForestLearner()
cv = CrossValidation(k=10)
results = cv(self.iris, [forest])
ca = CA(results)
self.assertGreater(ca, 0.9)
self.assertLess(ca, 0.99)
def test_LogisticRegression(self):
learn = LogisticRegressionLearner()
cv = CrossValidation(k=2)
results = cv(self.heart_disease, [learn])
ca = CA(results)
self.assertGreater(ca, 0.8)
self.assertLess(ca, 1.0)
def test_NN_model(self):
results = CrossValidation(self.iris, [self.learner], k=3)
self.assertGreater(CA(results), 0.90)
results = CrossValidation(self.housing, [self.learner], k=3)
mse = MSE()
res = mse(results)
self.assertLess(res[0], 35)
def test_KNN(self):
table = Table('iris')
learn = KNNLearner()
results = CrossValidation(table, [learn], k=10)
ca = CA(results)
self.assertGreater(ca, 0.8)
self.assertLess(ca, 0.99)
def test_missing_values(self):
data = Table("heart_disease")
booster = CatGBClassifier()
cv = CrossValidation(k=3)
results = cv(data, [booster])
ca = CA(results)
self.assertGreater(ca, 0.8)
self.assertLess(ca, 0.99)
def test_wrap_score_cls(self):
data = self.heart
model = RandomForestLearner(random_state=0)(data)
scorer = _wrap_score(CA(), _check_model(model, data))
mocked_model = Mock(wraps=model)
baseline_score = scorer(mocked_model, data)
mocked_model.assert_called_once()
self.assertAlmostEqual(baseline_score, 0.987, 3)
def test_adaboost_base_estimator(self):
np.random.seed(0)
stump_estimator = TreeLearner(max_depth=1)
tree_estimator = TreeLearner()
stump = SklAdaBoostLearner(base_estimator=stump_estimator)
tree = SklAdaBoostLearner(base_estimator=tree_estimator)
results = CrossValidation(self.iris, [stump, tree], k=4)
ca = CA(results)
self.assertLess(ca[0], ca[1])
def test_SoftmaxRegressionPreprocessors(self):
table = Table('iris')
table.X[:,2] = table.X[:,2] * 0.001
table.X[:,3] = table.X[:,3] * 0.001
learners = [SoftmaxRegressionLearner(preprocessors=[]),
SoftmaxRegressionLearner()]
results = CrossValidation(table, learners, k=10)
ca = CA(results)
self.assertTrue(ca[0] < ca[1])
class TestCA(unittest.TestCase):
def test_init(self):
res = Results(nmethods=2, nrows=100)
res.actual[:50] = 0
res.actual[50:] = 1
res.predicted = np.vstack((res.actual, res.actual))
np.testing.assert_almost_equal(CA(res), [1, 1])
res.predicted[0][0] = 1
np.testing.assert_almost_equal(CA(res), [0.99, 1])
res.predicted[1] = 1 - res.predicted[1]
np.testing.assert_almost_equal(CA(res), [0.99, 0])
def test_call(self):
res = Results(nmethods=2, nrows=100)
res.actual[:50] = 0
res.actual[50:] = 1
res.predicted = np.vstack((res.actual, res.actual))
ca = CA()
np.testing.assert_almost_equal(ca(res), [1, 1])
res.predicted[0][0] = 1
np.testing.assert_almost_equal(ca(res), [0.99, 1])
res.predicted[1] = 1 - res.predicted[1]
np.testing.assert_almost_equal(ca(res), [0.99, 0])
def test_bayes(self):
x = np.random.randint(2, size=(100, 5))
col = np.random.randint(5)
y = x[:, col].copy().reshape(100, 1)
t = Table(x, y)
t = Discretize(
method=discretize.EqualWidth(n=3))(t)
nb = NaiveBayesLearner()
res = TestOnTrainingData(t, [nb])
np.testing.assert_almost_equal(CA(res), [1])
t.Y[-20:] = 1 - t.Y[-20:]
res = TestOnTrainingData(t, [nb])
self.assertGreaterEqual(CA(res)[0], 0.75)
self.assertLess(CA(res)[0], 1)
def test_wrap_score_predict_cls(self):
data = self.titanic
model = NaiveBayesLearner()(data)
scorer = _wrap_score(CA(), _check_model(model, data))
mocked_model = Mock(wraps=model)
baseline_score = scorer(mocked_model, data)
# mocked_model.assert_not_called()
# mocked_model.predict.assert_called_once()
self.assertAlmostEqual(baseline_score, 0.778, 3)
def test_wrap_score_skl_predict_cls(self):
data = self.iris
model = RandomForestLearner(random_state=0)(data)
scorer = _wrap_score(CA(), _check_model(model, data))
mocked_model = Mock(wraps=model)
baseline_score = scorer(mocked_model, data)
mocked_model.assert_not_called()
mocked_model.predict.assert_not_called()
self.assertAlmostEqual(baseline_score, 0.993, 3)
def test_discrete_class(self):
data = self.iris
model = RandomForestLearner(random_state=0)(data)
res = permutation_feature_importance(model, data, CA(), self.n_repeats)
shape = len(data.domain.attributes), self.n_repeats
self.assertEqual(res[0].shape, shape)
self.assertEqual(res[1], [a.name for a in data.domain.attributes])
mean = np.array([0.013333, 0, 0.322667, 0.474667])
np.testing.assert_array_almost_equal(res[0].mean(axis=1), mean)
def test_sparse_data(self):
sparse_data = self.heart.to_sparse()
model = RandomForestLearner(random_state=0)(sparse_data)
res = permutation_feature_importance(model, sparse_data,
CA(), self.n_repeats)
shape = len(sparse_data.domain.attributes), self.n_repeats
self.assertEqual(res[0].shape, shape)
self.assertEqual(
res[1], [a.name for a in sparse_data.domain.attributes]
)
sparse_data = self.iris.to_sparse()
model = RandomForestLearner(random_state=0)(sparse_data)
res = permutation_feature_importance(model, sparse_data,
CA(), self.n_repeats)
shape = len(sparse_data.domain.attributes), self.n_repeats
self.assertEqual(res[0].shape, shape)
self.assertEqual(
res[1], [a.name for a in sparse_data.domain.attributes]
)
def test_SoftmaxRegressionPreprocessors(self):
table = self.iris.copy()
table.X[:, 2] = table.X[:, 2] * 0.001
table.X[:, 3] = table.X[:, 3] * 0.001
learners = [
SoftmaxRegressionLearner(preprocessors=[]),
SoftmaxRegressionLearner(),
]
results = CrossValidation(table, learners, k=10)
ca = CA(results)
self.assertLess(ca[0], ca[1])
def test_adaboost_base_estimator(self):
np.random.seed(0)
stump_estimator = SklTreeLearner(max_depth=1)
tree_estimator = SklTreeLearner()
stump = SklAdaBoostClassificationLearner(
base_estimator=stump_estimator, n_estimators=5)
tree = SklAdaBoostClassificationLearner(base_estimator=tree_estimator,
n_estimators=5)
cv = CrossValidation(k=4)
results = cv(self.iris, [stump, tree])
ca = CA(results)
self.assertLessEqual(ca[0], ca[1])
def test_SoftmaxRegressionPreprocessors(self):
table = self.iris.copy()
with table.unlocked():
table.X[:, 2] = table.X[:, 2] * 0.001
table.X[:, 3] = table.X[:, 3] * 0.001
learners = [
SoftmaxRegressionLearner(preprocessors=[]),
SoftmaxRegressionLearner()
]
cv = CrossValidation(k=10)
results = cv(table, learners)
ca = CA(results)
self.assertLess(ca[0], ca[1])