def test_store_models(self):
nrows, ncols = 1000, 10
t = random_data(nrows, ncols)
fitters = [naive_bayes.BayesLearner(), majority.MajorityFitter()]
cv = testing.CrossValidation(k=5)
res = cv(t, fitters)
self.assertIsNone(res.models)
cv = testing.CrossValidation(k=5, store_models=True)
res = cv(t, fitters)
self.assertEqual(len(res.models), 5)
for models in res.models:
self.assertEqual(len(models), 2)
self.assertIsInstance(models[0], naive_bayes.BayesClassifier)
self.assertIsInstance(models[1], majority.ConstantClassifier)
cv = testing.CrossValidation(k=5)
res = cv(t, fitters)
self.assertIsNone(res.models)
res = testing.CrossValidation(t, fitters, k=5, store_models=True)
self.assertEqual(len(res.models), 5)
for models in res.models:
self.assertEqual(len(models), 2)
self.assertIsInstance(models[0], naive_bayes.BayesClassifier)
self.assertIsInstance(models[1], majority.ConstantClassifier)
def test_miss_majority(self):
x = np.zeros((50, 3))
y = x[:, -1]
x[49] = 1
data = Table(x, y)
res = testing.CrossValidation(data, [majority.MajorityFitter()])
np.testing.assert_equal(res.predicted[0][:49], 0)
x[49] = 0
res = testing.CrossValidation(data, [majority.MajorityFitter()])
np.testing.assert_equal(res.predicted[0][:49], 0)
def main():
import sip
from PyQt4.QtGui import QApplication
from Orange.classification import logistic_regression, svm
from Orange.evaluation import testing
app = QApplication([])
w = OWLiftCurve()
w.show()
w.raise_()
data = Orange.data.Table("ionosphere")
results = testing.CrossValidation(
data,
[logistic_regression.LogisticRegressionLearner(penalty="l2"),
logistic_regression.LogisticRegressionLearner(penalty="l1"),
svm.SVMLearner(probability=True),
svm.NuSVMLearner(probability=True)
],
store_data=True
)
results.fitter_names = ["LR l2", "LR l1", "SVM", "Nu SVM"]
w.set_results(results)
rval = app.exec_()
sip.delete(w)
del w
app.processEvents()
del app
return rval
def test_NaiveBayes(self):
table = Orange.data.Table('titanic')
bayes = nb.BayesLearner()
results = testing.CrossValidation(table[::20], [bayes], k=10)
ca = scoring.CA(results)
self.assertGreater(ca, 0.7)
self.assertLess(ca, 0.9)
def test_RandomForest(self):
table = Orange.data.Table('iris')
forest = rf.RandomForestLearner()
results = testing.CrossValidation(table, [forest], k=10)
ca = scoring.CA(results)
self.assertGreater(ca, 0.9)
self.assertLess(ca, 0.99)
def test_folds(self):
nrows, ncols = 1000, 10
t = random_data(nrows, ncols)
res = testing.CrossValidation(t, [naive_bayes.BayesLearner()], k=5)
self.assertEqual(len(res.folds), 5)
for i, fold in enumerate(res.folds):
self.assertEqual(fold.start, i * 200)
self.assertEqual(fold.stop, (i + 1) * 200)
def test_store_data(self):
nrows, ncols = 1000, 10
t = random_data(nrows, ncols)
fitters = [naive_bayes.BayesLearner()]
cv = testing.CrossValidation()
res = cv(t, fitters)
self.assertIsNone(res.data)
cv = testing.CrossValidation(store_data=True)
res = cv(t, fitters)
self.assertIs(res.data, t)
res = testing.CrossValidation(t, fitters)
self.assertIsNone(res.data)
res = testing.CrossValidation(t, fitters, store_data=True)
self.assertIs(res.data, t)
def test_10_fold_probs(self):
data = Table('iris')[30:130]
fitters = [majority.MajorityFitter(), majority.MajorityFitter()]
results = testing.CrossValidation(k=10)(data, fitters)
self.assertEqual(results.predicted.shape, (2, len(data)))
np.testing.assert_equal(results.predicted, np.ones((2, 100)))
probs = results.probabilities
self.assertTrue((probs[:, :, 0] < probs[:, :, 2]).all())
self.assertTrue((probs[:, :, 2] < probs[:, :, 1]).all())
def update_results(self):
# items in need of an update
items = [(key, input) for key, input in self.learners.items()
if input.results is None]
learners = [input.learner for _, input in items]
self.setStatusMessage("Running")
# TODO: Test each learner individually
if self.resampling == OWTestLearners.KFold:
results = testing.CrossValidation(self.train_data,
learners,
k=self.k_folds,
store_data=True)
elif self.resampling == OWTestLearners.LeaveOneOut:
results = testing.LeaveOneOut(self.train_data,
learners,
store_data=True)
elif self.resampling == OWTestLearners.Bootstrap:
p = self.sample_p / 100.0
results = testing.Bootstrap(self.train_data,
learners,
n_resamples=self.n_repeat,
p=p,
store_data=True)
elif self.resampling == OWTestLearners.TestOnTrain:
results = testing.TestOnTrainingData(self.train_data,
learners,
store_data=True)
elif self.resampling == OWTestLearners.TestOnTest:
assert self.test_data is not None
results = testing.TestOnTestData(self.train_data,
self.test_data,
learners,
store_data=True)
else:
assert False
results = list(split_by_model(results))
class_var = self.train_data.domain.class_var
if is_discrete(class_var):
test_stats = classification_stats
else:
test_stats = regression_stats
stats = [test_stats(res) for res in results]
for (key, input), res, stat in zip(items, results, stats):
self.learners[key] = input._replace(results=res, stats=stat)
self.setStatusMessage("")
self._update_stats_model()
def test_results(self):
nrows, ncols = 1000, 10
t = random_data(nrows, ncols)
res = testing.CrossValidation(t, [naive_bayes.BayesLearner()])
y = t.Y
np.testing.assert_equal(res.actual, y[res.row_indices].reshape(nrows))
np.testing.assert_equal(res.predicted[0],
y[res.row_indices].reshape(nrows))
np.testing.assert_equal(np.argmax(res.probabilities[0], axis=1),
y[res.row_indices].reshape(nrows))
self.assertEqual(len(res.folds), 10)
for i, fold in enumerate(res.folds):
self.assertEqual(fold.start, i * 100)
self.assertEqual(fold.stop, (i + 1) * 100)
def update_results(self):
self.warning([1, 2])
self.error(2)
if self.train_data is None:
return
if self.resampling == OWTestLearners.TestOnTest:
if self.test_data is None:
self.warning(2, "Missing separate test data input")
return
elif self.test_data.domain.class_var != \
self.train_data.domain.class_var:
self.error(2, ("Inconsistent class variable between test " +
"and train data sets"))
return
# items in need of an update
items = [(key, input) for key, input in self.learners.items()
if input.results is None]
learners = [input.learner for _, input in items]
self.setStatusMessage("Running")
if self.test_data is not None and \
self.resampling != OWTestLearners.TestOnTest:
self.warning(1, "Test data is present but unused. "
"Select 'Test on test data' to use it.")
# TODO: Test each learner individually
if self.resampling == OWTestLearners.KFold:
results = testing.CrossValidation(
self.train_data, learners, k=self.k_folds, store_data=True
)
elif self.resampling == OWTestLearners.LeaveOneOut:
results = testing.LeaveOneOut(
self.train_data, learners, store_data=True
)
elif self.resampling == OWTestLearners.Bootstrap:
p = self.sample_p / 100.0
results = testing.Bootstrap(
self.train_data, learners, n_resamples=self.n_repeat, p=p,
store_data=True
)
elif self.resampling == OWTestLearners.TestOnTrain:
results = testing.TestOnTrainingData(
self.train_data, learners, store_data=True
)
elif self.resampling == OWTestLearners.TestOnTest:
if self.test_data is None:
return
results = testing.TestOnTestData(
self.train_data, self.test_data, learners, store_data=True
)
else:
assert False
results = list(split_by_model(results))
class_var = self.train_data.domain.class_var
if is_discrete(class_var):
test_stats = classification_stats
else:
test_stats = regression_stats
self._update_header()
stats = [test_stats(res) for res in results]
for (key, input), res, stat in zip(items, results, stats):
self.learners[key] = input._replace(results=res, stats=stat)
self.setStatusMessage("")
self._update_stats_model()
precompute=False,
max_iter=1000,
copy_X=True,
tol=0.0001,
warm_start=False,
positive=False,
preprocessors=None)
ElasticNet.name = "Elactic net Regression"
learners = [Linear, Ridge, Lasso, ElasticNet]
results = testing.CrossValidation(
data,
learners,
k=10,
#random_state=1,
#store_data=False,
#store_models=False,
#preprocessor=None,
#callback=None,
#warnings=None
)
#data information
print("------Data Information------")
print("File: %a" % data.name, "Data Instances: %a" % len(data),
"Attributes: %a" % len(data.domain.attributes), "Class:",
data.domain.class_var.name)
#regression part
for i in range(len(learners)):
print("Regularization: %a" % learners[i].name)
print("------Testing&Score------")
print("MSE: %.3f" % scoring.MSE(results)[i],
def setGeometry(self, rect):
super().setGeometry(rect)
def paintEvent(self, event):
painter = QPainter(self)
rect = self.geometry()
textRect = QRect(0, 0, rect.width(), rect.height())
painter.translate(textRect.bottomLeft())
painter.rotate(-90)
painter.drawText(
QRect(QPoint(0, 0), QSize(rect.height(), rect.width())),
Qt.AlignCenter, self.text())
painter.end()
if __name__ == "__main__":
from PyQt4.QtGui import QApplication
from Orange.evaluation import testing
from Orange.classification import naive_bayes
app = QApplication([])
w = OWConfusionMatrix()
w.show()
data = Orange.data.Table("iris")
res = testing.CrossValidation(data, [naive_bayes.BayesLearner()],
store_data=True)
w.set_results(res)
app.exec_()
def setGeometry(self, rect):
super().setGeometry(rect)
def paintEvent(self, event):
painter = QPainter(self)
rect = self.geometry()
textRect = QRect(0, 0, rect.width(), rect.height())
painter.translate(textRect.bottomLeft())
painter.rotate(-90)
painter.drawText(
QRect(QPoint(0, 0), QSize(rect.height(), rect.width())),
Qt.AlignCenter, self.text())
painter.end()
if __name__ == "__main__":
from PyQt4.QtGui import QApplication
from Orange.evaluation import testing
from Orange.classification import tree
app = QApplication([])
w = OWConfusionMatrix()
w.show()
data = Orange.data.Table("iris")
res = testing.CrossValidation(data, [tree.ClassificationTreeLearner()],
store_data=True)
w.set_results(res)
app.exec_()
