def cforange_proportion_test(input_dict):
import orange, orngTest, orngStat
learners = input_dict['learners']
data = input_dict['dataset']
learnProp = float(input_dict['learnProp'])
times = int(input_dict['times'])
results = orngTest.proportionTest(learners, data, learnProp, times=times)
output_dict = {}
output_dict['results']=results
return output_dict
def score(self, ids):
"""compute scores for the list of learners"""
if (not self.data):
for id in ids:
self.learners[id].results = None
return
# test which learners can accept the given data set
# e.g., regressions can't deal with classification data
learners = []
n = len(self.data.domain.attributes) * 2
indices = orange.MakeRandomIndices2(
p0=min(n, len(self.data)),
stratified=orange.MakeRandomIndices2.StratifiedIfPossible)
new = self.data.selectref(indices(self.data))
# new = self.data.selectref([1]*min(n, len(self.data)) +
# [0]*(len(self.data) - min(n, len(self.data))))
self.warning(0)
for l in [self.learners[id] for id in ids]:
learner = l.learner
if self.preprocessor:
learner = self.preprocessor.wrapLearner(learner)
try:
predictor = learner(new)
if predictor(new[0]).varType == new.domain.classVar.varType:
learners.append(learner)
else:
l.scores = []
except Exception as ex:
self.warning(
0,
"Learner %s ends with exception: %s" % (l.name, str(ex)))
l.scores = []
if not learners:
return
# computation of results (res, and cm if classification)
pb = None
if self.resampling == 0:
pb = OWGUI.ProgressBar(self, iterations=self.nFolds)
res = orngTest.crossValidation(
learners,
self.data,
folds=self.nFolds,
strat=orange.MakeRandomIndices.StratifiedIfPossible,
callback=pb.advance,
storeExamples=True)
pb.finish()
elif self.resampling == 1:
pb = OWGUI.ProgressBar(self, iterations=len(self.data))
res = orngTest.leaveOneOut(learners,
self.data,
callback=pb.advance,
storeExamples=True)
pb.finish()
elif self.resampling == 2:
pb = OWGUI.ProgressBar(self, iterations=self.pRepeat)
res = orngTest.proportionTest(learners,
self.data,
self.pLearning / 100.,
times=self.pRepeat,
callback=pb.advance,
storeExamples=True)
pb.finish()
elif self.resampling == 3:
pb = OWGUI.ProgressBar(self, iterations=len(learners))
res = orngTest.learnAndTestOnLearnData(learners,
self.data,
storeExamples=True,
callback=pb.advance)
pb.finish()
elif self.resampling == 4:
if not self.testdata:
for l in self.learners.values():
l.scores = []
return
pb = OWGUI.ProgressBar(self, iterations=len(learners))
res = orngTest.learnAndTestOnTestData(learners,
self.data,
self.testdata,
storeExamples=True,
callback=pb.advance)
pb.finish()
if self.isclassification():
cm = orngStat.computeConfusionMatrices(res,
classIndex=self.targetClass)
if self.preprocessor: # Unwrap learners
learners = [l.wrappedLearner for l in learners]
res.learners = learners
for l in [self.learners[id] for id in ids]:
if l.learner in learners:
l.results = res
self.error(list(range(len(self.stat))))
scores = []
for i, s in enumerate(self.stat):
try:
scores.append(eval("orngStat." + s.f))
except Exception as ex:
self.error(i, "An error occurred while evaluating orngStat." + s.f + "on %s due to %s" % \
(" ".join([l.name for l in learners]), ex))
scores.append([None] * len(self.learners))
for (i, l) in enumerate(learners):
self.learners[l.id].scores = [s[i] if s else None for s in scores]
self.sendResults()
tree = orange.TreeLearner(name="tree")
majority = orange.MajorityLearner(name="default")
learners = [bayes, tree, majority]
names = [x.name for x in learners]
def printResults(res):
CAs = orngStat.CA(res, reportSE=1)
for i in range(len(names)):
print "%s: %5.3f+-%5.3f " % (names[i], CAs[i][0], 1.96 * CAs[i][1]),
print
print "\nproportionsTest that will always give the same results"
for i in range(3):
res = orngTest.proportionTest(learners, data, 0.7)
printResults(res)
print "\nproportionsTest that will give different results, but the same each time the script is run"
myRandom = orange.RandomGenerator()
for i in range(3):
res = orngTest.proportionTest(learners,
data,
0.7,
randomGenerator=myRandom)
printResults(res)
if not vars().has_key("NO_RANDOMNESS"):
print "\nproportionsTest that will give different results each time it is run"
for i in range(3):
res = orngTest.proportionTest(learners,
bayes = orange.BayesLearner(name = "bayes")
tree = orange.TreeLearner(name = "tree")
majority = orange.MajorityLearner(name = "default")
learners = [bayes, tree, majority]
names = [x.name for x in learners]
def printResults(res):
CAs = orngStat.CA(res, reportSE=1)
for i in range(len(names)):
print "%s: %5.3f+-%5.3f " % (names[i], CAs[i][0], 1.96*CAs[i][1]),
print
print "\nproportionsTest that will always give the same results"
for i in range(3):
res = orngTest.proportionTest(learners, data, 0.7)
printResults(res)
print "\nproportionsTest that will give different results, but the same each time the script is run"
myRandom = orange.RandomGenerator()
for i in range(3):
res = orngTest.proportionTest(learners, data, 0.7, randomGenerator = myRandom)
printResults(res)
if not vars().has_key("NO_RANDOMNESS"):
print "\nproportionsTest that will give different results each time it is run"
for i in range(3):
res = orngTest.proportionTest(learners, data, 0.7, randseed = random.randint(0, 100))
printResults(res)
print "\nproportionsTest + storing classifiers"
