def oracle(self):
try:
return self._oracle
except AttributeError:
assert(len(self.speedups) == len(self.oracles))
self._oracle = self.speedup / labmath.geomean(self.oracles)
return self._oracle
def oracle(self):
try:
return self._oracle
except AttributeError:
assert (len(self.speedups) == len(self.oracles))
self._oracle = self.speedup / labmath.geomean(self.oracles)
return self._oracle
def speedup(self):
try:
return self._speedup
except AttributeError:
self._speedup = labmath.geomean(self.speedups)
return self._speedup
def visualise_classification_job(db, job):
basedir = "img/classification/{}/".format(job)
fs.mkdir(basedir)
fs.mkdir(basedir + "classifiers")
fs.mkdir(basedir + "err_fns")
visualise.err_fn_performance(db, basedir + "err_fns.png", job=job)
# Bar plot of all results.
visualise.classification(db, "img/classification/{}.png".format(job),
job=job)
# Per-classifier plots.
for i,classifier in enumerate(db.classification_classifiers):
visualise.classifier_speedups(db, classifier,
basedir + "classifiers/{}.png".format(i),
job=job)
# Per-err_fn plots.
for err_fn in db.err_fns:
visualise.err_fn_speedups(db, err_fn,
basedir + "err_fns/{}.png".format(err_fn),
job=job, sort=True)
# Results table.
query = db.execute(
"SELECT classifier,err_fn,Count(*) AS count\n"
"FROM classification_results\n"
"WHERE job=? GROUP BY classifier,err_fn",
(job,)
)
results = []
for classifier,err_fn,count in query:
correct, illegal, refused, performance, speedup = zip(*[
row for row in db.execute(
"SELECT correct,illegal,refused,performance,speedup\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(job, classifier, err_fn)
)
])
results.append([
classifier,
err_fn,
(sum(correct) / count) * 100,
(sum(illegal) / count) * 100,
(sum(refused) / count) * 100,
min(performance) * 100,
labmath.geomean(performance) * 100,
max(performance) * 100,
min(speedup),
labmath.geomean(speedup),
max(speedup)
])
str_args = {
"float_format": lambda f: "{:.2f}".format(f)
}
for i in range(len(results)):
results[i][0] = ml.classifier_basename(results[i][0])
columns=(
"CLASSIFIER",
"ERR_FN",
"ACC %",
"INV %",
"REF %",
"Omin %",
"Oavg %",
"Omax %",
"Smin",
"Savg",
"Smax",
)
latex.table(results, output=fs.path(experiment.TAB_ROOT, job + ".tex"),
columns=columns, **str_args)
def test_geomean(self):
self._test(1.81712059283, labmath.geomean([1,2,3]), approximate=True)
self._test(1.44224957031, labmath.geomean([1,1.5,2]), approximate=True)
self._test(2, labmath.geomean([2,2,2,2,2]))
self._test(2.2133638394, labmath.geomean([1,2,3,4]), approximate=True)
self._test(0, labmath.geomean([0,1,2,3,4]))
def test_geomean_single_item_array(self):
self._test(1, labmath.geomean([1]))
def test_geomean_empty_array(self):
self._test(0, labmath.geomean([]))
def speedup(self):
try:
return self._speedup
except AttributeError:
self._speedup = labmath.geomean(self.speedups)
return self._speedup
def visualise_classification_job(db, job):
basedir = "img/classification/{}/".format(job)
fs.mkdir(basedir)
fs.mkdir(basedir + "classifiers")
fs.mkdir(basedir + "err_fns")
visualise.err_fn_performance(db, basedir + "err_fns.png", job=job)
# Bar plot of all results.
visualise.classification(db,
"img/classification/{}.png".format(job),
job=job)
# Per-classifier plots.
for i, classifier in enumerate(db.classification_classifiers):
visualise.classifier_speedups(db,
classifier,
basedir + "classifiers/{}.png".format(i),
job=job)
# Per-err_fn plots.
for err_fn in db.err_fns:
visualise.err_fn_speedups(db,
err_fn,
basedir + "err_fns/{}.png".format(err_fn),
job=job,
sort=True)
# Results table.
query = db.execute(
"SELECT classifier,err_fn,Count(*) AS count\n"
"FROM classification_results\n"
"WHERE job=? GROUP BY classifier,err_fn", (job, ))
results = []
for classifier, err_fn, count in query:
correct, illegal, refused, performance, speedup = zip(*[
row for row in db.execute(
"SELECT correct,illegal,refused,performance,speedup\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?", (job, classifier,
err_fn))
])
results.append([
classifier, err_fn, (sum(correct) / count) * 100,
(sum(illegal) / count) * 100, (sum(refused) / count) * 100,
min(performance) * 100,
labmath.geomean(performance) * 100,
max(performance) * 100,
min(speedup),
labmath.geomean(speedup),
max(speedup)
])
str_args = {"float_format": lambda f: "{:.2f}".format(f)}
for i in range(len(results)):
results[i][0] = ml.classifier_basename(results[i][0])
columns = (
"CLASSIFIER",
"ERR_FN",
"ACC %",
"INV %",
"REF %",
"Omin %",
"Oavg %",
"Omax %",
"Smin",
"Savg",
"Smax",
)
latex.table(results,
output=fs.path(experiment.TAB_ROOT, job + ".tex"),
columns=columns,
**str_args)
