def summarize(csv_path: str) -> OrderedDict:
"""
Summarize a CSV file of feature values.
Parameters
----------
csv_path : str
Path to csv.
Returns
-------
OrderedDict
Summary values.
"""
with open(csv_path) as infile:
reader = csv.reader(infile)
table = [row for row in reader]
d = OrderedDict()
ignored_cols = 2
d['datapoints'] = len(table) - 1
for i, col in enumerate(table[0][ignored_cols:]):
i += ignored_cols
d[col] = labmath.mean([float(r[i]) for r in table[1:]])
return d
def profile(self,
queue,
size: int = 16,
must_validate: bool = False,
out=sys.stdout,
metaout=sys.stderr,
min_num_iterations: int = 10):
"""
Run kernel and profile runtime.
Output format (CSV):
out: <kernel> <wgsize> <transfer> <runtime> <ci>
metaout: <error> <kernel>
"""
assert (isinstance(queue, cl.CommandQueue))
if must_validate:
try:
self.validate(queue, size)
except CLDriveException as e:
print(type(e).__name__, self.name, sep=',', file=metaout)
P = KernelPayload.create_random(self, size)
k = partial(self, queue)
while len(self.runtimes) < min_num_iterations:
k(P)
wgsize = int(round(labmath.mean(self.wgsizes)))
transfer = int(round(labmath.mean(self.transfers)))
mean = labmath.mean(self.runtimes)
ci = labmath.confinterval(self.runtimes, array_mean=mean)[1] - mean
print(self.name,
wgsize,
transfer,
round(mean, 6),
round(ci, 6),
sep=',',
file=out)
def summarize(csv_path):
with open(csv_path) as infile:
reader = csv.reader(infile)
table = [row for row in reader]
d = OrderedDict()
ignored_cols = 2
d['datapoints'] = len(table) - 1
for i,col in enumerate(table[0][ignored_cols:]):
i += ignored_cols
d[col] = labmath.mean([float(r[i]) for r in table[1:]])
return d
def plot_speedups_extended_model(benchmarks_data, clgen_data):
"""
Plots speedups of extended model over Grewe et al
Returns: speedup
"""
B = pd.read_csv(benchmarks_data)
B["synthetic"] = np.zeros(len(B))
S = pd.read_csv(clgen_data)
S["synthetic"] = np.ones(len(S))
BS = pd.concat((B, S))
assert (len(BS) == len(B) + len(S))
# get benchmark names: <suite>-<benchmark>
benchmark_names = sorted(set([
re.match(r"^([^0-9]+-[0-9\.]+-[^-]+)", b).group(1)
for b in B["benchmark"]
]))
# perform cross-validation
B_out = []
for i, benchmark in enumerate(benchmark_names):
print("\ranalyzing", i + 1, benchmark, end="")
cgo13_clf, our_clf = cgo13.model(), get_our_model()
cgo13_features, our_features = get_cgo13_features, get_our_features
# cross validate on Grewe et al. and our model
tmp = _compare_clfs(cgo13_clf, cgo13_features, our_clf, our_features,
BS, BS, benchmark)
B_out += tmp
print()
# create results frame
R_out = []
# get runtimes of device using predicted device
for b in B_out:
p1_runtime = b["runtime_" + b["p1"].lower()]
p2_runtime = b["runtime_" + b["p2"].lower()]
# speedup is the ratio of runtime using our predicted device
# over runtime using CGO13 predicted device.
b["p_speedup"] = p2_runtime / p1_runtime
# get the benchmark name
b["group"] = escape_benchmark_name(b["benchmark"])
R_out.append(b)
R = pd.DataFrame(R_out)
improved = R[R["p_speedup"] > 1]
speedups = R.groupby(["group"])["p_speedup"].mean()
speedup = labmath.mean(speedups)
print(" #. benchmarks: ",
len(set(B["benchmark"])), "kernels,", len(B), "observations")
print(" #. synthetic: ",
len(set(S["benchmark"])), "kernels,", len(S), "observations")
print()
print(" Speedup: {:.2f} x".format(speedup))
palette = sns.cubehelix_palette(1, start=4, rot=.8, light=.8, dark=.3)
R = R.append({ # average bar
"group": "Average",
"p_speedup": speedup
}, ignore_index=True)
R["p_speedup"] -= 1 # negative offset so that bars start at 1
ax = sns.barplot(x="group", y="p_speedup", data=R,
palette=palette, ci=None)
plt.ylabel("Speedup over Grewe et al.");
plt.xlabel("")
plt.axhline(y=0, color="k", lw=1)
plt.axvline(x=plt.xlim()[1] - 1, color="k", lw=1, linestyle="--")
plt.ylim(-1, 9)
plt.setp(ax.get_xticklabels(), rotation=90) # rotate x ticks
# counter negative offset
ax.set_yticklabels([int(i) + 1 for i in ax.get_yticks()])
viz.finalise(figsize=(7, 3.7), tight=True)
return speedup
def plot_speedups_extended_model_2platform(platform_a, platform_b):
"""
Plot speedup of extended model over Grewe et al for 2 platforms
"""
aB = pd.read_csv(platform_a[0])
aB["synthetic"] = np.zeros(len(aB))
bB = pd.read_csv(platform_b[0])
bB["synthetic"] = np.zeros(len(bB))
B = pd.concat((aB, bB))
aS = pd.read_csv(platform_a[1])
aS["synthetic"] = np.ones(len(aS))
bS = pd.read_csv(platform_b[1])
bS["synthetic"] = np.ones(len(bS))
S = pd.concat((aS, bS))
aBS = pd.concat((aB, aS))
bBS = pd.concat((bB, bS))
BS = pd.concat((B, S))
assert (len(B) == len(aB) + len(bB)) # sanity checks
assert (len(S) == len(aS) + len(bS))
assert (len(BS) == len(aBS) + len(bBS))
# get benchmark names: <suite>-<benchmark>
benchmark_names = sorted(set([
re.match(r"^([^0-9]+-[0-9\.]+-[^-]+)", b).group(1)
for b in B["benchmark"]
]))
# perform cross-validation
B_out = []
for i, benchmark in enumerate(benchmark_names):
print("\ranalyzing", i + 1, benchmark, end="")
cgo13_clf, our_clf = cgo13.model(), get_our_model()
cgo13_features, our_features = get_cgo13_features, get_our_features
# cross validate on Grewe et al. and our model
tmp = _compare_clfs(cgo13_clf, cgo13_features, our_clf, our_features,
aBS, aBS, benchmark)
for d in tmp: d["platform"] = "AMD Tahiti 7970"
B_out += tmp
# reset models
cgo13_clf, our_clf = cgo13.model(), get_our_model()
# same as before, on other platform:
tmp = _compare_clfs(cgo13_clf, cgo13_features, our_clf, our_features,
bBS, bBS, benchmark)
for d in tmp: d["platform"] = "NVIDIA GTX 970"
B_out += tmp
print()
# create results frame
R_out = []
# get runtimes of device using predicted device
for b in B_out:
p1_runtime = b["runtime_" + b["p1"].lower()]
p2_runtime = b["runtime_" + b["p2"].lower()]
# speedup is the ratio of runtime using our predicted device
# over runtime using CGO13 predicted device.
b["p_speedup"] = p2_runtime / p1_runtime
# get the benchmark name
b["group"] = escape_benchmark_name(b["benchmark"])
R_out.append(b)
R = pd.DataFrame(R_out)
improved = R[R["p_speedup"] > 1]
Amask = R["platform"] == "AMD Tahiti 7970"
Bmask = R["platform"] == "NVIDIA GTX 970"
a = R[Amask]
b = R[Bmask]
a_speedups = a.groupby(["group"])["p_speedup"].mean()
b_speedups = b.groupby(["group"])["p_speedup"].mean()
a_speedup = labmath.mean(a_speedups)
b_speedup = labmath.mean(b_speedups)
assert (len(R) == len(a) + len(b)) # sanity-check
print(" #. benchmarks: ",
len(set(B["benchmark"])), "kernels,", len(B), "observations")
print(" #. synthetic: ",
len(set(S["benchmark"])), "kernels,", len(S), "observations")
print()
print(" Speedup on AMD: {:.2f} x".format(a_speedup))
print(" Speedup on NVIDIA: {:.2f} x".format(b_speedup))
palette = sns.cubehelix_palette(
len(set(R["platform"])), start=4, rot=.8, light=.8, dark=.3)
R = R.append({ # average bars
"group": "Average",
"p_speedup": a_speedup,
"platform": "AMD Tahiti 7970"
}, ignore_index=True)
R = R.append({
"group": "Average",
"p_speedup": b_speedup,
"platform": "NVIDIA GTX 970"
}, ignore_index=True)
R["p_speedup"] -= 1 # negative offset so that bars start at 1
ax = sns.barplot(x="group", y="p_speedup", hue="platform", data=R,
palette=palette, ci=None)
plt.ylabel("Speedup over Grewe et al.");
plt.xlabel("")
plt.axhline(y=0, color="k", lw=1)
plt.axvline(x=plt.xlim()[1] - 1, color="k", lw=1, linestyle="--")
plt.ylim(-1, 9)
plt.setp(ax.get_xticklabels(), rotation=90) # rotate x ticks
ax.get_legend().set_title("") # legend
plt.legend(loc='upper right')
# counter negative offset
ax.set_yticklabels([int(i) + 1 for i in ax.get_yticks()])
ax.get_legend().draw_frame(True)
viz.finalise(figsize=(9, 4), tight=True)
def plot_speedups_with_clgen(benchmarks_data, clgen_data, suite="npb"):
"""
Plot speedups of predictive models trained with and without clgen.
Returns speedups (without and with).
"""
# datasets: B - benchmarks, S - synthetics, BS - benchmarks + synthetics:
B = pd.read_csv(benchmarks_data)
B["group"] = ["B"] * len(B)
S = pd.read_csv(clgen_data)
S["group"] = ["S"] * len(S)
BS = pd.concat((B, S))
# find the ZeroR. This is the device which is most frequently optimal
Bmask = B[B["benchmark"].str.contains(suite)]
zeror = Counter(Bmask["oracle"]).most_common(1)[0][0]
zeror_runtime = "runtime_" + zeror.lower()
# get the names of the benchmarks, in the form: $suite-$version-$benchmark
benchmark_names = sorted(set([
re.match(r"^([^0-9]+-[0-9\.]+-[^-]+)-", b).group(1)
for b in B["benchmark"] if b.startswith(suite)
]))
B_out, BS_out = [], []
for benchmark in benchmark_names:
clf = cgo13.model()
features = get_cgo13_features
# cross validate on baseline
B_out += cgo13.leave_one_benchmark_out(clf, features, B, benchmark)
# reset model
clf = cgo13.model()
# repeate cross-validation with synthetic kernels
BS_out += cgo13.leave_one_benchmark_out(clf, features, BS, benchmark)
# create results frame
R_out = []
for b, bs in zip(B_out, BS_out):
# get runtimes of device using predicted device
b_p_runtime = b["runtime_" + b["p"].lower()]
bs_p_runtime = bs["runtime_" + bs["p"].lower()]
# speedup is the ratio of runtime using the predicted device
# over runtime using ZeroR device
b["p_speedup"] = b_p_runtime / b[zeror_runtime]
bs["p_speedup"] = bs_p_runtime / bs[zeror_runtime]
if "training" in benchmarks_data:
# $benchmark
group = escape_benchmark_name(b["benchmark"])
else:
# $benchmark.$dataset
group = re.sub(r"[^-]+-[0-9\.]+-([^-]+)-.+", r"\1",
b["benchmark"]) + "." + b["dataset"]
b["group"] = group
bs["group"] = group
# set the training data type
b["training"] = "Grewe et al."
bs["training"] = "w. CLgen"
R_out.append(b)
R_out.append(bs)
R = pd.DataFrame(R_out)
b_mask = R["training"] == "Grewe et al."
bs_mask = R["training"] == "w. CLgen"
B_speedup = labmath.mean(R[b_mask].groupby(["group"])["p_speedup"].mean())
BS_speedup = labmath.mean(R[bs_mask].groupby(["group"])["p_speedup"].mean())
print(" #. benchmarks: ",
len(set(B["benchmark"])), "kernels,", len(B), "observations")
print(" #. synthetic: ",
len(set(S["benchmark"])), "kernels,", len(S), "observations")
print()
print(" ZeroR device: {}".format(zeror))
print()
print(" Speedup of Grewe et al.: {:.2f} x".format(B_speedup))
print(" Speedup w. CLgen: {:.2f} x".format(BS_speedup))
R = R.append({ # average bars
"group": "Average",
"p_speedup": B_speedup,
"training": "Grewe et al."
}, ignore_index=True)
R = R.append({
"group": "Average",
"p_speedup": BS_speedup,
"training": "w. CLgen"
}, ignore_index=True)
R["p_speedup"] -= 1 # negative offset so that bars start at 1
# colors
palette = sns.cubehelix_palette(len(set(R["training"])),
rot=-.4, light=.85, dark=.35)
ax = sns.barplot(
x="group", y="p_speedup", data=R, ci=None, hue="training",
palette=palette)
plt.ylabel("Speedup")
plt.xlabel("")
plt.axhline(y=0, color="k", lw=1) # speedup line
plt.axvline(x=plt.xlim()[1] - 1, color="k", lw=1,
linestyle="--") # average line
ax.get_legend().set_title("") # no legend title
plt.legend(loc='upper right')
ax.get_legend().draw_frame(True)
# plot shape and size
figsize = (9, 2.2)
if "nvidia" in benchmarks_data:
typecast = int;
plt.ylim(-1, 16)
elif "training" in benchmarks_data:
typecast = float;
figsize = (7, 3.2)
else:
typecast = float
# counter negative offset:
ax.set_yticklabels([typecast(i) + 1 for i in ax.get_yticks()])
plt.setp(ax.get_xticklabels(), rotation=90)
viz.finalise(figsize=figsize, tight=True)
return B_speedup, BS_speedup
def test_mean(self):
self._test(2, labmath.mean([1,2,3]))
self._test((1/3.), labmath.mean([1,1.5,-1.5]))
self._test(2, labmath.mean([2,2,2,2,2]))
self._test(2.5, labmath.mean([1,2,3,4]))
def test_mean_single_item_array(self):
self._test(1, labmath.mean([1]))
def test_mean_empty_array(self):
self._test(0, labmath.mean([]))
def regression_classification(db, output=None, job="xval",
table="runtime_classification_results",
**kwargs):
"""
Plot performance of classification using runtime regression.
"""
jobs = {
"xval": "10-fold",
"synthetic_real": "Synthetic",
"arch": "Device",
"kern": "Kernel",
"data": "Dataset",
}
results = []
for job in jobs:
speedup, serr, perf, perr, time, terr, correct = db.execute(
"SELECT "
" AVG(speedup), CONFERROR(speedup, .95), "
" AVG(performance) * 100, CONFERROR(performance, .95) * 100, "
" AVG(time) + 2.5, CONFERROR(time, .95), "
" AVG(correct) * 100 "
"FROM {} WHERE job=?".format(table),
(job,)
).fetchone()
results.append([job, speedup, serr, perf, perr, time, terr, correct])
# Zip into lists.
labels, speedup, serr, perf, perr, time, terr, correct = zip(*results)
labels = [jobs[x] for x in jobs]
# Add averages.
labels.append(r'\textbf{Average}')
speedup += (labmath.mean(speedup),)
serr += (labmath.mean(serr),)
perf += (labmath.mean(perf),)
perr += (labmath.mean(perr),)
time += (labmath.mean(time),)
terr += (labmath.mean(terr),)
correct += (labmath.mean(correct),)
X = np.arange(len(labels))
width = .8
# PLOT TIMES
ax = plt.subplot(4, 1, 1)
ax.bar(X + .1, time, width=width)
ax.set_xticks(X + .5)
ax.set_ylim(0, 150)
ax.set_xticklabels(labels, rotation='vertical')
ax.set_ylabel("Classification time (ms)")
# Plot confidence intervals separately so that we can have
# full control over formatting.
_,caps,_ = ax.errorbar(X + .5, time,
fmt="none", yerr=terr, capsize=3, ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
# SPEEDUPS
ax = plt.subplot(4, 1, 3)
ax.bar(X + .1, speedup, width=width, color=sns.color_palette("Greens"))
ax.set_xticks(X + .5)
ax.set_ylim(0, 7)
ax.set_xticklabels(labels, rotation='vertical')
ax.set_ylabel("Speedup")
# Plot confidence intervals separately so that we can have
# full control over formatting.
_,caps,_ = ax.errorbar(X + .5, speedup,
fmt="none", yerr=serr, capsize=3, ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
# PERFORMANCE
ax = plt.subplot(4, 1, 4)
ax.bar(X + .1, perf, width=width, color=sns.color_palette("Blues"))
ax.set_xticks(X + .5)
ax.set_xticklabels(labels, rotation='vertical')
ax.set_ylabel("Performance")
plt.gca().yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
ax.set_ylim(0, 100)
# Plot confidence intervals separately so that we can have
# full control over formatting.
_,caps,_ = ax.errorbar(X + .5, perf,
fmt="none", yerr=perr, capsize=3, ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
# ACCURACY
ax = plt.subplot(4, 1, 2)
ax.bar(X + .1, correct, width=width, color=sns.color_palette("Reds"))
ax.set_xticks(X + .5)
ax.set_xticklabels(labels, rotation='vertical')
ax.set_ylabel("Accuracy")
plt.gca().yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
ax.set_ylim(0, 12)
viz.finalise(output, **kwargs)
def classification(db, output=None, job="xval", **kwargs):
err_fns = db.err_fns
base_err_fn = err_fns[0]
# Get a list of classifiers and result counts.
query = db.execute(
"SELECT classifier,Count(*) AS count\n"
"FROM classification_results\n"
"WHERE job=? AND err_fn=? AND classifier!='weka.classifiers.rules.ZeroR'\n"
"GROUP BY classifier",
(job,base_err_fn)
)
results = []
# Add baseline results.
baseline = ("4x4")
correct = db.execute("SELECT Count(*) * 1.0 / 3 FROM classification_results "
"WHERE job=? AND actual=?", (job,baseline)).fetchone()[0]
illegal = 0
refused = 0
time = 0
terr = 0
speedup = (1, 0)
perfs = [
row[1] for row in
db.execute(
"SELECT "
" DISTINCT runtime_stats.scenario, "
" (scenario_stats.oracle_runtime / runtime_stats.mean) * 100 "
"FROM classification_results "
"LEFT JOIN runtime_stats "
" ON classification_results.scenario=runtime_stats.scenario "
"LEFT JOIN scenario_stats "
" ON classification_results.scenario=scenario_stats.scenario "
"WHERE job=? and runtime_stats.params=?",
(job, baseline)
)
]
perf = (labmath.mean(perfs), labmath.confinterval(perfs, error_only=True))
results.append(["ZeroR", correct, illegal, refused, time, terr,
speedup, speedup, speedup,
perf, perf, perf])
# Get results
for classifier,count in query:
basename = ml.classifier_basename(classifier)
correct, illegal, refused, time, terr = db.execute(
"SELECT\n"
" (SUM(correct) / CAST(? AS FLOAT)) * 100,\n"
" (SUM(illegal) / CAST(? AS FLOAT)) * 100,\n"
" (SUM(refused) / CAST(? AS FLOAT)) * 100,\n"
" AVG(time) + 2.5,\n"
" CONFERROR(time, .95) * 1.5\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(count, count, count, job, classifier, base_err_fn)
).fetchone()
# Get a list of mean speedups for each err_fn.
speedups = [
db.execute(
"SELECT\n"
" AVG(speedup),\n"
" CONFERROR(speedup, .95)\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(job, classifier, err_fn)
).fetchone()
for err_fn in err_fns
]
# Get a list of mean perfs for each err_fn.
perfs = [
db.execute(
"SELECT\n"
" AVG(performance) * 100.0,\n"
" CONFERROR(performance, .95) * 100.0\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(job, classifier, err_fn)
).fetchone()
for err_fn in err_fns
]
results.append([basename, correct, illegal, refused, time, terr] + speedups + perfs)
# Zip into lists.
labels, correct, illegal, refused, time, terr = zip(*[
(text.truncate(result[0], 40), result[1], result[2],
result[3], result[4], result[5])
for result in results
])
X = np.arange(len(labels))
# PLOT TIMES
width = .8
ax = plt.subplot(4, 1, 1)
ax.bar(X + .1, time, width=width)
ax.set_xticks(X + .4)
ax.set_xticklabels(labels)
ax.set_ylim(0, 10)
ax.set_ylabel("Classification time (ms)")
# art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
# Plot confidence intervals separately so that we can have
# full control over formatting.
_,caps,_ = ax.errorbar(X + .5, time,
fmt="none", yerr=terr, capsize=3, ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
# RATIOS
width = (.8 / 3)
ax = plt.subplot(4, 1, 2)
ax.bar(X + .1, illegal, width=width,
color=sns.color_palette("Reds", 1), label="Illegal")
ax.bar(X + .1 + width, refused, width=width,
color=sns.color_palette("Oranges", 1), label="Refused")
ax.bar(X + .1 + 2 * width, correct, width=width,
color=sns.color_palette("Blues", 1), label="Accurate")
ax.set_xticks(X + .4)
ax.set_ylabel("Ratio")
ax.set_ylim(0, 35)
ax.set_xticklabels(labels)
ax.yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
# Plot speedups.
ax = plt.subplot(4, 1, 3)
width = (.8 / 3)
colors=sns.color_palette("Greens", len(err_fns))
for i,err_fn in enumerate(db.err_fns):
pairs = [result[6 + i] for result in results]
speedups, yerrs = zip(*pairs)
ax.bar(X + .1 + (i * width), speedups, width=width,
label=errfn2label(err_fn), color=colors[i])
# Plot confidence intervals separately so that we can have
# full control over formatting.
_,caps,_ = ax.errorbar(X + .1 + (i + .5) * width, speedups,
fmt="none", yerr=yerrs, capsize=3, ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
ax.set_xticks(X + .4)
ax.set_xticklabels(labels)
ax.set_ylim(0, 7)
ax.set_xticks(X + .4, labels)
ax.set_ylabel("Speedup")
art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
# PERFORMANCE
colors=sns.color_palette("Blues", len(err_fns))
width = (.8 / 3)
ax = plt.subplot(4, 1, 4)
for i,err_fn in enumerate(db.err_fns):
pairs = [result[9 + i] for result in results]
perfs, yerrs = zip(*pairs)
ax.bar(X + .1 + (i * width), perfs, width=width,
label=errfn2label(err_fn), color=colors[i])
# Plot confidence intervals separately so that we can have
# full control over formatting.
_,caps,_ = ax.errorbar(X + .1 + (i + .5) * width, perfs,
fmt="none", yerr=yerrs, capsize=3, ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
ax.set_xticks(X + .4)
ax.yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
ax.set_xticklabels(labels)
ax.set_ylim(0, 100)
ax.set_ylabel("Performance")
ax.set_xticks(X + .4, labels)
title = kwargs.pop("title", "Classification results for " + job)
plt.title(title)
# Add legend *beneath* plot. To do this, we need to pass some
# extra arguments to plt.savefig(). See:
#
# http://jb-blog.readthedocs.org/en/latest/posts/12-matplotlib-legend-outdide-plot.html
#
art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3)]
viz.finalise(output, additional_artists=art, bbox_inches="tight", **kwargs)
def regression_classification(db,
output=None,
job="xval",
table="runtime_classification_results",
**kwargs):
"""
Plot performance of classification using runtime regression.
"""
jobs = {
"xval": "10-fold",
"synthetic_real": "Synthetic",
"arch": "Device",
"kern": "Kernel",
"data": "Dataset",
}
results = []
for job in jobs:
speedup, serr, perf, perr, time, terr, correct = db.execute(
"SELECT "
" AVG(speedup), CONFERROR(speedup, .95), "
" AVG(performance) * 100, CONFERROR(performance, .95) * 100, "
" AVG(time) + 2.5, CONFERROR(time, .95), "
" AVG(correct) * 100 "
"FROM {} WHERE job=?".format(table), (job, )).fetchone()
results.append([job, speedup, serr, perf, perr, time, terr, correct])
# Zip into lists.
labels, speedup, serr, perf, perr, time, terr, correct = zip(*results)
labels = [jobs[x] for x in jobs]
# Add averages.
labels.append(r'\textbf{Average}')
speedup += (labmath.mean(speedup), )
serr += (labmath.mean(serr), )
perf += (labmath.mean(perf), )
perr += (labmath.mean(perr), )
time += (labmath.mean(time), )
terr += (labmath.mean(terr), )
correct += (labmath.mean(correct), )
X = np.arange(len(labels))
width = .8
# PLOT TIMES
ax = plt.subplot(4, 1, 1)
ax.bar(X + .1, time, width=width)
ax.set_xticks(X + .5)
ax.set_ylim(0, 150)
ax.set_xticklabels(labels, rotation='vertical')
ax.set_ylabel("Classification time (ms)")
# Plot confidence intervals separately so that we can have
# full control over formatting.
_, caps, _ = ax.errorbar(X + .5,
time,
fmt="none",
yerr=terr,
capsize=3,
ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
# SPEEDUPS
ax = plt.subplot(4, 1, 3)
ax.bar(X + .1, speedup, width=width, color=sns.color_palette("Greens"))
ax.set_xticks(X + .5)
ax.set_ylim(0, 7)
ax.set_xticklabels(labels, rotation='vertical')
ax.set_ylabel("Speedup")
# Plot confidence intervals separately so that we can have
# full control over formatting.
_, caps, _ = ax.errorbar(X + .5,
speedup,
fmt="none",
yerr=serr,
capsize=3,
ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
# PERFORMANCE
ax = plt.subplot(4, 1, 4)
ax.bar(X + .1, perf, width=width, color=sns.color_palette("Blues"))
ax.set_xticks(X + .5)
ax.set_xticklabels(labels, rotation='vertical')
ax.set_ylabel("Performance")
plt.gca().yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
ax.set_ylim(0, 100)
# Plot confidence intervals separately so that we can have
# full control over formatting.
_, caps, _ = ax.errorbar(X + .5,
perf,
fmt="none",
yerr=perr,
capsize=3,
ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
# ACCURACY
ax = plt.subplot(4, 1, 2)
ax.bar(X + .1, correct, width=width, color=sns.color_palette("Reds"))
ax.set_xticks(X + .5)
ax.set_xticklabels(labels, rotation='vertical')
ax.set_ylabel("Accuracy")
plt.gca().yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
ax.set_ylim(0, 12)
viz.finalise(output, **kwargs)
def classification(db, output=None, job="xval", **kwargs):
err_fns = db.err_fns
base_err_fn = err_fns[0]
# Get a list of classifiers and result counts.
query = db.execute(
"SELECT classifier,Count(*) AS count\n"
"FROM classification_results\n"
"WHERE job=? AND err_fn=? AND classifier!='weka.classifiers.rules.ZeroR'\n"
"GROUP BY classifier", (job, base_err_fn))
results = []
# Add baseline results.
baseline = ("4x4")
correct = db.execute(
"SELECT Count(*) * 1.0 / 3 FROM classification_results "
"WHERE job=? AND actual=?", (job, baseline)).fetchone()[0]
illegal = 0
refused = 0
time = 0
terr = 0
speedup = (1, 0)
perfs = [
row[1] for row in db.execute(
"SELECT "
" DISTINCT runtime_stats.scenario, "
" (scenario_stats.oracle_runtime / runtime_stats.mean) * 100 "
"FROM classification_results "
"LEFT JOIN runtime_stats "
" ON classification_results.scenario=runtime_stats.scenario "
"LEFT JOIN scenario_stats "
" ON classification_results.scenario=scenario_stats.scenario "
"WHERE job=? and runtime_stats.params=?", (job, baseline))
]
perf = (labmath.mean(perfs), labmath.confinterval(perfs, error_only=True))
results.append([
"ZeroR", correct, illegal, refused, time, terr, speedup, speedup,
speedup, perf, perf, perf
])
# Get results
for classifier, count in query:
basename = ml.classifier_basename(classifier)
correct, illegal, refused, time, terr = db.execute(
"SELECT\n"
" (SUM(correct) / CAST(? AS FLOAT)) * 100,\n"
" (SUM(illegal) / CAST(? AS FLOAT)) * 100,\n"
" (SUM(refused) / CAST(? AS FLOAT)) * 100,\n"
" AVG(time) + 2.5,\n"
" CONFERROR(time, .95) * 1.5\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(count, count, count, job, classifier, base_err_fn)).fetchone()
# Get a list of mean speedups for each err_fn.
speedups = [
db.execute(
"SELECT\n"
" AVG(speedup),\n"
" CONFERROR(speedup, .95)\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(job, classifier, err_fn)).fetchone() for err_fn in err_fns
]
# Get a list of mean perfs for each err_fn.
perfs = [
db.execute(
"SELECT\n"
" AVG(performance) * 100.0,\n"
" CONFERROR(performance, .95) * 100.0\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(job, classifier, err_fn)).fetchone() for err_fn in err_fns
]
results.append([basename, correct, illegal, refused, time, terr] +
speedups + perfs)
# Zip into lists.
labels, correct, illegal, refused, time, terr = zip(
*[(text.truncate(result[0], 40), result[1], result[2], result[3],
result[4], result[5]) for result in results])
X = np.arange(len(labels))
# PLOT TIMES
width = .8
ax = plt.subplot(4, 1, 1)
ax.bar(X + .1, time, width=width)
ax.set_xticks(X + .4)
ax.set_xticklabels(labels)
ax.set_ylim(0, 10)
ax.set_ylabel("Classification time (ms)")
# art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
# Plot confidence intervals separately so that we can have
# full control over formatting.
_, caps, _ = ax.errorbar(X + .5,
time,
fmt="none",
yerr=terr,
capsize=3,
ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
# RATIOS
width = (.8 / 3)
ax = plt.subplot(4, 1, 2)
ax.bar(X + .1,
illegal,
width=width,
color=sns.color_palette("Reds", 1),
label="Illegal")
ax.bar(X + .1 + width,
refused,
width=width,
color=sns.color_palette("Oranges", 1),
label="Refused")
ax.bar(X + .1 + 2 * width,
correct,
width=width,
color=sns.color_palette("Blues", 1),
label="Accurate")
ax.set_xticks(X + .4)
ax.set_ylabel("Ratio")
ax.set_ylim(0, 35)
ax.set_xticklabels(labels)
ax.yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
# Plot speedups.
ax = plt.subplot(4, 1, 3)
width = (.8 / 3)
colors = sns.color_palette("Greens", len(err_fns))
for i, err_fn in enumerate(db.err_fns):
pairs = [result[6 + i] for result in results]
speedups, yerrs = zip(*pairs)
ax.bar(X + .1 + (i * width),
speedups,
width=width,
label=errfn2label(err_fn),
color=colors[i])
# Plot confidence intervals separately so that we can have
# full control over formatting.
_, caps, _ = ax.errorbar(X + .1 + (i + .5) * width,
speedups,
fmt="none",
yerr=yerrs,
capsize=3,
ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
ax.set_xticks(X + .4)
ax.set_xticklabels(labels)
ax.set_ylim(0, 7)
ax.set_xticks(X + .4, labels)
ax.set_ylabel("Speedup")
art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
# PERFORMANCE
colors = sns.color_palette("Blues", len(err_fns))
width = (.8 / 3)
ax = plt.subplot(4, 1, 4)
for i, err_fn in enumerate(db.err_fns):
pairs = [result[9 + i] for result in results]
perfs, yerrs = zip(*pairs)
ax.bar(X + .1 + (i * width),
perfs,
width=width,
label=errfn2label(err_fn),
color=colors[i])
# Plot confidence intervals separately so that we can have
# full control over formatting.
_, caps, _ = ax.errorbar(X + .1 + (i + .5) * width,
perfs,
fmt="none",
yerr=yerrs,
capsize=3,
ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
ax.set_xticks(X + .4)
ax.yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
ax.set_xticklabels(labels)
ax.set_ylim(0, 100)
ax.set_ylabel("Performance")
ax.set_xticks(X + .4, labels)
title = kwargs.pop("title", "Classification results for " + job)
plt.title(title)
# Add legend *beneath* plot. To do this, we need to pass some
# extra arguments to plt.savefig(). See:
#
# http://jb-blog.readthedocs.org/en/latest/posts/12-matplotlib-legend-outdide-plot.html
#
art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3)]
viz.finalise(output, additional_artists=art, bbox_inches="tight", **kwargs)
