def EndEpisodeStep(self) -> random_opt_pb2.DelayedRewardStep:
start_ms = labdate.MillisecondsTimestamp()
step = random_opt_pb2.DelayedRewardStep(start_time_epoch_ms=start_ms, )
try:
clang.Compile([self.working_bytecode_path],
self.binary_path,
copts=['-O0'])
try:
runtimes = self.GetRuntimes()
self.episodes[-1].binary_runtime_ms.extend(runtimes)
if self.BinaryIsValid():
step.reward = self.runtime_reward(
sum(runtimes) / len(runtimes))
else:
self.episodes[-1].outcome = (
random_opt_pb2.DelayedRewardEpisode.EVAL_FAILED)
step.reward = self.eval_failed_reward
except ValueError as e:
self.episodes[-1].outcome = random_opt_pb2.Step.EXEC_FAILED
self.episodes[-1].outcome_error_msg = text.truncate(
str(e), 255)
step.reward = self.exec_failed_reward
except clang.ClangException as e:
self.episodes[-1].outcome = (
random_opt_pb2.DelayedRewardEpisode.COMPILE_FAILED)
self.episodes[-1].outcome_error_msg = text.truncate(str(e), 255)
step.reward = self.compile_failed_reward
obs = self.observation_space.sample()
step.total_step_runtime_ms = labdate.MillisecondsTimestamp() - start_ms
self.episodes[-1].step.extend([step])
return obs, step.reward, True, {}
def test_truncate(self):
self._test("foo", text.truncate("foo", 100))
self._test("1234567890", text.truncate("1234567890", 10))
self._test("12345...", text.truncate("1234567890", 8))
for i in range(10, 20):
self._test(i,
len(text.truncate("The quick brown fox jumped "
"over the slow lazy dog", i)))
def test_truncate():
assert "foo" == text.truncate("foo", 100)
assert "1234567890" == text.truncate("1234567890", 10)
assert "12345..." == text.truncate("1234567890", 8)
for i in range(10, 20):
assert i == len(
text.truncate(
"The quick brown fox jumped "
"over the slow lazy dog", i))
def ActionStep(self, action: int) -> random_opt_pb2.DelayedRewardStep:
if not self.action_space.contains(action):
raise ValueError(f"Unknown action: '{action}'")
start_ms = labdate.MillisecondsTimestamp()
obs = self.observation_space.sample()
opt_pass = self.config.candidate_pass[action]
step = random_opt_pb2.DelayedRewardStep(
start_time_epoch_ms=start_ms,
opt_pass=opt_pass,
)
# Run the full list of passes and update working_bytecode file.
try:
all_passes = [step.opt_pass for step in self.episodes[-1].step[1:]]
opt.RunOptPassOnBytecode(self.bytecode_path,
self.working_dir / 'temp.ll', all_passes)
step.bytecode_changed = BytecodesAreEqual(
self.working_dir / 'temp.ll', self.working_bytecode_path)
shutil.copyfile(self.working_dir / 'temp.ll',
self.working_bytecode_path)
step.reward = (self.bytecode_changed_reward
if step.bytecode_changed else
self.bytecode_unchanged_reward)
except llvm.LlvmError as e:
# Opt failed, set the error message.
step.reward = self.opt_failed_reward
step.opt_error_msg = text.truncate(str(e), 255)
step.total_step_runtime_ms = labdate.MillisecondsTimestamp() - start_ms
self.episodes[-1].step.extend([step])
return obs, step.reward, False, {}
def main():
db = _db.Database(experiment.ORACLE_PATH)
ml.start()
# Delete any old stuff.
fs.rm(experiment.IMG_ROOT + "/*")
fs.rm(experiment.TAB_ROOT + "/*")
# Make directories
fs.mkdir(experiment.TAB_ROOT)
fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/bars"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/heatmap"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/trisurf"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/devices"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/kernels"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/datasets"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/devices"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/kernels"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/datasets"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/devices"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/kernels"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/datasets"))
visualise.pie(db.num_scenarios_by_device,
fs.path(experiment.IMG_ROOT, "num_sceanrios_by_device"))
visualise.pie(db.num_runtime_stats_by_device,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_device"))
visualise.pie(db.num_scenarios_by_dataset,
fs.path(experiment.IMG_ROOT, "num_sceanrios_by_dataset"))
visualise.pie(db.num_runtime_stats_by_dataset,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_dataset"))
visualise.pie(db.num_runtime_stats_by_kernel,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel"))
visualise.pie(db.num_runtime_stats_by_kernel,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel"))
# Per-scenario plots
for row in db.scenario_properties:
scenario,device,kernel,north,south,east,west,max_wgsize,width,height,tout = row
title = ("{device}: {kernel}[{n},{s},{e},{w}]\n"
"{width} x {height} {type}s"
.format(device=text.truncate(device, 18), kernel=kernel,
n=north, s=south, e=east, w=west,
width=width, height=height, type=tout))
output = fs.path(experiment.IMG_ROOT,
"scenarios/heatmap/{id}.png".format(id=scenario))
space = _space.ParamSpace.from_dict(db.perf_scenario(scenario))
max_c = min(25, len(space.c))
max_r = min(25, len(space.r))
space.reshape(max_c=max_c, max_r=max_r)
# Heatmaps.
mask = _space.ParamSpace(space.c, space.r)
for j in range(len(mask.r)):
for i in range(len(mask.c)):
if space.matrix[j][i] == 0:
r, c = space.r[j], space.c[i]
# TODO: Get values from refused_params table.
if r * c >= max_wgsize:
# Illegal
mask.matrix[j][i] = -1
else:
# Refused
db.execute("INSERT OR IGNORE INTO refused_params VALUES(?,?)",
(scenario, hash_params(c, r)))
space.matrix[j][i] = -1
mask.matrix[j][i] = 1
db.commit()
new_order = list(reversed(range(space.matrix.shape[0])))
data = space.matrix[:][new_order]
figsize=(12,6)
_, ax = plt.subplots(1, 2, figsize=figsize, sharey=True)
sns.heatmap(data, ax=ax[0], vmin=-1, vmax=1,
xticklabels=space.c,
yticklabels=list(reversed(space.r)), square=True)
ax[0].set_title(title)
new_order = list(reversed(range(mask.matrix.shape[0])))
data = mask.matrix[:][new_order]
sns.heatmap(data, ax=ax[1], vmin=-1, vmax=1,
xticklabels=space.c,
yticklabels=list(reversed(space.r)), square=True)
# Set labels.
ax[0].set_ylabel("Rows")
ax[0].set_xlabel("Columns")
ax[1].set_ylabel("Rows")
ax[1].set_xlabel("Columns")
# plt.tight_layout()
# plt.gcf().set_size_inches(*figsize, dpi=300)
viz.finalise(output)
# 3D bars.
output = fs.path(experiment.IMG_ROOT,
"scenarios/bars/{id}.png".format(id=scenario))
space.bar3d(output=output, title=title, zlabel="Performance",
rotation=45)
# Trisurfs.
output = fs.path(experiment.IMG_ROOT,
"scenarios/trisurf/{id}.png".format(id=scenario))
space.trisurf(output=output, title=title, zlabel="Performance",
rotation=45)
#####################
# ML Visualisations #
#####################
#features_tab(db, experiment.TAB_ROOT)
visualise_classification_job(db, "xval")
visualise_classification_job(db, "arch")
visualise_classification_job(db, "xval_real")
visualise_classification_job(db, "synthetic_real")
# Runtime regression accuracy.
visualise_regression_job(db, "xval")
visualise_regression_job(db, "arch")
visualise_regression_job(db, "xval_real")
visualise_regression_job(db, "synthetic_real")
# Whole-dataset plots
visualise.runtimes_variance(db, fs.path(experiment.IMG_ROOT,
"runtime_variance.png"),
min_samples=30)
visualise.num_samples(db, fs.path(experiment.IMG_ROOT,
"num_samples.png"))
visualise.runtimes_range(db, fs.path(experiment.IMG_ROOT,
"runtimes_range.png"))
visualise.max_speedups(db, fs.path(experiment.IMG_ROOT,
"max_speedups.png"))
visualise.kernel_performance(db, fs.path(experiment.IMG_ROOT,
"kernel_performance.png"))
visualise.device_performance(db, fs.path(experiment.IMG_ROOT,
"device_performance.png"))
visualise.dataset_performance(db, fs.path(experiment.IMG_ROOT,
"dataset_performance.png"))
visualise.num_params_vs_accuracy(db, fs.path(experiment.IMG_ROOT,
"num_params_vs_accuracy.png"))
visualise.performance_vs_coverage(db,
fs.path(experiment.IMG_ROOT,
"performance_vs_coverage.png"))
visualise.performance_vs_max_wgsize(
db, fs.path(experiment.IMG_ROOT, "performance_vs_max_wgsize.png")
)
visualise.performance_vs_wgsize(db, fs.path(experiment.IMG_ROOT,
"performance_vs_wgsize.png"))
visualise.performance_vs_wg_c(db, fs.path(experiment.IMG_ROOT,
"performance_vs_wg_c.png"))
visualise.performance_vs_wg_r(db, fs.path(experiment.IMG_ROOT,
"performance_vs_wg_r.png"))
visualise.max_wgsizes(db, fs.path(experiment.IMG_ROOT, "max_wgsizes.png"))
visualise.oracle_speedups(db, fs.path(experiment.IMG_ROOT,
"oracle_speedups.png"))
visualise.coverage(db,
fs.path(experiment.IMG_ROOT, "coverage/coverage.png"))
visualise.safety(db, fs.path(experiment.IMG_ROOT, "safety/safety.png"))
visualise.oracle_wgsizes(db, fs.path(experiment.IMG_ROOT, "oracle/all.png"))
# Per-device plots
for i,device in enumerate(db.devices):
where = ("scenario IN "
"(SELECT id from scenarios WHERE device='{0}')"
.format(device))
output = fs.path(experiment.IMG_ROOT,
"coverage/devices/{0}.png".format(i))
visualise.coverage(db, output=output, where=where, title=device)
output = fs.path(experiment.IMG_ROOT,
"safety/devices/{0}.png".format(i))
visualise.safety(db, output, where=where, title=device)
output = fs.path(experiment.IMG_ROOT,
"oracle/devices/{0}.png".format(i))
visualise.oracle_wgsizes(db, output, where=where, title=device)
where = ("scenario IN (\n"
" SELECT id from scenarios WHERE device='{0}'\n"
") AND scenario IN (\n"
" SELECT id FROM scenarios WHERE kernel IN (\n"
" SELECT id FROM kernel_names WHERE synthetic=0\n"
" )\n"
")"
.format(device))
output = fs.path(experiment.IMG_ROOT,
"coverage/devices/{0}_real.png".format(i))
visualise.coverage(db, output=output, where=where,
title=device + ", real")
output = fs.path(experiment.IMG_ROOT,
"safety/devices/{0}_real.png".format(i))
visualise.safety(db, output, where=where,
title=device + ", real")
output = fs.path(experiment.IMG_ROOT,
"oracle/devices/{0}_real.png".format(i))
visualise.oracle_wgsizes(db, output, where=where,
title=device + ", real")
where = ("scenario IN (\n"
" SELECT id from scenarios WHERE device='{0}'\n"
") AND scenario IN (\n"
" SELECT id FROM scenarios WHERE kernel IN (\n"
" SELECT id FROM kernel_names WHERE synthetic=1\n"
" )\n"
")"
.format(device))
output = fs.path(experiment.IMG_ROOT,
"coverage/devices/{0}_synthetic.png".format(i))
visualise.coverage(db, output=output, where=where,
title=device + ", synthetic")
output = fs.path(experiment.IMG_ROOT,
"safety/devices/{0}_synthetic.png".format(i))
visualise.safety(db, output, where=where,
title=device + ", synthetic")
output = fs.path(experiment.IMG_ROOT,
"oracle/devices/{0}_synthetic.png".format(i))
visualise.oracle_wgsizes(db, output, where=where,
title=device + ", synthetic")
# Per-kernel plots
for kernel,ids in db.lookup_named_kernels().iteritems():
id_wrapped = ['"' + id + '"' for id in ids]
where = ("scenario IN "
"(SELECT id from scenarios WHERE kernel IN ({0}))"
.format(",".join(id_wrapped)))
output = fs.path(experiment.IMG_ROOT,
"coverage/kernels/{0}.png".format(kernel))
visualise.coverage(db, output=output, where=where, title=kernel)
output = fs.path(experiment.IMG_ROOT,
"safety/kernels/{0}.png".format(kernel))
visualise.safety(db, output=output, where=where, title=kernel)
output = fs.path(experiment.IMG_ROOT,
"oracle/kernels/{0}.png".format(kernel))
visualise.safety(db, output=output, where=where, title=kernel)
# Per-dataset plots
for i,dataset in enumerate(db.datasets):
where = ("scenario IN "
"(SELECT id from scenarios WHERE dataset='{0}')"
.format(dataset))
output = fs.path(experiment.IMG_ROOT,
"coverage/datasets/{0}.png".format(i))
visualise.coverage(db, output, where=where, title=dataset)
output = fs.path(experiment.IMG_ROOT,
"safety/datasets/{0}.png".format(i))
visualise.safety(db, output, where=where, title=dataset)
output = fs.path(experiment.IMG_ROOT,
"oracle/datasets/{0}.png".format(i))
visualise.safety(db, output, where=where, title=dataset)
ml.stop()
def test_truncate_bad_maxchar(self):
with self.assertRaises(text.TruncateError):
text.truncate("foo", -1)
text.truncate("foo", 3)
def test_truncate_bad_maxchar():
with pytest.raises(text.TruncateError):
text.truncate("foo", -1)
text.truncate("foo", 3)
def Step(self, step: random_opt_pb2.Step) -> random_opt_pb2.Step:
"""Run a step.
Args:
step: A step proto with field 'opt_pass' set.
Returns:
The input step.
"""
start_time = labdate.MillisecondsTimestamp()
step.start_time_epoch_ms = start_time
step.status = random_opt_pb2.Step.PASS
temp_bytecode = self.working_dir / 'temp_src.ll'
temp_binary = self.working_dir / 'temp_binary'
# Run the pass.
try:
opt.RunOptPassOnBytecode(self.working_bytecode_path, temp_bytecode,
list(step.opt_pass))
except llvm.LlvmError as e:
step.status = random_opt_pb2.Step.OPT_FAILED
step.status_msg = text.truncate(str(e), 255)
if step.status == random_opt_pb2.Step.PASS:
# Update bytecode file.
logging.debug('$ mv %s %s', temp_bytecode,
self.working_bytecode_path)
step.bytecode_changed = BytecodesAreEqual(
temp_bytecode, self.working_bytecode_path)
os.rename(str(temp_bytecode), str(self.working_bytecode_path))
# Compile a new binary.
try:
clang.Compile([self.working_bytecode_path],
temp_binary,
copts=['-O0'])
step.binary_changed = BinariesAreEqual(temp_binary,
self.binary_path)
os.rename(str(temp_binary), str(self.binary_path))
except llvm.LlvmError as e:
step.status = random_opt_pb2.Step.COMPILE_FAILED
step.status_msg = text.truncate(str(e), 255)
if step.status == random_opt_pb2.Step.PASS:
# Get the binary runtime.
try:
step.binary_runtime_ms.extend(self.GetRuntimes())
except ValueError as e:
step.status = random_opt_pb2.Step.EXEC_FAILED
step.status_msg = text.truncate(str(e), 255)
if step.status == random_opt_pb2.Step.PASS:
if self.BinaryIsValid():
step.speedup = (
(sum(self.episodes[-1].step[-1].binary_runtime_ms) /
len(self.episodes[-1].step[-1].binary_runtime_ms)) /
(sum(step.binary_runtime_ms) /
len(step.binary_runtime_ms)))
step.total_speedup = (
(sum(self.episodes[-1].step[0].binary_runtime_ms) /
len(self.episodes[-1].step[0].binary_runtime_ms)) /
(sum(step.binary_runtime_ms) /
len(step.binary_runtime_ms)))
else:
step.status = random_opt_pb2.Step.EVAL_FAILED
step.reward = self.Reward(step.status, step.speedup)
step.total_reward = self.episodes[-1].step[
-1].total_reward + step.reward
step.total_step_runtime_ms = labdate.MillisecondsTimestamp(
) - start_time
return step
def classification(db, output=None, job="xval", **kwargs):
err_fns = db.err_fns
base_err_fn = err_fns[0]
# Get a list of classifiers and result counts.
query = db.execute(
"SELECT classifier,Count(*) AS count\n"
"FROM classification_results\n"
"WHERE job=? AND err_fn=? AND classifier!='weka.classifiers.rules.ZeroR'\n"
"GROUP BY classifier",
(job,base_err_fn)
)
results = []
# Add baseline results.
baseline = ("4x4")
correct = db.execute("SELECT Count(*) * 1.0 / 3 FROM classification_results "
"WHERE job=? AND actual=?", (job,baseline)).fetchone()[0]
illegal = 0
refused = 0
time = 0
terr = 0
speedup = (1, 0)
perfs = [
row[1] for row in
db.execute(
"SELECT "
" DISTINCT runtime_stats.scenario, "
" (scenario_stats.oracle_runtime / runtime_stats.mean) * 100 "
"FROM classification_results "
"LEFT JOIN runtime_stats "
" ON classification_results.scenario=runtime_stats.scenario "
"LEFT JOIN scenario_stats "
" ON classification_results.scenario=scenario_stats.scenario "
"WHERE job=? and runtime_stats.params=?",
(job, baseline)
)
]
perf = (labmath.mean(perfs), labmath.confinterval(perfs, error_only=True))
results.append(["ZeroR", correct, illegal, refused, time, terr,
speedup, speedup, speedup,
perf, perf, perf])
# Get results
for classifier,count in query:
basename = ml.classifier_basename(classifier)
correct, illegal, refused, time, terr = db.execute(
"SELECT\n"
" (SUM(correct) / CAST(? AS FLOAT)) * 100,\n"
" (SUM(illegal) / CAST(? AS FLOAT)) * 100,\n"
" (SUM(refused) / CAST(? AS FLOAT)) * 100,\n"
" AVG(time) + 2.5,\n"
" CONFERROR(time, .95) * 1.5\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(count, count, count, job, classifier, base_err_fn)
).fetchone()
# Get a list of mean speedups for each err_fn.
speedups = [
db.execute(
"SELECT\n"
" AVG(speedup),\n"
" CONFERROR(speedup, .95)\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(job, classifier, err_fn)
).fetchone()
for err_fn in err_fns
]
# Get a list of mean perfs for each err_fn.
perfs = [
db.execute(
"SELECT\n"
" AVG(performance) * 100.0,\n"
" CONFERROR(performance, .95) * 100.0\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(job, classifier, err_fn)
).fetchone()
for err_fn in err_fns
]
results.append([basename, correct, illegal, refused, time, terr] + speedups + perfs)
# Zip into lists.
labels, correct, illegal, refused, time, terr = zip(*[
(text.truncate(result[0], 40), result[1], result[2],
result[3], result[4], result[5])
for result in results
])
X = np.arange(len(labels))
# PLOT TIMES
width = .8
ax = plt.subplot(4, 1, 1)
ax.bar(X + .1, time, width=width)
ax.set_xticks(X + .4)
ax.set_xticklabels(labels)
ax.set_ylim(0, 10)
ax.set_ylabel("Classification time (ms)")
# art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
# Plot confidence intervals separately so that we can have
# full control over formatting.
_,caps,_ = ax.errorbar(X + .5, time,
fmt="none", yerr=terr, capsize=3, ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
# RATIOS
width = (.8 / 3)
ax = plt.subplot(4, 1, 2)
ax.bar(X + .1, illegal, width=width,
color=sns.color_palette("Reds", 1), label="Illegal")
ax.bar(X + .1 + width, refused, width=width,
color=sns.color_palette("Oranges", 1), label="Refused")
ax.bar(X + .1 + 2 * width, correct, width=width,
color=sns.color_palette("Blues", 1), label="Accurate")
ax.set_xticks(X + .4)
ax.set_ylabel("Ratio")
ax.set_ylim(0, 35)
ax.set_xticklabels(labels)
ax.yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
# Plot speedups.
ax = plt.subplot(4, 1, 3)
width = (.8 / 3)
colors=sns.color_palette("Greens", len(err_fns))
for i,err_fn in enumerate(db.err_fns):
pairs = [result[6 + i] for result in results]
speedups, yerrs = zip(*pairs)
ax.bar(X + .1 + (i * width), speedups, width=width,
label=errfn2label(err_fn), color=colors[i])
# Plot confidence intervals separately so that we can have
# full control over formatting.
_,caps,_ = ax.errorbar(X + .1 + (i + .5) * width, speedups,
fmt="none", yerr=yerrs, capsize=3, ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
ax.set_xticks(X + .4)
ax.set_xticklabels(labels)
ax.set_ylim(0, 7)
ax.set_xticks(X + .4, labels)
ax.set_ylabel("Speedup")
art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
# PERFORMANCE
colors=sns.color_palette("Blues", len(err_fns))
width = (.8 / 3)
ax = plt.subplot(4, 1, 4)
for i,err_fn in enumerate(db.err_fns):
pairs = [result[9 + i] for result in results]
perfs, yerrs = zip(*pairs)
ax.bar(X + .1 + (i * width), perfs, width=width,
label=errfn2label(err_fn), color=colors[i])
# Plot confidence intervals separately so that we can have
# full control over formatting.
_,caps,_ = ax.errorbar(X + .1 + (i + .5) * width, perfs,
fmt="none", yerr=yerrs, capsize=3, ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
ax.set_xticks(X + .4)
ax.yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
ax.set_xticklabels(labels)
ax.set_ylim(0, 100)
ax.set_ylabel("Performance")
ax.set_xticks(X + .4, labels)
title = kwargs.pop("title", "Classification results for " + job)
plt.title(title)
# Add legend *beneath* plot. To do this, we need to pass some
# extra arguments to plt.savefig(). See:
#
# http://jb-blog.readthedocs.org/en/latest/posts/12-matplotlib-legend-outdide-plot.html
#
art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3)]
viz.finalise(output, additional_artists=art, bbox_inches="tight", **kwargs)
def classification(db, output=None, job="xval", **kwargs):
err_fns = db.err_fns
base_err_fn = err_fns[0]
# Get a list of classifiers and result counts.
query = db.execute(
"SELECT classifier,Count(*) AS count\n"
"FROM classification_results\n"
"WHERE job=? AND err_fn=? AND classifier!='weka.classifiers.rules.ZeroR'\n"
"GROUP BY classifier", (job, base_err_fn))
results = []
# Add baseline results.
baseline = ("4x4")
correct = db.execute(
"SELECT Count(*) * 1.0 / 3 FROM classification_results "
"WHERE job=? AND actual=?", (job, baseline)).fetchone()[0]
illegal = 0
refused = 0
time = 0
terr = 0
speedup = (1, 0)
perfs = [
row[1] for row in db.execute(
"SELECT "
" DISTINCT runtime_stats.scenario, "
" (scenario_stats.oracle_runtime / runtime_stats.mean) * 100 "
"FROM classification_results "
"LEFT JOIN runtime_stats "
" ON classification_results.scenario=runtime_stats.scenario "
"LEFT JOIN scenario_stats "
" ON classification_results.scenario=scenario_stats.scenario "
"WHERE job=? and runtime_stats.params=?", (job, baseline))
]
perf = (labmath.mean(perfs), labmath.confinterval(perfs, error_only=True))
results.append([
"ZeroR", correct, illegal, refused, time, terr, speedup, speedup,
speedup, perf, perf, perf
])
# Get results
for classifier, count in query:
basename = ml.classifier_basename(classifier)
correct, illegal, refused, time, terr = db.execute(
"SELECT\n"
" (SUM(correct) / CAST(? AS FLOAT)) * 100,\n"
" (SUM(illegal) / CAST(? AS FLOAT)) * 100,\n"
" (SUM(refused) / CAST(? AS FLOAT)) * 100,\n"
" AVG(time) + 2.5,\n"
" CONFERROR(time, .95) * 1.5\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(count, count, count, job, classifier, base_err_fn)).fetchone()
# Get a list of mean speedups for each err_fn.
speedups = [
db.execute(
"SELECT\n"
" AVG(speedup),\n"
" CONFERROR(speedup, .95)\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(job, classifier, err_fn)).fetchone() for err_fn in err_fns
]
# Get a list of mean perfs for each err_fn.
perfs = [
db.execute(
"SELECT\n"
" AVG(performance) * 100.0,\n"
" CONFERROR(performance, .95) * 100.0\n"
"FROM classification_results\n"
"WHERE job=? AND classifier=? AND err_fn=?",
(job, classifier, err_fn)).fetchone() for err_fn in err_fns
]
results.append([basename, correct, illegal, refused, time, terr] +
speedups + perfs)
# Zip into lists.
labels, correct, illegal, refused, time, terr = zip(
*[(text.truncate(result[0], 40), result[1], result[2], result[3],
result[4], result[5]) for result in results])
X = np.arange(len(labels))
# PLOT TIMES
width = .8
ax = plt.subplot(4, 1, 1)
ax.bar(X + .1, time, width=width)
ax.set_xticks(X + .4)
ax.set_xticklabels(labels)
ax.set_ylim(0, 10)
ax.set_ylabel("Classification time (ms)")
# art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
# Plot confidence intervals separately so that we can have
# full control over formatting.
_, caps, _ = ax.errorbar(X + .5,
time,
fmt="none",
yerr=terr,
capsize=3,
ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
# RATIOS
width = (.8 / 3)
ax = plt.subplot(4, 1, 2)
ax.bar(X + .1,
illegal,
width=width,
color=sns.color_palette("Reds", 1),
label="Illegal")
ax.bar(X + .1 + width,
refused,
width=width,
color=sns.color_palette("Oranges", 1),
label="Refused")
ax.bar(X + .1 + 2 * width,
correct,
width=width,
color=sns.color_palette("Blues", 1),
label="Accurate")
ax.set_xticks(X + .4)
ax.set_ylabel("Ratio")
ax.set_ylim(0, 35)
ax.set_xticklabels(labels)
ax.yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
# Plot speedups.
ax = plt.subplot(4, 1, 3)
width = (.8 / 3)
colors = sns.color_palette("Greens", len(err_fns))
for i, err_fn in enumerate(db.err_fns):
pairs = [result[6 + i] for result in results]
speedups, yerrs = zip(*pairs)
ax.bar(X + .1 + (i * width),
speedups,
width=width,
label=errfn2label(err_fn),
color=colors[i])
# Plot confidence intervals separately so that we can have
# full control over formatting.
_, caps, _ = ax.errorbar(X + .1 + (i + .5) * width,
speedups,
fmt="none",
yerr=yerrs,
capsize=3,
ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
ax.set_xticks(X + .4)
ax.set_xticklabels(labels)
ax.set_ylim(0, 7)
ax.set_xticks(X + .4, labels)
ax.set_ylabel("Speedup")
art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
# PERFORMANCE
colors = sns.color_palette("Blues", len(err_fns))
width = (.8 / 3)
ax = plt.subplot(4, 1, 4)
for i, err_fn in enumerate(db.err_fns):
pairs = [result[9 + i] for result in results]
perfs, yerrs = zip(*pairs)
ax.bar(X + .1 + (i * width),
perfs,
width=width,
label=errfn2label(err_fn),
color=colors[i])
# Plot confidence intervals separately so that we can have
# full control over formatting.
_, caps, _ = ax.errorbar(X + .1 + (i + .5) * width,
perfs,
fmt="none",
yerr=yerrs,
capsize=3,
ecolor="k")
for cap in caps:
cap.set_color('k')
cap.set_markeredgewidth(1)
ax.set_xticks(X + .4)
ax.yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
ax.set_xticklabels(labels)
ax.set_ylim(0, 100)
ax.set_ylabel("Performance")
ax.set_xticks(X + .4, labels)
title = kwargs.pop("title", "Classification results for " + job)
plt.title(title)
# Add legend *beneath* plot. To do this, we need to pass some
# extra arguments to plt.savefig(). See:
#
# http://jb-blog.readthedocs.org/en/latest/posts/12-matplotlib-legend-outdide-plot.html
#
art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3)]
viz.finalise(output, additional_artists=art, bbox_inches="tight", **kwargs)
def main():
db = _db.Database(experiment.ORACLE_PATH)
ml.start()
# Delete any old stuff.
fs.rm(experiment.IMG_ROOT + "/*")
fs.rm(experiment.TAB_ROOT + "/*")
# Make directories
fs.mkdir(experiment.TAB_ROOT)
fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/bars"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/heatmap"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/trisurf"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/devices"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/kernels"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/datasets"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/devices"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/kernels"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/datasets"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/devices"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/kernels"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/datasets"))
visualise.pie(db.num_scenarios_by_device,
fs.path(experiment.IMG_ROOT, "num_sceanrios_by_device"))
visualise.pie(db.num_runtime_stats_by_device,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_device"))
visualise.pie(db.num_scenarios_by_dataset,
fs.path(experiment.IMG_ROOT, "num_sceanrios_by_dataset"))
visualise.pie(db.num_runtime_stats_by_dataset,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_dataset"))
visualise.pie(db.num_runtime_stats_by_kernel,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel"))
visualise.pie(db.num_runtime_stats_by_kernel,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel"))
# Per-scenario plots
for row in db.scenario_properties:
scenario, device, kernel, north, south, east, west, max_wgsize, width, height, tout = row
title = ("{device}: {kernel}[{n},{s},{e},{w}]\n"
"{width} x {height} {type}s".format(device=text.truncate(
device, 18),
kernel=kernel,
n=north,
s=south,
e=east,
w=west,
width=width,
height=height,
type=tout))
output = fs.path(experiment.IMG_ROOT,
"scenarios/heatmap/{id}.png".format(id=scenario))
space = _space.ParamSpace.from_dict(db.perf_scenario(scenario))
max_c = min(25, len(space.c))
max_r = min(25, len(space.r))
space.reshape(max_c=max_c, max_r=max_r)
# Heatmaps.
mask = _space.ParamSpace(space.c, space.r)
for j in range(len(mask.r)):
for i in range(len(mask.c)):
if space.matrix[j][i] == 0:
r, c = space.r[j], space.c[i]
# TODO: Get values from refused_params table.
if r * c >= max_wgsize:
# Illegal
mask.matrix[j][i] = -1
else:
# Refused
db.execute(
"INSERT OR IGNORE INTO refused_params VALUES(?,?)",
(scenario, hash_params(c, r)))
space.matrix[j][i] = -1
mask.matrix[j][i] = 1
db.commit()
new_order = list(reversed(range(space.matrix.shape[0])))
data = space.matrix[:][new_order]
figsize = (12, 6)
_, ax = plt.subplots(1, 2, figsize=figsize, sharey=True)
sns.heatmap(data,
ax=ax[0],
vmin=-1,
vmax=1,
xticklabels=space.c,
yticklabels=list(reversed(space.r)),
square=True)
ax[0].set_title(title)
new_order = list(reversed(range(mask.matrix.shape[0])))
data = mask.matrix[:][new_order]
sns.heatmap(data,
ax=ax[1],
vmin=-1,
vmax=1,
xticklabels=space.c,
yticklabels=list(reversed(space.r)),
square=True)
# Set labels.
ax[0].set_ylabel("Rows")
ax[0].set_xlabel("Columns")
ax[1].set_ylabel("Rows")
ax[1].set_xlabel("Columns")
# plt.tight_layout()
# plt.gcf().set_size_inches(*figsize, dpi=300)
viz.finalise(output)
# 3D bars.
output = fs.path(experiment.IMG_ROOT,
"scenarios/bars/{id}.png".format(id=scenario))
space.bar3d(output=output,
title=title,
zlabel="Performance",
rotation=45)
# Trisurfs.
output = fs.path(experiment.IMG_ROOT,
"scenarios/trisurf/{id}.png".format(id=scenario))
space.trisurf(output=output,
title=title,
zlabel="Performance",
rotation=45)
#####################
# ML Visualisations #
#####################
#features_tab(db, experiment.TAB_ROOT)
visualise_classification_job(db, "xval")
visualise_classification_job(db, "arch")
visualise_classification_job(db, "xval_real")
visualise_classification_job(db, "synthetic_real")
# Runtime regression accuracy.
visualise_regression_job(db, "xval")
visualise_regression_job(db, "arch")
visualise_regression_job(db, "xval_real")
visualise_regression_job(db, "synthetic_real")
# Whole-dataset plots
visualise.runtimes_variance(db,
fs.path(experiment.IMG_ROOT,
"runtime_variance.png"),
min_samples=30)
visualise.num_samples(db, fs.path(experiment.IMG_ROOT, "num_samples.png"))
visualise.runtimes_range(
db, fs.path(experiment.IMG_ROOT, "runtimes_range.png"))
visualise.max_speedups(db, fs.path(experiment.IMG_ROOT,
"max_speedups.png"))
visualise.kernel_performance(
db, fs.path(experiment.IMG_ROOT, "kernel_performance.png"))
visualise.device_performance(
db, fs.path(experiment.IMG_ROOT, "device_performance.png"))
visualise.dataset_performance(
db, fs.path(experiment.IMG_ROOT, "dataset_performance.png"))
visualise.num_params_vs_accuracy(
db, fs.path(experiment.IMG_ROOT, "num_params_vs_accuracy.png"))
visualise.performance_vs_coverage(
db, fs.path(experiment.IMG_ROOT, "performance_vs_coverage.png"))
visualise.performance_vs_max_wgsize(
db, fs.path(experiment.IMG_ROOT, "performance_vs_max_wgsize.png"))
visualise.performance_vs_wgsize(
db, fs.path(experiment.IMG_ROOT, "performance_vs_wgsize.png"))
visualise.performance_vs_wg_c(
db, fs.path(experiment.IMG_ROOT, "performance_vs_wg_c.png"))
visualise.performance_vs_wg_r(
db, fs.path(experiment.IMG_ROOT, "performance_vs_wg_r.png"))
visualise.max_wgsizes(db, fs.path(experiment.IMG_ROOT, "max_wgsizes.png"))
visualise.oracle_speedups(
db, fs.path(experiment.IMG_ROOT, "oracle_speedups.png"))
visualise.coverage(db, fs.path(experiment.IMG_ROOT,
"coverage/coverage.png"))
visualise.safety(db, fs.path(experiment.IMG_ROOT, "safety/safety.png"))
visualise.oracle_wgsizes(db, fs.path(experiment.IMG_ROOT,
"oracle/all.png"))
# Per-device plots
for i, device in enumerate(db.devices):
where = (
"scenario IN "
"(SELECT id from scenarios WHERE device='{0}')".format(device))
output = fs.path(experiment.IMG_ROOT,
"coverage/devices/{0}.png".format(i))
visualise.coverage(db, output=output, where=where, title=device)
output = fs.path(experiment.IMG_ROOT,
"safety/devices/{0}.png".format(i))
visualise.safety(db, output, where=where, title=device)
output = fs.path(experiment.IMG_ROOT,
"oracle/devices/{0}.png".format(i))
visualise.oracle_wgsizes(db, output, where=where, title=device)
where = ("scenario IN (\n"
" SELECT id from scenarios WHERE device='{0}'\n"
") AND scenario IN (\n"
" SELECT id FROM scenarios WHERE kernel IN (\n"
" SELECT id FROM kernel_names WHERE synthetic=0\n"
" )\n"
")".format(device))
output = fs.path(experiment.IMG_ROOT,
"coverage/devices/{0}_real.png".format(i))
visualise.coverage(db,
output=output,
where=where,
title=device + ", real")
output = fs.path(experiment.IMG_ROOT,
"safety/devices/{0}_real.png".format(i))
visualise.safety(db, output, where=where, title=device + ", real")
output = fs.path(experiment.IMG_ROOT,
"oracle/devices/{0}_real.png".format(i))
visualise.oracle_wgsizes(db,
output,
where=where,
title=device + ", real")
where = ("scenario IN (\n"
" SELECT id from scenarios WHERE device='{0}'\n"
") AND scenario IN (\n"
" SELECT id FROM scenarios WHERE kernel IN (\n"
" SELECT id FROM kernel_names WHERE synthetic=1\n"
" )\n"
")".format(device))
output = fs.path(experiment.IMG_ROOT,
"coverage/devices/{0}_synthetic.png".format(i))
visualise.coverage(db,
output=output,
where=where,
title=device + ", synthetic")
output = fs.path(experiment.IMG_ROOT,
"safety/devices/{0}_synthetic.png".format(i))
visualise.safety(db, output, where=where, title=device + ", synthetic")
output = fs.path(experiment.IMG_ROOT,
"oracle/devices/{0}_synthetic.png".format(i))
visualise.oracle_wgsizes(db,
output,
where=where,
title=device + ", synthetic")
# Per-kernel plots
for kernel, ids in db.lookup_named_kernels().iteritems():
id_wrapped = ['"' + id + '"' for id in ids]
where = ("scenario IN "
"(SELECT id from scenarios WHERE kernel IN ({0}))".format(
",".join(id_wrapped)))
output = fs.path(experiment.IMG_ROOT,
"coverage/kernels/{0}.png".format(kernel))
visualise.coverage(db, output=output, where=where, title=kernel)
output = fs.path(experiment.IMG_ROOT,
"safety/kernels/{0}.png".format(kernel))
visualise.safety(db, output=output, where=where, title=kernel)
output = fs.path(experiment.IMG_ROOT,
"oracle/kernels/{0}.png".format(kernel))
visualise.safety(db, output=output, where=where, title=kernel)
# Per-dataset plots
for i, dataset in enumerate(db.datasets):
where = (
"scenario IN "
"(SELECT id from scenarios WHERE dataset='{0}')".format(dataset))
output = fs.path(experiment.IMG_ROOT,
"coverage/datasets/{0}.png".format(i))
visualise.coverage(db, output, where=where, title=dataset)
output = fs.path(experiment.IMG_ROOT,
"safety/datasets/{0}.png".format(i))
visualise.safety(db, output, where=where, title=dataset)
output = fs.path(experiment.IMG_ROOT,
"oracle/datasets/{0}.png".format(i))
visualise.safety(db, output, where=where, title=dataset)
ml.stop()