def eval_linear_models(db, models):
rows = db.execute(
"SELECT "
" scenario_stats.scenario, "
" kernels.max_wg_size, "
" scenario_stats.oracle_param "
"FROM scenarios "
"LEFT JOIN scenario_stats "
" ON scenarios.id=scenario_stats.scenario "
"LEFT JOIN kernels "
" ON scenarios.kernel=kernels.id"
).fetchall()
baseline = db.one_r()[0]
prof.start("Linear models")
for scenario,max_wgsize,oracle in rows:
for model in models:
wg_c, wg_r = model.predict(scenario, max_wgsize, oracle)
try:
prediction = hash_params(wg_c, wg_r)
illegal = 0 if wg_c * wg_r < max_wgsize else 1
correct = 1 if prediction == oracle else 0
db.runtime(scenario, prediction)
refused = 0
reshape_param = prediction
reshape_perf, reshape_speedup = perf_fn(db, scenario,
prediction, oracle,
baseline)
baseline_perf, baseline_speedup = reshape_perf, reshape_speedup
random_param = prediction
random_perf, random_speedup = reshape_perf, reshape_speedup
except lab.db.Error:
refused = not illegal
reshape_param = reshape(db, scenario, max_wgsize, wg_c, wg_r)
reshape_perf, reshape_speedup = perf_fn(db, scenario,
reshape_param,
oracle, baseline)
baseline_perf, baseline_speedup = perf_fn(db, scenario,
baseline, oracle,
baseline)
random_param = random.choice(db.W_legal(scenario))
random_perf, random_speedup = perf_fn(db, scenario,
random_param, oracle,
baseline)
db.add_model_result(model.id(), "reshape_fn", scenario, oracle,
reshape_param, correct, illegal, refused,
reshape_perf, reshape_speedup)
db.add_model_result(model.id(), "default_fn", scenario, oracle,
baseline, correct, illegal, refused,
baseline_perf, baseline_speedup)
db.add_model_result(model.id(), "random_fn", scenario, oracle,
random_param, correct, illegal, refused,
random_perf, random_speedup)
db.commit()
prof.stop("Linear models")
def main():
db = _db.Database(experiment.ORACLE_PATH)
ml.start()
# Delete any old stuff.
fs.rm(experiment.IMG_ROOT + "/*")
fs.rm(experiment.TAB_ROOT + "/*")
# Make directories
fs.mkdir(experiment.TAB_ROOT)
fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/bars"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/heatmap"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/trisurf"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/devices"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/kernels"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/datasets"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/devices"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/kernels"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/datasets"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/devices"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/kernels"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/datasets"))
visualise.pie(db.num_scenarios_by_device,
fs.path(experiment.IMG_ROOT, "num_sceanrios_by_device"))
visualise.pie(db.num_runtime_stats_by_device,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_device"))
visualise.pie(db.num_scenarios_by_dataset,
fs.path(experiment.IMG_ROOT, "num_sceanrios_by_dataset"))
visualise.pie(db.num_runtime_stats_by_dataset,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_dataset"))
visualise.pie(db.num_runtime_stats_by_kernel,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel"))
visualise.pie(db.num_runtime_stats_by_kernel,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel"))
# Per-scenario plots
for row in db.scenario_properties:
scenario,device,kernel,north,south,east,west,max_wgsize,width,height,tout = row
title = ("{device}: {kernel}[{n},{s},{e},{w}]\n"
"{width} x {height} {type}s"
.format(device=text.truncate(device, 18), kernel=kernel,
n=north, s=south, e=east, w=west,
width=width, height=height, type=tout))
output = fs.path(experiment.IMG_ROOT,
"scenarios/heatmap/{id}.png".format(id=scenario))
space = _space.ParamSpace.from_dict(db.perf_scenario(scenario))
max_c = min(25, len(space.c))
max_r = min(25, len(space.r))
space.reshape(max_c=max_c, max_r=max_r)
# Heatmaps.
mask = _space.ParamSpace(space.c, space.r)
for j in range(len(mask.r)):
for i in range(len(mask.c)):
if space.matrix[j][i] == 0:
r, c = space.r[j], space.c[i]
# TODO: Get values from refused_params table.
if r * c >= max_wgsize:
# Illegal
mask.matrix[j][i] = -1
else:
# Refused
db.execute("INSERT OR IGNORE INTO refused_params VALUES(?,?)",
(scenario, hash_params(c, r)))
space.matrix[j][i] = -1
mask.matrix[j][i] = 1
db.commit()
new_order = list(reversed(range(space.matrix.shape[0])))
data = space.matrix[:][new_order]
figsize=(12,6)
_, ax = plt.subplots(1, 2, figsize=figsize, sharey=True)
sns.heatmap(data, ax=ax[0], vmin=-1, vmax=1,
xticklabels=space.c,
yticklabels=list(reversed(space.r)), square=True)
ax[0].set_title(title)
new_order = list(reversed(range(mask.matrix.shape[0])))
data = mask.matrix[:][new_order]
sns.heatmap(data, ax=ax[1], vmin=-1, vmax=1,
xticklabels=space.c,
yticklabels=list(reversed(space.r)), square=True)
# Set labels.
ax[0].set_ylabel("Rows")
ax[0].set_xlabel("Columns")
ax[1].set_ylabel("Rows")
ax[1].set_xlabel("Columns")
# plt.tight_layout()
# plt.gcf().set_size_inches(*figsize, dpi=300)
viz.finalise(output)
# 3D bars.
output = fs.path(experiment.IMG_ROOT,
"scenarios/bars/{id}.png".format(id=scenario))
space.bar3d(output=output, title=title, zlabel="Performance",
rotation=45)
# Trisurfs.
output = fs.path(experiment.IMG_ROOT,
"scenarios/trisurf/{id}.png".format(id=scenario))
space.trisurf(output=output, title=title, zlabel="Performance",
rotation=45)
#####################
# ML Visualisations #
#####################
#features_tab(db, experiment.TAB_ROOT)
visualise_classification_job(db, "xval")
visualise_classification_job(db, "arch")
visualise_classification_job(db, "xval_real")
visualise_classification_job(db, "synthetic_real")
# Runtime regression accuracy.
visualise_regression_job(db, "xval")
visualise_regression_job(db, "arch")
visualise_regression_job(db, "xval_real")
visualise_regression_job(db, "synthetic_real")
# Whole-dataset plots
visualise.runtimes_variance(db, fs.path(experiment.IMG_ROOT,
"runtime_variance.png"),
min_samples=30)
visualise.num_samples(db, fs.path(experiment.IMG_ROOT,
"num_samples.png"))
visualise.runtimes_range(db, fs.path(experiment.IMG_ROOT,
"runtimes_range.png"))
visualise.max_speedups(db, fs.path(experiment.IMG_ROOT,
"max_speedups.png"))
visualise.kernel_performance(db, fs.path(experiment.IMG_ROOT,
"kernel_performance.png"))
visualise.device_performance(db, fs.path(experiment.IMG_ROOT,
"device_performance.png"))
visualise.dataset_performance(db, fs.path(experiment.IMG_ROOT,
"dataset_performance.png"))
visualise.num_params_vs_accuracy(db, fs.path(experiment.IMG_ROOT,
"num_params_vs_accuracy.png"))
visualise.performance_vs_coverage(db,
fs.path(experiment.IMG_ROOT,
"performance_vs_coverage.png"))
visualise.performance_vs_max_wgsize(
db, fs.path(experiment.IMG_ROOT, "performance_vs_max_wgsize.png")
)
visualise.performance_vs_wgsize(db, fs.path(experiment.IMG_ROOT,
"performance_vs_wgsize.png"))
visualise.performance_vs_wg_c(db, fs.path(experiment.IMG_ROOT,
"performance_vs_wg_c.png"))
visualise.performance_vs_wg_r(db, fs.path(experiment.IMG_ROOT,
"performance_vs_wg_r.png"))
visualise.max_wgsizes(db, fs.path(experiment.IMG_ROOT, "max_wgsizes.png"))
visualise.oracle_speedups(db, fs.path(experiment.IMG_ROOT,
"oracle_speedups.png"))
visualise.coverage(db,
fs.path(experiment.IMG_ROOT, "coverage/coverage.png"))
visualise.safety(db, fs.path(experiment.IMG_ROOT, "safety/safety.png"))
visualise.oracle_wgsizes(db, fs.path(experiment.IMG_ROOT, "oracle/all.png"))
# Per-device plots
for i,device in enumerate(db.devices):
where = ("scenario IN "
"(SELECT id from scenarios WHERE device='{0}')"
.format(device))
output = fs.path(experiment.IMG_ROOT,
"coverage/devices/{0}.png".format(i))
visualise.coverage(db, output=output, where=where, title=device)
output = fs.path(experiment.IMG_ROOT,
"safety/devices/{0}.png".format(i))
visualise.safety(db, output, where=where, title=device)
output = fs.path(experiment.IMG_ROOT,
"oracle/devices/{0}.png".format(i))
visualise.oracle_wgsizes(db, output, where=where, title=device)
where = ("scenario IN (\n"
" SELECT id from scenarios WHERE device='{0}'\n"
") AND scenario IN (\n"
" SELECT id FROM scenarios WHERE kernel IN (\n"
" SELECT id FROM kernel_names WHERE synthetic=0\n"
" )\n"
")"
.format(device))
output = fs.path(experiment.IMG_ROOT,
"coverage/devices/{0}_real.png".format(i))
visualise.coverage(db, output=output, where=where,
title=device + ", real")
output = fs.path(experiment.IMG_ROOT,
"safety/devices/{0}_real.png".format(i))
visualise.safety(db, output, where=where,
title=device + ", real")
output = fs.path(experiment.IMG_ROOT,
"oracle/devices/{0}_real.png".format(i))
visualise.oracle_wgsizes(db, output, where=where,
title=device + ", real")
where = ("scenario IN (\n"
" SELECT id from scenarios WHERE device='{0}'\n"
") AND scenario IN (\n"
" SELECT id FROM scenarios WHERE kernel IN (\n"
" SELECT id FROM kernel_names WHERE synthetic=1\n"
" )\n"
")"
.format(device))
output = fs.path(experiment.IMG_ROOT,
"coverage/devices/{0}_synthetic.png".format(i))
visualise.coverage(db, output=output, where=where,
title=device + ", synthetic")
output = fs.path(experiment.IMG_ROOT,
"safety/devices/{0}_synthetic.png".format(i))
visualise.safety(db, output, where=where,
title=device + ", synthetic")
output = fs.path(experiment.IMG_ROOT,
"oracle/devices/{0}_synthetic.png".format(i))
visualise.oracle_wgsizes(db, output, where=where,
title=device + ", synthetic")
# Per-kernel plots
for kernel,ids in db.lookup_named_kernels().iteritems():
id_wrapped = ['"' + id + '"' for id in ids]
where = ("scenario IN "
"(SELECT id from scenarios WHERE kernel IN ({0}))"
.format(",".join(id_wrapped)))
output = fs.path(experiment.IMG_ROOT,
"coverage/kernels/{0}.png".format(kernel))
visualise.coverage(db, output=output, where=where, title=kernel)
output = fs.path(experiment.IMG_ROOT,
"safety/kernels/{0}.png".format(kernel))
visualise.safety(db, output=output, where=where, title=kernel)
output = fs.path(experiment.IMG_ROOT,
"oracle/kernels/{0}.png".format(kernel))
visualise.safety(db, output=output, where=where, title=kernel)
# Per-dataset plots
for i,dataset in enumerate(db.datasets):
where = ("scenario IN "
"(SELECT id from scenarios WHERE dataset='{0}')"
.format(dataset))
output = fs.path(experiment.IMG_ROOT,
"coverage/datasets/{0}.png".format(i))
visualise.coverage(db, output, where=where, title=dataset)
output = fs.path(experiment.IMG_ROOT,
"safety/datasets/{0}.png".format(i))
visualise.safety(db, output, where=where, title=dataset)
output = fs.path(experiment.IMG_ROOT,
"oracle/datasets/{0}.png".format(i))
visualise.safety(db, output, where=where, title=dataset)
ml.stop()
def test_hash_params(self):
vals = range(4,40,4)
wgs = itertools.product(vals, vals)
checksums = [skelcl.hash_params(*wg) for wg in wgs]
print(checksums)
self._test(len(checksums), len(set(checksums)))
def test_hash_params(self):
vals = range(4, 40, 4)
wgs = itertools.product(vals, vals)
checksums = [skelcl.hash_params(*wg) for wg in wgs]
print(checksums)
self._test(len(checksums), len(set(checksums)))
def main():
db = _db.Database(experiment.ORACLE_PATH)
ml.start()
# Delete any old stuff.
fs.rm(experiment.IMG_ROOT + "/*")
fs.rm(experiment.TAB_ROOT + "/*")
# Make directories
fs.mkdir(experiment.TAB_ROOT)
fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/bars"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/heatmap"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/trisurf"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/devices"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/kernels"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/datasets"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/devices"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/kernels"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/datasets"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/devices"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/kernels"))
fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/datasets"))
visualise.pie(db.num_scenarios_by_device,
fs.path(experiment.IMG_ROOT, "num_sceanrios_by_device"))
visualise.pie(db.num_runtime_stats_by_device,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_device"))
visualise.pie(db.num_scenarios_by_dataset,
fs.path(experiment.IMG_ROOT, "num_sceanrios_by_dataset"))
visualise.pie(db.num_runtime_stats_by_dataset,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_dataset"))
visualise.pie(db.num_runtime_stats_by_kernel,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel"))
visualise.pie(db.num_runtime_stats_by_kernel,
fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel"))
# Per-scenario plots
for row in db.scenario_properties:
scenario, device, kernel, north, south, east, west, max_wgsize, width, height, tout = row
title = ("{device}: {kernel}[{n},{s},{e},{w}]\n"
"{width} x {height} {type}s".format(device=text.truncate(
device, 18),
kernel=kernel,
n=north,
s=south,
e=east,
w=west,
width=width,
height=height,
type=tout))
output = fs.path(experiment.IMG_ROOT,
"scenarios/heatmap/{id}.png".format(id=scenario))
space = _space.ParamSpace.from_dict(db.perf_scenario(scenario))
max_c = min(25, len(space.c))
max_r = min(25, len(space.r))
space.reshape(max_c=max_c, max_r=max_r)
# Heatmaps.
mask = _space.ParamSpace(space.c, space.r)
for j in range(len(mask.r)):
for i in range(len(mask.c)):
if space.matrix[j][i] == 0:
r, c = space.r[j], space.c[i]
# TODO: Get values from refused_params table.
if r * c >= max_wgsize:
# Illegal
mask.matrix[j][i] = -1
else:
# Refused
db.execute(
"INSERT OR IGNORE INTO refused_params VALUES(?,?)",
(scenario, hash_params(c, r)))
space.matrix[j][i] = -1
mask.matrix[j][i] = 1
db.commit()
new_order = list(reversed(range(space.matrix.shape[0])))
data = space.matrix[:][new_order]
figsize = (12, 6)
_, ax = plt.subplots(1, 2, figsize=figsize, sharey=True)
sns.heatmap(data,
ax=ax[0],
vmin=-1,
vmax=1,
xticklabels=space.c,
yticklabels=list(reversed(space.r)),
square=True)
ax[0].set_title(title)
new_order = list(reversed(range(mask.matrix.shape[0])))
data = mask.matrix[:][new_order]
sns.heatmap(data,
ax=ax[1],
vmin=-1,
vmax=1,
xticklabels=space.c,
yticklabels=list(reversed(space.r)),
square=True)
# Set labels.
ax[0].set_ylabel("Rows")
ax[0].set_xlabel("Columns")
ax[1].set_ylabel("Rows")
ax[1].set_xlabel("Columns")
# plt.tight_layout()
# plt.gcf().set_size_inches(*figsize, dpi=300)
viz.finalise(output)
# 3D bars.
output = fs.path(experiment.IMG_ROOT,
"scenarios/bars/{id}.png".format(id=scenario))
space.bar3d(output=output,
title=title,
zlabel="Performance",
rotation=45)
# Trisurfs.
output = fs.path(experiment.IMG_ROOT,
"scenarios/trisurf/{id}.png".format(id=scenario))
space.trisurf(output=output,
title=title,
zlabel="Performance",
rotation=45)
#####################
# ML Visualisations #
#####################
#features_tab(db, experiment.TAB_ROOT)
visualise_classification_job(db, "xval")
visualise_classification_job(db, "arch")
visualise_classification_job(db, "xval_real")
visualise_classification_job(db, "synthetic_real")
# Runtime regression accuracy.
visualise_regression_job(db, "xval")
visualise_regression_job(db, "arch")
visualise_regression_job(db, "xval_real")
visualise_regression_job(db, "synthetic_real")
# Whole-dataset plots
visualise.runtimes_variance(db,
fs.path(experiment.IMG_ROOT,
"runtime_variance.png"),
min_samples=30)
visualise.num_samples(db, fs.path(experiment.IMG_ROOT, "num_samples.png"))
visualise.runtimes_range(
db, fs.path(experiment.IMG_ROOT, "runtimes_range.png"))
visualise.max_speedups(db, fs.path(experiment.IMG_ROOT,
"max_speedups.png"))
visualise.kernel_performance(
db, fs.path(experiment.IMG_ROOT, "kernel_performance.png"))
visualise.device_performance(
db, fs.path(experiment.IMG_ROOT, "device_performance.png"))
visualise.dataset_performance(
db, fs.path(experiment.IMG_ROOT, "dataset_performance.png"))
visualise.num_params_vs_accuracy(
db, fs.path(experiment.IMG_ROOT, "num_params_vs_accuracy.png"))
visualise.performance_vs_coverage(
db, fs.path(experiment.IMG_ROOT, "performance_vs_coverage.png"))
visualise.performance_vs_max_wgsize(
db, fs.path(experiment.IMG_ROOT, "performance_vs_max_wgsize.png"))
visualise.performance_vs_wgsize(
db, fs.path(experiment.IMG_ROOT, "performance_vs_wgsize.png"))
visualise.performance_vs_wg_c(
db, fs.path(experiment.IMG_ROOT, "performance_vs_wg_c.png"))
visualise.performance_vs_wg_r(
db, fs.path(experiment.IMG_ROOT, "performance_vs_wg_r.png"))
visualise.max_wgsizes(db, fs.path(experiment.IMG_ROOT, "max_wgsizes.png"))
visualise.oracle_speedups(
db, fs.path(experiment.IMG_ROOT, "oracle_speedups.png"))
visualise.coverage(db, fs.path(experiment.IMG_ROOT,
"coverage/coverage.png"))
visualise.safety(db, fs.path(experiment.IMG_ROOT, "safety/safety.png"))
visualise.oracle_wgsizes(db, fs.path(experiment.IMG_ROOT,
"oracle/all.png"))
# Per-device plots
for i, device in enumerate(db.devices):
where = (
"scenario IN "
"(SELECT id from scenarios WHERE device='{0}')".format(device))
output = fs.path(experiment.IMG_ROOT,
"coverage/devices/{0}.png".format(i))
visualise.coverage(db, output=output, where=where, title=device)
output = fs.path(experiment.IMG_ROOT,
"safety/devices/{0}.png".format(i))
visualise.safety(db, output, where=where, title=device)
output = fs.path(experiment.IMG_ROOT,
"oracle/devices/{0}.png".format(i))
visualise.oracle_wgsizes(db, output, where=where, title=device)
where = ("scenario IN (\n"
" SELECT id from scenarios WHERE device='{0}'\n"
") AND scenario IN (\n"
" SELECT id FROM scenarios WHERE kernel IN (\n"
" SELECT id FROM kernel_names WHERE synthetic=0\n"
" )\n"
")".format(device))
output = fs.path(experiment.IMG_ROOT,
"coverage/devices/{0}_real.png".format(i))
visualise.coverage(db,
output=output,
where=where,
title=device + ", real")
output = fs.path(experiment.IMG_ROOT,
"safety/devices/{0}_real.png".format(i))
visualise.safety(db, output, where=where, title=device + ", real")
output = fs.path(experiment.IMG_ROOT,
"oracle/devices/{0}_real.png".format(i))
visualise.oracle_wgsizes(db,
output,
where=where,
title=device + ", real")
where = ("scenario IN (\n"
" SELECT id from scenarios WHERE device='{0}'\n"
") AND scenario IN (\n"
" SELECT id FROM scenarios WHERE kernel IN (\n"
" SELECT id FROM kernel_names WHERE synthetic=1\n"
" )\n"
")".format(device))
output = fs.path(experiment.IMG_ROOT,
"coverage/devices/{0}_synthetic.png".format(i))
visualise.coverage(db,
output=output,
where=where,
title=device + ", synthetic")
output = fs.path(experiment.IMG_ROOT,
"safety/devices/{0}_synthetic.png".format(i))
visualise.safety(db, output, where=where, title=device + ", synthetic")
output = fs.path(experiment.IMG_ROOT,
"oracle/devices/{0}_synthetic.png".format(i))
visualise.oracle_wgsizes(db,
output,
where=where,
title=device + ", synthetic")
# Per-kernel plots
for kernel, ids in db.lookup_named_kernels().iteritems():
id_wrapped = ['"' + id + '"' for id in ids]
where = ("scenario IN "
"(SELECT id from scenarios WHERE kernel IN ({0}))".format(
",".join(id_wrapped)))
output = fs.path(experiment.IMG_ROOT,
"coverage/kernels/{0}.png".format(kernel))
visualise.coverage(db, output=output, where=where, title=kernel)
output = fs.path(experiment.IMG_ROOT,
"safety/kernels/{0}.png".format(kernel))
visualise.safety(db, output=output, where=where, title=kernel)
output = fs.path(experiment.IMG_ROOT,
"oracle/kernels/{0}.png".format(kernel))
visualise.safety(db, output=output, where=where, title=kernel)
# Per-dataset plots
for i, dataset in enumerate(db.datasets):
where = (
"scenario IN "
"(SELECT id from scenarios WHERE dataset='{0}')".format(dataset))
output = fs.path(experiment.IMG_ROOT,
"coverage/datasets/{0}.png".format(i))
visualise.coverage(db, output, where=where, title=dataset)
output = fs.path(experiment.IMG_ROOT,
"safety/datasets/{0}.png".format(i))
visualise.safety(db, output, where=where, title=dataset)
output = fs.path(experiment.IMG_ROOT,
"oracle/datasets/{0}.png".format(i))
visualise.safety(db, output, where=where, title=dataset)
ml.stop()
