def main(): """ Evaluate dataset and omnitune performance. """ ml.start() db = migrate(_db.Database(experiment.ORACLE_PATH)) # linear_models(db) classification(db) runtime_regression(db) speedup_regression(db) ml.stop()
def main(): db = _db.Database(experiment.ORACLE_PATH) ml.start() # Delete any old stuff. fs.rm(experiment.IMG_ROOT + "/*") fs.rm(experiment.TAB_ROOT + "/*") # Make directories fs.mkdir(experiment.TAB_ROOT) fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/bars")) fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/heatmap")) fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/trisurf")) fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/devices")) fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/kernels")) fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/datasets")) fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/devices")) fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/kernels")) fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/datasets")) fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/devices")) fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/kernels")) fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/datasets")) visualise.pie(db.num_scenarios_by_device, fs.path(experiment.IMG_ROOT, "num_sceanrios_by_device")) visualise.pie(db.num_runtime_stats_by_device, fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_device")) visualise.pie(db.num_scenarios_by_dataset, fs.path(experiment.IMG_ROOT, "num_sceanrios_by_dataset")) visualise.pie(db.num_runtime_stats_by_dataset, fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_dataset")) visualise.pie(db.num_runtime_stats_by_kernel, fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel")) visualise.pie(db.num_runtime_stats_by_kernel, fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel")) # Per-scenario plots for row in db.scenario_properties: scenario,device,kernel,north,south,east,west,max_wgsize,width,height,tout = row title = ("{device}: {kernel}[{n},{s},{e},{w}]\n" "{width} x {height} {type}s" .format(device=text.truncate(device, 18), kernel=kernel, n=north, s=south, e=east, w=west, width=width, height=height, type=tout)) output = fs.path(experiment.IMG_ROOT, "scenarios/heatmap/{id}.png".format(id=scenario)) space = _space.ParamSpace.from_dict(db.perf_scenario(scenario)) max_c = min(25, len(space.c)) max_r = min(25, len(space.r)) space.reshape(max_c=max_c, max_r=max_r) # Heatmaps. mask = _space.ParamSpace(space.c, space.r) for j in range(len(mask.r)): for i in range(len(mask.c)): if space.matrix[j][i] == 0: r, c = space.r[j], space.c[i] # TODO: Get values from refused_params table. if r * c >= max_wgsize: # Illegal mask.matrix[j][i] = -1 else: # Refused db.execute("INSERT OR IGNORE INTO refused_params VALUES(?,?)", (scenario, hash_params(c, r))) space.matrix[j][i] = -1 mask.matrix[j][i] = 1 db.commit() new_order = list(reversed(range(space.matrix.shape[0]))) data = space.matrix[:][new_order] figsize=(12,6) _, ax = plt.subplots(1, 2, figsize=figsize, sharey=True) sns.heatmap(data, ax=ax[0], vmin=-1, vmax=1, xticklabels=space.c, yticklabels=list(reversed(space.r)), square=True) ax[0].set_title(title) new_order = list(reversed(range(mask.matrix.shape[0]))) data = mask.matrix[:][new_order] sns.heatmap(data, ax=ax[1], vmin=-1, vmax=1, xticklabels=space.c, yticklabels=list(reversed(space.r)), square=True) # Set labels. ax[0].set_ylabel("Rows") ax[0].set_xlabel("Columns") ax[1].set_ylabel("Rows") ax[1].set_xlabel("Columns") # plt.tight_layout() # plt.gcf().set_size_inches(*figsize, dpi=300) viz.finalise(output) # 3D bars. output = fs.path(experiment.IMG_ROOT, "scenarios/bars/{id}.png".format(id=scenario)) space.bar3d(output=output, title=title, zlabel="Performance", rotation=45) # Trisurfs. output = fs.path(experiment.IMG_ROOT, "scenarios/trisurf/{id}.png".format(id=scenario)) space.trisurf(output=output, title=title, zlabel="Performance", rotation=45) ##################### # ML Visualisations # ##################### #features_tab(db, experiment.TAB_ROOT) visualise_classification_job(db, "xval") visualise_classification_job(db, "arch") visualise_classification_job(db, "xval_real") visualise_classification_job(db, "synthetic_real") # Runtime regression accuracy. visualise_regression_job(db, "xval") visualise_regression_job(db, "arch") visualise_regression_job(db, "xval_real") visualise_regression_job(db, "synthetic_real") # Whole-dataset plots visualise.runtimes_variance(db, fs.path(experiment.IMG_ROOT, "runtime_variance.png"), min_samples=30) visualise.num_samples(db, fs.path(experiment.IMG_ROOT, "num_samples.png")) visualise.runtimes_range(db, fs.path(experiment.IMG_ROOT, "runtimes_range.png")) visualise.max_speedups(db, fs.path(experiment.IMG_ROOT, "max_speedups.png")) visualise.kernel_performance(db, fs.path(experiment.IMG_ROOT, "kernel_performance.png")) visualise.device_performance(db, fs.path(experiment.IMG_ROOT, "device_performance.png")) visualise.dataset_performance(db, fs.path(experiment.IMG_ROOT, "dataset_performance.png")) visualise.num_params_vs_accuracy(db, fs.path(experiment.IMG_ROOT, "num_params_vs_accuracy.png")) visualise.performance_vs_coverage(db, fs.path(experiment.IMG_ROOT, "performance_vs_coverage.png")) visualise.performance_vs_max_wgsize( db, fs.path(experiment.IMG_ROOT, "performance_vs_max_wgsize.png") ) visualise.performance_vs_wgsize(db, fs.path(experiment.IMG_ROOT, "performance_vs_wgsize.png")) visualise.performance_vs_wg_c(db, fs.path(experiment.IMG_ROOT, "performance_vs_wg_c.png")) visualise.performance_vs_wg_r(db, fs.path(experiment.IMG_ROOT, "performance_vs_wg_r.png")) visualise.max_wgsizes(db, fs.path(experiment.IMG_ROOT, "max_wgsizes.png")) visualise.oracle_speedups(db, fs.path(experiment.IMG_ROOT, "oracle_speedups.png")) visualise.coverage(db, fs.path(experiment.IMG_ROOT, "coverage/coverage.png")) visualise.safety(db, fs.path(experiment.IMG_ROOT, "safety/safety.png")) visualise.oracle_wgsizes(db, fs.path(experiment.IMG_ROOT, "oracle/all.png")) # Per-device plots for i,device in enumerate(db.devices): where = ("scenario IN " "(SELECT id from scenarios WHERE device='{0}')" .format(device)) output = fs.path(experiment.IMG_ROOT, "coverage/devices/{0}.png".format(i)) visualise.coverage(db, output=output, where=where, title=device) output = fs.path(experiment.IMG_ROOT, "safety/devices/{0}.png".format(i)) visualise.safety(db, output, where=where, title=device) output = fs.path(experiment.IMG_ROOT, "oracle/devices/{0}.png".format(i)) visualise.oracle_wgsizes(db, output, where=where, title=device) where = ("scenario IN (\n" " SELECT id from scenarios WHERE device='{0}'\n" ") AND scenario IN (\n" " SELECT id FROM scenarios WHERE kernel IN (\n" " SELECT id FROM kernel_names WHERE synthetic=0\n" " )\n" ")" .format(device)) output = fs.path(experiment.IMG_ROOT, "coverage/devices/{0}_real.png".format(i)) visualise.coverage(db, output=output, where=where, title=device + ", real") output = fs.path(experiment.IMG_ROOT, "safety/devices/{0}_real.png".format(i)) visualise.safety(db, output, where=where, title=device + ", real") output = fs.path(experiment.IMG_ROOT, "oracle/devices/{0}_real.png".format(i)) visualise.oracle_wgsizes(db, output, where=where, title=device + ", real") where = ("scenario IN (\n" " SELECT id from scenarios WHERE device='{0}'\n" ") AND scenario IN (\n" " SELECT id FROM scenarios WHERE kernel IN (\n" " SELECT id FROM kernel_names WHERE synthetic=1\n" " )\n" ")" .format(device)) output = fs.path(experiment.IMG_ROOT, "coverage/devices/{0}_synthetic.png".format(i)) visualise.coverage(db, output=output, where=where, title=device + ", synthetic") output = fs.path(experiment.IMG_ROOT, "safety/devices/{0}_synthetic.png".format(i)) visualise.safety(db, output, where=where, title=device + ", synthetic") output = fs.path(experiment.IMG_ROOT, "oracle/devices/{0}_synthetic.png".format(i)) visualise.oracle_wgsizes(db, output, where=where, title=device + ", synthetic") # Per-kernel plots for kernel,ids in db.lookup_named_kernels().iteritems(): id_wrapped = ['"' + id + '"' for id in ids] where = ("scenario IN " "(SELECT id from scenarios WHERE kernel IN ({0}))" .format(",".join(id_wrapped))) output = fs.path(experiment.IMG_ROOT, "coverage/kernels/{0}.png".format(kernel)) visualise.coverage(db, output=output, where=where, title=kernel) output = fs.path(experiment.IMG_ROOT, "safety/kernels/{0}.png".format(kernel)) visualise.safety(db, output=output, where=where, title=kernel) output = fs.path(experiment.IMG_ROOT, "oracle/kernels/{0}.png".format(kernel)) visualise.safety(db, output=output, where=where, title=kernel) # Per-dataset plots for i,dataset in enumerate(db.datasets): where = ("scenario IN " "(SELECT id from scenarios WHERE dataset='{0}')" .format(dataset)) output = fs.path(experiment.IMG_ROOT, "coverage/datasets/{0}.png".format(i)) visualise.coverage(db, output, where=where, title=dataset) output = fs.path(experiment.IMG_ROOT, "safety/datasets/{0}.png".format(i)) visualise.safety(db, output, where=where, title=dataset) output = fs.path(experiment.IMG_ROOT, "oracle/datasets/{0}.png".format(i)) visualise.safety(db, output, where=where, title=dataset) ml.stop()
def setUpClass(cls): # Calling start() multiple times should not be a problem. ml.start() ml.start()
def main(): db = _db.Database(experiment.ORACLE_PATH) ml.start() # Delete any old stuff. fs.rm(experiment.IMG_ROOT + "/*") fs.rm(experiment.TAB_ROOT + "/*") # Make directories fs.mkdir(experiment.TAB_ROOT) fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/bars")) fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/heatmap")) fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/trisurf")) fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/devices")) fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/kernels")) fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/datasets")) fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/devices")) fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/kernels")) fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/datasets")) fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/devices")) fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/kernels")) fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/datasets")) visualise.pie(db.num_scenarios_by_device, fs.path(experiment.IMG_ROOT, "num_sceanrios_by_device")) visualise.pie(db.num_runtime_stats_by_device, fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_device")) visualise.pie(db.num_scenarios_by_dataset, fs.path(experiment.IMG_ROOT, "num_sceanrios_by_dataset")) visualise.pie(db.num_runtime_stats_by_dataset, fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_dataset")) visualise.pie(db.num_runtime_stats_by_kernel, fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel")) visualise.pie(db.num_runtime_stats_by_kernel, fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel")) # Per-scenario plots for row in db.scenario_properties: scenario, device, kernel, north, south, east, west, max_wgsize, width, height, tout = row title = ("{device}: {kernel}[{n},{s},{e},{w}]\n" "{width} x {height} {type}s".format(device=text.truncate( device, 18), kernel=kernel, n=north, s=south, e=east, w=west, width=width, height=height, type=tout)) output = fs.path(experiment.IMG_ROOT, "scenarios/heatmap/{id}.png".format(id=scenario)) space = _space.ParamSpace.from_dict(db.perf_scenario(scenario)) max_c = min(25, len(space.c)) max_r = min(25, len(space.r)) space.reshape(max_c=max_c, max_r=max_r) # Heatmaps. mask = _space.ParamSpace(space.c, space.r) for j in range(len(mask.r)): for i in range(len(mask.c)): if space.matrix[j][i] == 0: r, c = space.r[j], space.c[i] # TODO: Get values from refused_params table. if r * c >= max_wgsize: # Illegal mask.matrix[j][i] = -1 else: # Refused db.execute( "INSERT OR IGNORE INTO refused_params VALUES(?,?)", (scenario, hash_params(c, r))) space.matrix[j][i] = -1 mask.matrix[j][i] = 1 db.commit() new_order = list(reversed(range(space.matrix.shape[0]))) data = space.matrix[:][new_order] figsize = (12, 6) _, ax = plt.subplots(1, 2, figsize=figsize, sharey=True) sns.heatmap(data, ax=ax[0], vmin=-1, vmax=1, xticklabels=space.c, yticklabels=list(reversed(space.r)), square=True) ax[0].set_title(title) new_order = list(reversed(range(mask.matrix.shape[0]))) data = mask.matrix[:][new_order] sns.heatmap(data, ax=ax[1], vmin=-1, vmax=1, xticklabels=space.c, yticklabels=list(reversed(space.r)), square=True) # Set labels. ax[0].set_ylabel("Rows") ax[0].set_xlabel("Columns") ax[1].set_ylabel("Rows") ax[1].set_xlabel("Columns") # plt.tight_layout() # plt.gcf().set_size_inches(*figsize, dpi=300) viz.finalise(output) # 3D bars. output = fs.path(experiment.IMG_ROOT, "scenarios/bars/{id}.png".format(id=scenario)) space.bar3d(output=output, title=title, zlabel="Performance", rotation=45) # Trisurfs. output = fs.path(experiment.IMG_ROOT, "scenarios/trisurf/{id}.png".format(id=scenario)) space.trisurf(output=output, title=title, zlabel="Performance", rotation=45) ##################### # ML Visualisations # ##################### #features_tab(db, experiment.TAB_ROOT) visualise_classification_job(db, "xval") visualise_classification_job(db, "arch") visualise_classification_job(db, "xval_real") visualise_classification_job(db, "synthetic_real") # Runtime regression accuracy. visualise_regression_job(db, "xval") visualise_regression_job(db, "arch") visualise_regression_job(db, "xval_real") visualise_regression_job(db, "synthetic_real") # Whole-dataset plots visualise.runtimes_variance(db, fs.path(experiment.IMG_ROOT, "runtime_variance.png"), min_samples=30) visualise.num_samples(db, fs.path(experiment.IMG_ROOT, "num_samples.png")) visualise.runtimes_range( db, fs.path(experiment.IMG_ROOT, "runtimes_range.png")) visualise.max_speedups(db, fs.path(experiment.IMG_ROOT, "max_speedups.png")) visualise.kernel_performance( db, fs.path(experiment.IMG_ROOT, "kernel_performance.png")) visualise.device_performance( db, fs.path(experiment.IMG_ROOT, "device_performance.png")) visualise.dataset_performance( db, fs.path(experiment.IMG_ROOT, "dataset_performance.png")) visualise.num_params_vs_accuracy( db, fs.path(experiment.IMG_ROOT, "num_params_vs_accuracy.png")) visualise.performance_vs_coverage( db, fs.path(experiment.IMG_ROOT, "performance_vs_coverage.png")) visualise.performance_vs_max_wgsize( db, fs.path(experiment.IMG_ROOT, "performance_vs_max_wgsize.png")) visualise.performance_vs_wgsize( db, fs.path(experiment.IMG_ROOT, "performance_vs_wgsize.png")) visualise.performance_vs_wg_c( db, fs.path(experiment.IMG_ROOT, "performance_vs_wg_c.png")) visualise.performance_vs_wg_r( db, fs.path(experiment.IMG_ROOT, "performance_vs_wg_r.png")) visualise.max_wgsizes(db, fs.path(experiment.IMG_ROOT, "max_wgsizes.png")) visualise.oracle_speedups( db, fs.path(experiment.IMG_ROOT, "oracle_speedups.png")) visualise.coverage(db, fs.path(experiment.IMG_ROOT, "coverage/coverage.png")) visualise.safety(db, fs.path(experiment.IMG_ROOT, "safety/safety.png")) visualise.oracle_wgsizes(db, fs.path(experiment.IMG_ROOT, "oracle/all.png")) # Per-device plots for i, device in enumerate(db.devices): where = ( "scenario IN " "(SELECT id from scenarios WHERE device='{0}')".format(device)) output = fs.path(experiment.IMG_ROOT, "coverage/devices/{0}.png".format(i)) visualise.coverage(db, output=output, where=where, title=device) output = fs.path(experiment.IMG_ROOT, "safety/devices/{0}.png".format(i)) visualise.safety(db, output, where=where, title=device) output = fs.path(experiment.IMG_ROOT, "oracle/devices/{0}.png".format(i)) visualise.oracle_wgsizes(db, output, where=where, title=device) where = ("scenario IN (\n" " SELECT id from scenarios WHERE device='{0}'\n" ") AND scenario IN (\n" " SELECT id FROM scenarios WHERE kernel IN (\n" " SELECT id FROM kernel_names WHERE synthetic=0\n" " )\n" ")".format(device)) output = fs.path(experiment.IMG_ROOT, "coverage/devices/{0}_real.png".format(i)) visualise.coverage(db, output=output, where=where, title=device + ", real") output = fs.path(experiment.IMG_ROOT, "safety/devices/{0}_real.png".format(i)) visualise.safety(db, output, where=where, title=device + ", real") output = fs.path(experiment.IMG_ROOT, "oracle/devices/{0}_real.png".format(i)) visualise.oracle_wgsizes(db, output, where=where, title=device + ", real") where = ("scenario IN (\n" " SELECT id from scenarios WHERE device='{0}'\n" ") AND scenario IN (\n" " SELECT id FROM scenarios WHERE kernel IN (\n" " SELECT id FROM kernel_names WHERE synthetic=1\n" " )\n" ")".format(device)) output = fs.path(experiment.IMG_ROOT, "coverage/devices/{0}_synthetic.png".format(i)) visualise.coverage(db, output=output, where=where, title=device + ", synthetic") output = fs.path(experiment.IMG_ROOT, "safety/devices/{0}_synthetic.png".format(i)) visualise.safety(db, output, where=where, title=device + ", synthetic") output = fs.path(experiment.IMG_ROOT, "oracle/devices/{0}_synthetic.png".format(i)) visualise.oracle_wgsizes(db, output, where=where, title=device + ", synthetic") # Per-kernel plots for kernel, ids in db.lookup_named_kernels().iteritems(): id_wrapped = ['"' + id + '"' for id in ids] where = ("scenario IN " "(SELECT id from scenarios WHERE kernel IN ({0}))".format( ",".join(id_wrapped))) output = fs.path(experiment.IMG_ROOT, "coverage/kernels/{0}.png".format(kernel)) visualise.coverage(db, output=output, where=where, title=kernel) output = fs.path(experiment.IMG_ROOT, "safety/kernels/{0}.png".format(kernel)) visualise.safety(db, output=output, where=where, title=kernel) output = fs.path(experiment.IMG_ROOT, "oracle/kernels/{0}.png".format(kernel)) visualise.safety(db, output=output, where=where, title=kernel) # Per-dataset plots for i, dataset in enumerate(db.datasets): where = ( "scenario IN " "(SELECT id from scenarios WHERE dataset='{0}')".format(dataset)) output = fs.path(experiment.IMG_ROOT, "coverage/datasets/{0}.png".format(i)) visualise.coverage(db, output, where=where, title=dataset) output = fs.path(experiment.IMG_ROOT, "safety/datasets/{0}.png".format(i)) visualise.safety(db, output, where=where, title=dataset) output = fs.path(experiment.IMG_ROOT, "oracle/datasets/{0}.png".format(i)) visualise.safety(db, output, where=where, title=dataset) ml.stop()