def main():
db = _db.Database(fs.path("joblist.db"))
data = [
row for row in db.execute("SELECT device,Count(*) AS count\n"
"FROM jobs\n"
"GROUP BY device\n"
"ORDER BY count")
]
io.info("Job list:")
print(fmt.table(data, columns=("Device", "Jobs")))
print()
jobs = [row for row in db.execute("SELECT * FROM jobs")]
fs.mkdir("jobs")
logs = {
"monza": open("jobs/monza.txt", "w"),
"whz5": open("jobs/whz5.txt", "w"),
"monza": open("jobs/monza.txt", "w"),
"cec": open("jobs/cec.txt", "w"),
"florence": open("jobs/florence.txt", "w"),
}
for job in jobs:
enum_job(logs, db, *job)
lab.exit()
def finalise(output=None, figsize=None, tight=True, **kwargs):
"""
Finalise a plot.
Display or show the plot, then close it.
Arguments:
output (str, optional): Path to save figure to. If not given,
show plot.
figsize ((float, float), optional): Figure size in inches.
**kwargs: Any additional arguments to pass to
plt.savefig(). Only required if output is not None.
"""
import matplotlib.pyplot as plt
# Set figure size.
if figsize is not None:
plt.gcf().set_size_inches(*figsize)
# Set plot layout.
if tight:
plt.tight_layout()
if output is None:
plt.show()
else:
plt.savefig(output, **kwargs)
io.info("Wrote", output)
plt.close()
def main():
db = _db.Database(fs.path("joblist.db"))
data = [row for row in
db.execute("SELECT device,Count(*) AS count\n"
"FROM jobs\n"
"GROUP BY device\n"
"ORDER BY count")]
io.info("Job list:")
print(fmt.table(data, columns=("Device", "Jobs")))
print()
jobs = [row for row in db.execute("SELECT * FROM jobs")]
fs.mkdir("jobs")
logs = {
"monza": open("jobs/monza.txt", "w"),
"whz5": open("jobs/whz5.txt", "w"),
"monza": open("jobs/monza.txt", "w"),
"cec": open("jobs/cec.txt", "w"),
"florence": open("jobs/florence.txt", "w"),
}
for job in jobs:
enum_job(logs, db, *job)
lab.exit()
def finalise(output=None, figsize=None, tight=True, **kwargs):
"""
Finalise a plot.
Display or show the plot, then close it.
Arguments:
output (str, optional): Path to save figure to. If not given,
show plot.
figsize ((float, float), optional): Figure size in inches.
**kwargs: Any additional arguments to pass to
plt.savefig(). Only required if output is not None.
"""
import matplotlib.pyplot as plt
# Set figure size.
if figsize is not None:
plt.gcf().set_size_inches(*figsize)
# Set plot layout.
if tight:
plt.tight_layout()
if output is None:
plt.show()
else:
plt.savefig(output, **kwargs)
io.info("Wrote", output)
plt.close()
def cp_loc(path, name):
"""
Copy database from local filesystem.
"""
path = fs.path(path)
dst = dst_path(name)
io.info("Copying", path, "->", name)
fs.cp(path, dst)
def cp_rmt(host, path="~/.omnitune/skelcl.db", name=None):
"""
Copy database from remote filesystem.
"""
name = name or host
dst = dst_path(name)
io.info("Copying {host}:{path}".format(host=host, path=path), "->", name)
system.scp(host, path, dst)
def migrate_3_to_4(old):
"""
SkelCL database migration script.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
# Create temporary database
fs.rm("/tmp/omnitune.skelcl.migration.db")
tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
tmp.attach(old.path, "rhs")
io.info("Migrating database to version 4.")
backup_path = old.path + ".3"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(old.path, backup_path)
tables = [
"kernels", "kernel_lookup", "kernel_names", "devices", "device_lookup",
"datasets", "dataset_lookup", "scenarios", "params", "runtimes",
"runtime_stats", "oracle_params"
]
for table in tables:
io.info("Copying data from '{}' ...".format(table))
tmp.execute("INSERT INTO {} SELECT * FROM rhs.{}".format(table, table))
tmp_path = tmp.path
old_path = old.path
tmp.execute("VACUUM")
# Sanity checks
bad = False
for table in tables:
old_count = tmp.num_rows("rhs." + table)
tmp_count = tmp.num_rows(table)
if old_count != tmp_count:
io.error("Bad rows count:", old_count, tmp_count)
bad = True
if bad:
io.fatal("Failed sanity check, aborting.")
else:
io.info("Passed sanity check.")
# Copy migrated database over the original one.
fs.cp(tmp_path, old_path)
fs.rm(tmp_path)
old.close()
tmp.close()
io.info("Migration completed.")
def main():
jobs = get_jobs()
io.info("Loaded", len(jobs), "jobs")
# Build example programs.
fs.cd(experiment.EXAMPLES_BUILD)
make.make()
for i,job in enumerate(jobs):
run_job(i, len(jobs), *job.split("\t"))
lab.exit()
def main():
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
bus = dbus.SystemBus()
name = dbus.service.BusName(SESSION_NAME, bus)
io.info("Launched session %s ..." % SESSION_NAME)
# Launch server.
Server(bus, OBJECT_PATH)
mainloop = gobject.MainLoop()
try:
mainloop.run()
except KeyboardInterrupt:
lab.exit()
def merge(old_oracle, dbs, path):
"""
Merge databases into one.
Arguments:
dbs (list of Database): Databases to merge.
path (str): Path to merged database.
Returns:
Database: merged database instance.
"""
print("Merging {n} databases:".format(n=len(dbs) + 1))
print(" ", old_oracle)
for db in dbs:
print(" ", db)
print()
# Make a copy of the old oracle database to work from.
io.info("Coping", old_oracle, "->", fs.basename(path))
fs.cp(old_oracle, path)
target = migrate(_db.Database(path=path))
for db in dbs + [target]:
try:
db.num_rows("runtimes")
except sqlite3.DatabaseError as e:
io.error("Broken db:", db.path)
io.fatal(e)
num_runtimes = [db.num_rows("runtimes") for db in dbs]
expected_total = target.num_rows("runtimes") + sum(num_runtimes)
target.merge(dbs)
total = target.num_rows("runtimes")
if total != expected_total:
io.fatal("Expected total", expected_total,
"!= actual total", total)
io.info(("Merged {num_db} databases, {n} rows"
.format(num_db=len(dbs), n=total)))
return target
def main():
"""
Evaluate dataset and omnitune performance.
"""
# Get the latest dataset from the oracle.
db = migrate(_db.Database(experiment.ORACLE_PATH))
backup_path = db.path + ".unprune"
io.info("Writing backup to", backup_path)
fs.cp(db.path, backup_path)
# Strip scenarios for which there isn't enough unique workgroup
# sizes.
db.prune_min_params_per_scenario(25)
# Strip scenarios so that there are at least a certain number of
# safe parameters.
db.prune_safe_params(3)
def merge(old_oracle, dbs, path):
"""
Merge databases into one.
Arguments:
dbs (list of Database): Databases to merge.
path (str): Path to merged database.
Returns:
Database: merged database instance.
"""
print("Merging {n} databases:".format(n=len(dbs) + 1))
print(" ", old_oracle)
for db in dbs:
print(" ", db)
print()
# Make a copy of the old oracle database to work from.
io.info("Coping", old_oracle, "->", fs.basename(path))
fs.cp(old_oracle, path)
target = migrate(_db.Database(path=path))
for db in dbs + [target]:
try:
db.num_rows("runtimes")
except sqlite3.DatabaseError as e:
io.error("Broken db:", db.path)
io.fatal(e)
num_runtimes = [db.num_rows("runtimes") for db in dbs]
expected_total = target.num_rows("runtimes") + sum(num_runtimes)
target.merge(dbs)
total = target.num_rows("runtimes")
if total != expected_total:
io.fatal("Expected total", expected_total, "!= actual total", total)
io.info(("Merged {num_db} databases, {n} rows".format(num_db=len(dbs),
n=total)))
return target
def __init__(self, *args, **kwargs):
"""
Construct a SkelCL server.
"""
# Fail if we can't find the path
if not fs.isdir(self.LLVM_PATH):
io.fatal("Could not find llvm path '{0}'".format(self.LLVM_PATH))
super(Server, self).__init__(*args, **kwargs)
io.info("Registered server %s/SkelCLServer ..." % SESSION_NAME)
# Setup persistent database.
self.db = migrate(Database())
self.db.status_report()
# Create an in-memory sample strategy cache.
self.strategies = cache.TransientCache()
def run_example_prog(prog, args):
"""
Run a SkelCL example program.
Arguments:
prog (str): The name of the program to run
args (list of str): Any arguments
"""
fs.cd(fs.path(experiment.EXAMPLES_BUILD, prog))
cmd = ["./" + prog] + args
cmd_str = " ".join(cmd)
io.info("COMMAND:", io.colourise(io.Colours.RED, cmd_str))
ret, _, _ = system.run(cmd, stdout=system.STDOUT, stderr=system.STDERR)
if ret:
system.echo(cmd_str, "/tmp/naughty.txt", append=True)
return ret
def run_job(i, n, wgsize, program, args):
wg_c, wg_r = unhash_params(wgsize)
# Set environment variable.
os.environ["OMNITUNE_OFFLINE_TRAINING"] = "1"
os.environ["OMNITUNE_STENCIL_WG_C"] = str(wg_c)
os.environ["OMNITUNE_STENCIL_WG_R"] = str(wg_r)
fs.cd(fs.path(experiment.EXAMPLES_BUILD, program))
cmd_str = "./{} {}".format(program, args.rstrip())
cmd = cmd_str.split()
io.info(i, "of", n, " - ", wgsize, "COMMAND:", io.colourise(io.Colours.RED, cmd_str))
ret, _, _ = system.run(cmd, stdout=system.STDOUT, stderr=system.STDERR)
if ret:
print(ret, wgsize, program, args, sep="\t", file=errlog)
else:
print(ret, wgsize, program, args, sep="\t", file=runlog)
def gather():
benchmarks = {
"canny": {},
"fdtd": {},
"gol": {},
"gaussian": {},
"heat": {},
"simple": {},
"simplecomplex": {}
}
for benchmark in benchmarks:
io.info("Benchmark %s" % benchmark)
fs.cd("/home/chris/src/msc-thesis/scraps/05-12/kernels/%s" % benchmark)
instcounts = []
for file in fs.ls():
instcounts.append(get_instcount(file))
benchmarks[benchmark] = merge_counts(instcounts)
return benchmarks
def migrate_5_to_6(db):
"""
SkelCL database migration script.
Database version 5 adds an additional "param_stats" table.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
io.info("Migrating database to version 6.")
backup_path = db.path + ".5"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(db.path, backup_path)
db.execute("DELETE FROM version")
db.execute("INSERT INTO version VALUES (6)")
db.execute("""
CREATE TABLE IF NOT EXISTS scenario_stats (
scenario CHAR(40), -- Key for scenarios
num_params INTEGER, -- The number of parameters in W_legal for scenario
oracle_param VARCHAR(255), -- The best parameter
oracle_runtime REAL, -- The runtime of the best parameter
worst_param VARCHAR(255), -- The worst parameter
worst_runtime REAL, -- The runtime of the worst parameter
mean_runtime REAL, -- The mean runtime of all parameters
PRIMARY KEY (scenario)
)
""")
db.populate_scenario_stats_table()
# Sanity checks
bad = False
if db.num_rows("scenario_stats") != len(db.scenarios):
io.error("Bad row count in scenario_stats table! Expected",
len(db.scenarios), "Observed:", db.num_rows("scenario_stats"))
bad = True
if bad:
io.fatal("Failed sanity check, aborting.")
else:
io.info("Passed sanity check.")
# Copy migrated database over the original one.
db.close()
io.info("Migration completed.")
def migrate_5_to_6(db):
"""
SkelCL database migration script.
Database version 5 adds an additional "param_stats" table.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
io.info("Migrating database to version 6.")
backup_path = db.path + ".5"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(db.path, backup_path)
db.execute("DELETE FROM version")
db.execute("INSERT INTO version VALUES (6)")
db.execute("""
CREATE TABLE IF NOT EXISTS scenario_stats (
scenario CHAR(40), -- Key for scenarios
num_params INTEGER, -- The number of parameters in W_legal for scenario
oracle_param VARCHAR(255), -- The best parameter
oracle_runtime REAL, -- The runtime of the best parameter
worst_param VARCHAR(255), -- The worst parameter
worst_runtime REAL, -- The runtime of the worst parameter
mean_runtime REAL, -- The mean runtime of all parameters
PRIMARY KEY (scenario)
)
""")
db.populate_scenario_stats_table()
# Sanity checks
bad = False
if db.num_rows("scenario_stats") != len(db.scenarios):
io.error("Bad row count in scenario_stats table! Expected",
len(db.scenarios), "Observed:", db.num_rows("scenario_stats"))
bad = True
if bad:
io.fatal("Failed sanity check, aborting.")
else:
io.info("Passed sanity check.")
# Copy migrated database over the original one.
db.close()
io.info("Migration completed.")
def gather():
benchmarks = {
"canny": {},
"fdtd": {},
"gol": {},
"gaussian": {},
"heat": {},
"simple": {},
"simplecomplex": {}
}
for benchmark in benchmarks:
io.info("Benchmark %s" % benchmark)
fs.cd("/home/chris/src/msc-thesis/scraps/05-12/kernels/%s" % benchmark)
instcounts = []
for file in fs.ls():
instcounts.append(get_instcount(file))
benchmarks[benchmark] = merge_counts(instcounts)
return benchmarks
def migrate_4_to_5(db):
"""
SkelCL database migration script.
Database version 5 adds an additional "param_stats" table.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
io.info("Migrating database to version 5.")
backup_path = db.path + ".4"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(db.path, backup_path)
db.execute("DELETE FROM version")
db.execute("INSERT INTO version VALUES (5)")
db.execute("""
-- Parameter stats table
CREATE TABLE IF NOT EXISTS param_stats (
params VARCHAR(255), -- Key for params
num_scenarios INTEGER, -- Number of scenarios for which param is legal, 0 < num_scenarios
coverage REAL, -- num_scenarios / total number of scenarios, 0 < coverage <= 1
performance REAL, -- Geometric mean of performance relative to the oracle for all scenarios for which param was legal, 0 < performance <= 1
PRIMARY KEY (params)
)
""")
db.populate_param_stats_table()
# Sanity checks
bad = False
if db.num_rows("param_stats") != len(db.params):
io.error("Bad row count in params table! Expected", len(db.params),
"Observed:", db.num_rows("param_stats"))
bad = True
if bad:
io.fatal("Failed sanity check, aborting.")
else:
io.info("Passed sanity check.")
# Copy migrated database over the original one.
db.close()
io.info("Migration completed.")
def migrate_1_to_2(old):
"""
SkelCL database migration script.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
# Create temporary database
fs.cp(old.path, "/tmp/omnitune.skelcl.migration.db")
tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
io.info("Migrating database to version 2.")
backup_path = old.path + ".1"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(old.path, backup_path)
# Update database version
tmp.drop_table("version")
tmp.create_table("version",
(("version", "integer"),))
tmp.execute("INSERT INTO version VALUES (2)")
# Rename table "data" to "datasets"
tmp.create_table("datasets",
(("id", "text primary key"),
("width", "integer"),
("height", "integer"),
("tin", "text"),
("tout", "text")))
tmp.execute("INSERT INTO datasets SELECT * FROM data")
tmp.drop_table("data")
# Rename column "scenarios.data" to "scenarios.dataset"
tmp.execute("ALTER TABLE scenarios RENAME TO old_scenarios")
tmp.create_table("scenarios",
(("id", "text primary key"),
("host", "text"),
("device", "text"),
("kernel", "text"),
("dataset", "text")))
tmp.execute("INSERT INTO scenarios SELECT * FROM old_scenarios")
tmp.drop_table("old_scenarios")
tmp.commit()
old_path = old.path
tmp_path = tmp.path
# Copy migrated database over the original one.
fs.cp(tmp_path, old_path)
fs.rm(tmp_path)
old.close()
tmp.close()
io.info("Migration completed.")
def migrate_4_to_5(db):
"""
SkelCL database migration script.
Database version 5 adds an additional "param_stats" table.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
io.info("Migrating database to version 5.")
backup_path = db.path + ".4"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(db.path, backup_path)
db.execute("DELETE FROM version")
db.execute("INSERT INTO version VALUES (5)")
db.execute("""
-- Parameter stats table
CREATE TABLE IF NOT EXISTS param_stats (
params VARCHAR(255), -- Key for params
num_scenarios INTEGER, -- Number of scenarios for which param is legal, 0 < num_scenarios
coverage REAL, -- num_scenarios / total number of scenarios, 0 < coverage <= 1
performance REAL, -- Geometric mean of performance relative to the oracle for all scenarios for which param was legal, 0 < performance <= 1
PRIMARY KEY (params)
)
""")
db.populate_param_stats_table()
# Sanity checks
bad = False
if db.num_rows("param_stats") != len(db.params):
io.error("Bad row count in params table! Expected", len(db.params),
"Observed:", db.num_rows("param_stats"))
bad = True
if bad:
io.fatal("Failed sanity check, aborting.")
else:
io.info("Passed sanity check.")
# Copy migrated database over the original one.
db.close()
io.info("Migration completed.")
def migrate_1_to_2(old):
"""
SkelCL database migration script.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
# Create temporary database
fs.cp(old.path, "/tmp/omnitune.skelcl.migration.db")
tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
io.info("Migrating database to version 2.")
backup_path = old.path + ".1"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(old.path, backup_path)
# Update database version
tmp.drop_table("version")
tmp.create_table("version",
(("version", "integer"),))
tmp.execute("INSERT INTO version VALUES (2)")
# Rename table "data" to "datasets"
tmp.create_table("datasets",
(("id", "text primary key"),
("width", "integer"),
("height", "integer"),
("tin", "text"),
("tout", "text")))
tmp.execute("INSERT INTO datasets SELECT * FROM data")
tmp.drop_table("data")
# Rename column "scenarios.data" to "scenarios.dataset"
tmp.execute("ALTER TABLE scenarios RENAME TO old_scenarios")
tmp.create_table("scenarios",
(("id", "text primary key"),
("host", "text"),
("device", "text"),
("kernel", "text"),
("dataset", "text")))
tmp.execute("INSERT INTO scenarios SELECT * FROM old_scenarios")
tmp.drop_table("old_scenarios")
tmp.commit()
old_path = old.path
tmp_path = tmp.path
# Copy migrated database over the original one.
fs.cp(tmp_path, old_path)
fs.rm(tmp_path)
old.close()
tmp.close()
io.info("Migration completed.")
def run_eval(db, dataset, eval_fn, eval_type="", nfolds=10):
# Cross validation using both synthetic and real data.
folds = dataset.folds(nfolds, seed=SEED)
print()
io.info("CROSS VALIDATION")
io.info("Size of training set:", folds[0][0].num_instances)
io.info("Size of testing set: ", folds[0][1].num_instances)
for i,fold in enumerate(folds):
training, testing = fold
io.debug("Cross-validating", eval_type, "- fold", i + 1, "of", nfolds)
eval_fn("xval", training, testing)
def table(rows, columns=None, output=None, data_args={}, **kwargs):
"""
Return a LaTeX formatted string of "list of list" table data.
See: http://pandas.pydata.org/pandas-docs/dev/generated/pandas.DataFrame.html
Requires the "booktabs" package to be included in LaTeX preamble:
\\usepackage{booktabs}
Examples:
>>> fmt.print([("foo", 1), ("bar", 2)])
0 1
0 foo 1
1 bar 2
>>> fmt.print([("foo", 1), ("bar", 2)], columns=("type", "value"))
type value
0 foo 1
1 bar 2
Arguments:
rows (list of list): Data to format, one row per element,
multiple columns per row.
columns (list of str, optional): Column names.
output (str, optional): Path to output file.
data_args (dict, optional): Any additional kwargs to pass to
pandas.DataFrame constructor.
**kwargs: Any additional arguments to pass to
pandas.DataFrame.to_latex().
Returns:
str: Formatted data as LaTeX table.
Raises:
Error: If number of columns (if provided) does not equal
number of columns in rows; or if number of columns is not
consistent across all rows.
"""
import pandas
# Number of columns.
num_columns = len(rows[0])
# Check that each row is the same length.
for i,row in enumerate(rows[1:]):
if len(row) != num_columns:
raise Error("Number of columns in row {i_row} ({c_row}) "
"does not match number of columns in row 0 ({z_row})"
.format(i_row=i, c_row=len(row), z_row=num_columns))
# Check that (if supplied), number of columns matches number of
# columns in rows.
if columns is not None and len(columns) != num_columns:
raise Error("Number of columns in header ({c_header}) does not "
"match the number of columns in the data ({c_rows})"
.format(c_header=len(columns), c_rows=num_columns))
# Default arguments.
if "index" not in kwargs:
kwargs["index"] = False
data_args["columns"] = columns
string = pandas.DataFrame(list(rows), **data_args).to_latex(**kwargs)
if output is None:
return string
else:
print(string, file=open(output, "w"))
io.info("Wrote", output)
def test_info(self):
out = StringIO()
io.info("foo", file=out)
self._test("INFO", re.search("INFO", out.getvalue()).group(0))
def migrate_0_to_1(old):
"""
SkelCL database migration script.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
def get_source(checksum):
query = old.execute("SELECT source FROM kernels WHERE checksum = ?",
(checksum,))
return query.fetchone()[0]
def get_device_attr(device_id, name, count):
query = old.execute("SELECT * FROM devices WHERE name = ?",
(name,))
attr = query.fetchone()
# Splice into the new
newattr = (device_id, attr[0], count) + attr[2:]
return newattr
def process_row(tmp, row):
# Get column values from row.
host = row[0]
dev_name = row[1]
dev_count = row[2]
kern_checksum = row[3]
north = row[4]
south = row[5]
east = row[6]
west = row[7]
data_width = row[8]
data_height = row[9]
max_wg_size = row[10]
wg_c = row[11]
wg_r = row[12]
runtime = row[13]
type_in = "float"
type_out = "float"
# Lookup source code.
source = get_source(kern_checksum)
user_source = get_user_source(source)
kernel_id = hash_kernel(north, south, east, west, max_wg_size, source)
device_id = hash_device(dev_name, dev_count)
data_id = hash_data(data_width, data_height, type_in, type_out)
scenario_id = hash_scenario(host, device_id, kernel_id, data_id)
params_id = hash_workgroup_size(wg_c, wg_r)
device_attr = get_device_attr(device_id, dev_name, dev_count)
# Add database entries.
tmp.execute("INSERT OR IGNORE INTO kernels VALUES (?,?,?,?,?,?,?)",
(kernel_id, north, south, east, west, max_wg_size, user_source))
placeholders = ",".join(["?"] * len(device_attr))
tmp.execute("INSERT OR IGNORE INTO devices VALUES (" + placeholders + ")",
device_attr)
tmp.execute("INSERT OR IGNORE INTO data VALUES (?,?,?,?,?)",
(data_id, data_width, data_height, type_in, type_out))
tmp.execute("INSERT OR IGNORE INTO params VALUES (?,?,?)",
(params_id, wg_c, wg_r))
tmp.execute("INSERT OR IGNORE INTO scenarios VALUES (?,?,?,?,?)",
(scenario_id, host, device_id, kernel_id, data_id))
tmp.execute("INSERT INTO runtimes VALUES (?,?,?)",
(scenario_id, params_id, runtime))
# Create temporary database
tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
# Clear anything that's already in the database.
for table in tmp.tables:
tmp.drop_table(table)
io.info("Migrating database to version 1.")
backup_path = old.path + ".0"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(old.path, backup_path)
io.debug("Migration: creating tables ...")
# Create table: kernels
tmp.create_table("version",
(("version", "integer"),))
# Set database version
tmp.execute("INSERT INTO version VALUES (1)")
# Create table: kernels
tmp.create_table("kernels",
(("id", "text primary key"),
("north", "integer"),
("south", "integer"),
("east", "integer"),
("west", "integer"),
("max_wg_size", "integer"),
("source", "text")))
# Create table: devices
tmp.create_table("devices",
(("id", "text primary key"),
("name", "text"),
("count", "integer"),
("address_bits", "integer"),
("double_fp_config", "integer"),
("endian_little", "integer"),
("execution_capabilities", "integer"),
("extensions", "text"),
("global_mem_cache_size", "integer"),
("global_mem_cache_type", "integer"),
("global_mem_cacheline_size", "integer"),
("global_mem_size", "integer"),
("host_unified_memory", "integer"),
("image2d_max_height", "integer"),
("image2d_max_width", "integer"),
("image3d_max_depth", "integer"),
("image3d_max_height", "integer"),
("image3d_max_width", "integer"),
("image_support", "integer"),
("local_mem_size", "integer"),
("local_mem_type", "integer"),
("max_clock_frequency", "integer"),
("max_compute_units", "integer"),
("max_constant_args", "integer"),
("max_constant_buffer_size", "integer"),
("max_mem_alloc_size", "integer"),
("max_parameter_size", "integer"),
("max_read_image_args", "integer"),
("max_samplers", "integer"),
("max_work_group_size", "integer"),
("max_work_item_dimensions", "integer"),
("max_work_item_sizes_0", "integer"),
("max_work_item_sizes_1", "integer"),
("max_work_item_sizes_2", "integer"),
("max_write_image_args", "integer"),
("mem_base_addr_align", "integer"),
("min_data_type_align_size", "integer"),
("native_vector_width_char", "integer"),
("native_vector_width_double", "integer"),
("native_vector_width_float", "integer"),
("native_vector_width_half", "integer"),
("native_vector_width_int", "integer"),
("native_vector_width_long", "integer"),
("native_vector_width_short", "integer"),
("preferred_vector_width_char", "integer"),
("preferred_vector_width_double", "integer"),
("preferred_vector_width_float", "integer"),
("preferred_vector_width_half", "integer"),
("preferred_vector_width_int", "integer"),
("preferred_vector_width_long", "integer"),
("preferred_vector_width_short", "integer"),
("queue_properties", "integer"),
("single_fp_config", "integer"),
("type", "integer"),
("vendor", "text"),
("vendor_id", "text"),
("version", "text")))
# Create table: data
tmp.create_table("data",
(("id", "text primary key"),
("width", "integer"),
("height", "integer"),
("tin", "text"),
("tout", "text")))
# Create table: params
tmp.create_table("params",
(("id", "text primary key"),
("wg_c", "integer"),
("wg_r", "integer")))
# Create table: scenarios
tmp.create_table("scenarios",
(("id", "text primary key"),
("host", "text"),
("device", "text"),
("kernel", "text"),
("data", "text")))
# Create table: runtimes
tmp.create_table("runtimes",
(("scenario", "text"),
("params", "text"),
("runtime", "real")))
i = 0
for row in old.execute("SELECT * from runtimes"):
process_row(tmp, row)
i += 1
if not i % 2500:
io.debug("Processed", i, "rows ...")
if not i % 5000:
tmp.commit()
tmp.commit()
old_path = old.path
tmp_path = tmp.path
# Copy migrated database over the original one.
fs.cp(tmp_path, old_path)
fs.rm(tmp_path)
old.close()
tmp.close()
io.info("Migration completed.")
def migrate_2_to_3(old):
"""
SkelCL database migration script.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
def _old_kernel2new(old_id):
kernel = old.execute("SELECT north,south,east,west,max_wg_size,source "
"FROM kernels WHERE id=?",
(old_id,)).fetchone()
if kernel:
return tmp.kernel_id(*kernel)
def _old_scenario2new(old_id):
device, old_kernel, dataset = old.execute("SELECT device,kernel,dataset "
"FROM scenarios WHERE id=?",
(old_id,)).fetchone()
kernel = _old_kernel2new(old_kernel)
return tmp.scenario_id(device, kernel, dataset)
# TODO: Un-comment out code!
# Create temporary database
fs.rm("/tmp/omnitune.skelcl.migration.db")
tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
tmp.attach(old.path, "rhs")
io.info("Migrating database to version 3.")
backup_path = old.path + ".2"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(old.path, backup_path)
tmp_path = tmp.path
old_path = old.path
tmp.run("create_tables")
# Populate feature and lookup tables.
for row in old.execute("SELECT * FROM devices"):
features = row[1:]
id = hash_device(*features)
io.debug("Features extracted for device", id)
row = (id,) + features
tmp.execute("INSERT INTO devices VALUES " +
placeholders(*row), row)
row = (features[0], features[1], id)
tmp.execute("INSERT INTO device_lookup VALUES " +
placeholders(*row), row)
tmp.commit()
for row in old.execute("SELECT * FROM kernels"):
args = row[1:]
tmp.kernel_id(*args)
for row in old.execute("SELECT * FROM datasets"):
features = row[1:]
id = hash_dataset(*features)
io.debug("Features extracted for dataset", id)
row = (id,) + features
tmp.execute("INSERT INTO datasets VALUES " +
placeholders(*row), row)
row = features + (id,)
tmp.execute("INSERT INTO dataset_lookup VALUES " +
placeholders(*row), row)
tmp.commit()
# Populate kernel_names table.
for row in old.execute("SELECT * FROM kernel_names"):
old_id = row[0]
synthetic, name = row[1:]
kernel = _old_kernel2new(old_id)
if kernel:
row = (kernel, synthetic, name)
tmp.execute("INSERT OR IGNORE INTO kernel_names VALUES " +
placeholders(*row), row)
tmp.commit()
# Populate scenarios table.
for row in old.execute("SELECT * FROM scenarios"):
old_id, _, device, old_kernel, dataset = row
kernel = _old_kernel2new(old_kernel)
new_id = hash_scenario(device, kernel, dataset)
row = (new_id, device, kernel, dataset)
tmp.execute("INSERT OR IGNORE INTO scenarios VALUES " +
placeholders(*row), row)
tmp.commit()
# Populate params table.
tmp.execute("INSERT INTO params SELECT * from rhs.params")
tmp.commit()
scenario_replacements = {
row[0]: _old_scenario2new(row[0])
for row in old.execute("SELECT * FROM scenarios")
}
tmp.execute("INSERT INTO runtimes SELECT * from rhs.runtimes")
for old_id, new_id in scenario_replacements.iteritems():
io.info("Runtimes", old_id, "->", new_id)
tmp.execute("UPDATE runtimes SET scenario=? WHERE scenario=?",
(new_id, old_id))
tmp.commit()
# Sanity checks
bad = False
for row in tmp.execute("SELECT DISTINCT scenario FROM runtimes"):
count = tmp.execute("SELECT Count(*) FROM scenarios WHERE id=?",
(row[0],)).fetchone()[0]
if count != 1:
io.error("Bad scenario count:", row[0], count)
bad = True
if bad:
io.fatal("Failed sanity check, aborting.")
else:
io.info("Passed sanity check.")
# Copy migrated database over the original one.
fs.cp(tmp_path, old_path)
fs.rm(tmp_path)
old.close()
tmp.close()
io.info("Migration completed.")
def test_info():
out = StringIO()
io.info("foo", file=out)
assert "INFO" == re.search("INFO", out.getvalue()).group(0)
def migrate_3_to_4(old):
"""
SkelCL database migration script.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
# Create temporary database
fs.rm("/tmp/omnitune.skelcl.migration.db")
tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
tmp.attach(old.path, "rhs")
io.info("Migrating database to version 4.")
backup_path = old.path + ".3"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(old.path, backup_path)
tables = [
"kernels",
"kernel_lookup",
"kernel_names",
"devices",
"device_lookup",
"datasets",
"dataset_lookup",
"scenarios",
"params",
"runtimes",
"runtime_stats",
"oracle_params"
]
for table in tables:
io.info("Copying data from '{}' ...".format(table))
tmp.execute("INSERT INTO {} SELECT * FROM rhs.{}".format(table, table))
tmp_path = tmp.path
old_path = old.path
tmp.execute("VACUUM")
# Sanity checks
bad = False
for table in tables:
old_count = tmp.num_rows("rhs." + table)
tmp_count = tmp.num_rows(table)
if old_count != tmp_count:
io.error("Bad rows count:", old_count, tmp_count)
bad = True
if bad:
io.fatal("Failed sanity check, aborting.")
else:
io.info("Passed sanity check.")
# Copy migrated database over the original one.
fs.cp(tmp_path, old_path)
fs.rm(tmp_path)
old.close()
tmp.close()
io.info("Migration completed.")
def migrate_2_to_3(old):
"""
SkelCL database migration script.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
def _old_kernel2new(old_id):
kernel = old.execute("SELECT north,south,east,west,max_wg_size,source "
"FROM kernels WHERE id=?",
(old_id,)).fetchone()
if kernel:
return tmp.kernel_id(*kernel)
def _old_scenario2new(old_id):
device, old_kernel, dataset = old.execute("SELECT device,kernel,dataset "
"FROM scenarios WHERE id=?",
(old_id,)).fetchone()
kernel = _old_kernel2new(old_kernel)
return tmp.scenario_id(device, kernel, dataset)
# TODO: Un-comment out code!
# Create temporary database
fs.rm("/tmp/omnitune.skelcl.migration.db")
tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
tmp.attach(old.path, "rhs")
io.info("Migrating database to version 3.")
backup_path = old.path + ".2"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(old.path, backup_path)
tmp_path = tmp.path
old_path = old.path
tmp.run("create_tables")
# Populate feature and lookup tables.
for row in old.execute("SELECT * FROM devices"):
features = row[1:]
id = hash_device(*features)
io.debug("Features extracted for device", id)
row = (id,) + features
tmp.execute("INSERT INTO devices VALUES " +
placeholders(*row), row)
row = (features[0], features[1], id)
tmp.execute("INSERT INTO device_lookup VALUES " +
placeholders(*row), row)
tmp.commit()
for row in old.execute("SELECT * FROM kernels"):
args = row[1:]
tmp.kernel_id(*args)
for row in old.execute("SELECT * FROM datasets"):
features = row[1:]
id = hash_dataset(*features)
io.debug("Features extracted for dataset", id)
row = (id,) + features
tmp.execute("INSERT INTO datasets VALUES " +
placeholders(*row), row)
row = features + (id,)
tmp.execute("INSERT INTO dataset_lookup VALUES " +
placeholders(*row), row)
tmp.commit()
# Populate kernel_names table.
for row in old.execute("SELECT * FROM kernel_names"):
old_id = row[0]
synthetic, name = row[1:]
kernel = _old_kernel2new(old_id)
if kernel:
row = (kernel, synthetic, name)
tmp.execute("INSERT OR IGNORE INTO kernel_names VALUES " +
placeholders(*row), row)
tmp.commit()
# Populate scenarios table.
for row in old.execute("SELECT * FROM scenarios"):
old_id, _, device, old_kernel, dataset = row
kernel = _old_kernel2new(old_kernel)
new_id = hash_scenario(device, kernel, dataset)
row = (new_id, device, kernel, dataset)
tmp.execute("INSERT OR IGNORE INTO scenarios VALUES " +
placeholders(*row), row)
tmp.commit()
# Populate params table.
tmp.execute("INSERT INTO params SELECT * from rhs.params")
tmp.commit()
scenario_replacements = {
row[0]: _old_scenario2new(row[0])
for row in old.execute("SELECT * FROM scenarios")
}
tmp.execute("INSERT INTO runtimes SELECT * from rhs.runtimes")
for old_id, new_id in scenario_replacements.iteritems():
io.info("Runtimes", old_id, "->", new_id)
tmp.execute("UPDATE runtimes SET scenario=? WHERE scenario=?",
(new_id, old_id))
tmp.commit()
# Sanity checks
bad = False
for row in tmp.execute("SELECT DISTINCT scenario FROM runtimes"):
count = tmp.execute("SELECT Count(*) FROM scenarios WHERE id=?",
(row[0],)).fetchone()[0]
if count != 1:
io.error("Bad scenario count:", row[0], count)
bad = True
if bad:
io.fatal("Failed sanity check, aborting.")
else:
io.info("Passed sanity check.")
# Copy migrated database over the original one.
fs.cp(tmp_path, old_path)
fs.rm(tmp_path)
old.close()
tmp.close()
io.info("Migration completed.")
def migrate_0_to_1(old):
"""
SkelCL database migration script.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
def get_source(checksum):
query = old.execute("SELECT source FROM kernels WHERE checksum = ?",
(checksum,))
return query.fetchone()[0]
def get_device_attr(device_id, name, count):
query = old.execute("SELECT * FROM devices WHERE name = ?",
(name,))
attr = query.fetchone()
# Splice into the new
newattr = (device_id, attr[0], count) + attr[2:]
return newattr
def process_row(tmp, row):
# Get column values from row.
host = row[0]
dev_name = row[1]
dev_count = row[2]
kern_checksum = row[3]
north = row[4]
south = row[5]
east = row[6]
west = row[7]
data_width = row[8]
data_height = row[9]
max_wg_size = row[10]
wg_c = row[11]
wg_r = row[12]
runtime = row[13]
type_in = "float"
type_out = "float"
# Lookup source code.
source = get_source(kern_checksum)
user_source = get_user_source(source)
kernel_id = hash_kernel(north, south, east, west, max_wg_size, source)
device_id = hash_device(dev_name, dev_count)
data_id = hash_data(data_width, data_height, type_in, type_out)
scenario_id = hash_scenario(host, device_id, kernel_id, data_id)
params_id = hash_workgroup_size(wg_c, wg_r)
device_attr = get_device_attr(device_id, dev_name, dev_count)
# Add database entries.
tmp.execute("INSERT OR IGNORE INTO kernels VALUES (?,?,?,?,?,?,?)",
(kernel_id,north,south,east,west,max_wg_size,user_source))
placeholders = ",".join(["?"] * len(device_attr))
tmp.execute("INSERT OR IGNORE INTO devices VALUES (" + placeholders + ")",
device_attr)
tmp.execute("INSERT OR IGNORE INTO data VALUES (?,?,?,?,?)",
(data_id, data_width, data_height, type_in, type_out))
tmp.execute("INSERT OR IGNORE INTO params VALUES (?,?,?)",
(params_id, wg_c, wg_r))
tmp.execute("INSERT OR IGNORE INTO scenarios VALUES (?,?,?,?,?)",
(scenario_id, host, device_id, kernel_id, data_id))
tmp.execute("INSERT INTO runtimes VALUES (?,?,?)",
(scenario_id, params_id, runtime))
# Create temporary database
tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
# Clear anything that's already in the database.
for table in tmp.tables:
tmp.drop_table(table)
io.info("Migrating database to version 1.")
backup_path = old.path + ".0"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(old.path, backup_path)
io.debug("Migration: creating tables ...")
# Create table: kernels
tmp.create_table("version",
(("version", "integer"),))
# Set database version
tmp.execute("INSERT INTO version VALUES (1)")
# Create table: kernels
tmp.create_table("kernels",
(("id", "text primary key"),
("north", "integer"),
("south", "integer"),
("east", "integer"),
("west", "integer"),
("max_wg_size", "integer"),
("source", "text")))
# Create table: devices
tmp.create_table("devices",
(("id", "text primary key"),
("name", "text"),
("count", "integer"),
("address_bits", "integer"),
("double_fp_config", "integer"),
("endian_little", "integer"),
("execution_capabilities", "integer"),
("extensions", "text"),
("global_mem_cache_size", "integer"),
("global_mem_cache_type", "integer"),
("global_mem_cacheline_size", "integer"),
("global_mem_size", "integer"),
("host_unified_memory", "integer"),
("image2d_max_height", "integer"),
("image2d_max_width", "integer"),
("image3d_max_depth", "integer"),
("image3d_max_height", "integer"),
("image3d_max_width", "integer"),
("image_support", "integer"),
("local_mem_size", "integer"),
("local_mem_type", "integer"),
("max_clock_frequency", "integer"),
("max_compute_units", "integer"),
("max_constant_args", "integer"),
("max_constant_buffer_size", "integer"),
("max_mem_alloc_size", "integer"),
("max_parameter_size", "integer"),
("max_read_image_args", "integer"),
("max_samplers", "integer"),
("max_work_group_size", "integer"),
("max_work_item_dimensions", "integer"),
("max_work_item_sizes_0", "integer"),
("max_work_item_sizes_1", "integer"),
("max_work_item_sizes_2", "integer"),
("max_write_image_args", "integer"),
("mem_base_addr_align", "integer"),
("min_data_type_align_size", "integer"),
("native_vector_width_char", "integer"),
("native_vector_width_double", "integer"),
("native_vector_width_float", "integer"),
("native_vector_width_half", "integer"),
("native_vector_width_int", "integer"),
("native_vector_width_long", "integer"),
("native_vector_width_short", "integer"),
("preferred_vector_width_char", "integer"),
("preferred_vector_width_double", "integer"),
("preferred_vector_width_float", "integer"),
("preferred_vector_width_half", "integer"),
("preferred_vector_width_int", "integer"),
("preferred_vector_width_long", "integer"),
("preferred_vector_width_short", "integer"),
("queue_properties", "integer"),
("single_fp_config", "integer"),
("type", "integer"),
("vendor", "text"),
("vendor_id", "text"),
("version", "text")))
# Create table: data
tmp.create_table("data",
(("id", "text primary key"),
("width", "integer"),
("height", "integer"),
("tin", "text"),
("tout", "text")))
# Create table: params
tmp.create_table("params",
(("id", "text primary key"),
("wg_c", "integer"),
("wg_r", "integer")))
# Create table: scenarios
tmp.create_table("scenarios",
(("id", "text primary key"),
("host", "text"),
("device", "text"),
("kernel", "text"),
("data", "text")))
# Create table: runtimes
tmp.create_table("runtimes",
(("scenario", "text"),
("params", "text"),
("runtime", "real")))
i = 0
for row in old.execute("SELECT * from runtimes"):
process_row(tmp, row)
i += 1
if not i % 2500:
io.debug("Processed", i, "rows ...")
if not i % 5000:
tmp.commit()
tmp.commit()
old_path = old.path
tmp_path = tmp.path
# Copy migrated database over the original one.
fs.cp(tmp_path, old_path)
fs.rm(tmp_path)
old.close()
tmp.close()
io.info("Migration completed.")
