def migrate_3_to_4(old):
"""
SkelCL database migration script.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
# Create temporary database
fs.rm("/tmp/omnitune.skelcl.migration.db")
tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
tmp.attach(old.path, "rhs")
io.info("Migrating database to version 4.")
backup_path = old.path + ".3"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(old.path, backup_path)
tables = [
"kernels", "kernel_lookup", "kernel_names", "devices", "device_lookup",
"datasets", "dataset_lookup", "scenarios", "params", "runtimes",
"runtime_stats", "oracle_params"
]
for table in tables:
io.info("Copying data from '{}' ...".format(table))
tmp.execute("INSERT INTO {} SELECT * FROM rhs.{}".format(table, table))
tmp_path = tmp.path
old_path = old.path
tmp.execute("VACUUM")
# Sanity checks
bad = False
for table in tables:
old_count = tmp.num_rows("rhs." + table)
tmp_count = tmp.num_rows(table)
if old_count != tmp_count:
io.error("Bad rows count:", old_count, tmp_count)
bad = True
if bad:
io.fatal("Failed sanity check, aborting.")
else:
io.info("Passed sanity check.")
# Copy migrated database over the original one.
fs.cp(tmp_path, old_path)
fs.rm(tmp_path)
old.close()
tmp.close()
io.info("Migration completed.")
def migrate_5_to_6(db):
"""
SkelCL database migration script.
Database version 5 adds an additional "param_stats" table.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
io.info("Migrating database to version 6.")
backup_path = db.path + ".5"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(db.path, backup_path)
db.execute("DELETE FROM version")
db.execute("INSERT INTO version VALUES (6)")
db.execute("""
CREATE TABLE IF NOT EXISTS scenario_stats (
scenario CHAR(40), -- Key for scenarios
num_params INTEGER, -- The number of parameters in W_legal for scenario
oracle_param VARCHAR(255), -- The best parameter
oracle_runtime REAL, -- The runtime of the best parameter
worst_param VARCHAR(255), -- The worst parameter
worst_runtime REAL, -- The runtime of the worst parameter
mean_runtime REAL, -- The mean runtime of all parameters
PRIMARY KEY (scenario)
)
""")
db.populate_scenario_stats_table()
# Sanity checks
bad = False
if db.num_rows("scenario_stats") != len(db.scenarios):
io.error("Bad row count in scenario_stats table! Expected",
len(db.scenarios), "Observed:", db.num_rows("scenario_stats"))
bad = True
if bad:
io.fatal("Failed sanity check, aborting.")
else:
io.info("Passed sanity check.")
# Copy migrated database over the original one.
db.close()
io.info("Migration completed.")
def migrate_4_to_5(db):
"""
SkelCL database migration script.
Database version 5 adds an additional "param_stats" table.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
io.info("Migrating database to version 5.")
backup_path = db.path + ".4"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(db.path, backup_path)
db.execute("DELETE FROM version")
db.execute("INSERT INTO version VALUES (5)")
db.execute("""
-- Parameter stats table
CREATE TABLE IF NOT EXISTS param_stats (
params VARCHAR(255), -- Key for params
num_scenarios INTEGER, -- Number of scenarios for which param is legal, 0 < num_scenarios
coverage REAL, -- num_scenarios / total number of scenarios, 0 < coverage <= 1
performance REAL, -- Geometric mean of performance relative to the oracle for all scenarios for which param was legal, 0 < performance <= 1
PRIMARY KEY (params)
)
""")
db.populate_param_stats_table()
# Sanity checks
bad = False
if db.num_rows("param_stats") != len(db.params):
io.error("Bad row count in params table! Expected", len(db.params),
"Observed:", db.num_rows("param_stats"))
bad = True
if bad:
io.fatal("Failed sanity check, aborting.")
else:
io.info("Passed sanity check.")
# Copy migrated database over the original one.
db.close()
io.info("Migration completed.")
def test_error():
out = StringIO()
io.error("foo", file=out)
assert "ERROR" == re.search("ERROR", out.getvalue()).group(0)
def trisurf(self,
output=None,
title=None,
figsize=(5, 4),
zlabel=None,
zticklabels=None,
rotation=None,
**kwargs):
import matplotlib.pyplot as plt
import matplotlib.cm as cm
num_vals = self.matrix.shape[0] * self.matrix.shape[1]
if num_vals < 3:
io.error("Cannot create trisurf of", num_vals, "values")
return
X = np.zeros((num_vals, ))
Y = np.zeros((num_vals, ))
Z = np.zeros((num_vals, ))
# Iterate over every point in space.
for j, i in product(range(self.matrix.shape[0]),
range(self.matrix.shape[1])):
# Convert point to list index.
index = j * self.matrix.shape[1] + i
X[index] = i
Y[index] = j
Z[index] = self.matrix[j][i]
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.plot_trisurf(X, Y, Z, cmap=cm.jet, **kwargs)
# Set X axis labels
xticks = []
xticklabels = []
for i, c in enumerate(self.c):
if not len(xticks) or c % 20 == 0:
xticks.append(i)
xticklabels.append(c)
ax.set_xticks(xticks)
ax.set_xticklabels(xticklabels)
ax.set_xlabel("$w_c$")
# Set Y axis labels
yticks = []
yticklabels = []
for i, c in enumerate(self.c):
if not len(yticks) or c % 20 == 0:
yticks.append(i)
yticklabels.append(c)
ax.set_yticks(yticks)
ax.set_yticklabels(yticklabels)
ax.set_ylabel("$w_r$")
# Set Z axis labels
if zlabel is not None:
ax.set_zlabel(zlabel)
if zticklabels is not None:
ax.set_zticks(np.arange(len(zticklabels)))
ax.set_zticklabels(zticklabels)
# Set plot rotation.
if rotation is not None: ax.view_init(azim=rotation)
# Set plot title.
if title: plt.title(title)
plt.tight_layout()
plt.gcf().set_size_inches(*figsize, dpi=300)
viz.finalise(output)
def migrate_2_to_3(old):
"""
SkelCL database migration script.
Arguments:
old (SkelCLDatabase): The database to migrate
"""
def _old_kernel2new(old_id):
kernel = old.execute(
"SELECT north,south,east,west,max_wg_size,source "
"FROM kernels WHERE id=?", (old_id, )).fetchone()
if kernel:
return tmp.kernel_id(*kernel)
def _old_scenario2new(old_id):
device, old_kernel, dataset = old.execute(
"SELECT device,kernel,dataset "
"FROM scenarios WHERE id=?", (old_id, )).fetchone()
kernel = _old_kernel2new(old_kernel)
return tmp.scenario_id(device, kernel, dataset)
# TODO: Un-comment out code!
# Create temporary database
fs.rm("/tmp/omnitune.skelcl.migration.db")
tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
tmp.attach(old.path, "rhs")
io.info("Migrating database to version 3.")
backup_path = old.path + ".2"
io.info("Creating backup of old database at '{0}'".format(backup_path))
fs.cp(old.path, backup_path)
tmp_path = tmp.path
old_path = old.path
tmp.run("create_tables")
# Populate feature and lookup tables.
for row in old.execute("SELECT * FROM devices"):
features = row[1:]
id = hash_device(*features)
io.debug("Features extracted for device", id)
row = (id, ) + features
tmp.execute("INSERT INTO devices VALUES " + placeholders(*row), row)
row = (features[0], features[1], id)
tmp.execute("INSERT INTO device_lookup VALUES " + placeholders(*row),
row)
tmp.commit()
for row in old.execute("SELECT * FROM kernels"):
args = row[1:]
tmp.kernel_id(*args)
for row in old.execute("SELECT * FROM datasets"):
features = row[1:]
id = hash_dataset(*features)
io.debug("Features extracted for dataset", id)
row = (id, ) + features
tmp.execute("INSERT INTO datasets VALUES " + placeholders(*row), row)
row = features + (id, )
tmp.execute("INSERT INTO dataset_lookup VALUES " + placeholders(*row),
row)
tmp.commit()
# Populate kernel_names table.
for row in old.execute("SELECT * FROM kernel_names"):
old_id = row[0]
synthetic, name = row[1:]
kernel = _old_kernel2new(old_id)
if kernel:
row = (kernel, synthetic, name)
tmp.execute(
"INSERT OR IGNORE INTO kernel_names VALUES " +
placeholders(*row), row)
tmp.commit()
# Populate scenarios table.
for row in old.execute("SELECT * FROM scenarios"):
old_id, _, device, old_kernel, dataset = row
kernel = _old_kernel2new(old_kernel)
new_id = hash_scenario(device, kernel, dataset)
row = (new_id, device, kernel, dataset)
tmp.execute(
"INSERT OR IGNORE INTO scenarios VALUES " + placeholders(*row),
row)
tmp.commit()
# Populate params table.
tmp.execute("INSERT INTO params SELECT * from rhs.params")
tmp.commit()
scenario_replacements = {
row[0]: _old_scenario2new(row[0])
for row in old.execute("SELECT * FROM scenarios")
}
tmp.execute("INSERT INTO runtimes SELECT * from rhs.runtimes")
for old_id, new_id in scenario_replacements.iteritems():
io.info("Runtimes", old_id, "->", new_id)
tmp.execute("UPDATE runtimes SET scenario=? WHERE scenario=?",
(new_id, old_id))
tmp.commit()
# Sanity checks
bad = False
for row in tmp.execute("SELECT DISTINCT scenario FROM runtimes"):
count = tmp.execute("SELECT Count(*) FROM scenarios WHERE id=?",
(row[0], )).fetchone()[0]
if count != 1:
io.error("Bad scenario count:", row[0], count)
bad = True
if bad:
io.fatal("Failed sanity check, aborting.")
else:
io.info("Passed sanity check.")
# Copy migrated database over the original one.
fs.cp(tmp_path, old_path)
fs.rm(tmp_path)
old.close()
tmp.close()
io.info("Migration completed.")