def migrate_3_to_4(old): """ SkelCL database migration script. Arguments: old (SkelCLDatabase): The database to migrate """ # Create temporary database fs.rm("/tmp/omnitune.skelcl.migration.db") tmp = _db.Database("/tmp/omnitune.skelcl.migration.db") tmp.attach(old.path, "rhs") io.info("Migrating database to version 4.") backup_path = old.path + ".3" io.info("Creating backup of old database at '{0}'".format(backup_path)) fs.cp(old.path, backup_path) tables = [ "kernels", "kernel_lookup", "kernel_names", "devices", "device_lookup", "datasets", "dataset_lookup", "scenarios", "params", "runtimes", "runtime_stats", "oracle_params" ] for table in tables: io.info("Copying data from '{}' ...".format(table)) tmp.execute("INSERT INTO {} SELECT * FROM rhs.{}".format(table, table)) tmp_path = tmp.path old_path = old.path tmp.execute("VACUUM") # Sanity checks bad = False for table in tables: old_count = tmp.num_rows("rhs." + table) tmp_count = tmp.num_rows(table) if old_count != tmp_count: io.error("Bad rows count:", old_count, tmp_count) bad = True if bad: io.fatal("Failed sanity check, aborting.") else: io.info("Passed sanity check.") # Copy migrated database over the original one. fs.cp(tmp_path, old_path) fs.rm(tmp_path) old.close() tmp.close() io.info("Migration completed.")
def migrate_5_to_6(db): """ SkelCL database migration script. Database version 5 adds an additional "param_stats" table. Arguments: old (SkelCLDatabase): The database to migrate """ io.info("Migrating database to version 6.") backup_path = db.path + ".5" io.info("Creating backup of old database at '{0}'".format(backup_path)) fs.cp(db.path, backup_path) db.execute("DELETE FROM version") db.execute("INSERT INTO version VALUES (6)") db.execute(""" CREATE TABLE IF NOT EXISTS scenario_stats ( scenario CHAR(40), -- Key for scenarios num_params INTEGER, -- The number of parameters in W_legal for scenario oracle_param VARCHAR(255), -- The best parameter oracle_runtime REAL, -- The runtime of the best parameter worst_param VARCHAR(255), -- The worst parameter worst_runtime REAL, -- The runtime of the worst parameter mean_runtime REAL, -- The mean runtime of all parameters PRIMARY KEY (scenario) ) """) db.populate_scenario_stats_table() # Sanity checks bad = False if db.num_rows("scenario_stats") != len(db.scenarios): io.error("Bad row count in scenario_stats table! Expected", len(db.scenarios), "Observed:", db.num_rows("scenario_stats")) bad = True if bad: io.fatal("Failed sanity check, aborting.") else: io.info("Passed sanity check.") # Copy migrated database over the original one. db.close() io.info("Migration completed.")
def migrate_4_to_5(db): """ SkelCL database migration script. Database version 5 adds an additional "param_stats" table. Arguments: old (SkelCLDatabase): The database to migrate """ io.info("Migrating database to version 5.") backup_path = db.path + ".4" io.info("Creating backup of old database at '{0}'".format(backup_path)) fs.cp(db.path, backup_path) db.execute("DELETE FROM version") db.execute("INSERT INTO version VALUES (5)") db.execute(""" -- Parameter stats table CREATE TABLE IF NOT EXISTS param_stats ( params VARCHAR(255), -- Key for params num_scenarios INTEGER, -- Number of scenarios for which param is legal, 0 < num_scenarios coverage REAL, -- num_scenarios / total number of scenarios, 0 < coverage <= 1 performance REAL, -- Geometric mean of performance relative to the oracle for all scenarios for which param was legal, 0 < performance <= 1 PRIMARY KEY (params) ) """) db.populate_param_stats_table() # Sanity checks bad = False if db.num_rows("param_stats") != len(db.params): io.error("Bad row count in params table! Expected", len(db.params), "Observed:", db.num_rows("param_stats")) bad = True if bad: io.fatal("Failed sanity check, aborting.") else: io.info("Passed sanity check.") # Copy migrated database over the original one. db.close() io.info("Migration completed.")
def merge(old_oracle, dbs, path): """ Merge databases into one. Arguments: dbs (list of Database): Databases to merge. path (str): Path to merged database. Returns: Database: merged database instance. """ print("Merging {n} databases:".format(n=len(dbs) + 1)) print(" ", old_oracle) for db in dbs: print(" ", db) print() # Make a copy of the old oracle database to work from. io.info("Coping", old_oracle, "->", fs.basename(path)) fs.cp(old_oracle, path) target = migrate(_db.Database(path=path)) for db in dbs + [target]: try: db.num_rows("runtimes") except sqlite3.DatabaseError as e: io.error("Broken db:", db.path) io.fatal(e) num_runtimes = [db.num_rows("runtimes") for db in dbs] expected_total = target.num_rows("runtimes") + sum(num_runtimes) target.merge(dbs) total = target.num_rows("runtimes") if total != expected_total: io.fatal("Expected total", expected_total, "!= actual total", total) io.info(("Merged {num_db} databases, {n} rows" .format(num_db=len(dbs), n=total))) return target
def merge(old_oracle, dbs, path): """ Merge databases into one. Arguments: dbs (list of Database): Databases to merge. path (str): Path to merged database. Returns: Database: merged database instance. """ print("Merging {n} databases:".format(n=len(dbs) + 1)) print(" ", old_oracle) for db in dbs: print(" ", db) print() # Make a copy of the old oracle database to work from. io.info("Coping", old_oracle, "->", fs.basename(path)) fs.cp(old_oracle, path) target = migrate(_db.Database(path=path)) for db in dbs + [target]: try: db.num_rows("runtimes") except sqlite3.DatabaseError as e: io.error("Broken db:", db.path) io.fatal(e) num_runtimes = [db.num_rows("runtimes") for db in dbs] expected_total = target.num_rows("runtimes") + sum(num_runtimes) target.merge(dbs) total = target.num_rows("runtimes") if total != expected_total: io.fatal("Expected total", expected_total, "!= actual total", total) io.info(("Merged {num_db} databases, {n} rows".format(num_db=len(dbs), n=total))) return target
def import_foreign(name, custom_name=None): """ Import a module with a custom name. NOTE this is only needed for Python2. For Python3, import the module using the "as" keyword to declare the custom name. For implementation details, see: http://stackoverflow.com/a/6032023 Example: To import the standard module "math" as "std_math": if labm8.is_python3(): import math as std_math else: std_math = modules.import_foreign("math", "std_math") Arguments: name (str): The name of the module to import. custom_name (str, optional): The custom name to assign the module to. Raises: ImportError: If the module is not found. """ if lab.is_python3(): io.error(("Ignoring attempt to import foreign module '{mod}' " "using python version {major}.{minor}" .format(mod=name, major=sys.version_info[0], minor=sys.version_info[1]))) return custom_name = custom_name or name f, pathname, desc = imp.find_module(name, sys.path[1:]) module = imp.load_module(custom_name, f, pathname, desc) f.close() return module
def migrate_2_to_3(old): """ SkelCL database migration script. Arguments: old (SkelCLDatabase): The database to migrate """ def _old_kernel2new(old_id): kernel = old.execute("SELECT north,south,east,west,max_wg_size,source " "FROM kernels WHERE id=?", (old_id,)).fetchone() if kernel: return tmp.kernel_id(*kernel) def _old_scenario2new(old_id): device, old_kernel, dataset = old.execute("SELECT device,kernel,dataset " "FROM scenarios WHERE id=?", (old_id,)).fetchone() kernel = _old_kernel2new(old_kernel) return tmp.scenario_id(device, kernel, dataset) # TODO: Un-comment out code! # Create temporary database fs.rm("/tmp/omnitune.skelcl.migration.db") tmp = _db.Database("/tmp/omnitune.skelcl.migration.db") tmp.attach(old.path, "rhs") io.info("Migrating database to version 3.") backup_path = old.path + ".2" io.info("Creating backup of old database at '{0}'".format(backup_path)) fs.cp(old.path, backup_path) tmp_path = tmp.path old_path = old.path tmp.run("create_tables") # Populate feature and lookup tables. for row in old.execute("SELECT * FROM devices"): features = row[1:] id = hash_device(*features) io.debug("Features extracted for device", id) row = (id,) + features tmp.execute("INSERT INTO devices VALUES " + placeholders(*row), row) row = (features[0], features[1], id) tmp.execute("INSERT INTO device_lookup VALUES " + placeholders(*row), row) tmp.commit() for row in old.execute("SELECT * FROM kernels"): args = row[1:] tmp.kernel_id(*args) for row in old.execute("SELECT * FROM datasets"): features = row[1:] id = hash_dataset(*features) io.debug("Features extracted for dataset", id) row = (id,) + features tmp.execute("INSERT INTO datasets VALUES " + placeholders(*row), row) row = features + (id,) tmp.execute("INSERT INTO dataset_lookup VALUES " + placeholders(*row), row) tmp.commit() # Populate kernel_names table. for row in old.execute("SELECT * FROM kernel_names"): old_id = row[0] synthetic, name = row[1:] kernel = _old_kernel2new(old_id) if kernel: row = (kernel, synthetic, name) tmp.execute("INSERT OR IGNORE INTO kernel_names VALUES " + placeholders(*row), row) tmp.commit() # Populate scenarios table. for row in old.execute("SELECT * FROM scenarios"): old_id, _, device, old_kernel, dataset = row kernel = _old_kernel2new(old_kernel) new_id = hash_scenario(device, kernel, dataset) row = (new_id, device, kernel, dataset) tmp.execute("INSERT OR IGNORE INTO scenarios VALUES " + placeholders(*row), row) tmp.commit() # Populate params table. tmp.execute("INSERT INTO params SELECT * from rhs.params") tmp.commit() scenario_replacements = { row[0]: _old_scenario2new(row[0]) for row in old.execute("SELECT * FROM scenarios") } tmp.execute("INSERT INTO runtimes SELECT * from rhs.runtimes") for old_id, new_id in scenario_replacements.iteritems(): io.info("Runtimes", old_id, "->", new_id) tmp.execute("UPDATE runtimes SET scenario=? WHERE scenario=?", (new_id, old_id)) tmp.commit() # Sanity checks bad = False for row in tmp.execute("SELECT DISTINCT scenario FROM runtimes"): count = tmp.execute("SELECT Count(*) FROM scenarios WHERE id=?", (row[0],)).fetchone()[0] if count != 1: io.error("Bad scenario count:", row[0], count) bad = True if bad: io.fatal("Failed sanity check, aborting.") else: io.info("Passed sanity check.") # Copy migrated database over the original one. fs.cp(tmp_path, old_path) fs.rm(tmp_path) old.close() tmp.close() io.info("Migration completed.")
def test_error(): out = StringIO() io.error("foo", file=out) assert "ERROR" == re.search("ERROR", out.getvalue()).group(0)
def test_error(self): out = StringIO() io.error("foo", file=out) self._test("ERROR", re.search("ERROR", out.getvalue()).group(0))
def trisurf(self, output=None, title=None, figsize=(5,4), zlabel=None, zticklabels=None, rotation=None, **kwargs): import matplotlib.pyplot as plt import matplotlib.cm as cm from mpl_toolkits.mplot3d import Axes3D num_vals = self.matrix.shape[0] * self.matrix.shape[1] if num_vals < 3: io.error("Cannot create trisurf of", num_vals, "values") return X = np.zeros((num_vals,)) Y = np.zeros((num_vals,)) Z = np.zeros((num_vals,)) # Iterate over every point in space. for j,i in product(range(self.matrix.shape[0]), range(self.matrix.shape[1])): # Convert point to list index. index = j * self.matrix.shape[1] + i X[index] = i Y[index] = j Z[index] = self.matrix[j][i] fig = plt.figure() ax = fig.add_subplot(111, projection='3d') ax.plot_trisurf(X, Y, Z, cmap=cm.jet, **kwargs) # Set X axis labels xticks = [] xticklabels = [] for i,c in enumerate(self.c): if not len(xticks) or c % 20 == 0: xticks.append(i) xticklabels.append(c) ax.set_xticks(xticks) ax.set_xticklabels(xticklabels) ax.set_xlabel("$w_c$") # Set Y axis labels yticks = [] yticklabels = [] for i,c in enumerate(self.c): if not len(yticks) or c % 20 == 0: yticks.append(i) yticklabels.append(c) ax.set_yticks(yticks) ax.set_yticklabels(yticklabels) ax.set_ylabel("$w_r$") # Set Z axis labels if zlabel is not None: ax.set_zlabel(zlabel) if zticklabels is not None: ax.set_zticks(np.arange(len(zticklabels))) ax.set_zticklabels(zticklabels) # Set plot rotation. if rotation is not None: ax.view_init(azim=rotation) # Set plot title. if title: plt.title(title) plt.tight_layout() plt.gcf().set_size_inches(*figsize, dpi=300) viz.finalise(output)