Esempio n. 1
0
def migrate_3_to_4(old):
    """
    SkelCL database migration script.

    Arguments:

        old (SkelCLDatabase): The database to migrate
    """
    # Create temporary database
    fs.rm("/tmp/omnitune.skelcl.migration.db")
    tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
    tmp.attach(old.path, "rhs")

    io.info("Migrating database to version 4.")

    backup_path = old.path + ".3"
    io.info("Creating backup of old database at '{0}'".format(backup_path))
    fs.cp(old.path, backup_path)

    tables = [
        "kernels", "kernel_lookup", "kernel_names", "devices", "device_lookup",
        "datasets", "dataset_lookup", "scenarios", "params", "runtimes",
        "runtime_stats", "oracle_params"
    ]

    for table in tables:
        io.info("Copying data from '{}' ...".format(table))
        tmp.execute("INSERT INTO {} SELECT * FROM rhs.{}".format(table, table))

    tmp_path = tmp.path
    old_path = old.path

    tmp.execute("VACUUM")

    # Sanity checks
    bad = False
    for table in tables:
        old_count = tmp.num_rows("rhs." + table)
        tmp_count = tmp.num_rows(table)

        if old_count != tmp_count:
            io.error("Bad rows count:", old_count, tmp_count)
            bad = True

    if bad:
        io.fatal("Failed sanity check, aborting.")
    else:
        io.info("Passed sanity check.")

    # Copy migrated database over the original one.
    fs.cp(tmp_path, old_path)
    fs.rm(tmp_path)

    old.close()
    tmp.close()
    io.info("Migration completed.")
Esempio n. 2
0
def migrate_5_to_6(db):
    """
    SkelCL database migration script.

    Database version 5 adds an additional "param_stats" table.

    Arguments:

        old (SkelCLDatabase): The database to migrate
    """
    io.info("Migrating database to version 6.")

    backup_path = db.path + ".5"
    io.info("Creating backup of old database at '{0}'".format(backup_path))
    fs.cp(db.path, backup_path)

    db.execute("DELETE FROM version")
    db.execute("INSERT INTO version VALUES (6)")

    db.execute("""
CREATE TABLE IF NOT EXISTS scenario_stats (
    scenario                        CHAR(40),     -- Key for scenarios
    num_params                      INTEGER,      -- The number of parameters in W_legal for scenario
    oracle_param                    VARCHAR(255), -- The best parameter
    oracle_runtime                  REAL,         -- The runtime of the best parameter
    worst_param                     VARCHAR(255), -- The worst parameter
    worst_runtime                   REAL,         -- The runtime of the worst parameter
    mean_runtime                    REAL,         -- The mean runtime of all parameters
    PRIMARY KEY (scenario)
)
""")

    db.populate_scenario_stats_table()

    # Sanity checks
    bad = False
    if db.num_rows("scenario_stats") != len(db.scenarios):
        io.error("Bad row count in scenario_stats table! Expected",
                 len(db.scenarios), "Observed:", db.num_rows("scenario_stats"))
        bad = True

    if bad:
        io.fatal("Failed sanity check, aborting.")
    else:
        io.info("Passed sanity check.")

    # Copy migrated database over the original one.
    db.close()
    io.info("Migration completed.")
Esempio n. 3
0
def migrate_5_to_6(db):
  """
  SkelCL database migration script.

  Database version 5 adds an additional "param_stats" table.

  Arguments:

      old (SkelCLDatabase): The database to migrate
  """
  io.info("Migrating database to version 6.")

  backup_path = db.path + ".5"
  io.info("Creating backup of old database at '{0}'".format(backup_path))
  fs.cp(db.path, backup_path)

  db.execute("DELETE FROM version")
  db.execute("INSERT INTO version VALUES (6)")

  db.execute("""
CREATE TABLE IF NOT EXISTS scenario_stats (
    scenario                        CHAR(40),     -- Key for scenarios
    num_params                      INTEGER,      -- The number of parameters in W_legal for scenario
    oracle_param                    VARCHAR(255), -- The best parameter
    oracle_runtime                  REAL,         -- The runtime of the best parameter
    worst_param                     VARCHAR(255), -- The worst parameter
    worst_runtime                   REAL,         -- The runtime of the worst parameter
    mean_runtime                    REAL,         -- The mean runtime of all parameters
    PRIMARY KEY (scenario)
)
""")

  db.populate_scenario_stats_table()

  # Sanity checks
  bad = False
  if db.num_rows("scenario_stats") != len(db.scenarios):
    io.error("Bad row count in scenario_stats table! Expected",
             len(db.scenarios), "Observed:", db.num_rows("scenario_stats"))
    bad = True

  if bad:
    io.fatal("Failed sanity check, aborting.")
  else:
    io.info("Passed sanity check.")

  # Copy migrated database over the original one.
  db.close()
  io.info("Migration completed.")
Esempio n. 4
0
def migrate_4_to_5(db):
    """
    SkelCL database migration script.

    Database version 5 adds an additional "param_stats" table.

    Arguments:

        old (SkelCLDatabase): The database to migrate
    """
    io.info("Migrating database to version 5.")

    backup_path = db.path + ".4"
    io.info("Creating backup of old database at '{0}'".format(backup_path))
    fs.cp(db.path, backup_path)

    db.execute("DELETE FROM version")
    db.execute("INSERT INTO version VALUES (5)")

    db.execute("""
-- Parameter stats table
CREATE TABLE IF NOT EXISTS param_stats (
    params                          VARCHAR(255), -- Key for params
    num_scenarios                   INTEGER,      -- Number of scenarios for which param is legal, 0 < num_scenarios
    coverage                        REAL,         -- num_scenarios / total number of scenarios, 0 < coverage <= 1
    performance                     REAL,         -- Geometric mean of performance relative to the oracle for all scenarios for which param was legal, 0 < performance <= 1
    PRIMARY KEY (params)
)
""")

    db.populate_param_stats_table()

    # Sanity checks
    bad = False
    if db.num_rows("param_stats") != len(db.params):
        io.error("Bad row count in params table! Expected", len(db.params),
                 "Observed:", db.num_rows("param_stats"))
        bad = True

    if bad:
        io.fatal("Failed sanity check, aborting.")
    else:
        io.info("Passed sanity check.")

    # Copy migrated database over the original one.
    db.close()
    io.info("Migration completed.")
Esempio n. 5
0
def migrate_4_to_5(db):
  """
  SkelCL database migration script.

  Database version 5 adds an additional "param_stats" table.

  Arguments:

      old (SkelCLDatabase): The database to migrate
  """
  io.info("Migrating database to version 5.")

  backup_path = db.path + ".4"
  io.info("Creating backup of old database at '{0}'".format(backup_path))
  fs.cp(db.path, backup_path)

  db.execute("DELETE FROM version")
  db.execute("INSERT INTO version VALUES (5)")

  db.execute("""
-- Parameter stats table
CREATE TABLE IF NOT EXISTS param_stats (
    params                          VARCHAR(255), -- Key for params
    num_scenarios                   INTEGER,      -- Number of scenarios for which param is legal, 0 < num_scenarios
    coverage                        REAL,         -- num_scenarios / total number of scenarios, 0 < coverage <= 1
    performance                     REAL,         -- Geometric mean of performance relative to the oracle for all scenarios for which param was legal, 0 < performance <= 1
    PRIMARY KEY (params)
)
""")

  db.populate_param_stats_table()

  # Sanity checks
  bad = False
  if db.num_rows("param_stats") != len(db.params):
    io.error("Bad row count in params table! Expected", len(db.params),
             "Observed:", db.num_rows("param_stats"))
    bad = True

  if bad:
    io.fatal("Failed sanity check, aborting.")
  else:
    io.info("Passed sanity check.")

  # Copy migrated database over the original one.
  db.close()
  io.info("Migration completed.")
Esempio n. 6
0
def merge(old_oracle, dbs, path):
    """
    Merge databases into one.

    Arguments:

        dbs (list of Database): Databases to merge.
        path (str): Path to merged database.

    Returns:

        Database: merged database instance.
    """
    print("Merging {n} databases:".format(n=len(dbs) + 1))
    print("   ", old_oracle)
    for db in dbs:
        print("   ", db)
    print()

    # Make a copy of the old oracle database to work from.
    io.info("Coping", old_oracle, "->", fs.basename(path))
    fs.cp(old_oracle, path)

    target = migrate(_db.Database(path=path))

    for db in dbs + [target]:
        try:
            db.num_rows("runtimes")
        except sqlite3.DatabaseError as e:
            io.error("Broken db:", db.path)
            io.fatal(e)

    num_runtimes = [db.num_rows("runtimes") for db in dbs]
    expected_total = target.num_rows("runtimes") + sum(num_runtimes)

    target.merge(dbs)

    total = target.num_rows("runtimes")

    if total != expected_total:
        io.fatal("Expected total", expected_total,
                 "!= actual total", total)

    io.info(("Merged {num_db} databases, {n} rows"
             .format(num_db=len(dbs), n=total)))

    return target
Esempio n. 7
0
def merge(old_oracle, dbs, path):
    """
    Merge databases into one.

    Arguments:

        dbs (list of Database): Databases to merge.
        path (str): Path to merged database.

    Returns:

        Database: merged database instance.
    """
    print("Merging {n} databases:".format(n=len(dbs) + 1))
    print("   ", old_oracle)
    for db in dbs:
        print("   ", db)
    print()

    # Make a copy of the old oracle database to work from.
    io.info("Coping", old_oracle, "->", fs.basename(path))
    fs.cp(old_oracle, path)

    target = migrate(_db.Database(path=path))

    for db in dbs + [target]:
        try:
            db.num_rows("runtimes")
        except sqlite3.DatabaseError as e:
            io.error("Broken db:", db.path)
            io.fatal(e)

    num_runtimes = [db.num_rows("runtimes") for db in dbs]
    expected_total = target.num_rows("runtimes") + sum(num_runtimes)

    target.merge(dbs)

    total = target.num_rows("runtimes")

    if total != expected_total:
        io.fatal("Expected total", expected_total, "!= actual total", total)

    io.info(("Merged {num_db} databases, {n} rows".format(num_db=len(dbs),
                                                          n=total)))

    return target
Esempio n. 8
0
def import_foreign(name, custom_name=None):
    """
    Import a module with a custom name.

    NOTE this is only needed for Python2. For Python3, import the
    module using the "as" keyword to declare the custom name.

    For implementation details, see:
    http://stackoverflow.com/a/6032023

    Example:

      To import the standard module "math" as "std_math":

          if labm8.is_python3():
            import math as std_math
          else:
            std_math = modules.import_foreign("math", "std_math")

    Arguments:

        name (str): The name of the module to import.
        custom_name (str, optional): The custom name to assign the module to.

    Raises:
        ImportError: If the module is not found.
    """
    if lab.is_python3():
        io.error(("Ignoring attempt to import foreign module '{mod}' "
                  "using python version {major}.{minor}"
                  .format(mod=name, major=sys.version_info[0],
                          minor=sys.version_info[1])))
        return

    custom_name = custom_name or name

    f, pathname, desc = imp.find_module(name, sys.path[1:])
    module = imp.load_module(custom_name, f, pathname, desc)
    f.close()

    return module
Esempio n. 9
0
def migrate_3_to_4(old):
    """
    SkelCL database migration script.

    Arguments:

        old (SkelCLDatabase): The database to migrate
    """
    # Create temporary database
    fs.rm("/tmp/omnitune.skelcl.migration.db")
    tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
    tmp.attach(old.path, "rhs")

    io.info("Migrating database to version 4.")

    backup_path = old.path + ".3"
    io.info("Creating backup of old database at '{0}'".format(backup_path))
    fs.cp(old.path, backup_path)

    tables = [
        "kernels",
        "kernel_lookup",
        "kernel_names",
        "devices",
        "device_lookup",
        "datasets",
        "dataset_lookup",
        "scenarios",
        "params",
        "runtimes",
        "runtime_stats",
        "oracle_params"
    ]

    for table in tables:
        io.info("Copying data from '{}' ...".format(table))
        tmp.execute("INSERT INTO {} SELECT * FROM rhs.{}".format(table, table))

    tmp_path = tmp.path
    old_path = old.path

    tmp.execute("VACUUM")

    # Sanity checks
    bad = False
    for table in tables:
        old_count = tmp.num_rows("rhs." + table)
        tmp_count = tmp.num_rows(table)

        if old_count != tmp_count:
            io.error("Bad rows count:", old_count, tmp_count)
            bad = True

    if bad:
        io.fatal("Failed sanity check, aborting.")
    else:
        io.info("Passed sanity check.")

    # Copy migrated database over the original one.
    fs.cp(tmp_path, old_path)
    fs.rm(tmp_path)

    old.close()
    tmp.close()
    io.info("Migration completed.")
Esempio n. 10
0
def migrate_2_to_3(old):
    """
    SkelCL database migration script.

    Arguments:

        old (SkelCLDatabase): The database to migrate
    """
    def _old_kernel2new(old_id):
        kernel = old.execute("SELECT north,south,east,west,max_wg_size,source "
                             "FROM kernels WHERE id=?",
                             (old_id,)).fetchone()
        if kernel:
            return tmp.kernel_id(*kernel)

    def _old_scenario2new(old_id):
        device, old_kernel, dataset = old.execute("SELECT device,kernel,dataset "
                                                  "FROM scenarios WHERE id=?",
                                                  (old_id,)).fetchone()
        kernel = _old_kernel2new(old_kernel)
        return tmp.scenario_id(device, kernel, dataset)

    # TODO: Un-comment out code!

    # Create temporary database
    fs.rm("/tmp/omnitune.skelcl.migration.db")
    tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
    tmp.attach(old.path, "rhs")

    io.info("Migrating database to version 3.")

    backup_path = old.path + ".2"
    io.info("Creating backup of old database at '{0}'".format(backup_path))
    fs.cp(old.path, backup_path)

    tmp_path = tmp.path
    old_path = old.path

    tmp.run("create_tables")

    # Populate feature and lookup tables.
    for row in old.execute("SELECT * FROM devices"):
        features = row[1:]
        id = hash_device(*features)
        io.debug("Features extracted for device", id)
        row = (id,) + features
        tmp.execute("INSERT INTO devices VALUES " +
                    placeholders(*row), row)

        row = (features[0], features[1], id)
        tmp.execute("INSERT INTO device_lookup VALUES " +
                    placeholders(*row), row)
        tmp.commit()

    for row in old.execute("SELECT * FROM kernels"):
        args = row[1:]
        tmp.kernel_id(*args)

    for row in old.execute("SELECT * FROM datasets"):
        features = row[1:]
        id = hash_dataset(*features)
        io.debug("Features extracted for dataset", id)
        row = (id,) + features
        tmp.execute("INSERT INTO datasets VALUES " +
                    placeholders(*row), row)

        row = features + (id,)
        tmp.execute("INSERT INTO dataset_lookup VALUES " +
                    placeholders(*row), row)
        tmp.commit()

    # Populate kernel_names table.
    for row in old.execute("SELECT * FROM kernel_names"):
        old_id = row[0]
        synthetic, name = row[1:]

        kernel = _old_kernel2new(old_id)
        if kernel:
            row = (kernel, synthetic, name)
            tmp.execute("INSERT OR IGNORE INTO kernel_names VALUES " +
                        placeholders(*row), row)
    tmp.commit()

    # Populate scenarios table.
    for row in old.execute("SELECT * FROM scenarios"):
        old_id, _, device, old_kernel, dataset = row
        kernel = _old_kernel2new(old_kernel)
        new_id = hash_scenario(device, kernel, dataset)

        row = (new_id, device, kernel, dataset)
        tmp.execute("INSERT OR IGNORE INTO scenarios VALUES " +
                    placeholders(*row), row)
    tmp.commit()

    # Populate params table.
    tmp.execute("INSERT INTO params SELECT * from rhs.params")
    tmp.commit()

    scenario_replacements = {
        row[0]: _old_scenario2new(row[0])
        for row in old.execute("SELECT * FROM scenarios")
    }

    tmp.execute("INSERT INTO runtimes SELECT * from rhs.runtimes")
    for old_id, new_id in scenario_replacements.iteritems():
        io.info("Runtimes", old_id, "->", new_id)
        tmp.execute("UPDATE runtimes SET scenario=? WHERE scenario=?",
                    (new_id, old_id))
    tmp.commit()

    # Sanity checks
    bad = False
    for row in tmp.execute("SELECT DISTINCT scenario FROM runtimes"):
        count = tmp.execute("SELECT Count(*) FROM scenarios WHERE id=?",
                            (row[0],)).fetchone()[0]
        if count != 1:
            io.error("Bad scenario count:", row[0], count)
            bad = True

    if bad:
        io.fatal("Failed sanity check, aborting.")
    else:
        io.info("Passed sanity check.")

    # Copy migrated database over the original one.
    fs.cp(tmp_path, old_path)
    fs.rm(tmp_path)

    old.close()
    tmp.close()
    io.info("Migration completed.")
Esempio n. 11
0
def test_error():
    out = StringIO()
    io.error("foo", file=out)
    assert "ERROR" == re.search("ERROR", out.getvalue()).group(0)
Esempio n. 12
0
def migrate_2_to_3(old):
  """
  SkelCL database migration script.

  Arguments:

      old (SkelCLDatabase): The database to migrate
  """

  def _old_kernel2new(old_id):
    kernel = old.execute("SELECT north,south,east,west,max_wg_size,source "
                         "FROM kernels WHERE id=?",
                         (old_id,)).fetchone()
    if kernel:
      return tmp.kernel_id(*kernel)

  def _old_scenario2new(old_id):
    device, old_kernel, dataset = old.execute("SELECT device,kernel,dataset "
                                              "FROM scenarios WHERE id=?",
                                              (old_id,)).fetchone()
    kernel = _old_kernel2new(old_kernel)
    return tmp.scenario_id(device, kernel, dataset)

  # TODO: Un-comment out code!

  # Create temporary database
  fs.rm("/tmp/omnitune.skelcl.migration.db")
  tmp = _db.Database("/tmp/omnitune.skelcl.migration.db")
  tmp.attach(old.path, "rhs")

  io.info("Migrating database to version 3.")

  backup_path = old.path + ".2"
  io.info("Creating backup of old database at '{0}'".format(backup_path))
  fs.cp(old.path, backup_path)

  tmp_path = tmp.path
  old_path = old.path

  tmp.run("create_tables")

  # Populate feature and lookup tables.
  for row in old.execute("SELECT * FROM devices"):
    features = row[1:]
    id = hash_device(*features)
    io.debug("Features extracted for device", id)
    row = (id,) + features
    tmp.execute("INSERT INTO devices VALUES " +
                placeholders(*row), row)

    row = (features[0], features[1], id)
    tmp.execute("INSERT INTO device_lookup VALUES " +
                placeholders(*row), row)
    tmp.commit()

  for row in old.execute("SELECT * FROM kernels"):
    args = row[1:]
    tmp.kernel_id(*args)

  for row in old.execute("SELECT * FROM datasets"):
    features = row[1:]
    id = hash_dataset(*features)
    io.debug("Features extracted for dataset", id)
    row = (id,) + features
    tmp.execute("INSERT INTO datasets VALUES " +
                placeholders(*row), row)

    row = features + (id,)
    tmp.execute("INSERT INTO dataset_lookup VALUES " +
                placeholders(*row), row)
    tmp.commit()

  # Populate kernel_names table.
  for row in old.execute("SELECT * FROM kernel_names"):
    old_id = row[0]
    synthetic, name = row[1:]

    kernel = _old_kernel2new(old_id)
    if kernel:
      row = (kernel, synthetic, name)
      tmp.execute("INSERT OR IGNORE INTO kernel_names VALUES " +
                  placeholders(*row), row)
  tmp.commit()

  # Populate scenarios table.
  for row in old.execute("SELECT * FROM scenarios"):
    old_id, _, device, old_kernel, dataset = row
    kernel = _old_kernel2new(old_kernel)
    new_id = hash_scenario(device, kernel, dataset)

    row = (new_id, device, kernel, dataset)
    tmp.execute("INSERT OR IGNORE INTO scenarios VALUES " +
                placeholders(*row), row)
  tmp.commit()

  # Populate params table.
  tmp.execute("INSERT INTO params SELECT * from rhs.params")
  tmp.commit()

  scenario_replacements = {
    row[0]: _old_scenario2new(row[0])
    for row in old.execute("SELECT * FROM scenarios")
  }

  tmp.execute("INSERT INTO runtimes SELECT * from rhs.runtimes")
  for old_id, new_id in scenario_replacements.iteritems():
    io.info("Runtimes", old_id, "->", new_id)
    tmp.execute("UPDATE runtimes SET scenario=? WHERE scenario=?",
                (new_id, old_id))
  tmp.commit()

  # Sanity checks
  bad = False
  for row in tmp.execute("SELECT DISTINCT scenario FROM runtimes"):
    count = tmp.execute("SELECT Count(*) FROM scenarios WHERE id=?",
                        (row[0],)).fetchone()[0]
    if count != 1:
      io.error("Bad scenario count:", row[0], count)
      bad = True

  if bad:
    io.fatal("Failed sanity check, aborting.")
  else:
    io.info("Passed sanity check.")

  # Copy migrated database over the original one.
  fs.cp(tmp_path, old_path)
  fs.rm(tmp_path)

  old.close()
  tmp.close()
  io.info("Migration completed.")
Esempio n. 13
0
 def test_error(self):
     out = StringIO()
     io.error("foo", file=out)
     self._test("ERROR", re.search("ERROR", out.getvalue()).group(0))
Esempio n. 14
0
    def trisurf(self, output=None, title=None, figsize=(5,4), zlabel=None,
                zticklabels=None, rotation=None, **kwargs):
        import matplotlib.pyplot as plt
        import matplotlib.cm as cm
        from mpl_toolkits.mplot3d import Axes3D

        num_vals = self.matrix.shape[0] * self.matrix.shape[1]
        if num_vals < 3:
            io.error("Cannot create trisurf of", num_vals, "values")
            return

        X = np.zeros((num_vals,))
        Y = np.zeros((num_vals,))
        Z = np.zeros((num_vals,))

        # Iterate over every point in space.
        for j,i in product(range(self.matrix.shape[0]),
                           range(self.matrix.shape[1])):
            # Convert point to list index.
            index = j * self.matrix.shape[1] + i
            X[index] = i
            Y[index] = j
            Z[index] = self.matrix[j][i]

        fig = plt.figure()
        ax = fig.add_subplot(111, projection='3d')
        ax.plot_trisurf(X, Y, Z, cmap=cm.jet, **kwargs)

        # Set X axis labels
        xticks = []
        xticklabels = []
        for i,c in enumerate(self.c):
            if not len(xticks) or c % 20 == 0:
                xticks.append(i)
                xticklabels.append(c)
        ax.set_xticks(xticks)
        ax.set_xticklabels(xticklabels)
        ax.set_xlabel("$w_c$")

        # Set Y axis labels
        yticks = []
        yticklabels = []
        for i,c in enumerate(self.c):
            if not len(yticks) or c % 20 == 0:
                yticks.append(i)
                yticklabels.append(c)
        ax.set_yticks(yticks)
        ax.set_yticklabels(yticklabels)
        ax.set_ylabel("$w_r$")

        # Set Z axis labels
        if zlabel is not None:
            ax.set_zlabel(zlabel)
        if zticklabels is not None:
            ax.set_zticks(np.arange(len(zticklabels)))
            ax.set_zticklabels(zticklabels)

        # Set plot rotation.
        if rotation is not None: ax.view_init(azim=rotation)
        # Set plot title.
        if title: plt.title(title)
        plt.tight_layout()
        plt.gcf().set_size_inches(*figsize, dpi=300)
        viz.finalise(output)