Exemplo n.º 1
0
    def EndEpisodeStep(self) -> random_opt_pb2.DelayedRewardStep:
        start_ms = labdate.MillisecondsTimestamp()
        step = random_opt_pb2.DelayedRewardStep(start_time_epoch_ms=start_ms, )
        try:
            clang.Compile([self.working_bytecode_path],
                          self.binary_path,
                          copts=['-O0'])
            try:
                runtimes = self.GetRuntimes()
                self.episodes[-1].binary_runtime_ms.extend(runtimes)
                if self.BinaryIsValid():
                    step.reward = self.runtime_reward(
                        sum(runtimes) / len(runtimes))
                else:
                    self.episodes[-1].outcome = (
                        random_opt_pb2.DelayedRewardEpisode.EVAL_FAILED)
                    step.reward = self.eval_failed_reward
            except ValueError as e:
                self.episodes[-1].outcome = random_opt_pb2.Step.EXEC_FAILED
                self.episodes[-1].outcome_error_msg = text.truncate(
                    str(e), 255)
                step.reward = self.exec_failed_reward
        except clang.ClangException as e:
            self.episodes[-1].outcome = (
                random_opt_pb2.DelayedRewardEpisode.COMPILE_FAILED)
            self.episodes[-1].outcome_error_msg = text.truncate(str(e), 255)
            step.reward = self.compile_failed_reward

        obs = self.observation_space.sample()
        step.total_step_runtime_ms = labdate.MillisecondsTimestamp() - start_ms
        self.episodes[-1].step.extend([step])
        return obs, step.reward, True, {}
Exemplo n.º 2
0
 def test_truncate(self):
     self._test("foo", text.truncate("foo", 100))
     self._test("1234567890", text.truncate("1234567890", 10))
     self._test("12345...", text.truncate("1234567890", 8))
     for i in range(10, 20):
         self._test(i,
                    len(text.truncate("The quick brown fox jumped "
                                      "over the slow lazy dog", i)))
Exemplo n.º 3
0
def test_truncate():
    assert "foo" == text.truncate("foo", 100)
    assert "1234567890" == text.truncate("1234567890", 10)
    assert "12345..." == text.truncate("1234567890", 8)
    for i in range(10, 20):
        assert i == len(
            text.truncate(
                "The quick brown fox jumped "
                "over the slow lazy dog", i))
Exemplo n.º 4
0
    def ActionStep(self, action: int) -> random_opt_pb2.DelayedRewardStep:
        if not self.action_space.contains(action):
            raise ValueError(f"Unknown action: '{action}'")
        start_ms = labdate.MillisecondsTimestamp()
        obs = self.observation_space.sample()
        opt_pass = self.config.candidate_pass[action]

        step = random_opt_pb2.DelayedRewardStep(
            start_time_epoch_ms=start_ms,
            opt_pass=opt_pass,
        )

        # Run the full list of passes and update working_bytecode file.
        try:
            all_passes = [step.opt_pass for step in self.episodes[-1].step[1:]]
            opt.RunOptPassOnBytecode(self.bytecode_path,
                                     self.working_dir / 'temp.ll', all_passes)
            step.bytecode_changed = BytecodesAreEqual(
                self.working_dir / 'temp.ll', self.working_bytecode_path)
            shutil.copyfile(self.working_dir / 'temp.ll',
                            self.working_bytecode_path)
            step.reward = (self.bytecode_changed_reward
                           if step.bytecode_changed else
                           self.bytecode_unchanged_reward)
        except llvm.LlvmError as e:
            # Opt failed, set the error message.
            step.reward = self.opt_failed_reward
            step.opt_error_msg = text.truncate(str(e), 255)

        step.total_step_runtime_ms = labdate.MillisecondsTimestamp() - start_ms
        self.episodes[-1].step.extend([step])
        return obs, step.reward, False, {}
Exemplo n.º 5
0
def main():
    db = _db.Database(experiment.ORACLE_PATH)
    ml.start()

    # Delete any old stuff.
    fs.rm(experiment.IMG_ROOT + "/*")
    fs.rm(experiment.TAB_ROOT + "/*")

    # Make directories
    fs.mkdir(experiment.TAB_ROOT)
    fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/bars"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/heatmap"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/trisurf"))

    fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/devices"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/kernels"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/datasets"))

    fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/devices"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/kernels"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/datasets"))

    fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/devices"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/kernels"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/datasets"))

    visualise.pie(db.num_scenarios_by_device,
                  fs.path(experiment.IMG_ROOT, "num_sceanrios_by_device"))
    visualise.pie(db.num_runtime_stats_by_device,
                  fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_device"))
    visualise.pie(db.num_scenarios_by_dataset,
                  fs.path(experiment.IMG_ROOT, "num_sceanrios_by_dataset"))
    visualise.pie(db.num_runtime_stats_by_dataset,
                  fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_dataset"))
    visualise.pie(db.num_runtime_stats_by_kernel,
                  fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel"))
    visualise.pie(db.num_runtime_stats_by_kernel,
                  fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel"))

    # Per-scenario plots
    for row in db.scenario_properties:
        scenario,device,kernel,north,south,east,west,max_wgsize,width,height,tout = row
        title = ("{device}: {kernel}[{n},{s},{e},{w}]\n"
                 "{width} x {height} {type}s"
                 .format(device=text.truncate(device, 18), kernel=kernel,
                         n=north, s=south, e=east, w=west,
                         width=width, height=height, type=tout))
        output = fs.path(experiment.IMG_ROOT,
                         "scenarios/heatmap/{id}.png".format(id=scenario))
        space = _space.ParamSpace.from_dict(db.perf_scenario(scenario))
        max_c = min(25, len(space.c))
        max_r = min(25, len(space.r))
        space.reshape(max_c=max_c, max_r=max_r)

        # Heatmaps.
        mask = _space.ParamSpace(space.c, space.r)
        for j in range(len(mask.r)):
            for i in range(len(mask.c)):
                if space.matrix[j][i] == 0:
                    r, c = space.r[j], space.c[i]
                    # TODO: Get values from refused_params table.
                    if r * c >= max_wgsize:
                        # Illegal
                        mask.matrix[j][i] = -1
                    else:
                        # Refused
                        db.execute("INSERT OR IGNORE INTO refused_params VALUES(?,?)",
                                   (scenario, hash_params(c, r)))
                        space.matrix[j][i] = -1
                        mask.matrix[j][i] = 1

        db.commit()
        new_order = list(reversed(range(space.matrix.shape[0])))
        data = space.matrix[:][new_order]

        figsize=(12,6)

        _, ax = plt.subplots(1, 2, figsize=figsize, sharey=True)
        sns.heatmap(data, ax=ax[0], vmin=-1, vmax=1,
                    xticklabels=space.c,
                    yticklabels=list(reversed(space.r)), square=True)

        ax[0].set_title(title)

        new_order = list(reversed(range(mask.matrix.shape[0])))
        data = mask.matrix[:][new_order]

        sns.heatmap(data, ax=ax[1], vmin=-1, vmax=1,
                    xticklabels=space.c,
                    yticklabels=list(reversed(space.r)), square=True)

        # Set labels.
        ax[0].set_ylabel("Rows")
        ax[0].set_xlabel("Columns")
        ax[1].set_ylabel("Rows")
        ax[1].set_xlabel("Columns")

        # plt.tight_layout()
        # plt.gcf().set_size_inches(*figsize, dpi=300)

        viz.finalise(output)

        # 3D bars.
        output = fs.path(experiment.IMG_ROOT,
                         "scenarios/bars/{id}.png".format(id=scenario))
        space.bar3d(output=output, title=title, zlabel="Performance",
                    rotation=45)

        # Trisurfs.
        output = fs.path(experiment.IMG_ROOT,
                         "scenarios/trisurf/{id}.png".format(id=scenario))
        space.trisurf(output=output, title=title, zlabel="Performance",
                      rotation=45)

    #####################
    # ML Visualisations #
    #####################
    #features_tab(db, experiment.TAB_ROOT)

    visualise_classification_job(db, "xval")
    visualise_classification_job(db, "arch")
    visualise_classification_job(db, "xval_real")
    visualise_classification_job(db, "synthetic_real")

    # Runtime regression accuracy.
    visualise_regression_job(db, "xval")
    visualise_regression_job(db, "arch")
    visualise_regression_job(db, "xval_real")
    visualise_regression_job(db, "synthetic_real")

    # Whole-dataset plots
    visualise.runtimes_variance(db, fs.path(experiment.IMG_ROOT,
                                            "runtime_variance.png"),
                                min_samples=30)
    visualise.num_samples(db, fs.path(experiment.IMG_ROOT,
                                      "num_samples.png"))
    visualise.runtimes_range(db, fs.path(experiment.IMG_ROOT,
                                         "runtimes_range.png"))
    visualise.max_speedups(db, fs.path(experiment.IMG_ROOT,
                                       "max_speedups.png"))
    visualise.kernel_performance(db, fs.path(experiment.IMG_ROOT,
                                             "kernel_performance.png"))
    visualise.device_performance(db, fs.path(experiment.IMG_ROOT,
                                             "device_performance.png"))
    visualise.dataset_performance(db, fs.path(experiment.IMG_ROOT,
                                              "dataset_performance.png"))
    visualise.num_params_vs_accuracy(db, fs.path(experiment.IMG_ROOT,
                                                 "num_params_vs_accuracy.png"))
    visualise.performance_vs_coverage(db,
                                      fs.path(experiment.IMG_ROOT,
                                              "performance_vs_coverage.png"))
    visualise.performance_vs_max_wgsize(
        db, fs.path(experiment.IMG_ROOT, "performance_vs_max_wgsize.png")
    )
    visualise.performance_vs_wgsize(db, fs.path(experiment.IMG_ROOT,
                                                "performance_vs_wgsize.png"))
    visualise.performance_vs_wg_c(db, fs.path(experiment.IMG_ROOT,
                                              "performance_vs_wg_c.png"))
    visualise.performance_vs_wg_r(db, fs.path(experiment.IMG_ROOT,
                                              "performance_vs_wg_r.png"))
    visualise.max_wgsizes(db, fs.path(experiment.IMG_ROOT, "max_wgsizes.png"))
    visualise.oracle_speedups(db, fs.path(experiment.IMG_ROOT,
                                          "oracle_speedups.png"))

    visualise.coverage(db,
                       fs.path(experiment.IMG_ROOT, "coverage/coverage.png"))
    visualise.safety(db, fs.path(experiment.IMG_ROOT, "safety/safety.png"))
    visualise.oracle_wgsizes(db, fs.path(experiment.IMG_ROOT, "oracle/all.png"))

    # Per-device plots
    for i,device in enumerate(db.devices):
        where = ("scenario IN "
                 "(SELECT id from scenarios WHERE device='{0}')"
                 .format(device))
        output = fs.path(experiment.IMG_ROOT,
                         "coverage/devices/{0}.png".format(i))
        visualise.coverage(db, output=output, where=where, title=device)
        output = fs.path(experiment.IMG_ROOT,
                         "safety/devices/{0}.png".format(i))
        visualise.safety(db, output, where=where, title=device)
        output = fs.path(experiment.IMG_ROOT,
                         "oracle/devices/{0}.png".format(i))
        visualise.oracle_wgsizes(db, output, where=where, title=device)

        where = ("scenario IN (\n"
                 "    SELECT id from scenarios WHERE device='{0}'\n"
                 ") AND scenario IN (\n"
                 "    SELECT id FROM scenarios WHERE kernel IN (\n"
                 "        SELECT id FROM kernel_names WHERE synthetic=0\n"
                 "    )\n"
                 ")"
                 .format(device))
        output = fs.path(experiment.IMG_ROOT,
                         "coverage/devices/{0}_real.png".format(i))
        visualise.coverage(db, output=output, where=where,
                           title=device + ", real")
        output = fs.path(experiment.IMG_ROOT,
                         "safety/devices/{0}_real.png".format(i))
        visualise.safety(db, output, where=where,
                         title=device + ", real")
        output = fs.path(experiment.IMG_ROOT,
                         "oracle/devices/{0}_real.png".format(i))
        visualise.oracle_wgsizes(db, output, where=where,
                                 title=device + ", real")


        where = ("scenario IN (\n"
                 "    SELECT id from scenarios WHERE device='{0}'\n"
                 ") AND scenario IN (\n"
                 "    SELECT id FROM scenarios WHERE kernel IN (\n"
                 "        SELECT id FROM kernel_names WHERE synthetic=1\n"
                 "    )\n"
                 ")"
                 .format(device))
        output = fs.path(experiment.IMG_ROOT,
                         "coverage/devices/{0}_synthetic.png".format(i))
        visualise.coverage(db, output=output, where=where,
                           title=device + ", synthetic")
        output = fs.path(experiment.IMG_ROOT,
                         "safety/devices/{0}_synthetic.png".format(i))
        visualise.safety(db, output, where=where,
                         title=device + ", synthetic")
        output = fs.path(experiment.IMG_ROOT,
                         "oracle/devices/{0}_synthetic.png".format(i))
        visualise.oracle_wgsizes(db, output, where=where,
                                 title=device + ", synthetic")

    # Per-kernel plots
    for kernel,ids in db.lookup_named_kernels().iteritems():
        id_wrapped = ['"' + id + '"' for id in ids]
        where = ("scenario IN "
                 "(SELECT id from scenarios WHERE kernel IN ({0}))"
                 .format(",".join(id_wrapped)))
        output = fs.path(experiment.IMG_ROOT,
                         "coverage/kernels/{0}.png".format(kernel))
        visualise.coverage(db, output=output, where=where, title=kernel)
        output = fs.path(experiment.IMG_ROOT,
                         "safety/kernels/{0}.png".format(kernel))
        visualise.safety(db, output=output, where=where, title=kernel)
        output = fs.path(experiment.IMG_ROOT,
                         "oracle/kernels/{0}.png".format(kernel))
        visualise.safety(db, output=output, where=where, title=kernel)

    # Per-dataset plots
    for i,dataset in enumerate(db.datasets):
        where = ("scenario IN "
                 "(SELECT id from scenarios WHERE dataset='{0}')"
                 .format(dataset))
        output = fs.path(experiment.IMG_ROOT,
                         "coverage/datasets/{0}.png".format(i))
        visualise.coverage(db, output, where=where, title=dataset)
        output = fs.path(experiment.IMG_ROOT,
                         "safety/datasets/{0}.png".format(i))
        visualise.safety(db, output, where=where, title=dataset)
        output = fs.path(experiment.IMG_ROOT,
                         "oracle/datasets/{0}.png".format(i))
        visualise.safety(db, output, where=where, title=dataset)

    ml.stop()
Exemplo n.º 6
0
 def test_truncate_bad_maxchar(self):
     with self.assertRaises(text.TruncateError):
         text.truncate("foo", -1)
         text.truncate("foo", 3)
Exemplo n.º 7
0
def test_truncate_bad_maxchar():
    with pytest.raises(text.TruncateError):
        text.truncate("foo", -1)
        text.truncate("foo", 3)
Exemplo n.º 8
0
    def Step(self, step: random_opt_pb2.Step) -> random_opt_pb2.Step:
        """Run a step.

    Args:
      step: A step proto with field 'opt_pass' set.

    Returns:
      The input step.
    """
        start_time = labdate.MillisecondsTimestamp()
        step.start_time_epoch_ms = start_time
        step.status = random_opt_pb2.Step.PASS
        temp_bytecode = self.working_dir / 'temp_src.ll'
        temp_binary = self.working_dir / 'temp_binary'

        # Run the pass.
        try:
            opt.RunOptPassOnBytecode(self.working_bytecode_path, temp_bytecode,
                                     list(step.opt_pass))
        except llvm.LlvmError as e:
            step.status = random_opt_pb2.Step.OPT_FAILED
            step.status_msg = text.truncate(str(e), 255)

        if step.status == random_opt_pb2.Step.PASS:
            # Update bytecode file.
            logging.debug('$ mv %s %s', temp_bytecode,
                          self.working_bytecode_path)
            step.bytecode_changed = BytecodesAreEqual(
                temp_bytecode, self.working_bytecode_path)
            os.rename(str(temp_bytecode), str(self.working_bytecode_path))
            # Compile a new binary.
            try:
                clang.Compile([self.working_bytecode_path],
                              temp_binary,
                              copts=['-O0'])
                step.binary_changed = BinariesAreEqual(temp_binary,
                                                       self.binary_path)
                os.rename(str(temp_binary), str(self.binary_path))
            except llvm.LlvmError as e:
                step.status = random_opt_pb2.Step.COMPILE_FAILED
                step.status_msg = text.truncate(str(e), 255)

        if step.status == random_opt_pb2.Step.PASS:
            # Get the binary runtime.
            try:
                step.binary_runtime_ms.extend(self.GetRuntimes())
            except ValueError as e:
                step.status = random_opt_pb2.Step.EXEC_FAILED
                step.status_msg = text.truncate(str(e), 255)

        if step.status == random_opt_pb2.Step.PASS:
            if self.BinaryIsValid():
                step.speedup = (
                    (sum(self.episodes[-1].step[-1].binary_runtime_ms) /
                     len(self.episodes[-1].step[-1].binary_runtime_ms)) /
                    (sum(step.binary_runtime_ms) /
                     len(step.binary_runtime_ms)))
                step.total_speedup = (
                    (sum(self.episodes[-1].step[0].binary_runtime_ms) /
                     len(self.episodes[-1].step[0].binary_runtime_ms)) /
                    (sum(step.binary_runtime_ms) /
                     len(step.binary_runtime_ms)))
            else:
                step.status = random_opt_pb2.Step.EVAL_FAILED

        step.reward = self.Reward(step.status, step.speedup)
        step.total_reward = self.episodes[-1].step[
            -1].total_reward + step.reward
        step.total_step_runtime_ms = labdate.MillisecondsTimestamp(
        ) - start_time
        return step
Exemplo n.º 9
0
def classification(db, output=None, job="xval", **kwargs):
    err_fns = db.err_fns
    base_err_fn = err_fns[0]
    # Get a list of classifiers and result counts.
    query = db.execute(
        "SELECT classifier,Count(*) AS count\n"
        "FROM classification_results\n"
        "WHERE job=? AND err_fn=? AND classifier!='weka.classifiers.rules.ZeroR'\n"
        "GROUP BY classifier",
        (job,base_err_fn)
    )
    results = []

    # Add baseline results.
    baseline = ("4x4")
    correct = db.execute("SELECT Count(*) * 1.0 / 3 FROM classification_results "
                         "WHERE job=? AND actual=?", (job,baseline)).fetchone()[0]
    illegal = 0
    refused = 0
    time = 0
    terr = 0
    speedup = (1, 0)
    perfs = [
        row[1] for row in
        db.execute(
            "SELECT "
            "  DISTINCT runtime_stats.scenario, "
            "  (scenario_stats.oracle_runtime / runtime_stats.mean) * 100 "
            "FROM classification_results "
            "LEFT JOIN runtime_stats "
            "  ON classification_results.scenario=runtime_stats.scenario "
            "LEFT JOIN scenario_stats "
            "  ON classification_results.scenario=scenario_stats.scenario "
            "WHERE job=? and runtime_stats.params=?",
            (job, baseline)
        )
    ]
    perf = (labmath.mean(perfs), labmath.confinterval(perfs, error_only=True))
    results.append(["ZeroR", correct, illegal, refused, time, terr,
                    speedup, speedup, speedup,
                    perf, perf, perf])

    # Get results
    for classifier,count in query:
        basename = ml.classifier_basename(classifier)
        correct, illegal, refused, time, terr = db.execute(
            "SELECT\n"
            "    (SUM(correct) / CAST(? AS FLOAT)) * 100,\n"
            "    (SUM(illegal) / CAST(? AS FLOAT)) * 100,\n"
            "    (SUM(refused) / CAST(? AS FLOAT)) * 100,\n"
            "    AVG(time) + 2.5,\n"
            "    CONFERROR(time, .95) * 1.5\n"
            "FROM classification_results\n"
            "WHERE job=? AND classifier=? AND err_fn=?",
            (count, count, count, job, classifier, base_err_fn)
        ).fetchone()
        # Get a list of mean speedups for each err_fn.
        speedups = [
            db.execute(
                "SELECT\n"
                "    AVG(speedup),\n"
                "    CONFERROR(speedup, .95)\n"
                "FROM classification_results\n"
                "WHERE job=? AND classifier=? AND err_fn=?",
                (job, classifier, err_fn)
            ).fetchone()
            for err_fn in err_fns
        ]
        # Get a list of mean perfs for each err_fn.
        perfs = [
            db.execute(
                "SELECT\n"
                "    AVG(performance) * 100.0,\n"
                "    CONFERROR(performance, .95) * 100.0\n"
                "FROM classification_results\n"
                "WHERE job=? AND classifier=? AND err_fn=?",
                (job, classifier, err_fn)
            ).fetchone()
            for err_fn in err_fns
        ]

        results.append([basename, correct, illegal, refused, time, terr] + speedups + perfs)

    # Zip into lists.
    labels, correct, illegal, refused, time, terr = zip(*[
        (text.truncate(result[0], 40), result[1], result[2],
         result[3], result[4], result[5])
        for result in results
    ])

    X = np.arange(len(labels))

    # PLOT TIMES
    width = .8
    ax = plt.subplot(4, 1, 1)
    ax.bar(X + .1, time, width=width)
    ax.set_xticks(X + .4)
    ax.set_xticklabels(labels)
    ax.set_ylim(0, 10)
    ax.set_ylabel("Classification time (ms)")
    # art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
    # Plot confidence intervals separately so that we can have
    # full control over formatting.
    _,caps,_ = ax.errorbar(X + .5, time,
                           fmt="none", yerr=terr, capsize=3, ecolor="k")
    for cap in caps:
        cap.set_color('k')
        cap.set_markeredgewidth(1)

    # RATIOS
    width = (.8 / 3)
    ax = plt.subplot(4, 1, 2)
    ax.bar(X + .1, illegal, width=width,
           color=sns.color_palette("Reds", 1), label="Illegal")
    ax.bar(X + .1 + width, refused, width=width,
           color=sns.color_palette("Oranges", 1), label="Refused")
    ax.bar(X + .1 + 2 * width, correct, width=width,
           color=sns.color_palette("Blues", 1), label="Accurate")
    ax.set_xticks(X + .4)
    ax.set_ylabel("Ratio")
    ax.set_ylim(0, 35)
    ax.set_xticklabels(labels)
    ax.yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
    art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]

    # Plot speedups.
    ax = plt.subplot(4, 1, 3)
    width = (.8 / 3)
    colors=sns.color_palette("Greens", len(err_fns))
    for i,err_fn in enumerate(db.err_fns):
        pairs = [result[6 + i] for result in results]
        speedups, yerrs = zip(*pairs)
        ax.bar(X + .1 + (i * width), speedups, width=width,
               label=errfn2label(err_fn), color=colors[i])

        # Plot confidence intervals separately so that we can have
        # full control over formatting.
        _,caps,_ = ax.errorbar(X + .1 + (i + .5) * width, speedups,
                               fmt="none", yerr=yerrs, capsize=3, ecolor="k")
        for cap in caps:
            cap.set_color('k')
            cap.set_markeredgewidth(1)
    ax.set_xticks(X + .4)
    ax.set_xticklabels(labels)
    ax.set_ylim(0, 7)
    ax.set_xticks(X + .4, labels)
    ax.set_ylabel("Speedup")
    art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]

    # PERFORMANCE
    colors=sns.color_palette("Blues", len(err_fns))
    width = (.8 / 3)
    ax = plt.subplot(4, 1, 4)
    for i,err_fn in enumerate(db.err_fns):
        pairs = [result[9 + i] for result in results]
        perfs, yerrs = zip(*pairs)
        ax.bar(X + .1 + (i * width), perfs, width=width,
               label=errfn2label(err_fn), color=colors[i])

        # Plot confidence intervals separately so that we can have
        # full control over formatting.
        _,caps,_ = ax.errorbar(X + .1 + (i + .5) * width, perfs,
                               fmt="none", yerr=yerrs, capsize=3, ecolor="k")
        for cap in caps:
            cap.set_color('k')
            cap.set_markeredgewidth(1)
    ax.set_xticks(X + .4)
    ax.yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
    ax.set_xticklabels(labels)
    ax.set_ylim(0, 100)
    ax.set_ylabel("Performance")
    ax.set_xticks(X + .4, labels)


    title = kwargs.pop("title", "Classification results for " + job)
    plt.title(title)

    # Add legend *beneath* plot. To do this, we need to pass some
    # extra arguments to plt.savefig(). See:
    #
    # http://jb-blog.readthedocs.org/en/latest/posts/12-matplotlib-legend-outdide-plot.html
    #
    art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3)]
    viz.finalise(output, additional_artists=art, bbox_inches="tight", **kwargs)
Exemplo n.º 10
0
def classification(db, output=None, job="xval", **kwargs):
    err_fns = db.err_fns
    base_err_fn = err_fns[0]
    # Get a list of classifiers and result counts.
    query = db.execute(
        "SELECT classifier,Count(*) AS count\n"
        "FROM classification_results\n"
        "WHERE job=? AND err_fn=? AND classifier!='weka.classifiers.rules.ZeroR'\n"
        "GROUP BY classifier", (job, base_err_fn))
    results = []

    # Add baseline results.
    baseline = ("4x4")
    correct = db.execute(
        "SELECT Count(*) * 1.0 / 3 FROM classification_results "
        "WHERE job=? AND actual=?", (job, baseline)).fetchone()[0]
    illegal = 0
    refused = 0
    time = 0
    terr = 0
    speedup = (1, 0)
    perfs = [
        row[1] for row in db.execute(
            "SELECT "
            "  DISTINCT runtime_stats.scenario, "
            "  (scenario_stats.oracle_runtime / runtime_stats.mean) * 100 "
            "FROM classification_results "
            "LEFT JOIN runtime_stats "
            "  ON classification_results.scenario=runtime_stats.scenario "
            "LEFT JOIN scenario_stats "
            "  ON classification_results.scenario=scenario_stats.scenario "
            "WHERE job=? and runtime_stats.params=?", (job, baseline))
    ]
    perf = (labmath.mean(perfs), labmath.confinterval(perfs, error_only=True))
    results.append([
        "ZeroR", correct, illegal, refused, time, terr, speedup, speedup,
        speedup, perf, perf, perf
    ])

    # Get results
    for classifier, count in query:
        basename = ml.classifier_basename(classifier)
        correct, illegal, refused, time, terr = db.execute(
            "SELECT\n"
            "    (SUM(correct) / CAST(? AS FLOAT)) * 100,\n"
            "    (SUM(illegal) / CAST(? AS FLOAT)) * 100,\n"
            "    (SUM(refused) / CAST(? AS FLOAT)) * 100,\n"
            "    AVG(time) + 2.5,\n"
            "    CONFERROR(time, .95) * 1.5\n"
            "FROM classification_results\n"
            "WHERE job=? AND classifier=? AND err_fn=?",
            (count, count, count, job, classifier, base_err_fn)).fetchone()
        # Get a list of mean speedups for each err_fn.
        speedups = [
            db.execute(
                "SELECT\n"
                "    AVG(speedup),\n"
                "    CONFERROR(speedup, .95)\n"
                "FROM classification_results\n"
                "WHERE job=? AND classifier=? AND err_fn=?",
                (job, classifier, err_fn)).fetchone() for err_fn in err_fns
        ]
        # Get a list of mean perfs for each err_fn.
        perfs = [
            db.execute(
                "SELECT\n"
                "    AVG(performance) * 100.0,\n"
                "    CONFERROR(performance, .95) * 100.0\n"
                "FROM classification_results\n"
                "WHERE job=? AND classifier=? AND err_fn=?",
                (job, classifier, err_fn)).fetchone() for err_fn in err_fns
        ]

        results.append([basename, correct, illegal, refused, time, terr] +
                       speedups + perfs)

    # Zip into lists.
    labels, correct, illegal, refused, time, terr = zip(
        *[(text.truncate(result[0], 40), result[1], result[2], result[3],
           result[4], result[5]) for result in results])

    X = np.arange(len(labels))

    # PLOT TIMES
    width = .8
    ax = plt.subplot(4, 1, 1)
    ax.bar(X + .1, time, width=width)
    ax.set_xticks(X + .4)
    ax.set_xticklabels(labels)
    ax.set_ylim(0, 10)
    ax.set_ylabel("Classification time (ms)")
    # art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]
    # Plot confidence intervals separately so that we can have
    # full control over formatting.
    _, caps, _ = ax.errorbar(X + .5,
                             time,
                             fmt="none",
                             yerr=terr,
                             capsize=3,
                             ecolor="k")
    for cap in caps:
        cap.set_color('k')
        cap.set_markeredgewidth(1)

    # RATIOS
    width = (.8 / 3)
    ax = plt.subplot(4, 1, 2)
    ax.bar(X + .1,
           illegal,
           width=width,
           color=sns.color_palette("Reds", 1),
           label="Illegal")
    ax.bar(X + .1 + width,
           refused,
           width=width,
           color=sns.color_palette("Oranges", 1),
           label="Refused")
    ax.bar(X + .1 + 2 * width,
           correct,
           width=width,
           color=sns.color_palette("Blues", 1),
           label="Accurate")
    ax.set_xticks(X + .4)
    ax.set_ylabel("Ratio")
    ax.set_ylim(0, 35)
    ax.set_xticklabels(labels)
    ax.yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
    art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]

    # Plot speedups.
    ax = plt.subplot(4, 1, 3)
    width = (.8 / 3)
    colors = sns.color_palette("Greens", len(err_fns))
    for i, err_fn in enumerate(db.err_fns):
        pairs = [result[6 + i] for result in results]
        speedups, yerrs = zip(*pairs)
        ax.bar(X + .1 + (i * width),
               speedups,
               width=width,
               label=errfn2label(err_fn),
               color=colors[i])

        # Plot confidence intervals separately so that we can have
        # full control over formatting.
        _, caps, _ = ax.errorbar(X + .1 + (i + .5) * width,
                                 speedups,
                                 fmt="none",
                                 yerr=yerrs,
                                 capsize=3,
                                 ecolor="k")
        for cap in caps:
            cap.set_color('k')
            cap.set_markeredgewidth(1)
    ax.set_xticks(X + .4)
    ax.set_xticklabels(labels)
    ax.set_ylim(0, 7)
    ax.set_xticks(X + .4, labels)
    ax.set_ylabel("Speedup")
    art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -.1), ncol=3)]

    # PERFORMANCE
    colors = sns.color_palette("Blues", len(err_fns))
    width = (.8 / 3)
    ax = plt.subplot(4, 1, 4)
    for i, err_fn in enumerate(db.err_fns):
        pairs = [result[9 + i] for result in results]
        perfs, yerrs = zip(*pairs)
        ax.bar(X + .1 + (i * width),
               perfs,
               width=width,
               label=errfn2label(err_fn),
               color=colors[i])

        # Plot confidence intervals separately so that we can have
        # full control over formatting.
        _, caps, _ = ax.errorbar(X + .1 + (i + .5) * width,
                                 perfs,
                                 fmt="none",
                                 yerr=yerrs,
                                 capsize=3,
                                 ecolor="k")
        for cap in caps:
            cap.set_color('k')
            cap.set_markeredgewidth(1)
    ax.set_xticks(X + .4)
    ax.yaxis.set_major_formatter(FormatStrFormatter('%d\\%%'))
    ax.set_xticklabels(labels)
    ax.set_ylim(0, 100)
    ax.set_ylabel("Performance")
    ax.set_xticks(X + .4, labels)

    title = kwargs.pop("title", "Classification results for " + job)
    plt.title(title)

    # Add legend *beneath* plot. To do this, we need to pass some
    # extra arguments to plt.savefig(). See:
    #
    # http://jb-blog.readthedocs.org/en/latest/posts/12-matplotlib-legend-outdide-plot.html
    #
    art = [plt.legend(loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3)]
    viz.finalise(output, additional_artists=art, bbox_inches="tight", **kwargs)
Exemplo n.º 11
0
def main():
    db = _db.Database(experiment.ORACLE_PATH)
    ml.start()

    # Delete any old stuff.
    fs.rm(experiment.IMG_ROOT + "/*")
    fs.rm(experiment.TAB_ROOT + "/*")

    # Make directories
    fs.mkdir(experiment.TAB_ROOT)
    fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/bars"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/heatmap"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "scenarios/trisurf"))

    fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/devices"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/kernels"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "coverage/datasets"))

    fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/devices"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/kernels"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "safety/datasets"))

    fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/devices"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/kernels"))
    fs.mkdir(fs.path(experiment.IMG_ROOT, "oracle/datasets"))

    visualise.pie(db.num_scenarios_by_device,
                  fs.path(experiment.IMG_ROOT, "num_sceanrios_by_device"))
    visualise.pie(db.num_runtime_stats_by_device,
                  fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_device"))
    visualise.pie(db.num_scenarios_by_dataset,
                  fs.path(experiment.IMG_ROOT, "num_sceanrios_by_dataset"))
    visualise.pie(db.num_runtime_stats_by_dataset,
                  fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_dataset"))
    visualise.pie(db.num_runtime_stats_by_kernel,
                  fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel"))
    visualise.pie(db.num_runtime_stats_by_kernel,
                  fs.path(experiment.IMG_ROOT, "num_runtime_stats_by_kernel"))

    # Per-scenario plots
    for row in db.scenario_properties:
        scenario, device, kernel, north, south, east, west, max_wgsize, width, height, tout = row
        title = ("{device}: {kernel}[{n},{s},{e},{w}]\n"
                 "{width} x {height} {type}s".format(device=text.truncate(
                     device, 18),
                                                     kernel=kernel,
                                                     n=north,
                                                     s=south,
                                                     e=east,
                                                     w=west,
                                                     width=width,
                                                     height=height,
                                                     type=tout))
        output = fs.path(experiment.IMG_ROOT,
                         "scenarios/heatmap/{id}.png".format(id=scenario))
        space = _space.ParamSpace.from_dict(db.perf_scenario(scenario))
        max_c = min(25, len(space.c))
        max_r = min(25, len(space.r))
        space.reshape(max_c=max_c, max_r=max_r)

        # Heatmaps.
        mask = _space.ParamSpace(space.c, space.r)
        for j in range(len(mask.r)):
            for i in range(len(mask.c)):
                if space.matrix[j][i] == 0:
                    r, c = space.r[j], space.c[i]
                    # TODO: Get values from refused_params table.
                    if r * c >= max_wgsize:
                        # Illegal
                        mask.matrix[j][i] = -1
                    else:
                        # Refused
                        db.execute(
                            "INSERT OR IGNORE INTO refused_params VALUES(?,?)",
                            (scenario, hash_params(c, r)))
                        space.matrix[j][i] = -1
                        mask.matrix[j][i] = 1

        db.commit()
        new_order = list(reversed(range(space.matrix.shape[0])))
        data = space.matrix[:][new_order]

        figsize = (12, 6)

        _, ax = plt.subplots(1, 2, figsize=figsize, sharey=True)
        sns.heatmap(data,
                    ax=ax[0],
                    vmin=-1,
                    vmax=1,
                    xticklabels=space.c,
                    yticklabels=list(reversed(space.r)),
                    square=True)

        ax[0].set_title(title)

        new_order = list(reversed(range(mask.matrix.shape[0])))
        data = mask.matrix[:][new_order]

        sns.heatmap(data,
                    ax=ax[1],
                    vmin=-1,
                    vmax=1,
                    xticklabels=space.c,
                    yticklabels=list(reversed(space.r)),
                    square=True)

        # Set labels.
        ax[0].set_ylabel("Rows")
        ax[0].set_xlabel("Columns")
        ax[1].set_ylabel("Rows")
        ax[1].set_xlabel("Columns")

        # plt.tight_layout()
        # plt.gcf().set_size_inches(*figsize, dpi=300)

        viz.finalise(output)

        # 3D bars.
        output = fs.path(experiment.IMG_ROOT,
                         "scenarios/bars/{id}.png".format(id=scenario))
        space.bar3d(output=output,
                    title=title,
                    zlabel="Performance",
                    rotation=45)

        # Trisurfs.
        output = fs.path(experiment.IMG_ROOT,
                         "scenarios/trisurf/{id}.png".format(id=scenario))
        space.trisurf(output=output,
                      title=title,
                      zlabel="Performance",
                      rotation=45)

    #####################
    # ML Visualisations #
    #####################
    #features_tab(db, experiment.TAB_ROOT)

    visualise_classification_job(db, "xval")
    visualise_classification_job(db, "arch")
    visualise_classification_job(db, "xval_real")
    visualise_classification_job(db, "synthetic_real")

    # Runtime regression accuracy.
    visualise_regression_job(db, "xval")
    visualise_regression_job(db, "arch")
    visualise_regression_job(db, "xval_real")
    visualise_regression_job(db, "synthetic_real")

    # Whole-dataset plots
    visualise.runtimes_variance(db,
                                fs.path(experiment.IMG_ROOT,
                                        "runtime_variance.png"),
                                min_samples=30)
    visualise.num_samples(db, fs.path(experiment.IMG_ROOT, "num_samples.png"))
    visualise.runtimes_range(
        db, fs.path(experiment.IMG_ROOT, "runtimes_range.png"))
    visualise.max_speedups(db, fs.path(experiment.IMG_ROOT,
                                       "max_speedups.png"))
    visualise.kernel_performance(
        db, fs.path(experiment.IMG_ROOT, "kernel_performance.png"))
    visualise.device_performance(
        db, fs.path(experiment.IMG_ROOT, "device_performance.png"))
    visualise.dataset_performance(
        db, fs.path(experiment.IMG_ROOT, "dataset_performance.png"))
    visualise.num_params_vs_accuracy(
        db, fs.path(experiment.IMG_ROOT, "num_params_vs_accuracy.png"))
    visualise.performance_vs_coverage(
        db, fs.path(experiment.IMG_ROOT, "performance_vs_coverage.png"))
    visualise.performance_vs_max_wgsize(
        db, fs.path(experiment.IMG_ROOT, "performance_vs_max_wgsize.png"))
    visualise.performance_vs_wgsize(
        db, fs.path(experiment.IMG_ROOT, "performance_vs_wgsize.png"))
    visualise.performance_vs_wg_c(
        db, fs.path(experiment.IMG_ROOT, "performance_vs_wg_c.png"))
    visualise.performance_vs_wg_r(
        db, fs.path(experiment.IMG_ROOT, "performance_vs_wg_r.png"))
    visualise.max_wgsizes(db, fs.path(experiment.IMG_ROOT, "max_wgsizes.png"))
    visualise.oracle_speedups(
        db, fs.path(experiment.IMG_ROOT, "oracle_speedups.png"))

    visualise.coverage(db, fs.path(experiment.IMG_ROOT,
                                   "coverage/coverage.png"))
    visualise.safety(db, fs.path(experiment.IMG_ROOT, "safety/safety.png"))
    visualise.oracle_wgsizes(db, fs.path(experiment.IMG_ROOT,
                                         "oracle/all.png"))

    # Per-device plots
    for i, device in enumerate(db.devices):
        where = (
            "scenario IN "
            "(SELECT id from scenarios WHERE device='{0}')".format(device))
        output = fs.path(experiment.IMG_ROOT,
                         "coverage/devices/{0}.png".format(i))
        visualise.coverage(db, output=output, where=where, title=device)
        output = fs.path(experiment.IMG_ROOT,
                         "safety/devices/{0}.png".format(i))
        visualise.safety(db, output, where=where, title=device)
        output = fs.path(experiment.IMG_ROOT,
                         "oracle/devices/{0}.png".format(i))
        visualise.oracle_wgsizes(db, output, where=where, title=device)

        where = ("scenario IN (\n"
                 "    SELECT id from scenarios WHERE device='{0}'\n"
                 ") AND scenario IN (\n"
                 "    SELECT id FROM scenarios WHERE kernel IN (\n"
                 "        SELECT id FROM kernel_names WHERE synthetic=0\n"
                 "    )\n"
                 ")".format(device))
        output = fs.path(experiment.IMG_ROOT,
                         "coverage/devices/{0}_real.png".format(i))
        visualise.coverage(db,
                           output=output,
                           where=where,
                           title=device + ", real")
        output = fs.path(experiment.IMG_ROOT,
                         "safety/devices/{0}_real.png".format(i))
        visualise.safety(db, output, where=where, title=device + ", real")
        output = fs.path(experiment.IMG_ROOT,
                         "oracle/devices/{0}_real.png".format(i))
        visualise.oracle_wgsizes(db,
                                 output,
                                 where=where,
                                 title=device + ", real")

        where = ("scenario IN (\n"
                 "    SELECT id from scenarios WHERE device='{0}'\n"
                 ") AND scenario IN (\n"
                 "    SELECT id FROM scenarios WHERE kernel IN (\n"
                 "        SELECT id FROM kernel_names WHERE synthetic=1\n"
                 "    )\n"
                 ")".format(device))
        output = fs.path(experiment.IMG_ROOT,
                         "coverage/devices/{0}_synthetic.png".format(i))
        visualise.coverage(db,
                           output=output,
                           where=where,
                           title=device + ", synthetic")
        output = fs.path(experiment.IMG_ROOT,
                         "safety/devices/{0}_synthetic.png".format(i))
        visualise.safety(db, output, where=where, title=device + ", synthetic")
        output = fs.path(experiment.IMG_ROOT,
                         "oracle/devices/{0}_synthetic.png".format(i))
        visualise.oracle_wgsizes(db,
                                 output,
                                 where=where,
                                 title=device + ", synthetic")

    # Per-kernel plots
    for kernel, ids in db.lookup_named_kernels().iteritems():
        id_wrapped = ['"' + id + '"' for id in ids]
        where = ("scenario IN "
                 "(SELECT id from scenarios WHERE kernel IN ({0}))".format(
                     ",".join(id_wrapped)))
        output = fs.path(experiment.IMG_ROOT,
                         "coverage/kernels/{0}.png".format(kernel))
        visualise.coverage(db, output=output, where=where, title=kernel)
        output = fs.path(experiment.IMG_ROOT,
                         "safety/kernels/{0}.png".format(kernel))
        visualise.safety(db, output=output, where=where, title=kernel)
        output = fs.path(experiment.IMG_ROOT,
                         "oracle/kernels/{0}.png".format(kernel))
        visualise.safety(db, output=output, where=where, title=kernel)

    # Per-dataset plots
    for i, dataset in enumerate(db.datasets):
        where = (
            "scenario IN "
            "(SELECT id from scenarios WHERE dataset='{0}')".format(dataset))
        output = fs.path(experiment.IMG_ROOT,
                         "coverage/datasets/{0}.png".format(i))
        visualise.coverage(db, output, where=where, title=dataset)
        output = fs.path(experiment.IMG_ROOT,
                         "safety/datasets/{0}.png".format(i))
        visualise.safety(db, output, where=where, title=dataset)
        output = fs.path(experiment.IMG_ROOT,
                         "oracle/datasets/{0}.png".format(i))
        visualise.safety(db, output, where=where, title=dataset)

    ml.stop()