def render_all_paths(self, output_port, transitivity=[1]): print(latex.subsection("Sampling paths"), file=output_port) untyped_cfg = "0" * self.get_num_modules() for trans in transitivity: weights = [self.max_runtime_over_typed(config.random_walk(untyped_cfg, trans)) for _ in range(self.sample_size)] print(latex.figure(self.graph_histogram(weights ,"%s-sample-paths-trans-%s.png" % (self.project_name, trans) ,"Sampled Paths in a %s-trans lattice" % trans ,"Max Overhead (runtime / typed runtime)")), file=output_port)
def render_graphs(self, output_port, cfgs, baseline, title="Module Graphs"): print(latex.subsection(title), file=output_port) for cfg in cfgs: mean = self.stats_of_config(cfg)["mean"] diff, txt = latex.difference(mean, baseline) g = self.graph_config( cfg, title="Config %s: %s %s than baseline" % (cfg, diff, txt), output="%s-graph-%s.png" % (self.project_name, cfg), ) print(latex.figure(g), file=output_port)
def render(self, output_port): title = "Ground Truth Results: %s" % self.project_name self.render_title(output_port, title) self.render_summary(output_port) best_cfgs = self.best_rows( config.is_gradual, lambda x, y: self.stats_by_config[x]["mean"] > self.stats_by_config[y]["mean"] ) worst_cfgs = self.best_rows( config.is_gradual, lambda x, y: self.stats_by_config[x]["mean"] < self.stats_by_config[y]["mean"] ) self.render_overall( output_port, ("untyped", config.is_untyped), ("gradual", config.is_gradual), ("fastest(%s)" % best_cfgs[0], lambda x: x == best_cfgs[0]), ("slowest(%s)" % worst_cfgs[0], lambda x: x == worst_cfgs[0]), ("typed", config.is_typed), ) print( "Num. within 2x: %s" % len( self.stats_of_predicate( lambda x: self.stats_by_config[x]["mean"] < 2 * self.stats_by_config["0" * self.get_num_modules()]["mean"] ) ), file=output_port, ) print(latex.subsection("Aggregate Figures"), file=output_port) self.render_normalized( output_port, ("untyped", config.is_untyped), ("gradual", config.is_gradual), ("top %s" % len(best_cfgs), lambda x: x in best_cfgs), ("bottom %s" % len(worst_cfgs), lambda x: x in worst_cfgs), ("typed", config.is_typed), ) self.render_absolute( output_port, *[(str(n), config.has_typed_modules(n)) for n in range(self.get_num_modules())] ) baseline = self.stats_of_config("0" * self.get_num_modules())["mean"] self.render_graphs( output_port, worst_cfgs, baseline, title="Top %s slowest gradually-typed configurations" % len(worst_cfgs) ) self.render_graphs( output_port, best_cfgs, baseline, title="Top %s fastest gradually-typed configurations" % len(best_cfgs) ) # self.render_all_paths(output_port, [1,2,3,4]) # self.render_cutoff_paths(output_port) print(latex.end(), file=output_port)
def render_cutoff_paths(self, output_port, xmax=None, ymax=None): print(latex.subsection("Paths with cutoff"), file=output_port) # Build a lattice for each cluster size {1 .. num_modules-1} print("Building lattice for %s" % self.project_name) lattice = self.make_lattice(transitivity=self.get_num_modules()) # xmax = max((e[2]["weight"] for e in lattice.edges_iter(data=True))) untyped_config = "0" * self.get_num_modules() untyped_mean = self.stats_of_config(untyped_config)["mean"] typed_config = "1" * self.get_num_modules() typed_mean = self.stats_of_config(typed_config)["mean"] rows = [] for group_size in range(1, self.get_num_modules()): # For each group size (freedom to type exactly N modules at once) # make a histogram of max overhead edges along each path # (Shows the number of paths that have 'really bad' overhead) print("Computing paths for group size '%s'" % group_size) cutoff = 1 + (self.get_num_modules() - group_size) paths = networkx.all_simple_paths(lattice, source=untyped_config, target=typed_config, cutoff=cutoff) weights = [self.max_weight(lattice, path) for path in paths if len(path) == 1 + cutoff] num_release_u = sum((1 for x in weights if x < (untyped_mean * constants.DELIVERABLE))) num_dev_u = sum((1 for x in weights if x < (untyped_mean * constants.USABLE))) num_x_u = sum((1 for x in weights if x < (untyped_mean * 15))) num_release_t = sum((1 for x in weights if x < (typed_mean * constants.DELIVERABLE))) num_dev_t = sum((1 for x in weights if x < (typed_mean * constants.USABLE))) num_x_t = sum((1 for x in weights if x < (typed_mean * 15))) num_paths = len(weights) rows.append( [str(cutoff), str(num_paths)] + [ "%s (%s\\%%)" % (x, round((x / num_paths) * 100, 2)) for x in [num_release_u, num_dev_u, num_x_u, num_release_t, num_dev_t, num_x_t] ] ) print( latex.table( [ "Path length", "Total", "$<$ 2x untyped", "$<$ 4x untyped", "$<$ 15x untyped", "$<$ 2x typed", "$<$ 4x typed", "$<$ 15x typed", ], rows, ), file=output_port, )
def render_all_paths(self, output_port, transitivity=[1]): """ Options: - transitivity : How many transitive edges to include. By default, edges are only between consecutive levels. If argument is a list, analyzes one graph of each transitivity """ print(latex.subsection("Lattices+Freedom"), file=output_port) typed_mean = self.stats_of_config("1" * self.get_num_modules())["mean"] untyped_mean = self.stats_of_config("0" * self.get_num_modules())["mean"] rows = [] for trans in transitivity: print("Building lattice for %s with transitivity %s" % (self.project_name, trans)) lattice = self.make_lattice(transitivity=trans) untyped_config = "0" * self.get_num_modules() typed_config = "1" * self.get_num_modules() paths = networkx.all_simple_paths(lattice, source=untyped_config, target=typed_config) weights = [self.max_weight(lattice, path) for path in paths] num_release_u = sum((1 for x in weights if x < (untyped_mean * constants.DELIVERABLE))) num_dev_u = sum((1 for x in weights if x < (untyped_mean * constants.USABLE))) num_x_u = sum((1 for x in weights if x < (untyped_mean * 15))) num_release_t = sum((1 for x in weights if x < (typed_mean * constants.DELIVERABLE))) num_dev_t = sum((1 for x in weights if x < (typed_mean * constants.USABLE))) num_x_t = sum((1 for x in weights if x < (typed_mean * 15))) num_paths = len(weights) rows.append( [str(trans), str(num_paths)] + [ "%s (%s\\%%)" % (x, round((x / num_paths) * 100, 2)) for x in [num_release_u, num_dev_u, num_x_u, num_release_t, num_dev_t, num_x_t] ] ) print( latex.table( [ "Fuel", "Total", "$<$ 2x untyped", "$<$ 4x untyped", "$<$ 15x untyped", "$<$ 2x typed", "$<$ 4x typed", "$<$ 15x typed", ], rows, ), file=output_port, )
def render_overall(self, output_port, *labeled_preds): labels = [k for (k,v) in labeled_preds] preds = [v for (k,v) in labeled_preds] results = [self.stats_of_predicate(p) for p in preds] baseline = (labels[0], results[0]) print(latex.subsection("Overall Runtimes"), file=output_port) print(latex.list([" ".join(["Average" ,"\\textbf{%s}" % tag ,"runtime" ,str(row["mean"]) ,"(%s times %s than %s)" % (latex.difference(row["mean"], baseline[1]["mean"])[0], latex.difference(row["mean"], baseline[1]["mean"])[1], baseline[0]) ,latex.list(["Median: %s" % row["median"] ,"Min: %s" % row["min"] ,"Max: %s" % row["max"] ,"95\\%% confidence: %s\\textendash~%s" % (row["ci"][0], row["ci"][1])])]) for (tag, row) in zip(labels, results)]), file=output_port)
def render_sample_results(self, output_port): print(latex.subsection("Notes"), file=output_port) num_sampled = sum((1 for v in self.stats_by_config.values() if v)) print("Sampled %s of %s configurations." % (num_sampled, self.get_num_configurations()), file=output_port) print(latex.table(["\# Typed", "\# Configs", "\# Samples", "Sample Mean", "Sample Variance", "95\% CI", "Standard Error", "Jarque-Bera"] ,[self.sample_stats((lambda cfg, nt=n: config.num_typed_modules(cfg) == n), n) for n in range(self.get_num_modules())]), file=output_port)
def render_summary(self, output_port): """ Print basic information that every summary should give. """ print(latex.subsection("Module Summary"), file=output_port) print(latex.list(["\\mono{%s}" % mn for mn in self.module_names], numbers=True), file=output_port)