def create_subsection_figures(props, dim_rows, dim_cols, exp_prefix): if len(props) == 0: print("No props: plots were not generated.") return section = reporting.Section("Figures", []) getter_mse = lambda p: float(p["result.best.trainMSE"]) predicate = lambda v, v_xaxis: v <= v_xaxis N = 50 # number of points per plot line r = (0.0, 1e0) xs = np.linspace(r[0], r[1], N) xticks = np.arange(r[0], r[1], r[1] / 10) savepath = "results/figures/ratioMSE.pdf" plotter.plot_ratio_meeting_predicate(props, getter_mse, predicate, xs=xs, xticks=xticks, show_plot=False, title="Ratio of solutions with MSE under the certain level", xlabel="MSE", series_dim=dim_method, xlogscale=False, savepath=savepath) section.add(reporting.FloatFigure(savepath.replace("results/", ""))) # Illustration of individual runs and their errors on training and validation sets savepath = "results/figures/progressionGrid.pdf" dim_rows = get_benchmarks_from_props(props, ignoreNumTests=True) dim_cols = (dim_methodGP * dim_all + dim_methodCDGP * dim_all + dim_methodCDGPprops * dim_weight) * \ dim_benchmarkNumTests # * dim_optThreshold plotter.plot_value_progression_grid_simple(props, dim_rows, dim_cols, ["cdgp.logTrainSet", "cdgp.logValidSet"], ["train", "valid"], plot_individual_runs=True, savepath=savepath) section.add(reporting.FloatFigure(savepath.replace("results/", ""))) return section
def create_section(title, desc, props, subsects, figures_list, exp_prefix): assert isinstance(title, str) assert isinstance(desc, str) assert isinstance(props, list) assert isinstance(figures_list, list) section = reporting.Section(title, []) section.add(reporting.BlockLatex(desc + "\n")) for s in subsects: section.add(s) # Create figures in the appropriate directory for f in figures_list: section.add(reporting.FloatFigure(f)) section.add(reporting.BlockLatex(r"\vspace{1cm}" + "\n")) return section
def create_section_with_results(title, desc, folders, numRuns=10, use_bench_simple_names=True, print_status_matrix=True): assert isinstance(title, str) assert isinstance(desc, str) assert isinstance(folders, list) print("\n*** Processing: {0}***".format(title)) for f in folders: print(f) if folders is None or len(folders) == 0: return None props = load_correct_props(folders, name=title) # Create figures in the appropriate directory plot_figures(props) # Uncomment this to print names of files with results of a certain configuration # print("\n(** {0} **) props_meeting the property:".format(title[:15])) # for p in props: # if float(p["cdgp.solverTimeMaxSec"]) >= 2.0: # print(p["thisFileName"] + ", " + p["cdgp.solverTimeMaxSec"]) def post(s): return s.replace("{ccccccccccccc}", "{rrrrrrrrrrrrr}").replace("{rrr", "{lrr").replace(r"\_{lex}", "_{lex}").replace(r"\_{", "_{") dim_benchmarks = Dim.from_dict(props, "benchmark") if print_status_matrix: d = dim_benchmarks * dim_methodCDGP * dim_testsRatio * dim_sa + \ dim_benchmarks * dim_methodGPR * dim_testsRatioGPR * dim_sa matrix = produce_status_matrix(d, props) print("\n****** Status matrix:") print(matrix + "\n") if use_bench_simple_names: configs = [Config(benchmarks_simple_names.get(c.get_caption(), c.get_caption()), c.filters[0][1]) for c in dim_benchmarks.configs] dim_benchmarks = Dim(configs) dim_benchmarks.sort() # --------------------- Shared stats --------------------- # -------------- Dimensions ----------------- dim_cols = dim_methodCDGP * dim_ea_type * dim_sel * dim_testsRatio +\ dim_methodGPR * dim_ea_type * dim_sel * dim_testsRatioGPR + \ dim_methodFormal dim_rows = dim_benchmarks.sort() + dim_true # dim_cols = dim_method * dim_sa # ------------------------------------------- vb = 1 # vertical border print("STATUS") text = post(printer.latex_table(props, dim_rows, dim_cols, get_num_computed, layered_headline=True, vertical_border=vb)) latex_status = printer.table_color_map(text, 0.0, numRuns/2, numRuns, "colorLow", "colorMedium", "colorHigh") print("SUCCESS RATES") text = post(printer.latex_table(props, dim_rows, dim_cols, fun_successRates, layered_headline=True, vertical_border=vb)) latex_successRates = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("AVG RUNTIME") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtime, layered_headline=True, vertical_border=vb)) latex_avgRuntime = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh") print("AVG RUNTIME (SUCCESSFUL)") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtimeOnlySuccessful, layered_headline=True, vertical_border=vb)) latex_avgRuntimeOnlySuccessful = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh") # print("SUCCESS RATES (FULL INFO)") # text = post(printer.latex_table(props, dim_rows, dim_cols, fun_successRates_full, layered_headline=True, vertical_border=vb)) # print("AVG SIZES") # text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_size, layered_headline=True, vertical_border=vb)) # latex_sizes = printer.table_color_map(text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh") print("AVG SIZES (SUCCESSFUL)") text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_sizeOnlySuccessful, layered_headline=True, vertical_border=vb)) latex_sizesOnlySuccessful = printer.table_color_map(text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh") # --------------------- CDGP stats --------------------- # -------------- Dimensions ----------------- dim_cols = dim_methodCDGP * dim_ea_type * dim_sel * dim_testsRatio + \ dim_methodGPR * dim_ea_type * dim_sel * dim_testsRatioGPR # ------------------------------------------- print("AVG BEST-OF-RUN FITNESS") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_fitness, layered_headline=True, vertical_border=vb)) latex_avgBestOfRunFitness = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("AVG TOTAL TESTS") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_totalTests, layered_headline=True, vertical_border=vb)) latex_avgTotalTests = printer.table_color_map(text, 0.0, 1000.0, 2000.0, "colorLow", "colorMedium", "colorHigh") print("AVG RUNTIME PER PROGRAM") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtimePerProgram, layered_headline=True, vertical_border=vb)) latex_avgRuntimePerProgram = printer.table_color_map(text, 0.01, 1.0, 2.0, "colorLow", "colorMedium", "colorHigh") print("AVG GENERATION") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_generation, layered_headline=True, vertical_border=vb)) latex_avgGeneration = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh") print("AVG GENERATION (SUCCESSFUL)") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_generationSuccessful, layered_headline=True, vertical_border=vb)) latex_avgGenerationSuccessful = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh") print("MAX SOLVER TIME") text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_maxSolverTime, layered_headline=True, vertical_border=vb)) latex_maxSolverTimes = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("AVG SOLVER TIME") text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_avgSolverTime, layered_headline=True, vertical_border=vb)) latex_avgSolverTimes = printer.table_color_map(text, 0.0, 0.015, 0.03, "colorLow", "colorMedium", "colorHigh") print("AVG NUM SOLVER CALLS") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avgSolverTotalCalls, layered_headline=True, vertical_border=vb)) latex_avgSolverTotalCalls = printer.table_color_map(text, 1e1, 1e2, 1e4, "colorLow", "colorMedium", "colorHigh") print("NUM SOLVER CALLS > 0.5s") text = post(printer.latex_table(props, dim_rows, dim_cols, get_numSolverCallsOverXs, layered_headline=True, vertical_border=vb)) latex_numSolverCallsOverXs = printer.table_color_map(text, 0, 50, 100, "colorLow", "colorMedium", "colorHigh") subsects_main = [ ("Status (correctly finished processes)", latex_status, reversed(reporting.color_scheme_red)), ("Success rates", latex_successRates, reporting.color_scheme_green), ("Average runtime [s]", latex_avgRuntime, reporting.color_scheme_violet), ("Average runtime (only successful) [s]", latex_avgRuntimeOnlySuccessful, reporting.color_scheme_violet), #("Average sizes of best of runs (number of nodes)", latex_sizes, reporting.color_scheme_yellow), ("Average sizes of best of runs (number of nodes) (only successful)", latex_sizesOnlySuccessful, reporting.color_scheme_yellow), ] subsects_cdgp = [ ("Average best-of-run ratio of passed tests", latex_avgBestOfRunFitness, reporting.color_scheme_green), ("Average sizes of $T_C$ (total tests in run)", latex_avgTotalTests, reporting.color_scheme_blue), ("Average generation (all)", latex_avgGeneration, reporting.color_scheme_teal), ("Average generation (only successful)", latex_avgGenerationSuccessful, reporting.color_scheme_teal), ("Approximate average runtime per program [s]", latex_avgRuntimePerProgram, reporting.color_scheme_brown), ("Max solver time per query [s]", latex_maxSolverTimes, reporting.color_scheme_violet), ("Avg solver time per query [s]", latex_avgSolverTimes, reporting.color_scheme_brown), ("Avg number of solver calls (in thousands; 1=1000)", latex_avgSolverTotalCalls, reporting.color_scheme_blue), ("Number of solver calls $>$ 0.5s", latex_numSolverCallsOverXs, reporting.color_scheme_blue), ] figures_cdgp = [ "figures/ratioEvaluated_correctVsAllRuns.pdf", "figures/ratioTime_correctVsAllCorrect.pdf", "figures/ratioTime_endedVsAllEnded.pdf", ] def get_content_of_subsections(subsects): content = [] vspace = reporting.BlockLatex(r"\vspace{0.75cm}"+"\n") for title, table, cs in subsects: if isinstance(cs, reporting.ColorScheme3): cs = cs.toBlockLatex() sub = reporting.SectionRelative(title, contents=[cs, reporting.BlockLatex(table + "\n"), vspace]) content.append(sub) return content section = reporting.Section(title, []) section.add(reporting.BlockLatex(desc + "\n")) section.add(reporting.Subsection("Shared Statistics", get_content_of_subsections(subsects_main))) section.add(reporting.Subsection("CDGP Statistics", get_content_of_subsections(subsects_cdgp))) for f in figures_cdgp: section.add(reporting.FloatFigure(f)) section.add(reporting.BlockLatex(r"\vspace{1cm}" + "\n")) return section
def print_table(props, dim_rows, dim_cols): def fun0(filtered): return str(len(filtered)) textStatus = printer.latex_table(props, dim_rows, dim_cols, fun0, latexize_underscores=False) textStatus = printer.table_color_map(textStatus, 0.0, 50.0, 100.0) print(textStatus) print("\n\n") def fun1(filtered): if len(filtered) == 0: return "-" num_opt = get_num_optimal(filtered) # return "{0}/{1}".format(str(num_opt), str(len(filtered))) return "{0}".format(str(num_opt)) textNumOptimal = printer.latex_table(props, dim_rows, dim_cols, fun1, latexize_underscores=False) textNumOptimal = printer.table_color_map(textNumOptimal, 0.0, 50.0, 100.0) print(textNumOptimal) print("\n\n") def fun2(filtered): if len(filtered) == 0: return "-" avgFit = round(get_stats_fitness(filtered)[0], 2) return "{0}".format(str(avgFit)) textAvgFitness = printer.latex_table(props, dim_rows, dim_cols, fun2, latexize_underscores=False) textAvgFitness = printer.table_color_map(textAvgFitness, 0.0, 25.0, 50.0) print(textAvgFitness) print("\n\n") def fun3(filtered): if len(filtered) == 0: return "-" avg_time = round(get_stats_duration(filtered)[0], 1) return "{0}".format(str(avg_time)) textAvgRuntime = printer.latex_table(props, dim_rows, dim_cols, fun3, latexize_underscores=False) textAvgRuntime = printer.table_color_map(textAvgRuntime, 0.0, 1000, 10000) print(textAvgRuntime) print("\n\n") def fun4(filtered): if len(filtered) == 0: return "-" evalSolver = get_sum(filtered, "result.stats.evaluatedSolver") evalSolverUnknown = get_sum(filtered, "result.stats.evaluatedSolverUnknown") evalSolverTimeout = get_sum(filtered, "result.stats.evaluatedSolverTimeout") if evalSolver > 0: percentUnsuccessful = float(evalSolverTimeout + evalSolverUnknown) / float(evalSolver) return str(round(percentUnsuccessful,3)) else: return "-" textRatioOfUnknowns = printer.latex_table(props, dim_rows, dim_cols, fun4, latexize_underscores=False) textRatioOfUnknowns = printer.table_color_map(textRatioOfUnknowns, 0.0, 0.3, 0.6) print(textRatioOfUnknowns) print("\n\n") report = reporting.ReportPDF() section1 = reporting.Section("Experiments", []) subsects = [("Status (correctly finished processes)", textStatus, reversed(reporting.color_scheme_red)), ("Number of optimal solutions (max=100)", textNumOptimal, reporting.color_scheme_green), ("Average fitness", textAvgFitness, reporting.color_scheme_green), ("Average runtime", textAvgRuntime, reporting.color_scheme_blue), ("Ratio of unknowns", textRatioOfUnknowns, reporting.color_scheme_yellow)] for title, table, cs in subsects: if isinstance(cs, reporting.ColorScheme3): cs = cs.toBlockLatex() sub = reporting.Subsection(title, [cs, reporting.BlockLatex(table + "\n")]) section1.add(sub) report.add(section1) report.save_and_compile("eps_results.tex")
def create_section_with_results(title, desc, folders, numRuns=10, use_bench_simple_names=True): assert isinstance(title, str) assert isinstance(desc, str) assert isinstance(folders, list) print("\n*** Processing: {0}***".format(title)) for f in folders: print(f) if folders is None or len(folders) == 0: return None props = load_correct_props(folders, name=title) def post(s): return s.replace("{ccccccccccccc}", "{rrrrrrrrrrrrr}").replace("{rrr", "{lrr").replace(r"\_{lex}", "_{lex}").replace(r"\_{7}", "_{7}").replace("other/", "").replace("sygus16/", "") dim_benchmarks = Dim.from_data(props, lambda p: p["benchmark"]) if use_bench_simple_names: configs = [Config(benchmarks_simple_names[c.get_caption()], c.filters[0][1]) for c in dim_benchmarks.configs] dim_benchmarks = Dim(configs) dim_benchmarks.sort() dim_cols = dim_method * dim_ea_type * dim_sel # dim_cols = dim_method * dim_sa print("STATUS") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_num_computed, layered_headline=True)) latex_status = printer.table_color_map(text, 0.0, numRuns/2, numRuns, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("SUCCESS RATES") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, fun_successRates, layered_headline=True)) latex_successRates = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") # print("SUCCESS RATES (FULL INFO)") # text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, fun_successRates_full, layered_headline=True)) # print(text + "\n\n") print("AVG BEST-OF-RUN FITNESS") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_fitness, layered_headline=True)) latex_avgBestOfRunFitness = printer.table_color_map(text, 0.6, 0.98, 1.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG TOTAL TESTS") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_totalTests, layered_headline=True)) latex_avgTotalTests = printer.table_color_map(text, 0.0, 1000.0, 2000.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG RUNTIME") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_runtime, layered_headline=True)) latex_avgRuntime = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG RUNTIME (SUCCESSFUL)") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_runtimeOnlySuccessful, layered_headline=True)) latex_avgRuntimeOnlySuccessful = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG RUNTIME PER PROGRAM") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_runtimePerProgram, layered_headline=True)) latex_avgRuntimePerProgram = printer.table_color_map(text, 0.01, 1.0, 2.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG GENERATION") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_generation, layered_headline=True)) latex_avgGeneration = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG GENERATION (SUCCESSFUL)") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_generationSuccessful, layered_headline=True)) latex_avgGenerationSuccessful = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG SIZES") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_stats_size, layered_headline=True)) latex_sizes = printer.table_color_map(text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") section = reporting.Section(title, []) subsects = [("Status (correctly finished processes)", latex_status, reversed(reporting.color_scheme_red)), ("Success rates", latex_successRates, reporting.color_scheme_green), ("Average best-of-run ratio of passed tests", latex_avgBestOfRunFitness, reporting.color_scheme_green), ("Average sizes of $T_C$ (total tests in run)", latex_avgTotalTests, reporting.color_scheme_blue), ("Average runtime [s]", latex_avgRuntime, reporting.color_scheme_violet), ("Average runtime (only successful) [s]", latex_avgRuntimeOnlySuccessful, reporting.color_scheme_violet), ("Average generation (optimal and nonoptimal bestOfRuns)", latex_avgGeneration, reporting.color_scheme_teal), ("Average generation (only optimal bestOfRuns)", latex_avgGenerationSuccessful, reporting.color_scheme_teal), ("Approximate average runtime per program [s]", latex_avgRuntimePerProgram, reporting.color_scheme_brown), ("Average sizes of best of runs (number of nodes)", latex_sizes, reporting.color_scheme_yellow)] bl_desc = reporting.BlockLatex(desc + "\n") section.add(bl_desc) for title, table, cs in subsects: if isinstance(cs, reporting.ColorScheme3): cs = cs.toBlockLatex() sub = reporting.Subsection(title, [cs, reporting.BlockLatex(table + "\n")]) section.add(sub) section.add(reporting.BlockLatex(r"\vspace{1cm}" + "\n")) return section