def create_subsection_ea_stats(props, title, dim_rows, dim_cols, headerRowNames): vb = 1 # vertical border variants = None # variants_benchmarkNumTests dim_rows_v2 = get_benchmarks_from_props(props, ignoreNumTests=True) # dim_rows_v2 += dim_true # TODO: within dict tables = [ TableGenerator(get_rankingOfBestSolutionsCDSR(ONLY_VISIBLE_SOLS=True, NUM_SHOWN=15), Dim(dim_cols.configs[:-1]), Dim(dim_rows.configs[:-1]), headerRowNames=headerRowNames, title="The best solutions found for each benchmark and their sizes. Format: solution (MSE on test set) (size)", color_scheme=reporting.color_scheme_violet, middle_col_align="l", default_color_thresholds=(0.0, 900.0, 1800.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), TableGenerator(get_stats_size, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Average sizes of best of runs (number of nodes)", color_scheme=reporting.color_scheme_yellow, default_color_thresholds=(0.0, 100.0, 200.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), # TableGenerator(get_stats_sizeOnlySuccessful, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Average sizes of best of runs (number of nodes) (only successful)", # color_scheme=reporting.color_scheme_yellow, # default_color_thresholds=(0.0, 100.0, 200.0), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), TableGenerator(get_avg_generation, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Average generation (all)", color_scheme=reporting.color_scheme_teal, default_color_thresholds=(0.0, 5000.0, 10000.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), TableGenerator(get_avg_evaluated, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Average number of evaluated solutions", color_scheme=reporting.color_scheme_brown, default_color_thresholds=(0.0, 5000.0, 10000.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), # TableGenerator(get_avg_evaluatedSuccessful, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Average number of evaluated solutions", # color_scheme=reporting.color_scheme_brown, # default_color_thresholds=(500.0, 25000.0, 100000.0), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), TableGenerator(get_avg_doneAlgRestarts, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Number of algorithm restarts (avg)", color_scheme=reporting.color_scheme_gray_light, default_color_thresholds=(0.0, 1e2, 1e4), vertical_border=vb, table_postprocessor=post, variants=variants, ), ] subsects_main = [] for t in tables: tup = (t.title, t.apply(props), t.color_scheme) subsects_main.append(tup) return reporting.Subsection(title, get_content_of_subsections(subsects_main))
def create_section_with_results(title, desc, folders, numRuns=10, use_bench_simple_names=True, print_status_matrix=True): assert isinstance(title, str) assert isinstance(desc, str) assert isinstance(folders, list) print("\n*** Processing: {0}***".format(title)) for f in folders: print(f) if folders is None or len(folders) == 0: return None props = load_correct_props(folders, name=title) # Create figures in the appropriate directory plot_figures(props) # Uncomment this to print names of files with results of a certain configuration # print("\n(** {0} **) props_meeting the property:".format(title[:15])) # for p in props: # if float(p["cdgp.solverTimeMaxSec"]) >= 2.0: # print(p["thisFileName"] + ", " + p["cdgp.solverTimeMaxSec"]) def post(s): return s.replace("{ccccccccccccc}", "{rrrrrrrrrrrrr}").replace("{rrr", "{lrr").replace(r"\_{lex}", "_{lex}").replace(r"\_{", "_{") dim_benchmarks = Dim.from_dict(props, "benchmark") if print_status_matrix: d = dim_benchmarks * dim_methodCDGP * dim_testsRatio * dim_sa + \ dim_benchmarks * dim_methodGPR * dim_testsRatioGPR * dim_sa matrix = produce_status_matrix(d, props) print("\n****** Status matrix:") print(matrix + "\n") if use_bench_simple_names: configs = [Config(benchmarks_simple_names.get(c.get_caption(), c.get_caption()), c.filters[0][1]) for c in dim_benchmarks.configs] dim_benchmarks = Dim(configs) dim_benchmarks.sort() # --------------------- Shared stats --------------------- # -------------- Dimensions ----------------- dim_cols = dim_methodCDGP * dim_ea_type * dim_sel * dim_testsRatio +\ dim_methodGPR * dim_ea_type * dim_sel * dim_testsRatioGPR + \ dim_methodFormal dim_rows = dim_benchmarks.sort() + dim_true # dim_cols = dim_method * dim_sa # ------------------------------------------- vb = 1 # vertical border print("STATUS") text = post(printer.latex_table(props, dim_rows, dim_cols, get_num_computed, layered_headline=True, vertical_border=vb)) latex_status = printer.table_color_map(text, 0.0, numRuns/2, numRuns, "colorLow", "colorMedium", "colorHigh") print("SUCCESS RATES") text = post(printer.latex_table(props, dim_rows, dim_cols, fun_successRates, layered_headline=True, vertical_border=vb)) latex_successRates = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("AVG RUNTIME") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtime, layered_headline=True, vertical_border=vb)) latex_avgRuntime = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh") print("AVG RUNTIME (SUCCESSFUL)") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtimeOnlySuccessful, layered_headline=True, vertical_border=vb)) latex_avgRuntimeOnlySuccessful = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh") # print("SUCCESS RATES (FULL INFO)") # text = post(printer.latex_table(props, dim_rows, dim_cols, fun_successRates_full, layered_headline=True, vertical_border=vb)) # print("AVG SIZES") # text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_size, layered_headline=True, vertical_border=vb)) # latex_sizes = printer.table_color_map(text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh") print("AVG SIZES (SUCCESSFUL)") text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_sizeOnlySuccessful, layered_headline=True, vertical_border=vb)) latex_sizesOnlySuccessful = printer.table_color_map(text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh") # --------------------- CDGP stats --------------------- # -------------- Dimensions ----------------- dim_cols = dim_methodCDGP * dim_ea_type * dim_sel * dim_testsRatio + \ dim_methodGPR * dim_ea_type * dim_sel * dim_testsRatioGPR # ------------------------------------------- print("AVG BEST-OF-RUN FITNESS") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_fitness, layered_headline=True, vertical_border=vb)) latex_avgBestOfRunFitness = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("AVG TOTAL TESTS") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_totalTests, layered_headline=True, vertical_border=vb)) latex_avgTotalTests = printer.table_color_map(text, 0.0, 1000.0, 2000.0, "colorLow", "colorMedium", "colorHigh") print("AVG RUNTIME PER PROGRAM") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtimePerProgram, layered_headline=True, vertical_border=vb)) latex_avgRuntimePerProgram = printer.table_color_map(text, 0.01, 1.0, 2.0, "colorLow", "colorMedium", "colorHigh") print("AVG GENERATION") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_generation, layered_headline=True, vertical_border=vb)) latex_avgGeneration = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh") print("AVG GENERATION (SUCCESSFUL)") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_generationSuccessful, layered_headline=True, vertical_border=vb)) latex_avgGenerationSuccessful = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh") print("MAX SOLVER TIME") text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_maxSolverTime, layered_headline=True, vertical_border=vb)) latex_maxSolverTimes = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("AVG SOLVER TIME") text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_avgSolverTime, layered_headline=True, vertical_border=vb)) latex_avgSolverTimes = printer.table_color_map(text, 0.0, 0.015, 0.03, "colorLow", "colorMedium", "colorHigh") print("AVG NUM SOLVER CALLS") text = post(printer.latex_table(props, dim_rows, dim_cols, get_avgSolverTotalCalls, layered_headline=True, vertical_border=vb)) latex_avgSolverTotalCalls = printer.table_color_map(text, 1e1, 1e2, 1e4, "colorLow", "colorMedium", "colorHigh") print("NUM SOLVER CALLS > 0.5s") text = post(printer.latex_table(props, dim_rows, dim_cols, get_numSolverCallsOverXs, layered_headline=True, vertical_border=vb)) latex_numSolverCallsOverXs = printer.table_color_map(text, 0, 50, 100, "colorLow", "colorMedium", "colorHigh") subsects_main = [ ("Status (correctly finished processes)", latex_status, reversed(reporting.color_scheme_red)), ("Success rates", latex_successRates, reporting.color_scheme_green), ("Average runtime [s]", latex_avgRuntime, reporting.color_scheme_violet), ("Average runtime (only successful) [s]", latex_avgRuntimeOnlySuccessful, reporting.color_scheme_violet), #("Average sizes of best of runs (number of nodes)", latex_sizes, reporting.color_scheme_yellow), ("Average sizes of best of runs (number of nodes) (only successful)", latex_sizesOnlySuccessful, reporting.color_scheme_yellow), ] subsects_cdgp = [ ("Average best-of-run ratio of passed tests", latex_avgBestOfRunFitness, reporting.color_scheme_green), ("Average sizes of $T_C$ (total tests in run)", latex_avgTotalTests, reporting.color_scheme_blue), ("Average generation (all)", latex_avgGeneration, reporting.color_scheme_teal), ("Average generation (only successful)", latex_avgGenerationSuccessful, reporting.color_scheme_teal), ("Approximate average runtime per program [s]", latex_avgRuntimePerProgram, reporting.color_scheme_brown), ("Max solver time per query [s]", latex_maxSolverTimes, reporting.color_scheme_violet), ("Avg solver time per query [s]", latex_avgSolverTimes, reporting.color_scheme_brown), ("Avg number of solver calls (in thousands; 1=1000)", latex_avgSolverTotalCalls, reporting.color_scheme_blue), ("Number of solver calls $>$ 0.5s", latex_numSolverCallsOverXs, reporting.color_scheme_blue), ] figures_cdgp = [ "figures/ratioEvaluated_correctVsAllRuns.pdf", "figures/ratioTime_correctVsAllCorrect.pdf", "figures/ratioTime_endedVsAllEnded.pdf", ] def get_content_of_subsections(subsects): content = [] vspace = reporting.BlockLatex(r"\vspace{0.75cm}"+"\n") for title, table, cs in subsects: if isinstance(cs, reporting.ColorScheme3): cs = cs.toBlockLatex() sub = reporting.SectionRelative(title, contents=[cs, reporting.BlockLatex(table + "\n"), vspace]) content.append(sub) return content section = reporting.Section(title, []) section.add(reporting.BlockLatex(desc + "\n")) section.add(reporting.Subsection("Shared Statistics", get_content_of_subsections(subsects_main))) section.add(reporting.Subsection("CDGP Statistics", get_content_of_subsections(subsects_cdgp))) for f in figures_cdgp: section.add(reporting.FloatFigure(f)) section.add(reporting.BlockLatex(r"\vspace{1cm}" + "\n")) return section
def create_subsection_cdgp_specific(props, dim_rows, dim_cols, headerRowNames): vb = 1 # vertical border # variants = variants_benchmarkNumTests variants = None props = [p for p in props if p["method"] != "GP"] print("AVG TOTAL TESTS") latex_avgTotalTests = create_single_table_bundle(props, dim_rows, dim_cols, get_avg_totalTests, headerRowNames, cv0=0.0, cv1=1000.0, cv2=2000.0, tableVariants=variants) # text = post( # printer.latex_table(props, dim_rows, dim_cols, get_avg_totalTests, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgTotalTests = printer.table_color_map(text, 0.0, 1000.0, 2000.0, "colorLow", "colorMedium", "colorHigh") # print("AVG RUNTIME PER PROGRAM") # text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtimePerProgram, layered_headline=True, # vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgRuntimePerProgram = printer.table_color_map(text, 0.01, 1.0, 2.0, "colorLow", "colorMedium", "colorHigh") # print("AVG GENERATION") # latex_avgGeneration = create_single_table_bundle(props, dim_rows, dim_cols, get_avg_generation, headerRowNames, # cv0=0.0, cv1=100.0, cv2=200.0, tableVariants=variants) # text = post( # printer.latex_table(props, dim_rows, dim_cols, get_avg_generation, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgGeneration = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh") # print("AVG EVALUATED SOLUTIONS") # text = post( # printer.latex_table(props, dim_rows, dim_cols, get_avg_evaluated, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgEvaluated = printer.table_color_map(text, 500.0, 25000.0, 100000.0, "colorLow", "colorMedium", "colorHigh") # print("AVG EVALUATED SOLUTIONS (SUCCESSFUL)") # text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_evaluatedSuccessful, layered_headline=True, # vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgEvaluatedSuccessful = printer.table_color_map(text, 500.0, 25000.0, 100000.0, "colorLow", "colorMedium", # "colorHigh") print("MAX SOLVER TIME") latex_maxSolverTimes = create_single_table_bundle(props, dim_rows, dim_cols, get_stats_maxSolverTime, headerRowNames, cv0=0.0, cv1=0.5, cv2=1.0, tableVariants=variants) # text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_maxSolverTime, layered_headline=True, # vertical_border=vb, headerRowNames=headerRowNames)) # latex_maxSolverTimes = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("AVG SOLVER TIME") latex_avgSolverTimes = create_single_table_bundle(props, dim_rows, dim_cols, get_stats_avgSolverTime, headerRowNames, cv0=0.0, cv1=0.015, cv2=0.03, tableVariants=variants) # text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_avgSolverTime, layered_headline=True, # vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgSolverTimes = printer.table_color_map(text, 0.0, 0.015, 0.03, "colorLow", "colorMedium", "colorHigh") print("AVG NUM SOLVER CALLS") latex_avgSolverTotalCalls = create_single_table_bundle( props, dim_rows, dim_cols, get_avgSolverTotalCalls, headerRowNames, cv0=1e1, cv1=1e2, cv2=1e4, tableVariants=variants) # text = post(printer.latex_table(props, dim_rows, dim_cols, get_avgSolverTotalCalls, layered_headline=True, # vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgSolverTotalCalls = printer.table_color_map(text, 1e1, 1e2, 1e4, "colorLow", "colorMedium", "colorHigh") print("NUM SOLVER CALLS > 0.5s") latex_numSolverCallsOverXs = create_single_table_bundle( props, dim_rows, dim_cols, get_numSolverCallsOverXs, headerRowNames, cv0=0, cv1=50, cv2=100, tableVariants=variants) # text = post(printer.latex_table(props, dim_rows, dim_cols, get_numSolverCallsOverXs, layered_headline=True, # vertical_border=vb, headerRowNames=headerRowNames)) # latex_numSolverCallsOverXs = printer.table_color_map(text, 0, 50, 100, "colorLow", "colorMedium", "colorHigh") print("MOST FREQUENTLY FOUND COUNTEREXAMPLE") latex_freqCounterexamples = create_single_table_bundle( props, dim_rows, dim_cols, get_freqCounterexamples, headerRowNames, cv0=0, cv1=50, cv2=100, tableVariants=variants, middle_col_align="l") subsects_cdgp = [ ("Average sizes of $T_C$ (total tests in run)", latex_avgTotalTests, reporting.color_scheme_blue), # ("Average generation (all)", latex_avgGeneration, reporting.color_scheme_teal), #("Average generation (only successful)", latex_avgGenerationSuccessful, reporting.color_scheme_teal), # ("Average evaluated solutions", latex_avgEvaluated, reporting.color_scheme_brown), # ("Average evaluated solutions (only successful)", latex_avgEvaluatedSuccessful, reporting.color_scheme_teal), # ("Approximate average runtime per program [s]", latex_avgRuntimePerProgram, reporting.color_scheme_brown), ("Max solver time per query [s]", latex_maxSolverTimes, reporting.color_scheme_violet), ("Avg solver time per query [s]", latex_avgSolverTimes, reporting.color_scheme_brown), ("Avg number of solver calls (in thousands; 1=1000)", latex_avgSolverTotalCalls, reporting.color_scheme_blue), ("Number of solver calls $>$ 0.5s", latex_numSolverCallsOverXs, reporting.color_scheme_blue), ("The most frequently found counterexamples for each benchmark and configuration", latex_freqCounterexamples, reporting.color_scheme_violet), ] return reporting.Subsection("CDGP Statistics", get_content_of_subsections(subsects_cdgp))
def create_subsection_shared_stats(props, dim_rows, dim_cols, numRuns, headerRowNames): vb = 1 # vertical border variants = None # variants_benchmarkNumTests dim_rows_v2 = get_benchmarks_from_props(props, simple_names=True, ignoreNumTests=True) dim_rows_v2 += dim_true # ---------------------------------------------------- # Cleaning experiment here, because dimension can be easily constructed. # dim_rows_v3 = get_benchmarks_from_props(props, simple_names=True, ignoreNumTests=True) # utils.reorganizeExperimentFiles(props, dim_rows_v3 * dim_benchmarkNumTests * dim_cols, target_dir="./exp3_final/", maxRuns=numRuns) # utils.deleteFilesByPredicate(props, lambda p: len(p["maxGenerations"]) > 7, simulate=False) # ---------------------------------------------------- def scNotColorValueExtractor(s): if s == "-" or "10^{" not in s: return s else: r = s.split("10^{") return r[1][:-2] tables = [ TableGenerator( get_num_computed, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Status (correctly finished runs)", color_scheme=reversed(reporting.color_scheme_red), default_color_thresholds=(0.0, numRuns / 2, numRuns), vertical_border=vb, table_postprocessor=post, variants=variants, ), # TableGenerator(fun_successRate, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Success rates (properties met + mse below thresh)", # color_scheme=reporting.color_scheme_darkgreen, # default_color_thresholds=(0.0, 0.5, 1.0), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), # TableGenerator(get_median_testMSEsuccessRateForThresh_onlyVerified, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Test set: success rate of accepted solutions with MSE below optThreshold (i.e., no overfitting)", # color_scheme=reporting.color_scheme_red2white2darkgreen, # default_color_thresholds=(0.0, 0.5, 1.0), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), # TableGenerator(fun_trainMseBelowThresh, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Training set: success rates (mse below thresh)", # color_scheme=reporting.color_scheme_green, # default_color_thresholds=(0.0, 0.5, 1.0), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), # TableGenerator(get_median_mseOptThresh, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Training set: optThreshold for MSE (median)", # color_scheme=reporting.color_scheme_violet, # default_color_thresholds=(-10.0, 0.0, 10.0), # color_value_extractor=scNotColorValueExtractor, # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), TableGenerator( get_median_trainMSE, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Training set: MSE (median)", color_scheme=reporting.color_scheme_gray_dark, default_color_thresholds=(-10.0, 0.0, 10.0), color_value_extractor=scNotColorValueExtractor, vertical_border=vb, table_postprocessor=post, variants=variants, ), # TableGenerator(get_avg_trainMSE, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Training set: MSE (avg)", # color_scheme=reporting.color_scheme_green, # default_color_thresholds=(0.0, 1e2, 1e4), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), TableGenerator( get_median_testMSE, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Test set: MSE (median)", color_scheme=reporting.color_scheme_gray_dark, default_color_thresholds=(-10.0, 0.0, 10.0), color_value_extractor=scNotColorValueExtractor, vertical_border=vb, table_postprocessor=post, variants=variants, ), # TableGenerator(get_median_testMSE_noScNot, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Test set: MSE (median) (noScNot)", # color_scheme=reporting.color_scheme_green, # default_color_thresholds=(0.0, 1e2, 1e4), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), # TableGenerator(get_avg_testMSE, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Test set: MSE (avg)", # color_scheme=reporting.color_scheme_green, # default_color_thresholds=(0.0, 1e2, 1e4), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), TableGenerator( fun_allPropertiesMet, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Success rates (properties met)", color_scheme=reporting.color_scheme_green, default_color_thresholds=(0.0, 0.5, 1.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), TableGenerator( get_avg_runtime, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Average runtime [s]", color_scheme=reporting.color_scheme_violet, default_color_thresholds=(0.0, 900.0, 1800.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), # TableGenerator(get_avg_runtimeOnlySuccessful, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Average runtime (only successful) [s]", # color_scheme=reporting.color_scheme_violet, # default_color_thresholds=(0.0, 900.0, 1800.0), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), TableGenerator( get_avg_doneAlgRestarts, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Number of algorithm restarts (avg)", color_scheme=reporting.color_scheme_gray_light, default_color_thresholds=(0.0, 1e2, 1e4), vertical_border=vb, table_postprocessor=post, variants=variants, ), # TableGenerator(get_stats_size, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Average sizes of best of runs (number of nodes)", # color_scheme=reporting.color_scheme_yellow, # default_color_thresholds=(0.0, 100.0, 200.0), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), # TableGenerator(get_stats_sizeOnlySuccessful, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Average sizes of best of runs (number of nodes) (only successful)", # color_scheme=reporting.color_scheme_yellow, # default_color_thresholds=(0.0, 100.0, 200.0), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), TableGenerator( get_avg_generation, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Average generation (all)", color_scheme=reporting.color_scheme_teal, default_color_thresholds=(0.0, 5000.0, 10000.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), TableGenerator( get_avg_evaluated, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Average number of evaluated solutions", color_scheme=reporting.color_scheme_brown, default_color_thresholds=(0.0, 5000.0, 10000.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), # TableGenerator(get_avg_evaluatedSuccessful, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Average number of evaluated solutions", # color_scheme=reporting.color_scheme_brown, # default_color_thresholds=(500.0, 25000.0, 100000.0), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), ] subsects_main = [] for t in tables: tup = (t.title, t.apply(props), t.color_scheme) subsects_main.append(tup) return reporting.Subsection("Shared Statistics", get_content_of_subsections(subsects_main))
def create_subsection_aggregation_tests(props, dim_rows, dim_cols, headerRowNames): vb = 1 # vertical border variants = None dim_rows_v2 = get_benchmarks_from_props(props, simple_names=True, ignoreNumTests=True) dim_rows_v2 += dim_true # By default: dim_cols = (dim_methodGP * dim_empty + dim_methodCDGP * dim_testsRatio) * dim_optThreshold tables = [ # TableGenerator(fun_successRateMseOnly, dim_rows_v2, # (dim_methodGP + dim_methodCDGP), # headerRowNames=[""], # title="Success rates (mse below thresh)", # color_scheme=reporting.color_scheme_violet, # default_color_thresholds=(0.0, 0.5, 1.0), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), TableGenerator( fun_successRate, dim_rows_v2, dim_benchmarkNumTests * dim_method, headerRowNames=["", ""], title="Success rates (mse below thresh + properties met)", color_scheme=reporting.color_scheme_darkgreen, default_color_thresholds=(0.0, 0.5, 1.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), TableGenerator( fun_successRate, dim_rows_v2, dim_method * dim_benchmarkNumTests, headerRowNames=["", ""], title="Success rates (mse below thresh + properties met)", color_scheme=reporting.color_scheme_darkgreen, default_color_thresholds=(0.0, 0.5, 1.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), TableGenerator( fun_successRate, dim_rows_v2, dim_optThreshold * dim_method, headerRowNames=["tolerance", ""], title="Success rates (mse below thresh + properties met)", color_scheme=reporting.color_scheme_darkgreen, default_color_thresholds=(0.0, 0.5, 1.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), TableGenerator( fun_successRate, dim_rows_v2, dim_method, headerRowNames=[""], title="Success rates (mse below thresh + properties met)", color_scheme=reporting.color_scheme_darkgreen, default_color_thresholds=(0.0, 0.5, 1.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), TableGenerator( fun_allPropertiesMet, dim_rows_v2, dim_benchmarkNumTests * dim_method, headerRowNames=["", ""], title="Success rates (properties met)", color_scheme=reporting.color_scheme_green, default_color_thresholds=(0.0, 0.5, 1.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), TableGenerator( fun_allPropertiesMet, dim_rows_v2, dim_method * dim_benchmarkNumTests, headerRowNames=["", ""], title="Success rates (properties met)", color_scheme=reporting.color_scheme_green, default_color_thresholds=(0.0, 0.5, 1.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), # TableGenerator(fun_allPropertiesMet, dim_rows_v2, # dim_benchmarkNumTests * dim_optThreshold * dim_method, # headerRowNames=["", "tolerance", ""], # title="Success rates (properties met)", # color_scheme=reporting.color_scheme_green, # default_color_thresholds=(0.0, 0.5, 1.0), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), TableGenerator( fun_allPropertiesMet, dim_rows_v2, dim_optThreshold * dim_method, headerRowNames=["tolerance", ""], title="Success rates (properties met)", color_scheme=reporting.color_scheme_green, default_color_thresholds=(0.0, 0.5, 1.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), TableGenerator( fun_allPropertiesMet, dim_rows_v2, dim_method, headerRowNames=[""], title="Success rates (properties met)", color_scheme=reporting.color_scheme_green, default_color_thresholds=(0.0, 0.5, 1.0), vertical_border=vb, table_postprocessor=post, variants=variants, ), ] subsects_main = [] for t in tables: tup = (t.title, t.apply(props), t.color_scheme) subsects_main.append(tup) return reporting.Subsection( "Tests of different data aggregation in tables", get_content_of_subsections(subsects_main))
def create_subsection_cdgp_specific(props, dim_rows, dim_cols, headerRowNames): vb = 1 # vertical border variants = [ p_benchmarkNumTests_equalTo("3"), p_benchmarkNumTests_equalTo("5"), p_benchmarkNumTests_equalTo("10") ] print("AVG BEST-OF-RUN FITNESS (MSE)") latex_avgBestOfRunFitness = create_single_table_bundle( props, dim_rows, dim_cols, get_avg_mse, headerRowNames, cv0=0.0, cv1=0.5, cv2=1.0, tableVariants=variants) # text = post( # printer.latex_table(props, dim_rows, dim_cols, get_avg_mse, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgBestOfRunFitness = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("AVG TOTAL TESTS") latex_avgTotalTests = create_single_table_bundle(props, dim_rows, dim_cols, get_avg_totalTests, headerRowNames, cv0=0.0, cv1=1000.0, cv2=2000.0, tableVariants=variants) # text = post( # printer.latex_table(props, dim_rows, dim_cols, get_avg_totalTests, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgTotalTests = printer.table_color_map(text, 0.0, 1000.0, 2000.0, "colorLow", "colorMedium", "colorHigh") # print("AVG RUNTIME PER PROGRAM") # text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtimePerProgram, layered_headline=True, # vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgRuntimePerProgram = printer.table_color_map(text, 0.01, 1.0, 2.0, "colorLow", "colorMedium", "colorHigh") print("AVG GENERATION") latex_avgGeneration = create_single_table_bundle(props, dim_rows, dim_cols, get_avg_generation, headerRowNames, cv0=0.0, cv1=100.0, cv2=200.0, tableVariants=variants) # text = post( # printer.latex_table(props, dim_rows, dim_cols, get_avg_generation, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgGeneration = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh") # print("AVG EVALUATED SOLUTIONS") # text = post( # printer.latex_table(props, dim_rows, dim_cols, get_avg_evaluated, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgEvaluated = printer.table_color_map(text, 500.0, 25000.0, 100000.0, "colorLow", "colorMedium", "colorHigh") # print("AVG EVALUATED SOLUTIONS (SUCCESSFUL)") # text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_evaluatedSuccessful, layered_headline=True, # vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgEvaluatedSuccessful = printer.table_color_map(text, 500.0, 25000.0, 100000.0, "colorLow", "colorMedium", # "colorHigh") print("MAX SOLVER TIME") latex_maxSolverTimes = create_single_table_bundle(props, dim_rows, dim_cols, get_stats_maxSolverTime, headerRowNames, cv0=0.0, cv1=0.5, cv2=1.0, tableVariants=variants) # text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_maxSolverTime, layered_headline=True, # vertical_border=vb, headerRowNames=headerRowNames)) # latex_maxSolverTimes = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("AVG SOLVER TIME") latex_avgSolverTimes = create_single_table_bundle(props, dim_rows, dim_cols, get_stats_avgSolverTime, headerRowNames, cv0=0.0, cv1=0.015, cv2=0.03, tableVariants=variants) # text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_avgSolverTime, layered_headline=True, # vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgSolverTimes = printer.table_color_map(text, 0.0, 0.015, 0.03, "colorLow", "colorMedium", "colorHigh") print("AVG NUM SOLVER CALLS") latex_avgSolverTotalCalls = create_single_table_bundle( props, dim_rows, dim_cols, get_avgSolverTotalCalls, headerRowNames, cv0=1e1, cv1=1e2, cv2=1e4, tableVariants=variants) # text = post(printer.latex_table(props, dim_rows, dim_cols, get_avgSolverTotalCalls, layered_headline=True, # vertical_border=vb, headerRowNames=headerRowNames)) # latex_avgSolverTotalCalls = printer.table_color_map(text, 1e1, 1e2, 1e4, "colorLow", "colorMedium", "colorHigh") print("NUM SOLVER CALLS > 0.5s") latex_numSolverCallsOverXs = create_single_table_bundle( props, dim_rows, dim_cols, get_numSolverCallsOverXs, headerRowNames, cv0=0, cv1=50, cv2=100, tableVariants=variants) # text = post(printer.latex_table(props, dim_rows, dim_cols, get_numSolverCallsOverXs, layered_headline=True, # vertical_border=vb, headerRowNames=headerRowNames)) # latex_numSolverCallsOverXs = printer.table_color_map(text, 0, 50, 100, "colorLow", "colorMedium", "colorHigh") subsects_cdgp = [ ("Average best-of-run MSE", latex_avgBestOfRunFitness, reporting.color_scheme_green), ("Average sizes of $T_C$ (total tests in run)", latex_avgTotalTests, reporting.color_scheme_blue), ("Average generation (all)", latex_avgGeneration, reporting.color_scheme_teal), #("Average generation (only successful)", latex_avgGenerationSuccessful, reporting.color_scheme_teal), # ("Average evaluated solutions", latex_avgEvaluated, reporting.color_scheme_teal), # ("Average evaluated solutions (only successful)", latex_avgEvaluatedSuccessful, reporting.color_scheme_teal), # ("Approximate average runtime per program [s]", latex_avgRuntimePerProgram, reporting.color_scheme_brown), ("Max solver time per query [s]", latex_maxSolverTimes, reporting.color_scheme_violet), ("Avg solver time per query [s]", latex_avgSolverTimes, reporting.color_scheme_brown), ("Avg number of solver calls (in thousands; 1=1000)", latex_avgSolverTotalCalls, reporting.color_scheme_blue), ("Number of solver calls $>$ 0.5s", latex_numSolverCallsOverXs, reporting.color_scheme_blue), ] return reporting.Subsection("CDGP Statistics", get_content_of_subsections(subsects_cdgp))
def create_subsection_shared_stats(props, dim_rows, dim_cols, numRuns, headerRowNames): vb = 1 # vertical border variants = [ p_benchmarkNumTests_equalTo("3"), p_benchmarkNumTests_equalTo("5"), p_benchmarkNumTests_equalTo("10") ] dim_rows_v2 = get_benchmarks_from_props(props, simple_names=True, ignoreNumTests=True) dim_rows_v2 += dim_true # ---------------------------------------------------- # Cleaning experiment. Here, because dimension can be easily constructed. # dim_rows_v3 = get_benchmarks_from_props(props, simple_names=True, ignoreNumTests=True) # utils.reorganizeExperimentFiles(props, dim_rows_v3 * dim_benchmarkNumTests * dim_cols, target_dir="./exp3_final/", maxRuns=numRuns) # ---------------------------------------------------- tables = [ TableGenerator( get_num_computed, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Status (correctly finished runs)", color_scheme=reversed(reporting.color_scheme_red), default_color_thresholds=(0.0, numRuns / 2, numRuns), vertical_border=vb, table_postprocessor=post, table_variants=variants, ), TableGenerator( fun_successRate, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Success rates (mse below thresh + properties met)", color_scheme=reporting.color_scheme_darkgreen, default_color_thresholds=(0.0, 0.5, 1.0), vertical_border=vb, table_postprocessor=post, table_variants=variants, ), TableGenerator( fun_allPropertiesMet, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Success rates (properties met)", color_scheme=reporting.color_scheme_green, default_color_thresholds=(0.0, 0.5, 1.0), vertical_border=vb, table_postprocessor=post, table_variants=variants, ), TableGenerator( get_avg_runtime, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Average runtime [s]", color_scheme=reporting.color_scheme_violet, default_color_thresholds=(0.0, 900.0, 1800.0), vertical_border=vb, table_postprocessor=post, table_variants=variants, ), TableGenerator( get_avg_runtimeOnlySuccessful, dim_rows, dim_cols, headerRowNames=headerRowNames, title="Average runtime (only successful) [s]", color_scheme=reporting.color_scheme_violet, default_color_thresholds=(0.0, 900.0, 1800.0), vertical_border=vb, table_postprocessor=post, table_variants=variants, ), # TableGenerator(get_stats_size, dim_rows, dim_cols, headerRowNames=headerRowNames, # title="Average sizes of best of runs (number of nodes)", # color_scheme=reporting.color_scheme_yellow, # default_color_thresholds=(0.0, 100.0, 200.0), # vertical_border=vb, table_postprocessor=post, table_variants=variants, # ), TableGenerator( get_stats_sizeOnlySuccessful, dim_rows, dim_cols, headerRowNames=headerRowNames, title= "Average sizes of best of runs (number of nodes) (only successful)", color_scheme=reporting.color_scheme_yellow, default_color_thresholds=(0.0, 100.0, 200.0), vertical_border=vb, table_postprocessor=post, table_variants=variants, ), ] subsects_main = [] for t in tables: tup = (t.title, t.apply(props), t.color_scheme) subsects_main.append(tup) return reporting.Subsection("Shared Statistics", get_content_of_subsections(subsects_main))
def print_table(props, dim_rows, dim_cols): def fun0(filtered): return str(len(filtered)) textStatus = printer.latex_table(props, dim_rows, dim_cols, fun0, latexize_underscores=False) textStatus = printer.table_color_map(textStatus, 0.0, 50.0, 100.0) print(textStatus) print("\n\n") def fun1(filtered): if len(filtered) == 0: return "-" num_opt = get_num_optimal(filtered) # return "{0}/{1}".format(str(num_opt), str(len(filtered))) return "{0}".format(str(num_opt)) textNumOptimal = printer.latex_table(props, dim_rows, dim_cols, fun1, latexize_underscores=False) textNumOptimal = printer.table_color_map(textNumOptimal, 0.0, 50.0, 100.0) print(textNumOptimal) print("\n\n") def fun2(filtered): if len(filtered) == 0: return "-" avgFit = round(get_stats_fitness(filtered)[0], 2) return "{0}".format(str(avgFit)) textAvgFitness = printer.latex_table(props, dim_rows, dim_cols, fun2, latexize_underscores=False) textAvgFitness = printer.table_color_map(textAvgFitness, 0.0, 25.0, 50.0) print(textAvgFitness) print("\n\n") def fun3(filtered): if len(filtered) == 0: return "-" avg_time = round(get_stats_duration(filtered)[0], 1) return "{0}".format(str(avg_time)) textAvgRuntime = printer.latex_table(props, dim_rows, dim_cols, fun3, latexize_underscores=False) textAvgRuntime = printer.table_color_map(textAvgRuntime, 0.0, 1000, 10000) print(textAvgRuntime) print("\n\n") def fun4(filtered): if len(filtered) == 0: return "-" evalSolver = get_sum(filtered, "result.stats.evaluatedSolver") evalSolverUnknown = get_sum(filtered, "result.stats.evaluatedSolverUnknown") evalSolverTimeout = get_sum(filtered, "result.stats.evaluatedSolverTimeout") if evalSolver > 0: percentUnsuccessful = float(evalSolverTimeout + evalSolverUnknown) / float(evalSolver) return str(round(percentUnsuccessful,3)) else: return "-" textRatioOfUnknowns = printer.latex_table(props, dim_rows, dim_cols, fun4, latexize_underscores=False) textRatioOfUnknowns = printer.table_color_map(textRatioOfUnknowns, 0.0, 0.3, 0.6) print(textRatioOfUnknowns) print("\n\n") report = reporting.ReportPDF() section1 = reporting.Section("Experiments", []) subsects = [("Status (correctly finished processes)", textStatus, reversed(reporting.color_scheme_red)), ("Number of optimal solutions (max=100)", textNumOptimal, reporting.color_scheme_green), ("Average fitness", textAvgFitness, reporting.color_scheme_green), ("Average runtime", textAvgRuntime, reporting.color_scheme_blue), ("Ratio of unknowns", textRatioOfUnknowns, reporting.color_scheme_yellow)] for title, table, cs in subsects: if isinstance(cs, reporting.ColorScheme3): cs = cs.toBlockLatex() sub = reporting.Subsection(title, [cs, reporting.BlockLatex(table + "\n")]) section1.add(sub) report.add(section1) report.save_and_compile("eps_results.tex")
def create_section_with_results(title, desc, folders, numRuns=10, use_bench_simple_names=True): assert isinstance(title, str) assert isinstance(desc, str) assert isinstance(folders, list) print("\n*** Processing: {0}***".format(title)) for f in folders: print(f) if folders is None or len(folders) == 0: return None props = load_correct_props(folders, name=title) def post(s): return s.replace("{ccccccccccccc}", "{rrrrrrrrrrrrr}").replace("{rrr", "{lrr").replace(r"\_{lex}", "_{lex}").replace(r"\_{7}", "_{7}").replace("other/", "").replace("sygus16/", "") dim_benchmarks = Dim.from_data(props, lambda p: p["benchmark"]) if use_bench_simple_names: configs = [Config(benchmarks_simple_names[c.get_caption()], c.filters[0][1]) for c in dim_benchmarks.configs] dim_benchmarks = Dim(configs) dim_benchmarks.sort() dim_cols = dim_method * dim_ea_type * dim_sel # dim_cols = dim_method * dim_sa print("STATUS") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_num_computed, layered_headline=True)) latex_status = printer.table_color_map(text, 0.0, numRuns/2, numRuns, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("SUCCESS RATES") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, fun_successRates, layered_headline=True)) latex_successRates = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") # print("SUCCESS RATES (FULL INFO)") # text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, fun_successRates_full, layered_headline=True)) # print(text + "\n\n") print("AVG BEST-OF-RUN FITNESS") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_fitness, layered_headline=True)) latex_avgBestOfRunFitness = printer.table_color_map(text, 0.6, 0.98, 1.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG TOTAL TESTS") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_totalTests, layered_headline=True)) latex_avgTotalTests = printer.table_color_map(text, 0.0, 1000.0, 2000.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG RUNTIME") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_runtime, layered_headline=True)) latex_avgRuntime = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG RUNTIME (SUCCESSFUL)") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_runtimeOnlySuccessful, layered_headline=True)) latex_avgRuntimeOnlySuccessful = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG RUNTIME PER PROGRAM") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_runtimePerProgram, layered_headline=True)) latex_avgRuntimePerProgram = printer.table_color_map(text, 0.01, 1.0, 2.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG GENERATION") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_generation, layered_headline=True)) latex_avgGeneration = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG GENERATION (SUCCESSFUL)") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_generationSuccessful, layered_headline=True)) latex_avgGenerationSuccessful = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") print("AVG SIZES") text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_stats_size, layered_headline=True)) latex_sizes = printer.table_color_map(text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh") # print(text + "\n\n") section = reporting.Section(title, []) subsects = [("Status (correctly finished processes)", latex_status, reversed(reporting.color_scheme_red)), ("Success rates", latex_successRates, reporting.color_scheme_green), ("Average best-of-run ratio of passed tests", latex_avgBestOfRunFitness, reporting.color_scheme_green), ("Average sizes of $T_C$ (total tests in run)", latex_avgTotalTests, reporting.color_scheme_blue), ("Average runtime [s]", latex_avgRuntime, reporting.color_scheme_violet), ("Average runtime (only successful) [s]", latex_avgRuntimeOnlySuccessful, reporting.color_scheme_violet), ("Average generation (optimal and nonoptimal bestOfRuns)", latex_avgGeneration, reporting.color_scheme_teal), ("Average generation (only optimal bestOfRuns)", latex_avgGenerationSuccessful, reporting.color_scheme_teal), ("Approximate average runtime per program [s]", latex_avgRuntimePerProgram, reporting.color_scheme_brown), ("Average sizes of best of runs (number of nodes)", latex_sizes, reporting.color_scheme_yellow)] bl_desc = reporting.BlockLatex(desc + "\n") section.add(bl_desc) for title, table, cs in subsects: if isinstance(cs, reporting.ColorScheme3): cs = cs.toBlockLatex() sub = reporting.Subsection(title, [cs, reporting.BlockLatex(table + "\n")]) section.add(sub) section.add(reporting.BlockLatex(r"\vspace{1cm}" + "\n")) return section
def create_subsection_shared_stats(props, dim_rows, dim_cols, numRuns): vb = 1 # vertical border print("STATUS") text = post( printer.latex_table(props, dim_rows, dim_cols, get_num_computed, layered_headline=True, vertical_border=vb)) latex_status = printer.table_color_map(text, 0.0, numRuns / 2, numRuns, "colorLow", "colorMedium", "colorHigh") print("SUCCESS RATES") print( printer.text_table(props, dim_rows, dim_cols, fun_successRate, d_cols=";")) text = post( printer.latex_table(props, dim_rows, dim_cols, fun_successRate, layered_headline=True, vertical_border=vb)) latex_successRates = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("FINAL PROPERTIES") print( printer.text_table(props, dim_rows, dim_cols, fun_propertiesMet, d_cols=";")) text = post( printer.latex_table(props, dim_rows, dim_cols, fun_propertiesMet, layered_headline=True, vertical_border=vb)) latex_propertiesMet = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("AVG RUNTIME") text = post( printer.latex_table(props, dim_rows, dim_cols, get_avg_runtime, layered_headline=True, vertical_border=vb)) latex_avgRuntime = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh") print("AVG RUNTIME (SUCCESSFUL)") text = post( printer.latex_table(props, dim_rows, dim_cols, get_avg_runtimeOnlySuccessful, layered_headline=True, vertical_border=vb)) latex_avgRuntimeOnlySuccessful = printer.table_color_map( text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh") # print("SUCCESS RATES (FULL INFO)") # text = post(printer.latex_table(props, dim_rows, dim_cols, fun_successRates_full, layered_headline=True, vertical_border=vb)) # print("AVG SIZES") # text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_size, layered_headline=True, vertical_border=vb)) # latex_sizes = printer.table_color_map(text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh") print("AVG SIZES (SUCCESSFUL)") text = post( printer.latex_table(props, dim_rows, dim_cols, get_stats_sizeOnlySuccessful, layered_headline=True, vertical_border=vb)) latex_sizesOnlySuccessful = printer.table_color_map( text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh") subsects_main = [ ("Status (correctly finished runs)", latex_status, reversed(reporting.color_scheme_red)), ("Success rates (mse below thresh (1.0e-25) + properties met)", latex_successRates, reporting.color_scheme_green), ("Success rates (properties met)", latex_propertiesMet, reporting.color_scheme_green), ("Average runtime [s]", latex_avgRuntime, reporting.color_scheme_violet), ("Average runtime (only successful) [s]", latex_avgRuntimeOnlySuccessful, reporting.color_scheme_violet), # ("Average sizes of best of runs (number of nodes)", latex_sizes, reporting.color_scheme_yellow), ("Average sizes of best of runs (number of nodes) (only successful)", latex_sizesOnlySuccessful, reporting.color_scheme_yellow), ] return reporting.Subsection("Shared Statistics", get_content_of_subsections(subsects_main))
def create_subsection_cdgp_specific(props, dim_rows, dim_cols, exp_prefix): vb = 1 # vertical border print("AVG BEST-OF-RUN FITNESS") text = post( printer.latex_table(props, dim_rows, dim_cols, get_avg_fitness, layered_headline=True, vertical_border=vb)) latex_avgBestOfRunFitness = printer.table_color_map( text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("AVG TOTAL TESTS") text = post( printer.latex_table(props, dim_rows, dim_cols, get_avg_totalTests, layered_headline=True, vertical_border=vb)) latex_avgTotalTests = printer.table_color_map(text, 0.0, 1000.0, 2000.0, "colorLow", "colorMedium", "colorHigh") print("AVG RUNTIME PER PROGRAM") text = post( printer.latex_table(props, dim_rows, dim_cols, get_avg_runtimePerProgram, layered_headline=True, vertical_border=vb)) latex_avgRuntimePerProgram = printer.table_color_map( text, 0.01, 1.0, 2.0, "colorLow", "colorMedium", "colorHigh") print("AVG GENERATION") text = post( printer.latex_table(props, dim_rows, dim_cols, get_avg_generation, layered_headline=True, vertical_border=vb)) latex_avgGeneration = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh") print("AVG EVALUATED SOLUTIONS") text = post( printer.latex_table(props, dim_rows, dim_cols, get_avg_evaluated, layered_headline=True, vertical_border=vb)) latex_avgEvaluated = printer.table_color_map(text, 500.0, 25000.0, 100000.0, "colorLow", "colorMedium", "colorHigh") print("AVG EVALUATED SOLUTIONS (SUCCESSFUL)") text = post( printer.latex_table(props, dim_rows, dim_cols, get_avg_evaluatedSuccessful, layered_headline=True, vertical_border=vb)) latex_avgEvaluatedSuccessful = printer.table_color_map( text, 500.0, 25000.0, 100000.0, "colorLow", "colorMedium", "colorHigh") print("MAX SOLVER TIME") text = post( printer.latex_table(props, dim_rows, dim_cols, get_stats_maxSolverTime, layered_headline=True, vertical_border=vb)) latex_maxSolverTimes = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh") print("AVG SOLVER TIME") text = post( printer.latex_table(props, dim_rows, dim_cols, get_stats_avgSolverTime, layered_headline=True, vertical_border=vb)) latex_avgSolverTimes = printer.table_color_map(text, 0.0, 0.015, 0.03, "colorLow", "colorMedium", "colorHigh") print("AVG NUM SOLVER CALLS") text = post( printer.latex_table(props, dim_rows, dim_cols, get_avgSolverTotalCalls, layered_headline=True, vertical_border=vb)) latex_avgSolverTotalCalls = printer.table_color_map( text, 1e1, 1e2, 1e4, "colorLow", "colorMedium", "colorHigh") print("NUM SOLVER CALLS > 0.5s") text = post( printer.latex_table(props, dim_rows, dim_cols, get_numSolverCallsOverXs, layered_headline=True, vertical_border=vb)) latex_numSolverCallsOverXs = printer.table_color_map( text, 0, 50, 100, "colorLow", "colorMedium", "colorHigh") plot_figures(props, exp_prefix=exp_prefix) subsects_cdgp = [ ("Average best-of-run ratio of passed tests", latex_avgBestOfRunFitness, reporting.color_scheme_green), ("Average sizes of $T_C$ (total tests in run)", latex_avgTotalTests, reporting.color_scheme_blue), ("Average generation (all)", latex_avgGeneration, reporting.color_scheme_teal), #("Average generation (only successful)", latex_avgGenerationSuccessful, reporting.color_scheme_teal), ("Average evaluated solutions", latex_avgEvaluated, reporting.color_scheme_teal), ("Average evaluated solutions (only successful)", latex_avgEvaluatedSuccessful, reporting.color_scheme_teal), ("Approximate average runtime per program [s]", latex_avgRuntimePerProgram, reporting.color_scheme_brown), ("Max solver time per query [s]", latex_maxSolverTimes, reporting.color_scheme_violet), ("Avg solver time per query [s]", latex_avgSolverTimes, reporting.color_scheme_brown), ("Avg number of solver calls (in thousands; 1=1000)", latex_avgSolverTotalCalls, reporting.color_scheme_blue), ("Number of solver calls $>$ 0.5s", latex_numSolverCallsOverXs, reporting.color_scheme_blue), ] return reporting.Subsection("CDGP Statistics", get_content_of_subsections(subsects_cdgp))