def create_subsection_ea_stats(props, title, dim_rows, dim_cols, headerRowNames):
vb = 1 # vertical border
variants = None # variants_benchmarkNumTests
dim_rows_v2 = get_benchmarks_from_props(props, ignoreNumTests=True)
# dim_rows_v2 += dim_true # TODO: within dict
tables = [
TableGenerator(get_rankingOfBestSolutionsCDSR(ONLY_VISIBLE_SOLS=True, NUM_SHOWN=15),
Dim(dim_cols.configs[:-1]), Dim(dim_rows.configs[:-1]),
headerRowNames=headerRowNames,
title="The best solutions found for each benchmark and their sizes. Format: solution (MSE on test set) (size)",
color_scheme=reporting.color_scheme_violet, middle_col_align="l",
default_color_thresholds=(0.0, 900.0, 1800.0),
vertical_border=vb, table_postprocessor=post, variants=variants,
),
TableGenerator(get_stats_size, dim_rows, dim_cols, headerRowNames=headerRowNames,
title="Average sizes of best of runs (number of nodes)",
color_scheme=reporting.color_scheme_yellow,
default_color_thresholds=(0.0, 100.0, 200.0),
vertical_border=vb, table_postprocessor=post, variants=variants,
),
# TableGenerator(get_stats_sizeOnlySuccessful, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Average sizes of best of runs (number of nodes) (only successful)",
# color_scheme=reporting.color_scheme_yellow,
# default_color_thresholds=(0.0, 100.0, 200.0),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
TableGenerator(get_avg_generation, dim_rows, dim_cols, headerRowNames=headerRowNames,
title="Average generation (all)",
color_scheme=reporting.color_scheme_teal,
default_color_thresholds=(0.0, 5000.0, 10000.0),
vertical_border=vb, table_postprocessor=post, variants=variants,
),
TableGenerator(get_avg_evaluated, dim_rows, dim_cols, headerRowNames=headerRowNames,
title="Average number of evaluated solutions",
color_scheme=reporting.color_scheme_brown,
default_color_thresholds=(0.0, 5000.0, 10000.0),
vertical_border=vb, table_postprocessor=post, variants=variants,
),
# TableGenerator(get_avg_evaluatedSuccessful, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Average number of evaluated solutions",
# color_scheme=reporting.color_scheme_brown,
# default_color_thresholds=(500.0, 25000.0, 100000.0),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
TableGenerator(get_avg_doneAlgRestarts, dim_rows, dim_cols, headerRowNames=headerRowNames,
title="Number of algorithm restarts (avg)",
color_scheme=reporting.color_scheme_gray_light,
default_color_thresholds=(0.0, 1e2, 1e4),
vertical_border=vb, table_postprocessor=post, variants=variants,
),
]
subsects_main = []
for t in tables:
tup = (t.title, t.apply(props), t.color_scheme)
subsects_main.append(tup)
return reporting.Subsection(title, get_content_of_subsections(subsects_main))
def create_section_with_results(title, desc, folders, numRuns=10, use_bench_simple_names=True, print_status_matrix=True):
assert isinstance(title, str)
assert isinstance(desc, str)
assert isinstance(folders, list)
print("\n*** Processing: {0}***".format(title))
for f in folders:
print(f)
if folders is None or len(folders) == 0:
return None
props = load_correct_props(folders, name=title)
# Create figures in the appropriate directory
plot_figures(props)
# Uncomment this to print names of files with results of a certain configuration
# print("\n(** {0} **) props_meeting the property:".format(title[:15]))
# for p in props:
# if float(p["cdgp.solverTimeMaxSec"]) >= 2.0:
# print(p["thisFileName"] + ", " + p["cdgp.solverTimeMaxSec"])
def post(s):
return s.replace("{ccccccccccccc}", "{rrrrrrrrrrrrr}").replace("{rrr", "{lrr").replace(r"\_{lex}", "_{lex}").replace(r"\_{", "_{")
dim_benchmarks = Dim.from_dict(props, "benchmark")
if print_status_matrix:
d = dim_benchmarks * dim_methodCDGP * dim_testsRatio * dim_sa + \
dim_benchmarks * dim_methodGPR * dim_testsRatioGPR * dim_sa
matrix = produce_status_matrix(d, props)
print("\n****** Status matrix:")
print(matrix + "\n")
if use_bench_simple_names:
configs = [Config(benchmarks_simple_names.get(c.get_caption(), c.get_caption()), c.filters[0][1]) for c in dim_benchmarks.configs]
dim_benchmarks = Dim(configs)
dim_benchmarks.sort()
# --------------------- Shared stats ---------------------
# -------------- Dimensions -----------------
dim_cols = dim_methodCDGP * dim_ea_type * dim_sel * dim_testsRatio +\
dim_methodGPR * dim_ea_type * dim_sel * dim_testsRatioGPR + \
dim_methodFormal
dim_rows = dim_benchmarks.sort() + dim_true
# dim_cols = dim_method * dim_sa
# -------------------------------------------
vb = 1 # vertical border
print("STATUS")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_num_computed, layered_headline=True, vertical_border=vb))
latex_status = printer.table_color_map(text, 0.0, numRuns/2, numRuns, "colorLow", "colorMedium", "colorHigh")
print("SUCCESS RATES")
text = post(printer.latex_table(props, dim_rows, dim_cols, fun_successRates, layered_headline=True, vertical_border=vb))
latex_successRates = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh")
print("AVG RUNTIME")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtime, layered_headline=True, vertical_border=vb))
latex_avgRuntime = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh")
print("AVG RUNTIME (SUCCESSFUL)")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtimeOnlySuccessful, layered_headline=True, vertical_border=vb))
latex_avgRuntimeOnlySuccessful = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh")
# print("SUCCESS RATES (FULL INFO)")
# text = post(printer.latex_table(props, dim_rows, dim_cols, fun_successRates_full, layered_headline=True, vertical_border=vb))
# print("AVG SIZES")
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_size, layered_headline=True, vertical_border=vb))
# latex_sizes = printer.table_color_map(text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh")
print("AVG SIZES (SUCCESSFUL)")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_sizeOnlySuccessful, layered_headline=True, vertical_border=vb))
latex_sizesOnlySuccessful = printer.table_color_map(text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh")
# --------------------- CDGP stats ---------------------
# -------------- Dimensions -----------------
dim_cols = dim_methodCDGP * dim_ea_type * dim_sel * dim_testsRatio + \
dim_methodGPR * dim_ea_type * dim_sel * dim_testsRatioGPR
# -------------------------------------------
print("AVG BEST-OF-RUN FITNESS")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_fitness, layered_headline=True, vertical_border=vb))
latex_avgBestOfRunFitness = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh")
print("AVG TOTAL TESTS")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_totalTests, layered_headline=True, vertical_border=vb))
latex_avgTotalTests = printer.table_color_map(text, 0.0, 1000.0, 2000.0, "colorLow", "colorMedium", "colorHigh")
print("AVG RUNTIME PER PROGRAM")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtimePerProgram, layered_headline=True, vertical_border=vb))
latex_avgRuntimePerProgram = printer.table_color_map(text, 0.01, 1.0, 2.0, "colorLow", "colorMedium", "colorHigh")
print("AVG GENERATION")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_generation, layered_headline=True, vertical_border=vb))
latex_avgGeneration = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh")
print("AVG GENERATION (SUCCESSFUL)")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_generationSuccessful, layered_headline=True, vertical_border=vb))
latex_avgGenerationSuccessful = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh")
print("MAX SOLVER TIME")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_maxSolverTime, layered_headline=True, vertical_border=vb))
latex_maxSolverTimes = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh")
print("AVG SOLVER TIME")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_avgSolverTime, layered_headline=True, vertical_border=vb))
latex_avgSolverTimes = printer.table_color_map(text, 0.0, 0.015, 0.03, "colorLow", "colorMedium", "colorHigh")
print("AVG NUM SOLVER CALLS")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_avgSolverTotalCalls, layered_headline=True, vertical_border=vb))
latex_avgSolverTotalCalls = printer.table_color_map(text, 1e1, 1e2, 1e4, "colorLow", "colorMedium", "colorHigh")
print("NUM SOLVER CALLS > 0.5s")
text = post(printer.latex_table(props, dim_rows, dim_cols, get_numSolverCallsOverXs, layered_headline=True, vertical_border=vb))
latex_numSolverCallsOverXs = printer.table_color_map(text, 0, 50, 100, "colorLow", "colorMedium", "colorHigh")
subsects_main = [
("Status (correctly finished processes)", latex_status, reversed(reporting.color_scheme_red)),
("Success rates", latex_successRates, reporting.color_scheme_green),
("Average runtime [s]", latex_avgRuntime, reporting.color_scheme_violet),
("Average runtime (only successful) [s]", latex_avgRuntimeOnlySuccessful, reporting.color_scheme_violet),
#("Average sizes of best of runs (number of nodes)", latex_sizes, reporting.color_scheme_yellow),
("Average sizes of best of runs (number of nodes) (only successful)", latex_sizesOnlySuccessful, reporting.color_scheme_yellow),
]
subsects_cdgp = [
("Average best-of-run ratio of passed tests", latex_avgBestOfRunFitness, reporting.color_scheme_green),
("Average sizes of $T_C$ (total tests in run)", latex_avgTotalTests, reporting.color_scheme_blue),
("Average generation (all)", latex_avgGeneration, reporting.color_scheme_teal),
("Average generation (only successful)", latex_avgGenerationSuccessful, reporting.color_scheme_teal),
("Approximate average runtime per program [s]", latex_avgRuntimePerProgram, reporting.color_scheme_brown),
("Max solver time per query [s]", latex_maxSolverTimes, reporting.color_scheme_violet),
("Avg solver time per query [s]", latex_avgSolverTimes, reporting.color_scheme_brown),
("Avg number of solver calls (in thousands; 1=1000)", latex_avgSolverTotalCalls, reporting.color_scheme_blue),
("Number of solver calls $>$ 0.5s", latex_numSolverCallsOverXs, reporting.color_scheme_blue),
]
figures_cdgp = [
"figures/ratioEvaluated_correctVsAllRuns.pdf",
"figures/ratioTime_correctVsAllCorrect.pdf",
"figures/ratioTime_endedVsAllEnded.pdf",
]
def get_content_of_subsections(subsects):
content = []
vspace = reporting.BlockLatex(r"\vspace{0.75cm}"+"\n")
for title, table, cs in subsects:
if isinstance(cs, reporting.ColorScheme3):
cs = cs.toBlockLatex()
sub = reporting.SectionRelative(title, contents=[cs, reporting.BlockLatex(table + "\n"), vspace])
content.append(sub)
return content
section = reporting.Section(title, [])
section.add(reporting.BlockLatex(desc + "\n"))
section.add(reporting.Subsection("Shared Statistics", get_content_of_subsections(subsects_main)))
section.add(reporting.Subsection("CDGP Statistics", get_content_of_subsections(subsects_cdgp)))
for f in figures_cdgp:
section.add(reporting.FloatFigure(f))
section.add(reporting.BlockLatex(r"\vspace{1cm}" + "\n"))
return section
def create_subsection_cdgp_specific(props, dim_rows, dim_cols, headerRowNames):
vb = 1 # vertical border
# variants = variants_benchmarkNumTests
variants = None
props = [p for p in props if p["method"] != "GP"]
print("AVG TOTAL TESTS")
latex_avgTotalTests = create_single_table_bundle(props,
dim_rows,
dim_cols,
get_avg_totalTests,
headerRowNames,
cv0=0.0,
cv1=1000.0,
cv2=2000.0,
tableVariants=variants)
# text = post(
# printer.latex_table(props, dim_rows, dim_cols, get_avg_totalTests, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgTotalTests = printer.table_color_map(text, 0.0, 1000.0, 2000.0, "colorLow", "colorMedium", "colorHigh")
# print("AVG RUNTIME PER PROGRAM")
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtimePerProgram, layered_headline=True,
# vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgRuntimePerProgram = printer.table_color_map(text, 0.01, 1.0, 2.0, "colorLow", "colorMedium", "colorHigh")
# print("AVG GENERATION")
# latex_avgGeneration = create_single_table_bundle(props, dim_rows, dim_cols, get_avg_generation, headerRowNames,
# cv0=0.0, cv1=100.0, cv2=200.0, tableVariants=variants)
# text = post(
# printer.latex_table(props, dim_rows, dim_cols, get_avg_generation, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgGeneration = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh")
# print("AVG EVALUATED SOLUTIONS")
# text = post(
# printer.latex_table(props, dim_rows, dim_cols, get_avg_evaluated, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgEvaluated = printer.table_color_map(text, 500.0, 25000.0, 100000.0, "colorLow", "colorMedium", "colorHigh")
# print("AVG EVALUATED SOLUTIONS (SUCCESSFUL)")
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_evaluatedSuccessful, layered_headline=True,
# vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgEvaluatedSuccessful = printer.table_color_map(text, 500.0, 25000.0, 100000.0, "colorLow", "colorMedium",
# "colorHigh")
print("MAX SOLVER TIME")
latex_maxSolverTimes = create_single_table_bundle(props,
dim_rows,
dim_cols,
get_stats_maxSolverTime,
headerRowNames,
cv0=0.0,
cv1=0.5,
cv2=1.0,
tableVariants=variants)
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_maxSolverTime, layered_headline=True,
# vertical_border=vb, headerRowNames=headerRowNames))
# latex_maxSolverTimes = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh")
print("AVG SOLVER TIME")
latex_avgSolverTimes = create_single_table_bundle(props,
dim_rows,
dim_cols,
get_stats_avgSolverTime,
headerRowNames,
cv0=0.0,
cv1=0.015,
cv2=0.03,
tableVariants=variants)
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_avgSolverTime, layered_headline=True,
# vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgSolverTimes = printer.table_color_map(text, 0.0, 0.015, 0.03, "colorLow", "colorMedium", "colorHigh")
print("AVG NUM SOLVER CALLS")
latex_avgSolverTotalCalls = create_single_table_bundle(
props,
dim_rows,
dim_cols,
get_avgSolverTotalCalls,
headerRowNames,
cv0=1e1,
cv1=1e2,
cv2=1e4,
tableVariants=variants)
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_avgSolverTotalCalls, layered_headline=True,
# vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgSolverTotalCalls = printer.table_color_map(text, 1e1, 1e2, 1e4, "colorLow", "colorMedium", "colorHigh")
print("NUM SOLVER CALLS > 0.5s")
latex_numSolverCallsOverXs = create_single_table_bundle(
props,
dim_rows,
dim_cols,
get_numSolverCallsOverXs,
headerRowNames,
cv0=0,
cv1=50,
cv2=100,
tableVariants=variants)
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_numSolverCallsOverXs, layered_headline=True,
# vertical_border=vb, headerRowNames=headerRowNames))
# latex_numSolverCallsOverXs = printer.table_color_map(text, 0, 50, 100, "colorLow", "colorMedium", "colorHigh")
print("MOST FREQUENTLY FOUND COUNTEREXAMPLE")
latex_freqCounterexamples = create_single_table_bundle(
props,
dim_rows,
dim_cols,
get_freqCounterexamples,
headerRowNames,
cv0=0,
cv1=50,
cv2=100,
tableVariants=variants,
middle_col_align="l")
subsects_cdgp = [
("Average sizes of $T_C$ (total tests in run)", latex_avgTotalTests,
reporting.color_scheme_blue),
# ("Average generation (all)", latex_avgGeneration, reporting.color_scheme_teal),
#("Average generation (only successful)", latex_avgGenerationSuccessful, reporting.color_scheme_teal),
# ("Average evaluated solutions", latex_avgEvaluated, reporting.color_scheme_brown),
# ("Average evaluated solutions (only successful)", latex_avgEvaluatedSuccessful, reporting.color_scheme_teal),
# ("Approximate average runtime per program [s]", latex_avgRuntimePerProgram, reporting.color_scheme_brown),
("Max solver time per query [s]", latex_maxSolverTimes,
reporting.color_scheme_violet),
("Avg solver time per query [s]", latex_avgSolverTimes,
reporting.color_scheme_brown),
("Avg number of solver calls (in thousands; 1=1000)",
latex_avgSolverTotalCalls, reporting.color_scheme_blue),
("Number of solver calls $>$ 0.5s", latex_numSolverCallsOverXs,
reporting.color_scheme_blue),
("The most frequently found counterexamples for each benchmark and configuration",
latex_freqCounterexamples, reporting.color_scheme_violet),
]
return reporting.Subsection("CDGP Statistics",
get_content_of_subsections(subsects_cdgp))
def create_subsection_shared_stats(props, dim_rows, dim_cols, numRuns,
headerRowNames):
vb = 1 # vertical border
variants = None # variants_benchmarkNumTests
dim_rows_v2 = get_benchmarks_from_props(props,
simple_names=True,
ignoreNumTests=True)
dim_rows_v2 += dim_true
# ----------------------------------------------------
# Cleaning experiment here, because dimension can be easily constructed.
# dim_rows_v3 = get_benchmarks_from_props(props, simple_names=True, ignoreNumTests=True)
# utils.reorganizeExperimentFiles(props, dim_rows_v3 * dim_benchmarkNumTests * dim_cols, target_dir="./exp3_final/", maxRuns=numRuns)
# utils.deleteFilesByPredicate(props, lambda p: len(p["maxGenerations"]) > 7, simulate=False)
# ----------------------------------------------------
def scNotColorValueExtractor(s):
if s == "-" or "10^{" not in s:
return s
else:
r = s.split("10^{")
return r[1][:-2]
tables = [
TableGenerator(
get_num_computed,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Status (correctly finished runs)",
color_scheme=reversed(reporting.color_scheme_red),
default_color_thresholds=(0.0, numRuns / 2, numRuns),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
# TableGenerator(fun_successRate, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Success rates (properties met + mse below thresh)",
# color_scheme=reporting.color_scheme_darkgreen,
# default_color_thresholds=(0.0, 0.5, 1.0),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
# TableGenerator(get_median_testMSEsuccessRateForThresh_onlyVerified, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Test set: success rate of accepted solutions with MSE below optThreshold (i.e., no overfitting)",
# color_scheme=reporting.color_scheme_red2white2darkgreen,
# default_color_thresholds=(0.0, 0.5, 1.0),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
# TableGenerator(fun_trainMseBelowThresh, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Training set: success rates (mse below thresh)",
# color_scheme=reporting.color_scheme_green,
# default_color_thresholds=(0.0, 0.5, 1.0),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
# TableGenerator(get_median_mseOptThresh, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Training set: optThreshold for MSE (median)",
# color_scheme=reporting.color_scheme_violet,
# default_color_thresholds=(-10.0, 0.0, 10.0),
# color_value_extractor=scNotColorValueExtractor,
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
TableGenerator(
get_median_trainMSE,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Training set: MSE (median)",
color_scheme=reporting.color_scheme_gray_dark,
default_color_thresholds=(-10.0, 0.0, 10.0),
color_value_extractor=scNotColorValueExtractor,
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
# TableGenerator(get_avg_trainMSE, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Training set: MSE (avg)",
# color_scheme=reporting.color_scheme_green,
# default_color_thresholds=(0.0, 1e2, 1e4),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
TableGenerator(
get_median_testMSE,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Test set: MSE (median)",
color_scheme=reporting.color_scheme_gray_dark,
default_color_thresholds=(-10.0, 0.0, 10.0),
color_value_extractor=scNotColorValueExtractor,
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
# TableGenerator(get_median_testMSE_noScNot, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Test set: MSE (median) (noScNot)",
# color_scheme=reporting.color_scheme_green,
# default_color_thresholds=(0.0, 1e2, 1e4),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
# TableGenerator(get_avg_testMSE, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Test set: MSE (avg)",
# color_scheme=reporting.color_scheme_green,
# default_color_thresholds=(0.0, 1e2, 1e4),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
TableGenerator(
fun_allPropertiesMet,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Success rates (properties met)",
color_scheme=reporting.color_scheme_green,
default_color_thresholds=(0.0, 0.5, 1.0),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
TableGenerator(
get_avg_runtime,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Average runtime [s]",
color_scheme=reporting.color_scheme_violet,
default_color_thresholds=(0.0, 900.0, 1800.0),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
# TableGenerator(get_avg_runtimeOnlySuccessful, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Average runtime (only successful) [s]",
# color_scheme=reporting.color_scheme_violet,
# default_color_thresholds=(0.0, 900.0, 1800.0),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
TableGenerator(
get_avg_doneAlgRestarts,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Number of algorithm restarts (avg)",
color_scheme=reporting.color_scheme_gray_light,
default_color_thresholds=(0.0, 1e2, 1e4),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
# TableGenerator(get_stats_size, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Average sizes of best of runs (number of nodes)",
# color_scheme=reporting.color_scheme_yellow,
# default_color_thresholds=(0.0, 100.0, 200.0),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
# TableGenerator(get_stats_sizeOnlySuccessful, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Average sizes of best of runs (number of nodes) (only successful)",
# color_scheme=reporting.color_scheme_yellow,
# default_color_thresholds=(0.0, 100.0, 200.0),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
TableGenerator(
get_avg_generation,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Average generation (all)",
color_scheme=reporting.color_scheme_teal,
default_color_thresholds=(0.0, 5000.0, 10000.0),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
TableGenerator(
get_avg_evaluated,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Average number of evaluated solutions",
color_scheme=reporting.color_scheme_brown,
default_color_thresholds=(0.0, 5000.0, 10000.0),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
# TableGenerator(get_avg_evaluatedSuccessful, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Average number of evaluated solutions",
# color_scheme=reporting.color_scheme_brown,
# default_color_thresholds=(500.0, 25000.0, 100000.0),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
]
subsects_main = []
for t in tables:
tup = (t.title, t.apply(props), t.color_scheme)
subsects_main.append(tup)
return reporting.Subsection("Shared Statistics",
get_content_of_subsections(subsects_main))
def create_subsection_aggregation_tests(props, dim_rows, dim_cols,
headerRowNames):
vb = 1 # vertical border
variants = None
dim_rows_v2 = get_benchmarks_from_props(props,
simple_names=True,
ignoreNumTests=True)
dim_rows_v2 += dim_true
# By default: dim_cols = (dim_methodGP * dim_empty + dim_methodCDGP * dim_testsRatio) * dim_optThreshold
tables = [
# TableGenerator(fun_successRateMseOnly, dim_rows_v2,
# (dim_methodGP + dim_methodCDGP),
# headerRowNames=[""],
# title="Success rates (mse below thresh)",
# color_scheme=reporting.color_scheme_violet,
# default_color_thresholds=(0.0, 0.5, 1.0),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
TableGenerator(
fun_successRate,
dim_rows_v2,
dim_benchmarkNumTests * dim_method,
headerRowNames=["", ""],
title="Success rates (mse below thresh + properties met)",
color_scheme=reporting.color_scheme_darkgreen,
default_color_thresholds=(0.0, 0.5, 1.0),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
TableGenerator(
fun_successRate,
dim_rows_v2,
dim_method * dim_benchmarkNumTests,
headerRowNames=["", ""],
title="Success rates (mse below thresh + properties met)",
color_scheme=reporting.color_scheme_darkgreen,
default_color_thresholds=(0.0, 0.5, 1.0),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
TableGenerator(
fun_successRate,
dim_rows_v2,
dim_optThreshold * dim_method,
headerRowNames=["tolerance", ""],
title="Success rates (mse below thresh + properties met)",
color_scheme=reporting.color_scheme_darkgreen,
default_color_thresholds=(0.0, 0.5, 1.0),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
TableGenerator(
fun_successRate,
dim_rows_v2,
dim_method,
headerRowNames=[""],
title="Success rates (mse below thresh + properties met)",
color_scheme=reporting.color_scheme_darkgreen,
default_color_thresholds=(0.0, 0.5, 1.0),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
TableGenerator(
fun_allPropertiesMet,
dim_rows_v2,
dim_benchmarkNumTests * dim_method,
headerRowNames=["", ""],
title="Success rates (properties met)",
color_scheme=reporting.color_scheme_green,
default_color_thresholds=(0.0, 0.5, 1.0),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
TableGenerator(
fun_allPropertiesMet,
dim_rows_v2,
dim_method * dim_benchmarkNumTests,
headerRowNames=["", ""],
title="Success rates (properties met)",
color_scheme=reporting.color_scheme_green,
default_color_thresholds=(0.0, 0.5, 1.0),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
# TableGenerator(fun_allPropertiesMet, dim_rows_v2,
# dim_benchmarkNumTests * dim_optThreshold * dim_method,
# headerRowNames=["", "tolerance", ""],
# title="Success rates (properties met)",
# color_scheme=reporting.color_scheme_green,
# default_color_thresholds=(0.0, 0.5, 1.0),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
TableGenerator(
fun_allPropertiesMet,
dim_rows_v2,
dim_optThreshold * dim_method,
headerRowNames=["tolerance", ""],
title="Success rates (properties met)",
color_scheme=reporting.color_scheme_green,
default_color_thresholds=(0.0, 0.5, 1.0),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
TableGenerator(
fun_allPropertiesMet,
dim_rows_v2,
dim_method,
headerRowNames=[""],
title="Success rates (properties met)",
color_scheme=reporting.color_scheme_green,
default_color_thresholds=(0.0, 0.5, 1.0),
vertical_border=vb,
table_postprocessor=post,
variants=variants,
),
]
subsects_main = []
for t in tables:
tup = (t.title, t.apply(props), t.color_scheme)
subsects_main.append(tup)
return reporting.Subsection(
"Tests of different data aggregation in tables",
get_content_of_subsections(subsects_main))
def create_subsection_cdgp_specific(props, dim_rows, dim_cols, headerRowNames):
vb = 1 # vertical border
variants = [
p_benchmarkNumTests_equalTo("3"),
p_benchmarkNumTests_equalTo("5"),
p_benchmarkNumTests_equalTo("10")
]
print("AVG BEST-OF-RUN FITNESS (MSE)")
latex_avgBestOfRunFitness = create_single_table_bundle(
props,
dim_rows,
dim_cols,
get_avg_mse,
headerRowNames,
cv0=0.0,
cv1=0.5,
cv2=1.0,
tableVariants=variants)
# text = post(
# printer.latex_table(props, dim_rows, dim_cols, get_avg_mse, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgBestOfRunFitness = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh")
print("AVG TOTAL TESTS")
latex_avgTotalTests = create_single_table_bundle(props,
dim_rows,
dim_cols,
get_avg_totalTests,
headerRowNames,
cv0=0.0,
cv1=1000.0,
cv2=2000.0,
tableVariants=variants)
# text = post(
# printer.latex_table(props, dim_rows, dim_cols, get_avg_totalTests, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgTotalTests = printer.table_color_map(text, 0.0, 1000.0, 2000.0, "colorLow", "colorMedium", "colorHigh")
# print("AVG RUNTIME PER PROGRAM")
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_runtimePerProgram, layered_headline=True,
# vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgRuntimePerProgram = printer.table_color_map(text, 0.01, 1.0, 2.0, "colorLow", "colorMedium", "colorHigh")
print("AVG GENERATION")
latex_avgGeneration = create_single_table_bundle(props,
dim_rows,
dim_cols,
get_avg_generation,
headerRowNames,
cv0=0.0,
cv1=100.0,
cv2=200.0,
tableVariants=variants)
# text = post(
# printer.latex_table(props, dim_rows, dim_cols, get_avg_generation, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgGeneration = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh")
# print("AVG EVALUATED SOLUTIONS")
# text = post(
# printer.latex_table(props, dim_rows, dim_cols, get_avg_evaluated, layered_headline=True, vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgEvaluated = printer.table_color_map(text, 500.0, 25000.0, 100000.0, "colorLow", "colorMedium", "colorHigh")
# print("AVG EVALUATED SOLUTIONS (SUCCESSFUL)")
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_avg_evaluatedSuccessful, layered_headline=True,
# vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgEvaluatedSuccessful = printer.table_color_map(text, 500.0, 25000.0, 100000.0, "colorLow", "colorMedium",
# "colorHigh")
print("MAX SOLVER TIME")
latex_maxSolverTimes = create_single_table_bundle(props,
dim_rows,
dim_cols,
get_stats_maxSolverTime,
headerRowNames,
cv0=0.0,
cv1=0.5,
cv2=1.0,
tableVariants=variants)
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_maxSolverTime, layered_headline=True,
# vertical_border=vb, headerRowNames=headerRowNames))
# latex_maxSolverTimes = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh")
print("AVG SOLVER TIME")
latex_avgSolverTimes = create_single_table_bundle(props,
dim_rows,
dim_cols,
get_stats_avgSolverTime,
headerRowNames,
cv0=0.0,
cv1=0.015,
cv2=0.03,
tableVariants=variants)
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_avgSolverTime, layered_headline=True,
# vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgSolverTimes = printer.table_color_map(text, 0.0, 0.015, 0.03, "colorLow", "colorMedium", "colorHigh")
print("AVG NUM SOLVER CALLS")
latex_avgSolverTotalCalls = create_single_table_bundle(
props,
dim_rows,
dim_cols,
get_avgSolverTotalCalls,
headerRowNames,
cv0=1e1,
cv1=1e2,
cv2=1e4,
tableVariants=variants)
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_avgSolverTotalCalls, layered_headline=True,
# vertical_border=vb, headerRowNames=headerRowNames))
# latex_avgSolverTotalCalls = printer.table_color_map(text, 1e1, 1e2, 1e4, "colorLow", "colorMedium", "colorHigh")
print("NUM SOLVER CALLS > 0.5s")
latex_numSolverCallsOverXs = create_single_table_bundle(
props,
dim_rows,
dim_cols,
get_numSolverCallsOverXs,
headerRowNames,
cv0=0,
cv1=50,
cv2=100,
tableVariants=variants)
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_numSolverCallsOverXs, layered_headline=True,
# vertical_border=vb, headerRowNames=headerRowNames))
# latex_numSolverCallsOverXs = printer.table_color_map(text, 0, 50, 100, "colorLow", "colorMedium", "colorHigh")
subsects_cdgp = [
("Average best-of-run MSE", latex_avgBestOfRunFitness,
reporting.color_scheme_green),
("Average sizes of $T_C$ (total tests in run)", latex_avgTotalTests,
reporting.color_scheme_blue),
("Average generation (all)", latex_avgGeneration,
reporting.color_scheme_teal),
#("Average generation (only successful)", latex_avgGenerationSuccessful, reporting.color_scheme_teal),
# ("Average evaluated solutions", latex_avgEvaluated, reporting.color_scheme_teal),
# ("Average evaluated solutions (only successful)", latex_avgEvaluatedSuccessful, reporting.color_scheme_teal),
# ("Approximate average runtime per program [s]", latex_avgRuntimePerProgram, reporting.color_scheme_brown),
("Max solver time per query [s]", latex_maxSolverTimes,
reporting.color_scheme_violet),
("Avg solver time per query [s]", latex_avgSolverTimes,
reporting.color_scheme_brown),
("Avg number of solver calls (in thousands; 1=1000)",
latex_avgSolverTotalCalls, reporting.color_scheme_blue),
("Number of solver calls $>$ 0.5s", latex_numSolverCallsOverXs,
reporting.color_scheme_blue),
]
return reporting.Subsection("CDGP Statistics",
get_content_of_subsections(subsects_cdgp))
def create_subsection_shared_stats(props, dim_rows, dim_cols, numRuns,
headerRowNames):
vb = 1 # vertical border
variants = [
p_benchmarkNumTests_equalTo("3"),
p_benchmarkNumTests_equalTo("5"),
p_benchmarkNumTests_equalTo("10")
]
dim_rows_v2 = get_benchmarks_from_props(props,
simple_names=True,
ignoreNumTests=True)
dim_rows_v2 += dim_true
# ----------------------------------------------------
# Cleaning experiment. Here, because dimension can be easily constructed.
# dim_rows_v3 = get_benchmarks_from_props(props, simple_names=True, ignoreNumTests=True)
# utils.reorganizeExperimentFiles(props, dim_rows_v3 * dim_benchmarkNumTests * dim_cols, target_dir="./exp3_final/", maxRuns=numRuns)
# ----------------------------------------------------
tables = [
TableGenerator(
get_num_computed,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Status (correctly finished runs)",
color_scheme=reversed(reporting.color_scheme_red),
default_color_thresholds=(0.0, numRuns / 2, numRuns),
vertical_border=vb,
table_postprocessor=post,
table_variants=variants,
),
TableGenerator(
fun_successRate,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Success rates (mse below thresh + properties met)",
color_scheme=reporting.color_scheme_darkgreen,
default_color_thresholds=(0.0, 0.5, 1.0),
vertical_border=vb,
table_postprocessor=post,
table_variants=variants,
),
TableGenerator(
fun_allPropertiesMet,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Success rates (properties met)",
color_scheme=reporting.color_scheme_green,
default_color_thresholds=(0.0, 0.5, 1.0),
vertical_border=vb,
table_postprocessor=post,
table_variants=variants,
),
TableGenerator(
get_avg_runtime,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Average runtime [s]",
color_scheme=reporting.color_scheme_violet,
default_color_thresholds=(0.0, 900.0, 1800.0),
vertical_border=vb,
table_postprocessor=post,
table_variants=variants,
),
TableGenerator(
get_avg_runtimeOnlySuccessful,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title="Average runtime (only successful) [s]",
color_scheme=reporting.color_scheme_violet,
default_color_thresholds=(0.0, 900.0, 1800.0),
vertical_border=vb,
table_postprocessor=post,
table_variants=variants,
),
# TableGenerator(get_stats_size, dim_rows, dim_cols, headerRowNames=headerRowNames,
# title="Average sizes of best of runs (number of nodes)",
# color_scheme=reporting.color_scheme_yellow,
# default_color_thresholds=(0.0, 100.0, 200.0),
# vertical_border=vb, table_postprocessor=post, table_variants=variants,
# ),
TableGenerator(
get_stats_sizeOnlySuccessful,
dim_rows,
dim_cols,
headerRowNames=headerRowNames,
title=
"Average sizes of best of runs (number of nodes) (only successful)",
color_scheme=reporting.color_scheme_yellow,
default_color_thresholds=(0.0, 100.0, 200.0),
vertical_border=vb,
table_postprocessor=post,
table_variants=variants,
),
]
subsects_main = []
for t in tables:
tup = (t.title, t.apply(props), t.color_scheme)
subsects_main.append(tup)
return reporting.Subsection("Shared Statistics",
get_content_of_subsections(subsects_main))
def print_table(props, dim_rows, dim_cols):
def fun0(filtered):
return str(len(filtered))
textStatus = printer.latex_table(props, dim_rows, dim_cols, fun0, latexize_underscores=False)
textStatus = printer.table_color_map(textStatus, 0.0, 50.0, 100.0)
print(textStatus)
print("\n\n")
def fun1(filtered):
if len(filtered) == 0:
return "-"
num_opt = get_num_optimal(filtered)
# return "{0}/{1}".format(str(num_opt), str(len(filtered)))
return "{0}".format(str(num_opt))
textNumOptimal = printer.latex_table(props, dim_rows, dim_cols, fun1, latexize_underscores=False)
textNumOptimal = printer.table_color_map(textNumOptimal, 0.0, 50.0, 100.0)
print(textNumOptimal)
print("\n\n")
def fun2(filtered):
if len(filtered) == 0:
return "-"
avgFit = round(get_stats_fitness(filtered)[0], 2)
return "{0}".format(str(avgFit))
textAvgFitness = printer.latex_table(props, dim_rows, dim_cols, fun2, latexize_underscores=False)
textAvgFitness = printer.table_color_map(textAvgFitness, 0.0, 25.0, 50.0)
print(textAvgFitness)
print("\n\n")
def fun3(filtered):
if len(filtered) == 0:
return "-"
avg_time = round(get_stats_duration(filtered)[0], 1)
return "{0}".format(str(avg_time))
textAvgRuntime = printer.latex_table(props, dim_rows, dim_cols, fun3, latexize_underscores=False)
textAvgRuntime = printer.table_color_map(textAvgRuntime, 0.0, 1000, 10000)
print(textAvgRuntime)
print("\n\n")
def fun4(filtered):
if len(filtered) == 0:
return "-"
evalSolver = get_sum(filtered, "result.stats.evaluatedSolver")
evalSolverUnknown = get_sum(filtered, "result.stats.evaluatedSolverUnknown")
evalSolverTimeout = get_sum(filtered, "result.stats.evaluatedSolverTimeout")
if evalSolver > 0:
percentUnsuccessful = float(evalSolverTimeout + evalSolverUnknown) / float(evalSolver)
return str(round(percentUnsuccessful,3))
else:
return "-"
textRatioOfUnknowns = printer.latex_table(props, dim_rows, dim_cols, fun4, latexize_underscores=False)
textRatioOfUnknowns = printer.table_color_map(textRatioOfUnknowns, 0.0, 0.3, 0.6)
print(textRatioOfUnknowns)
print("\n\n")
report = reporting.ReportPDF()
section1 = reporting.Section("Experiments", [])
subsects = [("Status (correctly finished processes)", textStatus, reversed(reporting.color_scheme_red)),
("Number of optimal solutions (max=100)", textNumOptimal, reporting.color_scheme_green),
("Average fitness", textAvgFitness, reporting.color_scheme_green),
("Average runtime", textAvgRuntime, reporting.color_scheme_blue),
("Ratio of unknowns", textRatioOfUnknowns, reporting.color_scheme_yellow)]
for title, table, cs in subsects:
if isinstance(cs, reporting.ColorScheme3):
cs = cs.toBlockLatex()
sub = reporting.Subsection(title, [cs, reporting.BlockLatex(table + "\n")])
section1.add(sub)
report.add(section1)
report.save_and_compile("eps_results.tex")
def create_section_with_results(title, desc, folders, numRuns=10, use_bench_simple_names=True):
assert isinstance(title, str)
assert isinstance(desc, str)
assert isinstance(folders, list)
print("\n*** Processing: {0}***".format(title))
for f in folders:
print(f)
if folders is None or len(folders) == 0:
return None
props = load_correct_props(folders, name=title)
def post(s):
return s.replace("{ccccccccccccc}", "{rrrrrrrrrrrrr}").replace("{rrr", "{lrr").replace(r"\_{lex}", "_{lex}").replace(r"\_{7}", "_{7}").replace("other/", "").replace("sygus16/", "")
dim_benchmarks = Dim.from_data(props, lambda p: p["benchmark"])
if use_bench_simple_names:
configs = [Config(benchmarks_simple_names[c.get_caption()], c.filters[0][1]) for c in dim_benchmarks.configs]
dim_benchmarks = Dim(configs)
dim_benchmarks.sort()
dim_cols = dim_method * dim_ea_type * dim_sel
# dim_cols = dim_method * dim_sa
print("STATUS")
text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_num_computed, layered_headline=True))
latex_status = printer.table_color_map(text, 0.0, numRuns/2, numRuns, "colorLow", "colorMedium", "colorHigh")
# print(text + "\n\n")
print("SUCCESS RATES")
text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, fun_successRates, layered_headline=True))
latex_successRates = printer.table_color_map(text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh")
# print(text + "\n\n")
# print("SUCCESS RATES (FULL INFO)")
# text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, fun_successRates_full, layered_headline=True))
# print(text + "\n\n")
print("AVG BEST-OF-RUN FITNESS")
text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_fitness, layered_headline=True))
latex_avgBestOfRunFitness = printer.table_color_map(text, 0.6, 0.98, 1.0, "colorLow", "colorMedium", "colorHigh")
# print(text + "\n\n")
print("AVG TOTAL TESTS")
text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_totalTests, layered_headline=True))
latex_avgTotalTests = printer.table_color_map(text, 0.0, 1000.0, 2000.0, "colorLow", "colorMedium", "colorHigh")
# print(text + "\n\n")
print("AVG RUNTIME")
text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_runtime, layered_headline=True))
latex_avgRuntime = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh")
# print(text + "\n\n")
print("AVG RUNTIME (SUCCESSFUL)")
text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_runtimeOnlySuccessful, layered_headline=True))
latex_avgRuntimeOnlySuccessful = printer.table_color_map(text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh")
# print(text + "\n\n")
print("AVG RUNTIME PER PROGRAM")
text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_runtimePerProgram, layered_headline=True))
latex_avgRuntimePerProgram = printer.table_color_map(text, 0.01, 1.0, 2.0, "colorLow", "colorMedium", "colorHigh")
# print(text + "\n\n")
print("AVG GENERATION")
text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_generation, layered_headline=True))
latex_avgGeneration = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh")
# print(text + "\n\n")
print("AVG GENERATION (SUCCESSFUL)")
text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_avg_generationSuccessful, layered_headline=True))
latex_avgGenerationSuccessful = printer.table_color_map(text, 0.0, 50.0, 100.0, "colorLow", "colorMedium", "colorHigh")
# print(text + "\n\n")
print("AVG SIZES")
text = post(printer.latex_table(props, dim_benchmarks.sort(), dim_cols, get_stats_size, layered_headline=True))
latex_sizes = printer.table_color_map(text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh")
# print(text + "\n\n")
section = reporting.Section(title, [])
subsects = [("Status (correctly finished processes)", latex_status, reversed(reporting.color_scheme_red)),
("Success rates", latex_successRates, reporting.color_scheme_green),
("Average best-of-run ratio of passed tests", latex_avgBestOfRunFitness, reporting.color_scheme_green),
("Average sizes of $T_C$ (total tests in run)", latex_avgTotalTests, reporting.color_scheme_blue),
("Average runtime [s]", latex_avgRuntime, reporting.color_scheme_violet),
("Average runtime (only successful) [s]", latex_avgRuntimeOnlySuccessful, reporting.color_scheme_violet),
("Average generation (optimal and nonoptimal bestOfRuns)", latex_avgGeneration, reporting.color_scheme_teal),
("Average generation (only optimal bestOfRuns)", latex_avgGenerationSuccessful, reporting.color_scheme_teal),
("Approximate average runtime per program [s]", latex_avgRuntimePerProgram, reporting.color_scheme_brown),
("Average sizes of best of runs (number of nodes)", latex_sizes, reporting.color_scheme_yellow)]
bl_desc = reporting.BlockLatex(desc + "\n")
section.add(bl_desc)
for title, table, cs in subsects:
if isinstance(cs, reporting.ColorScheme3):
cs = cs.toBlockLatex()
sub = reporting.Subsection(title, [cs, reporting.BlockLatex(table + "\n")])
section.add(sub)
section.add(reporting.BlockLatex(r"\vspace{1cm}" + "\n"))
return section
def create_subsection_shared_stats(props, dim_rows, dim_cols, numRuns):
vb = 1 # vertical border
print("STATUS")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_num_computed,
layered_headline=True,
vertical_border=vb))
latex_status = printer.table_color_map(text, 0.0, numRuns / 2, numRuns,
"colorLow", "colorMedium",
"colorHigh")
print("SUCCESS RATES")
print(
printer.text_table(props,
dim_rows,
dim_cols,
fun_successRate,
d_cols=";"))
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
fun_successRate,
layered_headline=True,
vertical_border=vb))
latex_successRates = printer.table_color_map(text, 0.0, 0.5, 1.0,
"colorLow", "colorMedium",
"colorHigh")
print("FINAL PROPERTIES")
print(
printer.text_table(props,
dim_rows,
dim_cols,
fun_propertiesMet,
d_cols=";"))
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
fun_propertiesMet,
layered_headline=True,
vertical_border=vb))
latex_propertiesMet = printer.table_color_map(text, 0.0, 0.5, 1.0,
"colorLow", "colorMedium",
"colorHigh")
print("AVG RUNTIME")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_avg_runtime,
layered_headline=True,
vertical_border=vb))
latex_avgRuntime = printer.table_color_map(text, 0.0, 1800.0, 3600.0,
"colorLow", "colorMedium",
"colorHigh")
print("AVG RUNTIME (SUCCESSFUL)")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_avg_runtimeOnlySuccessful,
layered_headline=True,
vertical_border=vb))
latex_avgRuntimeOnlySuccessful = printer.table_color_map(
text, 0.0, 1800.0, 3600.0, "colorLow", "colorMedium", "colorHigh")
# print("SUCCESS RATES (FULL INFO)")
# text = post(printer.latex_table(props, dim_rows, dim_cols, fun_successRates_full, layered_headline=True, vertical_border=vb))
# print("AVG SIZES")
# text = post(printer.latex_table(props, dim_rows, dim_cols, get_stats_size, layered_headline=True, vertical_border=vb))
# latex_sizes = printer.table_color_map(text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh")
print("AVG SIZES (SUCCESSFUL)")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_stats_sizeOnlySuccessful,
layered_headline=True,
vertical_border=vb))
latex_sizesOnlySuccessful = printer.table_color_map(
text, 0.0, 100.0, 200.0, "colorLow", "colorMedium", "colorHigh")
subsects_main = [
("Status (correctly finished runs)", latex_status,
reversed(reporting.color_scheme_red)),
("Success rates (mse below thresh (1.0e-25) + properties met)",
latex_successRates, reporting.color_scheme_green),
("Success rates (properties met)", latex_propertiesMet,
reporting.color_scheme_green),
("Average runtime [s]", latex_avgRuntime,
reporting.color_scheme_violet),
("Average runtime (only successful) [s]",
latex_avgRuntimeOnlySuccessful, reporting.color_scheme_violet),
# ("Average sizes of best of runs (number of nodes)", latex_sizes, reporting.color_scheme_yellow),
("Average sizes of best of runs (number of nodes) (only successful)",
latex_sizesOnlySuccessful, reporting.color_scheme_yellow),
]
return reporting.Subsection("Shared Statistics",
get_content_of_subsections(subsects_main))
def create_subsection_cdgp_specific(props, dim_rows, dim_cols, exp_prefix):
vb = 1 # vertical border
print("AVG BEST-OF-RUN FITNESS")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_avg_fitness,
layered_headline=True,
vertical_border=vb))
latex_avgBestOfRunFitness = printer.table_color_map(
text, 0.0, 0.5, 1.0, "colorLow", "colorMedium", "colorHigh")
print("AVG TOTAL TESTS")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_avg_totalTests,
layered_headline=True,
vertical_border=vb))
latex_avgTotalTests = printer.table_color_map(text, 0.0, 1000.0, 2000.0,
"colorLow", "colorMedium",
"colorHigh")
print("AVG RUNTIME PER PROGRAM")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_avg_runtimePerProgram,
layered_headline=True,
vertical_border=vb))
latex_avgRuntimePerProgram = printer.table_color_map(
text, 0.01, 1.0, 2.0, "colorLow", "colorMedium", "colorHigh")
print("AVG GENERATION")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_avg_generation,
layered_headline=True,
vertical_border=vb))
latex_avgGeneration = printer.table_color_map(text, 0.0, 50.0, 100.0,
"colorLow", "colorMedium",
"colorHigh")
print("AVG EVALUATED SOLUTIONS")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_avg_evaluated,
layered_headline=True,
vertical_border=vb))
latex_avgEvaluated = printer.table_color_map(text, 500.0, 25000.0,
100000.0, "colorLow",
"colorMedium", "colorHigh")
print("AVG EVALUATED SOLUTIONS (SUCCESSFUL)")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_avg_evaluatedSuccessful,
layered_headline=True,
vertical_border=vb))
latex_avgEvaluatedSuccessful = printer.table_color_map(
text, 500.0, 25000.0, 100000.0, "colorLow", "colorMedium", "colorHigh")
print("MAX SOLVER TIME")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_stats_maxSolverTime,
layered_headline=True,
vertical_border=vb))
latex_maxSolverTimes = printer.table_color_map(text, 0.0, 0.5, 1.0,
"colorLow", "colorMedium",
"colorHigh")
print("AVG SOLVER TIME")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_stats_avgSolverTime,
layered_headline=True,
vertical_border=vb))
latex_avgSolverTimes = printer.table_color_map(text, 0.0, 0.015, 0.03,
"colorLow", "colorMedium",
"colorHigh")
print("AVG NUM SOLVER CALLS")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_avgSolverTotalCalls,
layered_headline=True,
vertical_border=vb))
latex_avgSolverTotalCalls = printer.table_color_map(
text, 1e1, 1e2, 1e4, "colorLow", "colorMedium", "colorHigh")
print("NUM SOLVER CALLS > 0.5s")
text = post(
printer.latex_table(props,
dim_rows,
dim_cols,
get_numSolverCallsOverXs,
layered_headline=True,
vertical_border=vb))
latex_numSolverCallsOverXs = printer.table_color_map(
text, 0, 50, 100, "colorLow", "colorMedium", "colorHigh")
plot_figures(props, exp_prefix=exp_prefix)
subsects_cdgp = [
("Average best-of-run ratio of passed tests",
latex_avgBestOfRunFitness, reporting.color_scheme_green),
("Average sizes of $T_C$ (total tests in run)", latex_avgTotalTests,
reporting.color_scheme_blue),
("Average generation (all)", latex_avgGeneration,
reporting.color_scheme_teal),
#("Average generation (only successful)", latex_avgGenerationSuccessful, reporting.color_scheme_teal),
("Average evaluated solutions", latex_avgEvaluated,
reporting.color_scheme_teal),
("Average evaluated solutions (only successful)",
latex_avgEvaluatedSuccessful, reporting.color_scheme_teal),
("Approximate average runtime per program [s]",
latex_avgRuntimePerProgram, reporting.color_scheme_brown),
("Max solver time per query [s]", latex_maxSolverTimes,
reporting.color_scheme_violet),
("Avg solver time per query [s]", latex_avgSolverTimes,
reporting.color_scheme_brown),
("Avg number of solver calls (in thousands; 1=1000)",
latex_avgSolverTotalCalls, reporting.color_scheme_blue),
("Number of solver calls $>$ 0.5s", latex_numSolverCallsOverXs,
reporting.color_scheme_blue),
]
return reporting.Subsection("CDGP Statistics",
get_content_of_subsections(subsects_cdgp))
