def show_abstraction1_vs_abstraction2_wrt_sum_of_query_times(
exp_result_dirname,
abstraction_1="_A_complete_",
abstraction_2="_A_heuristic_alg2_",
xscale=None,
yscale=None):
cegarabou_methods = copy.deepcopy(BEST_CEGARABOU_METHODS)
if "A" in cegarabou_methods:
cegarabou_methods.pop("A")
cegarabou_df = read_cegarabou_df(exp_result_dirname=exp_result_dirname,
best_cegarabou_methods=cegarabou_methods)
cegarabou_df["refinement_sequence_length"] = cegarabou_df.filenames.apply(
lambda s: int(s.split("__RS_")[1].split("__AS_")[0]))
print_num_of_refine_sequences(df=cegarabou_df)
cegarabou_df["num_of_refine_steps"] = cegarabou_df[
"num_of_refine_steps"] * cegarabou_df["refinement_sequence_length"]
print(f"cegarabou_df.columns={cegarabou_df.columns}")
print_abstraction_time(df=cegarabou_df)
print_refinement_time(df=cegarabou_df)
show_col_vs_col_wrt_category(df=cegarabou_df,
left_col=abstraction_1,
right_col=abstraction_2,
category="ar_times",
cat_aggregation_func=get_sum_times_func(),
aggregated_name="Sum of Query Times",
xscale=xscale,
yscale=yscale,
min_x_limit=0,
max_x_limit=None,
min_y_limit=0,
max_y_limit=None)
def show_rs_vs_rs_wrt_sum_of_query_times(exp_result_dirname,
rs_1=USED_RS_1,
rs_2=USED_RS_2,
xscale=None,
yscale=None):
cegarabou_methods = copy.deepcopy(BEST_CEGARABOU_METHODS)
if "RS" in cegarabou_methods:
cegarabou_methods.pop("RS")
cegarabou_df = read_cegarabou_df(exp_result_dirname=exp_result_dirname,
best_cegarabou_methods=cegarabou_methods)
refinement_sequence_length = 100
cegarabou_df["num_of_refine_steps"] = cegarabou_df[
"num_of_refine_steps"].apply(lambda x: refinement_sequence_length * x)
show_col_vs_col_wrt_category(df=cegarabou_df,
left_col="_RS_{}_".format(rs_1),
right_col="_RS_{}_".format(rs_2),
category="ar_times",
cat_aggregation_func=get_sum_times_func(),
aggregated_name="Sum of Query Times",
xscale=xscale,
yscale=yscale,
min_x_limit=0,
max_x_limit=None,
min_y_limit=0,
max_y_limit=None)
def show_cegar_vs_cetar_wrt_last_time(exp_result_dirname):
cegarabou_df = read_cegarabou_df(exp_result_dirname)
print(cegarabou_df.columns)
cegar_df = cegarabou_df[cegarabou_df.filenames.str.contains("__R_cegar__")]
cetar_df = cegarabou_df[cegarabou_df.filenames.str.contains("__R_cetar__")]
print(cegar_df.shape)
print(cetar_df.shape)
assert (cegar_df.shape[0] + cetar_df.shape[0] == cegarabou_df.shape[0])
replace_func = lambda filename: filename.replace("__R_cegar__",
"__R_cetar__")
cegar_df["replace_filenames"] = cegar_df.filenames.apply(replace_func)
cegar_cetar_df = cegar_df.merge(right=cetar_df,
left_on="replace_filenames",
right_on="filenames",
suffixes=["_cegar", "_cetar"])
col_x = "last_query_time_cegar"
col_y = "last_query_time_cetar"
show_df_col_vs_col(cegar_cetar_df,
col_x=col_x,
col_y=col_y,
color="b",
min_x_limit=0,
max_x_limit=100,
min_y_limit=0,
max_y_limit=75,
title="scatter {} vs {}".format(col_x, col_y),
set_timeouts_line=True)
def show_complete_vs_heuristic_last_time(exp_result_dirname):
cegarabou_df = read_cegarabou_df(exp_result_dirname)
complete_df = cegarabou_df[cegarabou_df.filenames.str.contains(
"__A_complete_")]
heuristic_df = cegarabou_df[cegarabou_df.filenames.str.contains(
"__A_heuristic_")]
print(complete_df.shape)
print(heuristic_df.shape)
print(complete_df.shape[0] +
heuristic_df.shape[0] == cegarabou_df.shape[0])
replace_func = lambda filename: filename.replace("__A_complete_",
"__A_heuristic_")
complete_df["replace_filenames"] = complete_df.filenames.apply(
replace_func)
complete_heuristic_df = complete_df.merge(
right=heuristic_df,
left_on="replace_filenames",
right_on="filenames",
suffixes=["_complete", "_heuristic"])
col_x = "last_query_time_complete"
col_y = "last_query_time_heuristic"
show_df_col_vs_col(complete_heuristic_df,
col_x=col_x,
col_y=col_y,
color="b",
min_x_limit=0,
max_x_limit=1000,
min_y_limit=0,
max_y_limit=1000,
xscale="log",
yscale="log",
title="scatter {} vs {}".format(col_x, col_y))
def show_complete_vs_heuristic_wrt_num_of_queries(exp_result_dirname,
xscale=None,
yscale=None):
cegarabou_methods = copy.deepcopy(BEST_CEGARABOU_METHODS)
if "A" in cegarabou_methods:
cegarabou_methods.pop("A")
cegarabou_df = read_cegarabou_df(exp_result_dirname=exp_result_dirname,
best_cegarabou_methods=cegarabou_methods)
cegarabou_df["refinement_sequence_length"] = cegarabou_df.filenames.apply(
lambda s: int(s.split("__RS_")[1].split("__AS_")[0]))
cegarabou_df["num_of_refine_steps"] = cegarabou_df[
"num_of_refine_steps"] * cegarabou_df["refinement_sequence_length"]
show_col_vs_col_wrt_category(df=cegarabou_df,
left_col="_A_complete_",
right_col="_A_heuristic_",
category="num_of_refine_steps",
aggregated_name="#Refinement Steps",
xscale=xscale,
yscale=yscale,
min_x_limit=0,
max_x_limit=None,
min_y_limit=0,
max_y_limit=None)
def show_cegar_vs_cetar_wrt_num_of_queries(exp_result_dirname,
xscale=None,
yscale=None):
cegarabou_methods = copy.deepcopy(BEST_CEGARABOU_METHODS)
if "R" in cegarabou_methods:
cegarabou_methods.pop("R")
cegarabou_df = read_cegarabou_df(exp_result_dirname=exp_result_dirname,
best_cegarabou_methods=cegarabou_methods,
exp_params=EXP_PARAMS)
refinement_sequence_length = 100
cegarabou_df["num_of_refine_steps"] = cegarabou_df[
"num_of_refine_steps"].apply(lambda x: refinement_sequence_length * x)
show_col_vs_col_wrt_category(df=cegarabou_df,
left_col="_R_cegar_",
right_col="_R_cetar_",
category="num_of_refine_steps",
aggregated_name="#Refinement Steps",
xscale=xscale,
yscale=yscale,
min_x_limit=0,
max_x_limit=None,
min_y_limit=0,
max_y_limit=None)
def show_as_vs_as_wrt_num_of_queries(exp_result_dirname,
as_1=USED_AS_1,
as_2=USED_AS_2,
xscale=None,
yscale=None):
cegarabou_methods = copy.deepcopy(BEST_CEGARABOU_METHODS)
if "AS" in cegarabou_methods:
cegarabou_methods.pop("AS")
cegarabou_df = read_cegarabou_df(exp_result_dirname=exp_result_dirname,
best_cegarabou_methods=cegarabou_methods)
refinement_sequence_length = 100
cegarabou_df["num_of_refine_steps"] = cegarabou_df[
"num_of_refine_steps"].apply(lambda x: refinement_sequence_length * x)
show_col_vs_col_wrt_category(df=cegarabou_df,
left_col="_AS_{}_".format(as_1),
right_col="_AS_{}_".format(as_2),
category="num_of_refine_steps",
aggregated_name="#Refinement Steps",
xscale=xscale,
yscale=yscale,
min_x_limit=0,
max_x_limit=None,
min_y_limit=0,
max_y_limit=None)
def validate_equal_results_marabou_cegarabou(exp_name="2020-01-24"):
df_marabou = read_marabou_df(exp_result_dirname=exp_name)
df_cegarabou = read_cegarabou_df(exp_result_dirname=exp_name)
verify_equal_query_results(marabou_df=df_marabou,
cegarabou_df=df_cegarabou)