def pictures_from_stats(args):
# plot_pareto_fronts()
logger = logging.getLogger(__name__)
logger.debug("pictures from stats")
boot_size = int(args["--bootstrap"])
results_dir = args["--dir"]
plots_dir = Path(args["-o"])
results = collections.defaultdict(list)
with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"):
for problem_name, problem_mod, algorithms in serialization.each_result(
BudgetResultsExtractor(), results_dir
):
for algo_name, budgets in algorithms:
for result in budgets:
_, _, cost_data = next(result["analysis"])
cost_data = list(x() for x in cost_data)
cost_analysis = yield_analysis(cost_data, boot_size)
budget = cost_analysis["btstrpd"]["metrics"]
budget_err = cost_analysis["stdev"]
for metric_name, metric_name_long, data_process in result[
"analysis"
]:
if metric_name in best_func:
if metric_name == "dst from pareto":
metric_name = "dst"
data_process = list(x() for x in data_process)
data_analysis = yield_analysis(data_process, boot_size)
score = data_analysis["btstrpd"]["metrics"]
score_err = data_analysis["stdev"]
keys = [
(problem_name, algo_name, metric_name, group)
for group in algos_groups[algo_name]
]
value = (budget, budget_err, score, score_err)
print("PLOT: " + str(value))
for key in keys:
results[key].append(value)
plot_results(results, plots_dir, (500, 4500))
def pictures_summary(args):
logger = logging.getLogger(__name__)
logger.debug("pictures_summary")
selected = set(args["--selected"].upper().split(","))
boot_size = int(args["--bootstrap"])
results_dir = args["--dir"]
plots_dir = Path(args["-o"])
logger.debug("Plotting summary with selected algos: " + ",".join(selected))
scoring = collections.defaultdict(lambda: collections.defaultdict(dict))
problems = set()
with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"):
for problem_name, problem_mod, algorithms in serialization.each_result(
BudgetResultsExtractor(), results_dir
):
problems.add(problem_name)
problem_score = collections.defaultdict(list)
algos = list(algorithms)
for algo_name, results in algos:
max_result = find_acceptable_result_for_budget(list(results), boot_size)
if max_result:
print(
"{}, {} , budget={}".format(
problem_name, algo_name, max_result["budget"]
)
)
for metric_name, metric_name_long, data_process in max_result[
"analysis"
]:
if metric_name in ranking.best_func:
data_process = list(x() for x in data_process)
data_analysis = yield_analysis(data_process, boot_size)
score = math.log(
math.fabs(data_analysis["btstrpd"]["metrics"]) + 1.0
)
scoring[metric_name][algo_name][problem_name] = score
problem_score[metric_name].append((algo_name, score))
else:
print("{}, {}, NO BUDGET".format(problem_name, algo_name))
for metric_name in scoring:
if metric_name != "pdi":
max_score = (
max(x for algo, x in problem_score[metric_name]) + 0.0001
)
for algo_name, _ in algos:
if (
algo_name in scoring[metric_name]
and problem_name in scoring[metric_name][algo_name]
):
scoring[metric_name][algo_name][problem_name] /= max_score
plot_results_summary(problems, scoring, selected, plots_dir)
def rank(args):
# plot_pareto_fronts()
logger = logging.getLogger(__name__)
logger.debug("ranking")
boot_size = int(args["--bootstrap"])
scoring = collections.defaultdict(list)
with log_time(process_time, logger,
"Preparing data done in {time_res:.3f}"):
for problem_name, problem_mod, algorithms in serialization.each_result(
BudgetResultsExtractor(), RESULTS_DIR):
for algo_name, results in algorithms:
max_budget_result = find_acceptable_result_for_budget(
list(results), boot_size)
if max_budget_result:
for (
metric_name,
metric_name_long,
data_process,
) in max_budget_result["analysis"]:
if metric_name in best_func:
data_process = list(x() for x in data_process)
data_analysis = yield_analysis(
data_process, boot_size)
score = data_analysis["btstrpd"]["metrics"]
scoring[(problem_name, metric_name)].append(
(algo_name, score))
global_scoring = collections.defaultdict(collections.Counter)
print("Problem ranking\n################")
for problem_name, metric_name in scoring:
metric_scoring = scoring[(problem_name, metric_name)]
algo_win, score = best_func[metric_name](metric_scoring,
key=lambda x: x[1])
print("{}, {} : {}".format(problem_name, metric_name, algo_win))
weak_winners = get_weak_winners(metric_scoring, (algo_win, score),
winner_tolerance[metric_name])
# if not weak_winners:
# global_scoring[metric_name].update([algo_win])
if not weak_winners:
global_scoring[metric_name].update([algo_win, algo_win])
else:
global_scoring[metric_name].update(
[algo_win] + [algo for algo, score in weak_winners])
print("\nGlobal ranking\n##############")
for metric_name in global_scoring:
print("{} : ".format(metric_name) + ", ".join(
"{} ({})".format(score[0], score[1])
for score in global_scoring[metric_name].most_common()))
def pictures_from_stats(args, queue):
# plot_pareto_fronts()
logger = logging.getLogger(__name__)
logger.debug("pictures from stats")
boot_size = int(args['--bootstrap'])
results = collections.defaultdict(list)
with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"):
for problem_name, problem_mod, algorithms in RunResult.each_result(RESULTS_DIR):
for algo_name, budgets in algorithms:
for result in budgets:
_, _, cost_data = next(result["analysis"])
cost_data = list(x() for x in cost_data)
cost_analysis = yield_analysis(cost_data, boot_size)
budget = cost_analysis["btstrpd"]["metrics"]
budget_err = cost_analysis["stdev"]
for metric_name, metric_name_long, data_process in result["analysis"]:
if metric_name in best_func:
if metric_name == 'dst from pareto':
metric_name = 'dst'
data_process = list(x() for x in data_process)
data_analysis = yield_analysis(data_process, boot_size)
score = data_analysis["btstrpd"]["metrics"]
score_err = data_analysis["stdev"]
keys = [(problem_name, algo_name, metric_name, group) for group in
algos_groups[algo_name]]
value = (budget, budget_err, score, score_err)
for key in keys:
results[key].append(value)
plot_results(results)
def detailed_rank(args):
# plot_pareto_fronts()
logger = logging.getLogger(__name__)
logger.debug("detailed ranking")
boot_size = int(args["--bootstrap"])
for result_set in result_dirs:
print("***{}***".format(result_set))
scoring = collections.defaultdict(list)
with log_time(process_time, logger,
"Preparing data done in {time_res:.3f}"):
for problem_name, problem_mod, algorithms in serialization.each_result(
BudgetResultsExtractor(), result_set):
for algo_name, results in algorithms:
for result in results:
if validate_cost(result, boot_size):
for metric_name, metric_name_long, data_process in result[
"analysis"]:
if metric_name in best_func:
data_process = list(x()
for x in data_process)
data_analysis = yield_analysis(
data_process, boot_size)
score = data_analysis["btstrpd"]["metrics"]
scoring[(problem_name, result["budget"],
metric_name)].append(
(algo_name, score))
global_scoring = collections.defaultdict(collections.Counter)
for problem_name, budget, metric_name in scoring:
algo_win, score = best_func[metric_name](scoring[(problem_name,
budget,
metric_name)],
key=lambda x: x[1])
global_scoring[(budget, metric_name)].update([algo_win])
for budget, metric_name in sorted(global_scoring):
print("{} {} : ".format(budget, metric_name) +
", ".join("{} ({})".format(score[0], score[1])
for score in global_scoring[
(budget, metric_name)].most_common()))
def pictures_summary(args, queue):
logger = logging.getLogger(__name__)
logger.debug("pictures_summary")
selected = set(args['--selected'].upper().split(','))
boot_size = int(args['--bootstrap'])
logger.debug('Plotting summary with selected algos: ' + ','.join(selected))
scoring = collections.defaultdict(lambda: collections.defaultdict(dict))
problems = set()
with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"):
for problem_name, problem_mod, algorithms in RunResult.each_result(RESULTS_DIR):
problems.add(problem_name)
problem_score = collections.defaultdict(list)
algos = list(algorithms)
for algo_name, results in algos:
max_result = find_acceptable_result_for_budget(list(results), boot_size)
if max_result:
print('{}, {} , budget={}'.format(problem_name, algo_name, max_result['budget']))
for metric_name, metric_name_long, data_process in max_result["analysis"]:
if metric_name in ranking.best_func:
data_process = list(x() for x in data_process)
data_analysis = yield_analysis(data_process, boot_size)
score = math.log(math.fabs(data_analysis["btstrpd"]["metrics"]) + 1.0)
scoring[metric_name][algo_name][problem_name] = score
problem_score[metric_name].append((algo_name, score))
else:
print('{}, {}, NO BUDGET'.format(problem_name, algo_name))
for metric_name in scoring:
if metric_name != 'pdi':
max_score = max(x for algo, x in problem_score[metric_name]) + 0.0001
for algo_name, _ in algos:
if algo_name in scoring[metric_name] and problem_name in scoring[metric_name][algo_name]:
scoring[metric_name][algo_name][problem_name] /= max_score
plot_results_summary(problems, scoring, selected)
def pictures_time(args):
logger = logging.getLogger(__name__)
logger.debug("pictures from stats")
boot_size = int(args["--bootstrap"])
results_dir = args["--dir"]
plots_dir = Path(args["-o"])
plot_data = collections.defaultdict(list)
with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"):
for problem_name, problem_mod, algorithms in serialization.each_result(
TimeResultsExtractor(), results_dir
):
for algo_name, results in algorithms:
for result in results:
time = result["time"]
for metric_name, metric_name_long, data_process in result[
"analysis"
]:
if metric_name in best_func:
if metric_name == "dst from pareto":
metric_name = "dst"
data_process = list(x() for x in data_process)
data_analysis = yield_analysis(data_process, boot_size)
score = data_analysis["btstrpd"]["metrics"]
score_err = data_analysis["stdev"]
keys = [
(problem_name, algo_name, metric_name, group)
for group in algos_groups[algo_name]
]
value = (time, 0, score, score_err)
for key in keys:
plot_data[key].append(value)
max_time = max(list(plot_data.values())[0])[0]
plot_results(plot_data, plots_dir, (0, max_time))
def table_rank(args):
logger = logging.getLogger(__name__)
logger.debug("table ranking")
boot_size = int(args["--bootstrap"])
results = collections.defaultdict(
lambda: collections.defaultdict(collections.Counter))
for result_set in result_dirs:
print("***{}***".format(result_set))
with log_time(process_time, logger,
"Preparing data done in {time_res:.3f}"):
for problem_name, problem_mod, algorithms in serialization.each_result(
BudgetResultsExtractor(), result_set):
print(result_set, problem_name)
scoring = collections.defaultdict(list)
for algo_name, results_data in algorithms:
results_data = list(results_data)
for i in range(len(results_data)):
original_budget = results_data[i]["budget"]
if original_budget == 40:
result = results_data[i]
# result = find_acceptable_result_for_budget(
# results_data[: i + 1], boot_size
# )
if result:
print("{} {} {} -> {}".format(
problem_name,
algo_name,
original_budget,
result["budget"],
))
for metric_name, metric_name_long, data_process in result[
"analysis"]:
if metric_name in best_func:
data_process = list(
x() for x in data_process)
data_analysis = yield_analysis(
data_process, boot_size)
score = data_analysis["btstrpd"][
"metrics"]
scoring[(original_budget,
metric_name)].append(
(algo_name, score))
print("****{}****".format(problem_name))
for budget, metric_name in sorted(scoring):
metric_scoring = scoring[(budget, metric_name)]
algo_win, score = best_func[metric_name](
metric_scoring, key=lambda x: x[1])
weak_winners = get_weak_winners(
metric_scoring, (algo_win, score),
winner_tolerance[metric_name])
# # Only strong
# if not weak_winners:
# results[(budget, metric_name)][result_set].update([algo_win])
# Strong = 2 points, Weak or Winner = 1 point
if not weak_winners:
results[(budget, metric_name)][result_set].update(
[algo_win, algo_win])
else:
results[(budget, metric_name)][result_set].update(
[algo_win] +
[algo for algo, score in weak_winners])
# print('{} {} {}'.format(budget, metric_name, scoring[(budget, metric_name)]))
print("*****{} {}".format(budget, metric_name))
if not weak_winners:
print("*****Strong winner: {} :{}".format(
algo_win, score))
else:
print("*****Winner: {} :{}".format(algo_win, score))
print("*****Weak winners: {}".format(weak_winners))
print("""\\begin{table}[ht]
\\centering
\\caption{Final results}
\\label{tab:results"}
\\resizebox{\\textwidth}{!}{%
\\begin{tabular}{ r@{ }l | c | c | c | }
\multicolumn{2}{c}{}
& $K_0$
& $K_1$
& $K_2$
\\\\ \\hline""")
prevous_budget = None
for budget, metric_name in sorted(
sorted(results.keys(), key=lambda x: metrics_order.index(x[1])),
key=lambda x: x[0],
):
budget_label = str(budget) + " "
if prevous_budget and prevous_budget != budget:
print("\\hdashline")
elif prevous_budget:
budget_label = ""
score_str = ""
for result_set in result_dirs:
results_counter = results[(budget, metric_name)][result_set]
algo_ranking = results_counter.most_common(2)
values = list(results_counter.values())
if len(algo_ranking) == 2:
winner = format_result(algo_ranking[0], values, 2)
second = format_result(
algo_ranking[1],
values,
1 if algo_ranking[0][1] != algo_ranking[1][1] else 2,
)
score_str += "& {}, {}".format(winner, second)
elif len(algo_ranking) == 1:
winner = format_result(algo_ranking[0], values, 2)
score_str += "& {}".format(winner)
else:
score_str += "& "
print("{}& {} {}\\\\".format(budget_label, metric_name, score_str))
prevous_budget = budget
print(""" \\end{tabular}}\n\\end{table}""")
def force_data(args):
boot_size, (metric_name, metric_name_long, data_process) = args
data_process = list(x() for x in data_process)
force_analysis = yield_analysis(data_process, boot_size)
return metric_name, metric_name_long, data_process, force_analysis
def force_data(args):
boot_size, (metric_name, metric_name_long, data_process) = args
data_process = list(x() for x in data_process)
force_analysis = yield_analysis(data_process, boot_size)
return metric_name, metric_name_long, data_process, force_analysis
