def main_worker(queue, configurer): configurer(queue) logger = logging.getLogger(__name__) logger.debug("Starting the evogil. Parsing arguments.") with log_time(system_time, logger, "Parsing done in {time_res}s"): argv = docopt(__doc__, version='EvoGIL 3.0') logger.debug("Parsing result: %s", argv) run_dict = { 'run': simulation.run_parallel.run_parallel, 'statistics': statistic.stats.statistics, 'stats': statistic.stats.statistics, 'rank': statistic.ranking.rank, 'table': statistic.ranking.table_rank, 'rank_details': statistic.ranking.detailed_rank, 'pictures': plots.pictures.pictures_from_stats, 'pictures_summary': plots.pictures.pictures_summary, 'best_fronts': plots.best_fronts.best_fronts, 'violin': plots.violin.violin, 'summary': statistic.summary.analyse_results, 'list': all_algos_problems, } for k, v in run_dict.items(): logger.debug("run_dict: k,v = %s,%s", k, v) if argv[k]: logger.debug("run_dict match. argv[k]=%s", argv[k]) v(argv, queue) break
def run_parallel(args): worker_factory, simulation_cases = factory.resolve_configuration(args) logger.debug("Shuffling the job queue") random.shuffle(simulation_cases) logger.debug("Creating the pool") processes_no = int(args["-j"]) rxtools.configure_default_executor(processes_no) wall_time = [] start_time = datetime.now() results = [] logger.debug("Simulation cases: %s", simulation_cases) logger.debug("Work will be divided into %d processes", processes_no) sys = ActorSystem("multiprocTCPBase", logDefs=log_helper.EVOGIL_LOG_CONFIG) with log_time(system_time, logger, "Pool evaluated in {time_res}s", out=wall_time): def process_result(subres): results.append(subres) log_simulation_stats(start_time, subres[-1], len(simulation_cases)) rx.from_iterable(range(len(simulation_cases))).pipe( ops.map(lambda i: worker_factory(simulation_cases[i], i)), # ops.map(lambda w: rxtools.from_process(w.run)), ops.map(lambda w : w.run()), # ops.merge(max_concurrent=1) ops.do_action(on_next=process_result) ).run() log_summary(args, results, simulation_cases, wall_time) rxtools.shutdown_default_executor() sys.shutdown()
def main_worker(): logger = logging.getLogger(__name__) logger.debug("Starting the evogil. Parsing arguments.") with log_time(system_time, logger, "Parsing done in {time_res}s"): argv = docopt(__doc__, version="EvoGIL 3.0") logger.debug("Parsing result: %s", argv) run_dict = { "run": simulation.run_parallel.run_parallel, "statistics": statistic.stats.statistics, "stats": statistic.stats.statistics, "rank": statistic.ranking.rank, "table": statistic.ranking.table_rank, "rank_details": statistic.ranking.detailed_rank, "pictures": plots.pictures.pictures_time, "pictures_summary": plots.pictures.pictures_summary, "best_fronts": plots.best_fronts.best_fronts, "violin": plots.violin.violin, "summary": statistic.summary.analyse_results, "list": all_algos_problems, } set_default_options(argv) for k, v in run_dict.items(): logger.debug("run_dict: k,v = %s,%s", k, v) if argv[k]: logger.debug("run_dict match. argv[k]=%s", argv[k]) v(argv) break
def pictures_summary(args): logger = logging.getLogger(__name__) logger.debug("pictures_summary") selected = set(args["--selected"].upper().split(",")) boot_size = int(args["--bootstrap"]) results_dir = args["--dir"] plots_dir = Path(args["-o"]) logger.debug("Plotting summary with selected algos: " + ",".join(selected)) scoring = collections.defaultdict(lambda: collections.defaultdict(dict)) problems = set() with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"): for problem_name, problem_mod, algorithms in serialization.each_result( BudgetResultsExtractor(), results_dir ): problems.add(problem_name) problem_score = collections.defaultdict(list) algos = list(algorithms) for algo_name, results in algos: max_result = find_acceptable_result_for_budget(list(results), boot_size) if max_result: print( "{}, {} , budget={}".format( problem_name, algo_name, max_result["budget"] ) ) for metric_name, metric_name_long, data_process in max_result[ "analysis" ]: if metric_name in ranking.best_func: data_process = list(x() for x in data_process) data_analysis = yield_analysis(data_process, boot_size) score = math.log( math.fabs(data_analysis["btstrpd"]["metrics"]) + 1.0 ) scoring[metric_name][algo_name][problem_name] = score problem_score[metric_name].append((algo_name, score)) else: print("{}, {}, NO BUDGET".format(problem_name, algo_name)) for metric_name in scoring: if metric_name != "pdi": max_score = ( max(x for algo, x in problem_score[metric_name]) + 0.0001 ) for algo_name, _ in algos: if ( algo_name in scoring[metric_name] and problem_name in scoring[metric_name][algo_name] ): scoring[metric_name][algo_name][problem_name] /= max_score plot_results_summary(problems, scoring, selected, plots_dir)
def rank(args): # plot_pareto_fronts() logger = logging.getLogger(__name__) logger.debug("ranking") boot_size = int(args["--bootstrap"]) scoring = collections.defaultdict(list) with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"): for problem_name, problem_mod, algorithms in serialization.each_result( BudgetResultsExtractor(), RESULTS_DIR): for algo_name, results in algorithms: max_budget_result = find_acceptable_result_for_budget( list(results), boot_size) if max_budget_result: for ( metric_name, metric_name_long, data_process, ) in max_budget_result["analysis"]: if metric_name in best_func: data_process = list(x() for x in data_process) data_analysis = yield_analysis( data_process, boot_size) score = data_analysis["btstrpd"]["metrics"] scoring[(problem_name, metric_name)].append( (algo_name, score)) global_scoring = collections.defaultdict(collections.Counter) print("Problem ranking\n################") for problem_name, metric_name in scoring: metric_scoring = scoring[(problem_name, metric_name)] algo_win, score = best_func[metric_name](metric_scoring, key=lambda x: x[1]) print("{}, {} : {}".format(problem_name, metric_name, algo_win)) weak_winners = get_weak_winners(metric_scoring, (algo_win, score), winner_tolerance[metric_name]) # if not weak_winners: # global_scoring[metric_name].update([algo_win]) if not weak_winners: global_scoring[metric_name].update([algo_win, algo_win]) else: global_scoring[metric_name].update( [algo_win] + [algo for algo, score in weak_winners]) print("\nGlobal ranking\n##############") for metric_name in global_scoring: print("{} : ".format(metric_name) + ", ".join( "{} ({})".format(score[0], score[1]) for score in global_scoring[metric_name].most_common()))
def run(self): log_helper.init() logger = logging.getLogger(__name__) logger.info( "Starting the worker. PID: %d, simulation case: %s", os.getpid(), self.simulation, ) if self.simulation.renice and os.name == "posix": logger.debug("Renice the process PID:%s by %s", os.getpid(), self.simulation) os.nice(int(self.simulation.renice)) # pylint: disable=no-member self._init_random_seed(logger) driver = None try: final_driver, problem_mod = factory.prepare( self.simulation.algorithm_name, self.simulation.problem_name) logger.debug("Creating the driver used to perform computation") driver = final_driver() proc_time = [] logger.debug("Beginning processing of %s, simulation: %s", driver, self.simulation) with log_time( process_time, logger, "Processing done in {time_res}s CPU time", out=proc_time, ): results = self.run_driver(driver, problem_mod, logger) return results, proc_time[-1], self.simulation_no except NotViableConfiguration as e: reason = inspect.trace()[-1] logger.info( "Configuration disabled by %s:%d:%s. simulation case:%s", reason[1], reason[2], reason[3], self.simulation, ) logger.debug("Configuration disabled args:%s. Stack:", exc_info=e) except Exception as e: logger.exception("Some error", exc_info=e) finally: if driver: driver.shutdown() logger.debug("Finished processing. simulation case:%s", self.simulation)
def pictures_from_stats(args): # plot_pareto_fronts() logger = logging.getLogger(__name__) logger.debug("pictures from stats") boot_size = int(args["--bootstrap"]) results_dir = args["--dir"] plots_dir = Path(args["-o"]) results = collections.defaultdict(list) with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"): for problem_name, problem_mod, algorithms in serialization.each_result( BudgetResultsExtractor(), results_dir ): for algo_name, budgets in algorithms: for result in budgets: _, _, cost_data = next(result["analysis"]) cost_data = list(x() for x in cost_data) cost_analysis = yield_analysis(cost_data, boot_size) budget = cost_analysis["btstrpd"]["metrics"] budget_err = cost_analysis["stdev"] for metric_name, metric_name_long, data_process in result[ "analysis" ]: if metric_name in best_func: if metric_name == "dst from pareto": metric_name = "dst" data_process = list(x() for x in data_process) data_analysis = yield_analysis(data_process, boot_size) score = data_analysis["btstrpd"]["metrics"] score_err = data_analysis["stdev"] keys = [ (problem_name, algo_name, metric_name, group) for group in algos_groups[algo_name] ] value = (budget, budget_err, score, score_err) print("PLOT: " + str(value)) for key in keys: results[key].append(value) plot_results(results, plots_dir, (500, 4500))
def detailed_rank(args): # plot_pareto_fronts() logger = logging.getLogger(__name__) logger.debug("detailed ranking") boot_size = int(args["--bootstrap"]) for result_set in result_dirs: print("***{}***".format(result_set)) scoring = collections.defaultdict(list) with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"): for problem_name, problem_mod, algorithms in serialization.each_result( BudgetResultsExtractor(), result_set): for algo_name, results in algorithms: for result in results: if validate_cost(result, boot_size): for metric_name, metric_name_long, data_process in result[ "analysis"]: if metric_name in best_func: data_process = list(x() for x in data_process) data_analysis = yield_analysis( data_process, boot_size) score = data_analysis["btstrpd"]["metrics"] scoring[(problem_name, result["budget"], metric_name)].append( (algo_name, score)) global_scoring = collections.defaultdict(collections.Counter) for problem_name, budget, metric_name in scoring: algo_win, score = best_func[metric_name](scoring[(problem_name, budget, metric_name)], key=lambda x: x[1]) global_scoring[(budget, metric_name)].update([algo_win]) for budget, metric_name in sorted(global_scoring): print("{} {} : ".format(budget, metric_name) + ", ".join("{} ({})".format(score[0], score[1]) for score in global_scoring[ (budget, metric_name)].most_common()))
def pictures_summary(args, queue): logger = logging.getLogger(__name__) logger.debug("pictures_summary") selected = set(args['--selected'].upper().split(',')) boot_size = int(args['--bootstrap']) logger.debug('Plotting summary with selected algos: ' + ','.join(selected)) scoring = collections.defaultdict(lambda: collections.defaultdict(dict)) problems = set() with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"): for problem_name, problem_mod, algorithms in RunResult.each_result(RESULTS_DIR): problems.add(problem_name) problem_score = collections.defaultdict(list) algos = list(algorithms) for algo_name, results in algos: max_result = find_acceptable_result_for_budget(list(results), boot_size) if max_result: print('{}, {} , budget={}'.format(problem_name, algo_name, max_result['budget'])) for metric_name, metric_name_long, data_process in max_result["analysis"]: if metric_name in ranking.best_func: data_process = list(x() for x in data_process) data_analysis = yield_analysis(data_process, boot_size) score = math.log(math.fabs(data_analysis["btstrpd"]["metrics"]) + 1.0) scoring[metric_name][algo_name][problem_name] = score problem_score[metric_name].append((algo_name, score)) else: print('{}, {}, NO BUDGET'.format(problem_name, algo_name)) for metric_name in scoring: if metric_name != 'pdi': max_score = max(x for algo, x in problem_score[metric_name]) + 0.0001 for algo_name, _ in algos: if algo_name in scoring[metric_name] and problem_name in scoring[metric_name][algo_name]: scoring[metric_name][algo_name][problem_name] /= max_score plot_results_summary(problems, scoring, selected)
def pictures_time(args): logger = logging.getLogger(__name__) logger.debug("pictures from stats") boot_size = int(args["--bootstrap"]) results_dir = args["--dir"] plots_dir = Path(args["-o"]) plot_data = collections.defaultdict(list) with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"): for problem_name, problem_mod, algorithms in serialization.each_result( TimeResultsExtractor(), results_dir ): for algo_name, results in algorithms: for result in results: time = result["time"] for metric_name, metric_name_long, data_process in result[ "analysis" ]: if metric_name in best_func: if metric_name == "dst from pareto": metric_name = "dst" data_process = list(x() for x in data_process) data_analysis = yield_analysis(data_process, boot_size) score = data_analysis["btstrpd"]["metrics"] score_err = data_analysis["stdev"] keys = [ (problem_name, algo_name, metric_name, group) for group in algos_groups[algo_name] ] value = (time, 0, score, score_err) for key in keys: plot_data[key].append(value) max_time = max(list(plot_data.values())[0])[0] plot_results(plot_data, plots_dir, (0, max_time))
def pictures_from_stats(args, queue): # plot_pareto_fronts() logger = logging.getLogger(__name__) logger.debug("pictures from stats") boot_size = int(args['--bootstrap']) results = collections.defaultdict(list) with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"): for problem_name, problem_mod, algorithms in RunResult.each_result(RESULTS_DIR): for algo_name, budgets in algorithms: for result in budgets: _, _, cost_data = next(result["analysis"]) cost_data = list(x() for x in cost_data) cost_analysis = yield_analysis(cost_data, boot_size) budget = cost_analysis["btstrpd"]["metrics"] budget_err = cost_analysis["stdev"] for metric_name, metric_name_long, data_process in result["analysis"]: if metric_name in best_func: if metric_name == 'dst from pareto': metric_name = 'dst' data_process = list(x() for x in data_process) data_analysis = yield_analysis(data_process, boot_size) score = data_analysis["btstrpd"]["metrics"] score_err = data_analysis["stdev"] keys = [(problem_name, algo_name, metric_name, group) for group in algos_groups[algo_name]] value = (budget, budget_err, score, score_err) for key in keys: results[key].append(value) plot_results(results)
def worker(args): logger = logging.getLogger(__name__) logger.debug("Starting the worker. args:%s", args) (problem, algo), budgets, runid, renice = args if renice: logger.debug("Renice the process PID:%s by %s", os.getpid(), renice) os.nice(int(renice)) logger.debug("Getting random seed") # basically we duplicate the code of https://github.com/python/cpython/blob/master/Lib/random.py#L111 because # in case os.urandom is not available, random.seed defaults to epoch time. That would set the seed equal in each # process, which is not acceptable. try: random_seed = int.from_bytes(os.urandom(2500), 'big') except NotImplementedError: random_seed = int(time.time() * 256 + os.getpid()) # that's not enough for MT, but will have to do for now. random.seed(random_seed) drivers = algo.split('+') runres = RunResult(algo, problem, runid=runid, results_path=RESULTS_DIR) try: final_driver, problem_mod = None, None for driver_pos, driver in list(enumerate(drivers))[::-1]: final_driver, problem_mod = prepare(driver, problem, final_driver, drivers, driver_pos ) logger.debug("Creating the driver used to perform computation") driver = final_driver() total_cost, result = 0, None proc_time = [] results = [] logger.debug("Beginning processing of %s, args: %s", driver, args) with log_time(process_time, logger, "Processing done in {time_res}s CPU time", out=proc_time): if isinstance(driver, DriverGen): logger.debug("The driver %s is DriverGen-based", show_partial(driver)) driver.max_budget = budgets[-1] gen = driver.population_generator() proxy = None logger.debug("Starting processing") for budget in budgets: logger.debug("Curr budget step is %d", budget) while total_cost < budget: logger.debug("Waiting for next proxy") proxy = gen.send(proxy) logger.debug("Proxy.cost: %d", proxy.cost) total_cost += proxy.cost logger.debug("total_cost: %d", total_cost) logger.debug("Cost %d equals/overpasses next budget step %d. Storing finalized population", total_cost, budget) finalpop = proxy.finalized_population() finalpop_fit = [[fit(x) for fit in problem_mod.fitnesses] for x in finalpop] runres.store(budget, total_cost, finalpop, finalpop_fit) results.append((total_cost, finalpop)) logger.debug("End loop, total_cost:%d", total_cost) logger.debug("Final population: %s", proxy.finalized_population()) elif isinstance(driver, DriverLegacy): logger.debug("The driver %s is DriverLegacy-based", show_partial(driver)) with log_time(process_time, logger, "All iterations in {time_res}s CPU time"): for budget in budgets: logger.debug("Re-creating the driver used to perform computation") driver = final_driver() driver.budget = budget with log_time(process_time, logger, "Iteration with budget {0} in {{time_res}}s CPU time".format(budget)): logger.debug("Running with budget=%d", budget) total_cost = driver.steps(count(), budget) finalpop = driver.finish() finalpop_fit = [[fit(x) for fit in problem_mod.fitnesses] for x in finalpop] runres.store(budget, total_cost, finalpop, finalpop_fit) results.append((budget, finalpop)) else: e = NotImplementedError() logger.exception("Oops. The driver type is not recognized, got %s", show_partial(driver), exc_info=e) raise e return results, proc_time[-1] except NotViableConfiguration as e: reason = inspect.trace()[-1] logger.info("Configuartion disabled by %s:%d:%s. args:%s", reason[1], reason[2], reason[3], args) logger.debug("Configuration disabled args:%s. Stack:", exc_info=e) except Exception as e: logger.exception("Some error", exc_info=e) finally: logger.debug("Finished processing. args:%s", args)
def table_rank(args): logger = logging.getLogger(__name__) logger.debug("table ranking") boot_size = int(args["--bootstrap"]) results = collections.defaultdict( lambda: collections.defaultdict(collections.Counter)) for result_set in result_dirs: print("***{}***".format(result_set)) with log_time(process_time, logger, "Preparing data done in {time_res:.3f}"): for problem_name, problem_mod, algorithms in serialization.each_result( BudgetResultsExtractor(), result_set): print(result_set, problem_name) scoring = collections.defaultdict(list) for algo_name, results_data in algorithms: results_data = list(results_data) for i in range(len(results_data)): original_budget = results_data[i]["budget"] if original_budget == 40: result = results_data[i] # result = find_acceptable_result_for_budget( # results_data[: i + 1], boot_size # ) if result: print("{} {} {} -> {}".format( problem_name, algo_name, original_budget, result["budget"], )) for metric_name, metric_name_long, data_process in result[ "analysis"]: if metric_name in best_func: data_process = list( x() for x in data_process) data_analysis = yield_analysis( data_process, boot_size) score = data_analysis["btstrpd"][ "metrics"] scoring[(original_budget, metric_name)].append( (algo_name, score)) print("****{}****".format(problem_name)) for budget, metric_name in sorted(scoring): metric_scoring = scoring[(budget, metric_name)] algo_win, score = best_func[metric_name]( metric_scoring, key=lambda x: x[1]) weak_winners = get_weak_winners( metric_scoring, (algo_win, score), winner_tolerance[metric_name]) # # Only strong # if not weak_winners: # results[(budget, metric_name)][result_set].update([algo_win]) # Strong = 2 points, Weak or Winner = 1 point if not weak_winners: results[(budget, metric_name)][result_set].update( [algo_win, algo_win]) else: results[(budget, metric_name)][result_set].update( [algo_win] + [algo for algo, score in weak_winners]) # print('{} {} {}'.format(budget, metric_name, scoring[(budget, metric_name)])) print("*****{} {}".format(budget, metric_name)) if not weak_winners: print("*****Strong winner: {} :{}".format( algo_win, score)) else: print("*****Winner: {} :{}".format(algo_win, score)) print("*****Weak winners: {}".format(weak_winners)) print("""\\begin{table}[ht] \\centering \\caption{Final results} \\label{tab:results"} \\resizebox{\\textwidth}{!}{% \\begin{tabular}{ r@{ }l | c | c | c | } \multicolumn{2}{c}{} & $K_0$ & $K_1$ & $K_2$ \\\\ \\hline""") prevous_budget = None for budget, metric_name in sorted( sorted(results.keys(), key=lambda x: metrics_order.index(x[1])), key=lambda x: x[0], ): budget_label = str(budget) + " " if prevous_budget and prevous_budget != budget: print("\\hdashline") elif prevous_budget: budget_label = "" score_str = "" for result_set in result_dirs: results_counter = results[(budget, metric_name)][result_set] algo_ranking = results_counter.most_common(2) values = list(results_counter.values()) if len(algo_ranking) == 2: winner = format_result(algo_ranking[0], values, 2) second = format_result( algo_ranking[1], values, 1 if algo_ranking[0][1] != algo_ranking[1][1] else 2, ) score_str += "& {}, {}".format(winner, second) elif len(algo_ranking) == 1: winner = format_result(algo_ranking[0], values, 2) score_str += "& {}".format(winner) else: score_str += "& " print("{}& {} {}\\\\".format(budget_label, metric_name, score_str)) prevous_budget = budget print(""" \\end{tabular}}\n\\end{table}""")
def statistics(args, queue): logger = logging.getLogger(__name__) badbench = [] cost_badbench = [] boot_size = int(args['--bootstrap']) # pretty format screen_width = sum(y for x in fields for y in x[1]) + 4 * len(fields) [err_prefix] = [ i for i, (name, lens, fmt) in enumerate(fields) if name == "RESULT, confidence interval" ] err_prefix = sum(y for x in fields[:err_prefix] for y in x[1]) + 4 * err_prefix - 2 err_prefix = " " * err_prefix def print_header(): print() print('..'.join('[{0:^{1}}]'.format(head, sum(width)) for head, width, var in fields) + "..", flush=True) return True with close_and_join(multiprocessing.Pool(min(int(args['-j']), 8))) as p: for problem_name, problem_mod, algorithms in RunResult.each_result(): for algo_name, budgets in algorithms: header_just_printed = print_header() for result in budgets: len_data = len(result["results"]) first_budget_line = True avg_pop_len = average([len(x.population) for x in result["results"]]) with log_time(process_time, logger, "Calculating metrics for {} :: {} :: {} in {{time_res:.3f}}s".format( problem_name, algo_name, result["budget"] )): results_precalc = p.map(force_data, zip(repeat(boot_size), result["analysis"]), chunksize=1) for metric_name, metric_name_long, data_process, analysis in results_precalc: if first_budget_line and not header_just_printed: if screen_width % 2 == 1: print("-" + " -" * (screen_width // 2)) else: print(" -" * (screen_width // 2)) first_budget_line = False columns = [] for i, (head, width, var) in enumerate(fields): columns.append(var.format(*width, **locals())) # the data print("", " :: ".join(columns), ":: ", flush=True) header_just_printed = False if analysis["goodbench"] != "✓": lower_process = analysis["lower"] upper_process = analysis["upper"] low_out_fence_process = analysis["low_out_fence"] upp_out_fence_process = analysis["upp_out_fence"] stdev_process = analysis["stdev"] mean_process = analysis["mean"] outliers = len([x for x in data_process if lower_process <= x <= upper_process]) print( "{err_prefix}:: Suspicious result analysis:\n" "{err_prefix}:: {0:>2} / {1:2} ({4:7.3f}%) out of [ {2:>18.13} ; {3:<18.13} ]\n" "{err_prefix}:: Δ {7:<18.13}\n" "{err_prefix}:: Bounds: [ {5:>18.13} ; {6:<18.13} ]\n" "{err_prefix}:: Δ {8:<18.13}".format( outliers, len(data_process), lower_process, upper_process, 100.0 * outliers / len(data_process), min(data_process), max(data_process), upper_process - lower_process, max(data_process) - min(data_process), err_prefix=err_prefix) ) print("{err_prefix}:: Values".format(err_prefix=err_prefix)) def aux(x): try: return abs(x - mean_process) * 100.0 / stdev_process except ZeroDivisionError: return float("inf") print(''.join( "{err_prefix}:: {0:>30.20} = avg {1:<+30} = avg {3:+8.3f}% ⨉ σ | {2:17} {4:17} {5:17}\n".format( x, x - mean_process, (lower_process <= x <= upper_process) and "(out of mean±3σ)" or "", aux(x), ((low_out_fence_process <= x < analysis["low_inn_fence"]) or ( analysis[ "upp_inn_fence"] <= x < upp_out_fence_process)) and " (mild outlier)" or "", ((x < low_out_fence_process) or ( upp_out_fence_process < x)) and "(EXTREME outlier)" or "", err_prefix=err_prefix ) for x in data_process), end='' ) if abs(analysis["mean_nooutliers_diff"]) > 10.: badbench.append([problem_name, algo_name, result["budget"], metric_name_long]) print(err_prefix + "::", "#"*22, "#"*67, "#"*22) print(err_prefix + "::", "#"*22, "Mean of results changed a lot (> 10%), so probably UNTRUSTED result", "#"*22) print(err_prefix + "::", "#"*22, "#"*67, "#"*22) else: print(err_prefix + "::", "Mean of results changed a little (< 10%), so probably that's all okay") if badbench: print("#" * 237) for i in badbench: print(">>> " + " :: ".join(str(x) for x in i))
def statistics(args): logger = logging.getLogger(__name__) badbench = [] cost_badbench = [] boot_size = int(args["--bootstrap"]) # pretty format screen_width = sum(y for x in fields for y in x[1]) + 4 * len(fields) [err_prefix] = [ i for i, (name, lens, fmt) in enumerate(fields) if name == "RESULT, confidence interval" ] err_prefix = sum(y for x in fields[:err_prefix] for y in x[1]) + 4 * err_prefix - 2 err_prefix = " " * err_prefix def print_header(): print() print( "..".join("[{0:^{1}}]".format(head, sum(width)) for head, width, var in fields) + "..", flush=True, ) return True with close_and_join(multiprocessing.Pool(min(int(args["-j"]), 8))) as p: for problem_name, problem_mod, algorithms in serialization.each_result( BudgetResultsExtractor()): for algo_name, budgets in algorithms: header_just_printed = print_header() for result in budgets: len_data = len(result["results"]) first_budget_line = True avg_pop_len = average( [len(x.population) for x in result["results"]]) with log_time( process_time, logger, "Calculating metrics for {} :: {} :: {} in {{time_res:.3f}}s" .format(problem_name, algo_name, result["budget"]), ): results_precalc = p.map( force_data, zip(repeat(boot_size), result["analysis"]), chunksize=1, ) for ( metric_name, metric_name_long, data_process, analysis, ) in results_precalc: if first_budget_line and not header_just_printed: if screen_width % 2 == 1: print("-" + " -" * (screen_width // 2)) else: print(" -" * (screen_width // 2)) first_budget_line = False columns = [] for i, (head, width, var) in enumerate(fields): columns.append(var.format(*width, **locals())) # the data print("", " :: ".join(columns), ":: ", flush=True) header_just_printed = False if analysis["goodbench"] != "✓": lower_process = analysis["lower"] upper_process = analysis["upper"] low_out_fence_process = analysis[ "low_out_fence"] upp_out_fence_process = analysis[ "upp_out_fence"] stdev_process = analysis["stdev"] mean_process = analysis["mean"] outliers = len([ x for x in data_process if lower_process <= x <= upper_process ]) print( "{err_prefix}:: Suspicious result analysis:\n" "{err_prefix}:: {0:>2} / {1:2} ({4:7.3f}%) out of [ {2:>18.13} ; {3:<18.13} ]\n" "{err_prefix}:: Δ {7:<18.13}\n" "{err_prefix}:: Bounds: [ {5:>18.13} ; {6:<18.13} ]\n" "{err_prefix}:: Δ {8:<18.13}" .format( outliers, len(data_process), lower_process, upper_process, 100.0 * outliers / len(data_process), min(data_process), max(data_process), upper_process - lower_process, max(data_process) - min(data_process), err_prefix=err_prefix, )) print("{err_prefix}:: Values".format( err_prefix=err_prefix)) def aux(x): try: return (abs(x - mean_process) * 100.0 / stdev_process) except ZeroDivisionError: return float("inf") print( "".join( "{err_prefix}:: {0:>30.20} = avg {1:<+30} = avg {3:+8.3f}% ⨉ σ | {2:17} {4:17} {5:17}\n" .format( x, x - mean_process, (lower_process <= x <= upper_process) and "(out of mean±3σ)" or "", aux(x), ((low_out_fence_process <= x < analysis["low_inn_fence"]) or (analysis["upp_inn_fence"] <= x < upp_out_fence_process)) and " (mild outlier)" or "", ((x < low_out_fence_process) or (upp_out_fence_process < x)) and "(EXTREME outlier)" or "", err_prefix=err_prefix, ) for x in data_process), end="", ) if abs(analysis["mean_nooutliers_diff"] ) > 10.0: badbench.append([ problem_name, algo_name, result["budget"], metric_name_long, ]) print(err_prefix + "::", "#" * 22, "#" * 67, "#" * 22) print( err_prefix + "::", "#" * 22, "Mean of results changed a lot (> 10%), so probably UNTRUSTED result", "#" * 22, ) print(err_prefix + "::", "#" * 22, "#" * 67, "#" * 22) else: print( err_prefix + "::", "Mean of results changed a little (< 10%), so probably that's all okay", ) if badbench: print("#" * 237) for i in badbench: print(">>> " + " :: ".join(str(x) for x in i))
def run_parallel(args, queue): logger = logging.getLogger(__name__) budgets = sorted([int(budget) for budget in args['<budget>'].split(',')]) logger.debug("Budgets: %s", budgets) order = list(product(run_config.problems, run_config.algorithms)) logger.debug("Problems * algorithms: %s", order) algorithms = run_config.algorithms if args['--algo']: algorithms_filter = args['--algo'].lower().split(',') logger.debug("Selecting algorithms by name: %s", algorithms) algorithms = [ algo for algo in algorithms if algo.lower() in algorithms_filter ] logger.debug("Selected: %s", algorithms) problems = run_config.problems if args['--problem']: problems_filter = args['--problem'].lower().split(',') logger.debug("Selecting problems by name: %s", problems) problems = [ problem for problem in problems if problem.lower() in problems_filter ] logger.debug("Selected: %s", problems) order = list(product(problems, algorithms)) logger.info("Selected following tests:") for problem, algo in order: logger.info(" {problem:12} :: {algo:12}".format(problem=problem, algo=algo)) logger.debug("Duplicating problems (-N flag)") order = [(test, budgets, runid, args["--renice"]) for test in order for runid in range(int(args['-N']))] logger.debug("Shuffling the job queue") random.shuffle(order) logger.debug("Creating the pool") with close_and_join(multiprocessing.Pool(int(args['-j']))) as p: wall_time = [] start_time = datetime.now() results = [] with log_time(system_time, logger, "Pool evaluated in {time_res}s", out=wall_time): for i, subres in enumerate(p.imap(worker, order, chunksize=1)): results.append(subres) current_time = datetime.now() diff_time = current_time - start_time ratio = i * 1. / len(order) try: est_delivery_time = start_time + diff_time / ratio time_to_delivery = est_delivery_time - current_time logging.info( "Job queue progress: %.3f%%. Est. finish in %02d:%02d:%02d (at %s)", ratio * 100, time_to_delivery.days * 24 + time_to_delivery.seconds // 3600, time_to_delivery.seconds // 60, time_to_delivery.seconds % 60, est_delivery_time.strftime("%Y-%m-%d %H:%M:%S.%f")) except ZeroDivisionError: logging.info( "Job queue progress: %.3f%%. Est. finish: unknown yet", ratio) proc_times = sum(subres[1] for subres in results if subres is not None) errors = [(test, budgets, runid) for comp_result, (test, budgets, runid, renice) in zip(results, order) if comp_result is None] speedup = proc_times / wall_time[0] logger.info("SUMMARY:") logger.info(" wall time: %7.3f", wall_time[0]) logger.info(" CPU+user time: %7.3f", proc_times) logger.info(" est. speedup: %7.3f", speedup) if errors: logger.error("Errors encountered:") for (probl, algo), budgets, runid in errors: logger.error(" %9s :: %14s :: runID=%d :: budgets=%s", probl, algo, runid, ','.join(str(x) for x in budgets)) summary = collections.defaultdict(float) for (bench, _, _, _), subres in zip(order, results): if subres: summary[bench] += subres[1] if logger.isEnabledFor(logging.INFO): logger.info("Running time:") res = [] for (prob, alg), timesum in sorted(summary.items(), key=operator.itemgetter(1), reverse=True): prob_show = "'" + prob + "'" alg_show = "'" + alg + "'" avg_time = timesum / float(args['-N']) logger.info( " prob:{prob_show:16} algo:{alg_show:16}) time:{avg_time:>8.3f}s" .format(**locals()))
def worker(args): logger = logging.getLogger(__name__) logger.debug("Starting the worker. args:%s", args) (problem, algo), budgets, runid, renice = args if renice: logger.debug("Renice the process PID:%s by %s", os.getpid(), renice) os.nice(int(renice)) logger.debug("Getting random seed") # basically we duplicate the code of https://github.com/python/cpython/blob/master/Lib/random.py#L111 because # in case os.urandom is not available, random.seed defaults to epoch time. That would set the seed equal in each # process, which is not acceptable. try: random_seed = int.from_bytes(os.urandom(2500), 'big') except NotImplementedError: random_seed = int( time.time() * 256 + os.getpid() ) # that's not enough for MT, but will have to do for now. random.seed(random_seed) drivers = algo.split('+') runres = RunResult(algo, problem, runid=runid, results_path=RESULTS_DIR) try: final_driver, problem_mod = None, None for driver_pos, driver in list(enumerate(drivers))[::-1]: final_driver, problem_mod = prepare(driver, problem, final_driver, drivers, driver_pos) logger.debug("Creating the driver used to perform computation") driver = final_driver() total_cost, result = 0, None proc_time = [] results = [] logger.debug("Beginning processing of %s, args: %s", driver, args) with log_time(process_time, logger, "Processing done in {time_res}s CPU time", out=proc_time): if isinstance(driver, DriverGen): logger.debug("The driver %s is DriverGen-based", show_partial(driver)) driver.max_budget = budgets[-1] gen = driver.population_generator() proxy = None logger.debug("Starting processing") for budget in budgets: logger.debug("Curr budget step is %d", budget) while total_cost < budget: logger.debug("Waiting for next proxy") proxy = gen.send(proxy) logger.debug("Proxy.cost: %d", proxy.cost) total_cost += proxy.cost logger.debug("total_cost: %d", total_cost) logger.debug( "Cost %d equals/overpasses next budget step %d. Storing finalized population", total_cost, budget) finalpop = proxy.finalized_population() finalpop_fit = [[fit(x) for fit in problem_mod.fitnesses] for x in finalpop] runres.store(budget, total_cost, finalpop, finalpop_fit) results.append((total_cost, finalpop)) logger.debug("End loop, total_cost:%d", total_cost) logger.debug("Final population: %s", proxy.finalized_population()) elif isinstance(driver, DriverLegacy): logger.debug("The driver %s is DriverLegacy-based", show_partial(driver)) with log_time(process_time, logger, "All iterations in {time_res}s CPU time"): for budget in budgets: logger.debug( "Re-creating the driver used to perform computation" ) driver = final_driver() driver.budget = budget with log_time( process_time, logger, "Iteration with budget {0} in {{time_res}}s CPU time" .format(budget)): logger.debug("Running with budget=%d", budget) total_cost = driver.steps(count(), budget) finalpop = driver.finish() finalpop_fit = [[ fit(x) for fit in problem_mod.fitnesses ] for x in finalpop] runres.store(budget, total_cost, finalpop, finalpop_fit) results.append((budget, finalpop)) else: e = NotImplementedError() logger.exception( "Oops. The driver type is not recognized, got %s", show_partial(driver), exc_info=e) raise e return results, proc_time[-1] except NotViableConfiguration as e: reason = inspect.trace()[-1] logger.info("Configuartion disabled by %s:%d:%s. args:%s", reason[1], reason[2], reason[3], args) logger.debug("Configuration disabled args:%s. Stack:", exc_info=e) except Exception as e: logger.exception("Some error", exc_info=e) finally: logger.debug("Finished processing. args:%s", args)
def run_parallel(args, queue): logger = logging.getLogger(__name__) budgets = sorted([int(budget) for budget in args['<budget>'].split(',')]) logger.debug("Budgets: %s", budgets) order = list(product(run_config.problems, run_config.algorithms)) logger.debug("Problems * algorithms: %s", order) algorithms = run_config.algorithms if args['--algo']: algorithms_filter = args['--algo'].lower().split(',') logger.debug("Selecting algorithms by name: %s", algorithms) algorithms = [ algo for algo in algorithms if algo.lower() in algorithms_filter ] logger.debug("Selected: %s", algorithms) problems = run_config.problems if args['--problem']: problems_filter = args['--problem'].lower().split(',') logger.debug("Selecting problems by name: %s", problems) problems = [ problem for problem in problems if problem.lower() in problems_filter ] logger.debug("Selected: %s", problems) order = list(product(problems, algorithms)) logger.info("Selected following tests:") for problem, algo in order: logger.info(" {problem:12} :: {algo:12}".format(problem=problem, algo=algo)) logger.debug("Duplicating problems (-N flag)") order = [ (test, budgets, runid, args["--renice"]) for test in order for runid in range(int(args['-N'])) ] logger.debug("Shuffling the job queue") random.shuffle(order) logger.debug("Creating the pool") with close_and_join(multiprocessing.Pool(int(args['-j']))) as p: wall_time = [] start_time = datetime.now() results = [] with log_time(system_time, logger, "Pool evaluated in {time_res}s", out=wall_time): for i, subres in enumerate(p.imap(worker, order, chunksize=1)): results.append(subres) current_time = datetime.now() diff_time = current_time - start_time ratio = i * 1. / len(order) try: est_delivery_time = start_time + diff_time / ratio time_to_delivery = est_delivery_time - current_time logging.info("Job queue progress: %.3f%%. Est. finish in %02d:%02d:%02d (at %s)", ratio * 100, time_to_delivery.days * 24 + time_to_delivery.seconds // 3600, time_to_delivery.seconds // 60, time_to_delivery.seconds % 60, est_delivery_time.strftime("%Y-%m-%d %H:%M:%S.%f") ) except ZeroDivisionError: logging.info("Job queue progress: %.3f%%. Est. finish: unknown yet", ratio) proc_times = sum(subres[1] for subres in results if subres is not None) errors = [(test, budgets, runid) for comp_result, (test, budgets, runid, renice) in zip(results, order) if comp_result is None ] speedup = proc_times / wall_time[0] logger.info("SUMMARY:") logger.info(" wall time: %7.3f", wall_time[0]) logger.info(" CPU+user time: %7.3f", proc_times) logger.info(" est. speedup: %7.3f", speedup) if errors: logger.error("Errors encountered:") for (probl, algo), budgets, runid in errors: logger.error(" %9s :: %14s :: runID=%d :: budgets=%s", probl, algo, runid, ','.join(str(x) for x in budgets)) summary = collections.defaultdict(float) for (bench, _, _, _), subres in zip(order, results): if subres: summary[bench] += subres[1] if logger.isEnabledFor(logging.INFO): logger.info("Running time:") res = [] for (prob, alg), timesum in sorted(summary.items(), key=operator.itemgetter(1), reverse=True): prob_show = "'" + prob + "'" alg_show = "'" + alg + "'" avg_time = timesum / float(args['-N']) logger.info(" prob:{prob_show:16} algo:{alg_show:16}) time:{avg_time:>8.3f}s".format(**locals()))