def test_get_best(self): res = plot_util.get_best(branin_experiment) self.assertAlmostEqual(0.72639128754424731, res) res = plot_util.get_best(branin_experiment, cut=22) self.assertAlmostEqual(0.72639128754424731, res) res = plot_util.get_best(branin_experiment, cut=10) self.assertAlmostEqual(12.779763397329571, res) self.assertRaises(ValueError, plot_util.get_best, branin_experiment, 0.5) self.assertRaises(ValueError, plot_util.get_best, branin_experiment, 0)
def main(pkl_list, name_list, title="", save="", y_min=0, y_max=0, cut=sys.maxint): best_trials = list() for i in range(len(name_list)): best_trials.append(list()) for pkl in pkl_list[i]: fh = open(pkl, "r") trials = cPickle.load(fh) fh.close() best_trials[i].append(plot_util.get_best(trials, cut=cut)) plot_box_whisker(best_trials=best_trials, name_list=name_list, title=title, save=save, y_min=y_min, y_max=y_max) if save != "": sys.stdout.write("Saving plot to " + save + "\n") else: sys.stdout.write("..Done\n")
def calculate_rankings(trial_list, name_list, bootstrap_samples=500, cut=50): bootstrap_samples = int(bootstrap_samples) optimizers = [name[0] for name in name_list] pickles = plot_util.load_pickles(name_list, trial_list) rankings = dict() rs = np.random.RandomState(1) combinations = [] for i in range(bootstrap_samples): combination = [] target = len(optimizers) maximum = [len(pickles[name]) for name in optimizers] for idx in range(target): combination.append(rs.randint(maximum[idx])) combinations.append(np.array(combination)) for optimizer in optimizers: rankings[optimizer] = np.zeros((cut + 1, ), dtype=np.float64) rankings[optimizer][0] = np.mean(range(1, len(optimizers) + 1)) for i in range(1, cut + 1): num_products = 0 for combination in combinations: ranks = scipy.stats.rankdata([ np.round( plot_util.get_best(pickles[optimizers[idx]][number], i), 5) for idx, number in enumerate(combination) ]) num_products += 1 for j, optimizer in enumerate(optimizers): rankings[optimizer][i] += ranks[j] for optimizer in optimizers: rankings[optimizer][i] = rankings[optimizer][i] / num_products return rankings
def calculate_rankings(trial_list, name_list, bootstrap_samples=500, cut=50): bootstrap_samples = int(bootstrap_samples) optimizers = [name[0] for name in name_list] pickles = plot_util.load_pickles(name_list, trial_list) rankings = dict() rs = np.random.RandomState(1) combinations = [] for i in range(bootstrap_samples): combination = [] target = len(optimizers) maximum = [len(pickles[name]) for name in optimizers] for idx in range(target): combination.append(rs.randint(maximum[idx])) combinations.append(np.array(combination)) for optimizer in optimizers: rankings[optimizer] = np.zeros((cut+1,), dtype=np.float64) rankings[optimizer][0] = np.mean(range(1, len(optimizers) + 1)) for i in range(1, cut+1): num_products = 0 for combination in combinations: ranks = scipy.stats.rankdata( [np.round( plot_util.get_best(pickles[optimizers[idx]][number], i), 5) for idx, number in enumerate(combination)]) num_products += 1 for j, optimizer in enumerate(optimizers): rankings[optimizer][i] += ranks[j] for optimizer in optimizers: rankings[optimizer][i] = rankings[optimizer][i] / num_products return rankings
def collect_results(directory): locker = Locker() errors = [] sio = StringIO.StringIO() sio.write("Statistics for %s\n" % directory) sio.write("%30s | %6s | %7s/%7s/%7s/%7s/%7s/%7s/%7s/%7s | %10s | %10s\n" % ("Optimizer", "Seed", "#conf", "#runs", "#compl", "#incom", "#crashs", "#run", "#notrun", "#NaNs", "best", "AvgRunTime")) subdirs = os.listdir(directory) subdirs.sort() results = defaultdict(list) # Look for all sub-experiments for subdir in subdirs: subdir = os.path.join(directory, subdir) if os.path.isdir(subdir): # Get the experiment pickle... for possible_experiment_pickle in os.listdir(subdir): # Some simple checks for an experiment pickle if possible_experiment_pickle[-4:] == ".pkl" and \ possible_experiment_pickle[:-4] in subdir: exp_pkl = os.path.join(subdir, possible_experiment_pickle) locker.lock(exp_pkl) with open(exp_pkl) as fh: try: pkl = cPickle.load(fh) except Exception as e: errors.append(exp_pkl + ' ' + str(type(e))) continue locker.unlock(exp_pkl) cfg = ConfigParser.ConfigParser() cfg.read(os.path.join(subdir, "config.cfg")) optimizer = pkl["experiment_name"] optimizer = optimizer.split("/")[-1] configurations = len(pkl["trials"]) instance_runs = len(pkl["instance_order"]) # HPOlib < 0.1 don't have a seed in the config, try to # infer it try: seed = cfg.getint("HPOLIB", "seed") except: seed = subdir.replace(possible_experiment_pickle[:-4], "") seed = seed[1:].split("_")[0] seed = int(seed) crashs = get_num_runs_for_state(BROKEN_STATE, pkl) candidates = get_num_runs_for_state(CANDIDATE_STATE, pkl) running = get_num_runs_for_state(RUNNING_STATE, pkl) incomplete = get_num_runs_for_state(INCOMPLETE_STATE, pkl) complete = get_num_runs_for_state(COMPLETE_STATE, pkl) nans = get_num_nans(pkl) try: best_performance = plot_util.get_best(pkl) if best_performance == sys.maxint: # There is not at least one evaluated config best_performance = np.nan except Exception as e: errors.append(str(e) + ' ' + exp_pkl) continue instance_durations = get_instance_durations(pkl) mean_instance_durations = np.mean(instance_durations) #total_runtime = get_total_time(pkl) results[optimizer].append([ optimizer, int(seed), configurations, instance_runs, complete, incomplete, crashs, running, candidates, nans, best_performance, mean_instance_durations ]) def comparator(left, right): if left[0] < right[0]: return -1 elif left[0] > right[0]: return 1 else: if left[1] < right[1]: return -1 elif left[1] > right[1]: return 1 else: return 0 for optimizer in sorted(results): results[optimizer].sort(cmp=comparator) results_for_mean = [] runtimes_for_mean = [] total_times_for_mean = [] for result in results[optimizer]: results_for_mean.append(float(result[10])) runtimes_for_mean.append(float(result[11])) sio.write( "%30s | %6d | %7s/%7s/%7s/%7s/%7s/%7s/%7s/%7s | %10f | %10f\n" % (result[0], result[1], result[2], result[3], result[4], result[5], result[6], result[7], result[8], result[9], result[10], result[11])) sio.write("#NumRuns %5d | Mean %5f | Std %5f | Best %5f | Median %5f " "| AvgRunTime %10f | AvgTotTime %10f\n" % ( len(results_for_mean), np.mean(results_for_mean), np.std(results_for_mean), np.min(results_for_mean), np.median(results_for_mean), np.mean(runtimes_for_mean), np.mean(total_times_for_mean), )) if len(errors) > 0: sio.write("\nCouldn't read the following .pkl files\n") for error in errors: sio.write(error) sio.write("\n") return sio
def collect_results(directory): locker = Locker() errors = [] sio = StringIO.StringIO() sio.write("Statistics for %s\n" % directory) sio.write("%30s | %6s | %7s/%7s/%7s/%7s/%7s/%7s/%7s/%7s | %10s | %10s\n" % ("Optimizer", "Seed", "#conf", "#runs", "#compl", "#incom", "#crashs", "#run", "#notrun", "#NaNs", "best", "AvgRunTime")) subdirs = os.listdir(directory) subdirs.sort() results = defaultdict(list) # Look for all sub-experiments for subdir in subdirs: subdir = os.path.join(directory, subdir) if os.path.isdir(subdir): # Get the experiment pickle... for possible_experiment_pickle in os.listdir(subdir): # Some simple checks for an experiment pickle if possible_experiment_pickle[-4:] == ".pkl" and \ possible_experiment_pickle[:-4] in subdir: exp_pkl = os.path.join(subdir, possible_experiment_pickle) locker.lock(exp_pkl) with open(exp_pkl) as fh: try: pkl = cPickle.load(fh) except Exception as e: errors.append(exp_pkl + ' ' + str(type(e))) continue locker.unlock(exp_pkl) cfg = ConfigParser.ConfigParser() cfg.read(os.path.join(subdir, "config.cfg")) optimizer = pkl["experiment_name"] optimizer = optimizer.split("/")[-1] configurations = len(pkl["trials"]) instance_runs = len(pkl["instance_order"]) # HPOlib < 0.1 don't have a seed in the config, try to # infer it try: seed = cfg.getint("HPOLIB", "seed") except: seed = subdir.replace(possible_experiment_pickle[:-4], "") seed = seed[1:].split("_")[0] seed = int(seed) crashs = get_num_runs_for_state(BROKEN_STATE, pkl) candidates = get_num_runs_for_state(CANDIDATE_STATE, pkl) running = get_num_runs_for_state(RUNNING_STATE, pkl) incomplete = get_num_runs_for_state(INCOMPLETE_STATE, pkl) complete = get_num_runs_for_state(COMPLETE_STATE, pkl) nans = get_num_nans(pkl) try: best_performance = plot_util.get_best(pkl) if best_performance == sys.maxint: # There is not at least one evaluated config best_performance = np.nan except Exception as e: errors.append(str(e) + ' ' + exp_pkl) continue instance_durations = get_instance_durations(pkl) mean_instance_durations = np.mean(instance_durations) #total_runtime = get_total_time(pkl) results[optimizer].append([optimizer, int(seed), configurations, instance_runs, complete, incomplete, crashs, running, candidates, nans, best_performance, mean_instance_durations]) def comparator(left, right): if left[0] < right[0]: return -1 elif left[0] > right[0]: return 1 else: if left[1] < right[1]: return -1 elif left[1] > right[1]: return 1 else: return 0 for optimizer in sorted(results): results[optimizer].sort(cmp=comparator) results_for_mean = [] runtimes_for_mean = [] total_times_for_mean = [] for result in results[optimizer]: results_for_mean.append(float(result[10])) runtimes_for_mean.append(float(result[11])) sio.write("%30s | %6d | %7s/%7s/%7s/%7s/%7s/%7s/%7s/%7s | %10f | %10f\n" % (result[0], result[1], result[2], result[3], result[4], result[5], result[6], result[7], result[8], result[9], result[10], result[11])) sio.write("#NumRuns %5d | Mean %5f | Std %5f | Best %5f | Median %5f " "| AvgRunTime %10f | AvgTotTime %10f\n" % (len(results_for_mean), np.mean(results_for_mean), np.std(results_for_mean), np.min(results_for_mean), np.median(results_for_mean), np.mean(runtimes_for_mean), np.mean(total_times_for_mean),)) if len(errors) > 0: sio.write("\nCouldn't read the following .pkl files\n") for error in errors: sio.write(error) sio.write("\n") return sio