Example #1
0
    def test_get_best(self):
        res = plot_util.get_best(branin_experiment)
        self.assertAlmostEqual(0.72639128754424731, res)

        res = plot_util.get_best(branin_experiment, cut=22)
        self.assertAlmostEqual(0.72639128754424731, res)

        res = plot_util.get_best(branin_experiment, cut=10)
        self.assertAlmostEqual(12.779763397329571, res)

        self.assertRaises(ValueError, plot_util.get_best, branin_experiment,
                          0.5)

        self.assertRaises(ValueError, plot_util.get_best, branin_experiment, 0)
Example #2
0
    def test_get_best(self):
        res = plot_util.get_best(branin_experiment)
        self.assertAlmostEqual(0.72639128754424731, res)

        res = plot_util.get_best(branin_experiment, cut=22)
        self.assertAlmostEqual(0.72639128754424731, res)

        res = plot_util.get_best(branin_experiment, cut=10)
        self.assertAlmostEqual(12.779763397329571, res)

        self.assertRaises(ValueError, plot_util.get_best,
                         branin_experiment, 0.5)

        self.assertRaises(ValueError, plot_util.get_best,
                          branin_experiment, 0)
Example #3
0
def main(pkl_list,
         name_list,
         title="",
         save="",
         y_min=0,
         y_max=0,
         cut=sys.maxint):

    best_trials = list()
    for i in range(len(name_list)):
        best_trials.append(list())
        for pkl in pkl_list[i]:
            fh = open(pkl, "r")
            trials = cPickle.load(fh)
            fh.close()
            best_trials[i].append(plot_util.get_best(trials, cut=cut))

    plot_box_whisker(best_trials=best_trials,
                     name_list=name_list,
                     title=title,
                     save=save,
                     y_min=y_min,
                     y_max=y_max)

    if save != "":
        sys.stdout.write("Saving plot to " + save + "\n")
    else:
        sys.stdout.write("..Done\n")
Example #4
0
def main(pkl_list, name_list, title="", save="", y_min=0, y_max=0, cut=sys.maxint):

    best_trials = list()
    for i in range(len(name_list)):
        best_trials.append(list())
        for pkl in pkl_list[i]:
            fh = open(pkl, "r")
            trials = cPickle.load(fh)
            fh.close()
            best_trials[i].append(plot_util.get_best(trials, cut=cut))

    plot_box_whisker(best_trials=best_trials, name_list=name_list,
                     title=title, save=save, y_min=y_min, y_max=y_max)

    if save != "":
        sys.stdout.write("Saving plot to " + save + "\n")
    else:
        sys.stdout.write("..Done\n")
def calculate_rankings(trial_list, name_list, bootstrap_samples=500, cut=50):
    bootstrap_samples = int(bootstrap_samples)
    optimizers = [name[0] for name in name_list]
    pickles = plot_util.load_pickles(name_list, trial_list)
    rankings = dict()

    rs = np.random.RandomState(1)

    combinations = []
    for i in range(bootstrap_samples):
        combination = []
        target = len(optimizers)
        maximum = [len(pickles[name]) for name in optimizers]
        for idx in range(target):
            combination.append(rs.randint(maximum[idx]))
        combinations.append(np.array(combination))

    for optimizer in optimizers:
        rankings[optimizer] = np.zeros((cut + 1, ), dtype=np.float64)
        rankings[optimizer][0] = np.mean(range(1, len(optimizers) + 1))

    for i in range(1, cut + 1):
        num_products = 0

        for combination in combinations:

            ranks = scipy.stats.rankdata([
                np.round(
                    plot_util.get_best(pickles[optimizers[idx]][number], i), 5)
                for idx, number in enumerate(combination)
            ])
            num_products += 1
            for j, optimizer in enumerate(optimizers):
                rankings[optimizer][i] += ranks[j]

        for optimizer in optimizers:
            rankings[optimizer][i] = rankings[optimizer][i] / num_products

    return rankings
Example #6
0
def calculate_rankings(trial_list, name_list, bootstrap_samples=500, cut=50):
    bootstrap_samples = int(bootstrap_samples)
    optimizers = [name[0] for name in name_list]
    pickles = plot_util.load_pickles(name_list, trial_list)
    rankings = dict()

    rs = np.random.RandomState(1)

    combinations = []
    for i in range(bootstrap_samples):
        combination = []
        target = len(optimizers)
        maximum = [len(pickles[name]) for name in optimizers]
        for idx in range(target):
            combination.append(rs.randint(maximum[idx]))
        combinations.append(np.array(combination))

    for optimizer in optimizers:
        rankings[optimizer] = np.zeros((cut+1,), dtype=np.float64)
        rankings[optimizer][0] = np.mean(range(1, len(optimizers) + 1))

    for i in range(1, cut+1):
        num_products = 0

        for combination in combinations:

            ranks = scipy.stats.rankdata(
                [np.round(
                    plot_util.get_best(pickles[optimizers[idx]][number], i), 5)
                 for idx, number in enumerate(combination)])
            num_products += 1
            for j, optimizer in enumerate(optimizers):
                rankings[optimizer][i] += ranks[j]

        for optimizer in optimizers:
            rankings[optimizer][i] = rankings[optimizer][i] / num_products

    return rankings
Example #7
0
def collect_results(directory):
    locker = Locker()
    errors = []
    sio = StringIO.StringIO()
    sio.write("Statistics for %s\n" % directory)
    sio.write("%30s | %6s | %7s/%7s/%7s/%7s/%7s/%7s/%7s/%7s | %10s | %10s\n" %
              ("Optimizer", "Seed", "#conf", "#runs", "#compl", "#incom",
               "#crashs", "#run", "#notrun", "#NaNs", "best", "AvgRunTime"))

    subdirs = os.listdir(directory)
    subdirs.sort()
    results = defaultdict(list)
    # Look for all sub-experiments
    for subdir in subdirs:
        subdir = os.path.join(directory, subdir)
        if os.path.isdir(subdir):
            # Get the experiment pickle...
            for possible_experiment_pickle in os.listdir(subdir):
                # Some simple checks for an experiment pickle
                if possible_experiment_pickle[-4:] == ".pkl" and \
                        possible_experiment_pickle[:-4] in subdir:
                    exp_pkl = os.path.join(subdir, possible_experiment_pickle)
                    locker.lock(exp_pkl)
                    with open(exp_pkl) as fh:
                        try:
                            pkl = cPickle.load(fh)
                        except Exception as e:
                            errors.append(exp_pkl + ' ' + str(type(e)))
                            continue
                    locker.unlock(exp_pkl)

                    cfg = ConfigParser.ConfigParser()
                    cfg.read(os.path.join(subdir, "config.cfg"))

                    optimizer = pkl["experiment_name"]
                    optimizer = optimizer.split("/")[-1]

                    configurations = len(pkl["trials"])
                    instance_runs = len(pkl["instance_order"])

                    # HPOlib < 0.1 don't have a seed in the config, try to
                    # infer it
                    try:
                        seed = cfg.getint("HPOLIB", "seed")
                    except:
                        seed = subdir.replace(possible_experiment_pickle[:-4],
                                              "")
                        seed = seed[1:].split("_")[0]
                        seed = int(seed)

                    crashs = get_num_runs_for_state(BROKEN_STATE, pkl)
                    candidates = get_num_runs_for_state(CANDIDATE_STATE, pkl)
                    running = get_num_runs_for_state(RUNNING_STATE, pkl)
                    incomplete = get_num_runs_for_state(INCOMPLETE_STATE, pkl)
                    complete = get_num_runs_for_state(COMPLETE_STATE, pkl)
                    nans = get_num_nans(pkl)

                    try:
                        best_performance = plot_util.get_best(pkl)
                        if best_performance == sys.maxint:
                            # There is not at least one evaluated config
                            best_performance = np.nan
                    except Exception as e:
                        errors.append(str(e) + ' ' + exp_pkl)
                        continue

                    instance_durations = get_instance_durations(pkl)
                    mean_instance_durations = np.mean(instance_durations)

                    #total_runtime = get_total_time(pkl)
                    results[optimizer].append([
                        optimizer,
                        int(seed), configurations, instance_runs, complete,
                        incomplete, crashs, running, candidates, nans,
                        best_performance, mean_instance_durations
                    ])

    def comparator(left, right):
        if left[0] < right[0]:
            return -1
        elif left[0] > right[0]:
            return 1
        else:
            if left[1] < right[1]:
                return -1
            elif left[1] > right[1]:
                return 1
            else:
                return 0

    for optimizer in sorted(results):
        results[optimizer].sort(cmp=comparator)

        results_for_mean = []
        runtimes_for_mean = []
        total_times_for_mean = []

        for result in results[optimizer]:
            results_for_mean.append(float(result[10]))
            runtimes_for_mean.append(float(result[11]))
            sio.write(
                "%30s | %6d | %7s/%7s/%7s/%7s/%7s/%7s/%7s/%7s | %10f | %10f\n"
                % (result[0], result[1], result[2], result[3], result[4],
                   result[5], result[6], result[7], result[8], result[9],
                   result[10], result[11]))

        sio.write("#NumRuns %5d | Mean %5f | Std %5f | Best %5f | Median %5f "
                  "| AvgRunTime %10f | AvgTotTime %10f\n" % (
                      len(results_for_mean),
                      np.mean(results_for_mean),
                      np.std(results_for_mean),
                      np.min(results_for_mean),
                      np.median(results_for_mean),
                      np.mean(runtimes_for_mean),
                      np.mean(total_times_for_mean),
                  ))

    if len(errors) > 0:
        sio.write("\nCouldn't read the following .pkl files\n")
        for error in errors:
            sio.write(error)
            sio.write("\n")
    return sio
Example #8
0
def collect_results(directory):
    locker = Locker()
    errors = []
    sio = StringIO.StringIO()
    sio.write("Statistics for %s\n" % directory)
    sio.write("%30s | %6s | %7s/%7s/%7s/%7s/%7s/%7s/%7s/%7s | %10s | %10s\n" %
             ("Optimizer", "Seed", "#conf", "#runs", "#compl", "#incom",
              "#crashs", "#run", "#notrun", "#NaNs", "best", "AvgRunTime"))


    subdirs = os.listdir(directory)
    subdirs.sort()
    results = defaultdict(list)
    # Look for all sub-experiments
    for subdir in subdirs:
        subdir = os.path.join(directory, subdir)
        if os.path.isdir(subdir):
            # Get the experiment pickle...
            for possible_experiment_pickle in os.listdir(subdir):
                # Some simple checks for an experiment pickle
                if possible_experiment_pickle[-4:] == ".pkl" and \
                        possible_experiment_pickle[:-4] in subdir:
                    exp_pkl = os.path.join(subdir, possible_experiment_pickle)
                    locker.lock(exp_pkl)
                    with open(exp_pkl) as fh:
                        try:
                            pkl = cPickle.load(fh)
                        except Exception as e:
                            errors.append(exp_pkl + ' ' +  str(type(e)))
                            continue
                    locker.unlock(exp_pkl)

                    cfg = ConfigParser.ConfigParser()
                    cfg.read(os.path.join(subdir, "config.cfg"))

                    optimizer = pkl["experiment_name"]
                    optimizer = optimizer.split("/")[-1]

                    configurations = len(pkl["trials"])
                    instance_runs = len(pkl["instance_order"])

                    # HPOlib < 0.1 don't have a seed in the config, try to
                    # infer it
                    try:
                        seed = cfg.getint("HPOLIB", "seed")
                    except:
                        seed = subdir.replace(possible_experiment_pickle[:-4],
                                              "")
                        seed = seed[1:].split("_")[0]
                        seed = int(seed)

                    crashs = get_num_runs_for_state(BROKEN_STATE, pkl)
                    candidates = get_num_runs_for_state(CANDIDATE_STATE, pkl)
                    running = get_num_runs_for_state(RUNNING_STATE, pkl)
                    incomplete = get_num_runs_for_state(INCOMPLETE_STATE, pkl)
                    complete = get_num_runs_for_state(COMPLETE_STATE, pkl)
                    nans = get_num_nans(pkl)

                    try:
                        best_performance = plot_util.get_best(pkl)
                        if best_performance == sys.maxint:
                            # There is not at least one evaluated config
                            best_performance = np.nan
                    except Exception as e:
                        errors.append(str(e) + ' ' + exp_pkl)
                        continue

                    instance_durations = get_instance_durations(pkl)
                    mean_instance_durations = np.mean(instance_durations)

                    #total_runtime = get_total_time(pkl)
                    results[optimizer].append([optimizer, int(seed),
                        configurations, instance_runs, complete, incomplete,
                        crashs, running, candidates, nans, best_performance,
                        mean_instance_durations])

    def comparator(left, right):
        if left[0] < right[0]:
            return -1
        elif left[0] > right[0]:
            return 1
        else:
            if left[1] < right[1]:
                return -1
            elif left[1] > right[1]:
                return 1
            else:
                return 0

    for optimizer in sorted(results):
        results[optimizer].sort(cmp=comparator)

        results_for_mean = []
        runtimes_for_mean = []
        total_times_for_mean = []

        for result in results[optimizer]:
            results_for_mean.append(float(result[10]))
            runtimes_for_mean.append(float(result[11]))
            sio.write("%30s | %6d | %7s/%7s/%7s/%7s/%7s/%7s/%7s/%7s | %10f | %10f\n"
                      % (result[0], result[1], result[2], result[3], result[4],
                         result[5], result[6], result[7], result[8], result[9],
                         result[10], result[11]))

        sio.write("#NumRuns %5d | Mean %5f | Std %5f | Best %5f | Median %5f "
                  "| AvgRunTime %10f | AvgTotTime %10f\n"
                  % (len(results_for_mean), np.mean(results_for_mean),
                     np.std(results_for_mean), np.min(results_for_mean),
                     np.median(results_for_mean), np.mean(runtimes_for_mean), np.mean(total_times_for_mean),))

    if len(errors) > 0:
        sio.write("\nCouldn't read the following .pkl files\n")
        for error in errors:
            sio.write(error)
            sio.write("\n")
    return sio