Esempio n. 1
0
 def test_pareto(self):
     test_arr = np.asarray([[5, 5], [2, 2], [1, 4], [3, 2]])
     mins = test_arr[pareto(test_arr, maximize=False)]
     maxes = test_arr[pareto(test_arr, maximize=True)]
     self.assertTrue([2, 2] in mins)
     self.assertTrue([1, 4] in mins)
     self.assertFalse(([3, 2] in maxes))
     self.assertTrue([5, 5] in maxes)
Esempio n. 2
0
 def test_pareto(self):
     test_arr = np.asarray([[5, 5], [2, 2], [1, 4], [3, 2]])
     mins = test_arr[pareto(test_arr, maximize=False)]
     maxes = test_arr[pareto(test_arr, maximize=True)]
     self.assertTrue([2, 2] in mins)
     self.assertTrue([1, 4] in mins)
     self.assertFalse(([3, 2] in maxes))
     self.assertTrue([5, 5] in maxes)
Esempio n. 3
0
def visualize(collection,
              maximize=False,
              showbest=True,
              showmean=True,
              latexify=False,
              fontfamily="serif",
              scale='linear',
              analysis=True,
              print_pareto=False):
    """
    Visualize the progress of an optimization.

    Args:
        collection (pymongo Collection): The pymongo colllection containing your
            rocketsled optimization. For example, if you set you opt_label to
            'opt123', and used the same db as your LaunchPad, you could use
            lpad.db.opt123
        maximize (bool): Whether to plot optimizing for minimum or maximum.
        showbest (bool): Point out the best point on legend and on plot. If more
            than one best point (i.e., multiple equal maxima), show them all. If
            multiobjective, shows best for each objective, and prints the best
            value and x for each objective.
        showmean (bool): Show the mean and standard deviation for the guesses
            as the computations are carried out.
        latexify (bool): Use LaTeX for formatting.
        fontfamily (str): The font family to use for rendering. Choose from
            'serif', 'sans-serif', 'fantasy', 'monospace', or 'cursive'.
        scale (str): Whether to scale the plot's y axis according to log ('log')
            or 'linear' scale.
        analysis (bool): If True, stdouts info from analyze().
        print_pareto (bool): If True, display all Pareto-optimal objective
            values.

    Returns:
        Either None, a matplotlib plot, or a pymongo iterator. See 'mode' for
        details.
    """

    dtypes = Dtypes()
    fxstr = "$f(x)$" if latexify else "f(x)"
    opt = max if maximize else min
    objs = collection.find_one({'index': {'$exists': 1}})['y']
    n_objs = len(objs) if isinstance(objs, (list, tuple)) else 1

    dt = datetime.datetime.now()
    dtdata = [dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second]
    timestr = "{}-{}-{} {}:{}.{}".format(*dtdata)
    t0 = time.time()
    if latexify:
        plt.rc('text', usetex=True)
    else:
        plt.rc('text', usetex=False)
    plt.rc('font', family=fontfamily, size=9)

    N_COLS = 3
    # print(int(math.ceil(float(n_objs)/float(N_COLS))))

    if n_objs < N_COLS:
        f, axarr = plt.subplots(n_objs, squeeze=False)
    else:
        f, axarr = plt.subplots(N_COLS,
                                int(math.ceil(n_objs / N_COLS)),
                                squeeze=False)

    docset = collection.find({'index': {'$exists': 1}})
    docs = [None] * docset.count()
    for i, doc in enumerate(docset):
        docs[i] = {'y': doc['y'], 'index': doc['index'], 'x': doc['x']}

    if n_objs > 1:
        Y = np.asarray([doc['y'] for doc in docs])
        pareto_set = Y[pareto(Y, maximize=maximize)].tolist()
        pareto_graph = [(i + 1, doc['y']) for i, doc in enumerate(docs)
                        if doc['y'] in pareto_set]
        pareto_i = [i[0] for i in pareto_graph]

    for obj in range(n_objs):
        ax = axarr[obj % N_COLS, int(math.floor(obj / N_COLS))]

        i = []
        fx = []
        best = []
        mean = []
        std = []
        n = collection.find().count() - 2

        for doc in docs:
            fx.append(doc['y'] if n_objs == 1 else doc['y'][obj])
            i.append(doc['index'])
            best.append(opt(fx))
            mean.append(np.mean(fx))
            std.append(np.std(fx))

        if time.time() - t0 > 60:
            warnings.warn(
                "Gathering data from the db is taking a while. Ensure"
                "the latency to your db is low and the bandwidth"
                "is as high as possible!")

        mean = np.asarray(mean)
        std = np.asarray(std)

        ax.scatter(i, fx, color='blue', label=fxstr, s=10)
        ax.plot(i,
                best,
                color='orange',
                label="best {} value found so far"
                "".format(fxstr))
        if showmean:
            ax.plot(i,
                    mean,
                    color='grey',
                    label="mean {} value (with std "
                    "dev.)".format(fxstr))
            ax.fill_between(i, mean + std, mean - std, color='grey', alpha=0.3)

        ax.set_xlabel("{} evaluation".format(fxstr))
        ax.set_ylabel("{} value".format(fxstr))
        best_val = opt(best)

        if showbest:
            if latexify:
                best_label = "Best value: $f(x) = {}$" \
                             "".format(latex_float(best_val))
            else:
                best_label = "Best value: f(x) = {:.2E}".format(best_val)
            best = collection.find({'y': best_val})
            for b in best:
                bl = None if n_objs > 1 else best_label
                ax.scatter([b['index']], [best_val],
                           color='darkgreen',
                           s=50,
                           linewidth=3,
                           label=bl,
                           facecolors='none',
                           edgecolors='darkgreen')

                artext = "$x = $ [" if latexify else "x = ["
                for i, xi in enumerate(b['x']):
                    if i > 0:
                        artext += ". \mbox{~~~~~}" if latexify else "     "
                    if type(xi) in dtypes.floats:
                        if latexify:
                            artext += "${}$,\n".format(latex_float(xi))
                        else:
                            artext += "{:.2E},\n".format(xi)
                    else:
                        artext += str(xi) + ",\n"

                artext = artext[:-2] + "]"
                objstr = "objective {}".format(obj) if n_objs > 1 else ""
                if maximize:
                    print("max(f(x)) {} is {} at x = {}".format(
                        objstr, best_val, b['x']))
                else:
                    print("min(f(x)) {} is {} at x = {}".format(
                        objstr, best_val, b['x']))
                ax.annotate(artext,
                            xy=(b['index'] + 0.5, best_val),
                            xytext=(b['index'] + float(n) / 12.0, best_val),
                            arrowprops=dict(color='green'),
                            color='darkgreen',
                            bbox=dict(facecolor='white', alpha=1.0))
        else:
            best_label = ""

        if n_objs > 1:
            pareto_fx = [i[1][obj] for i in pareto_graph]
            ax.scatter(pareto_i,
                       pareto_fx,
                       color='red',
                       label="Pareto optimal",
                       s=20)

        if n_objs > 1:
            ax.set_title("Objective {}: {}".format(obj + 1, best_label))
        ax.set_yscale(scale)

    plt.gcf().set_size_inches(10, 10)
    if analysis:
        print(analyze(collection))

    if print_pareto and n_objs > 1:
        print("Pareto Frontier: {} points".format(len(pareto_set)))
        pareto_y = [doc['y'] for doc in docs if doc['y'] in pareto_set]
        pareto_x = [doc['x'] for doc in docs if doc['y'] in pareto_set]

        for i, _ in enumerate(pareto_set):
            print("f(x) = {} @ x = {}".format(pareto_y[i], pareto_x[i]))

    if n_objs % N_COLS != 0 and n_objs > N_COLS:
        for i in range(n_objs % N_COLS, N_COLS):
            plt.delaxes(axarr[i, -1])

    plt.legend()
    # plt.tight_layout(pad=0.01, w_pad=0.01, h_pad=0.01)
    plt.subplots_adjust(wspace=0.3, hspace=0.5)
    plt.suptitle("Rocketsled optimization results for {} - "
                 "{}".format(collection.name, timestr),
                 y=0.99)
    plt.show()
Esempio n. 4
0
    def plot(self,
             show_best=True,
             show_mean=True,
             latexify=False,
             font_family="serif",
             scale='linear',
             summarize=True,
             print_pareto=False):
        """
        Visualize the progress of an optimization.

        Args:
            maximize (bool): Whether to plot optimizing for minimum or maximum.
            show_best (bool): Point out the best point on legend and on plot. If
                more than one best point (i.e., multiple equal maxima), show
                them all. If multiobjective, shows best for each objective, and
                prints the best value and x for each objective.
            show_mean (bool): Show the mean and standard deviation for the
                guesses as the computations are carried out.
            latexify (bool): Use LaTeX for formatting.
            font_family (str): The font family to use for rendering. Choose from
                'serif', 'sans-serif', 'fantasy', 'monospace', or 'cursive'.
            scale (str): Whether to scale the plot's y axis according to log
                ('log') or 'linear' scale.
            summarize (bool): If True, stdouts summary from .summarize.
            print_pareto (bool): If True, display all Pareto-optimal objective
                values.

        Returns:
            A matplotlib plot object handle
        """
        if not self.is_configured:
            raise NotConfiguredError(
                "Use MissionControl.configure to configure"
                "your optimization collection before "
                "plotting!")
        maximize = self.config["maximize"]
        fxstr = "$f(x)$" if latexify else "f(x)"
        opt = max if maximize else min
        objs = self.c.find_one({'index': {'$exists': 1}})['y']
        n_objs = len(objs) if isinstance(objs, (list, tuple)) else 1
        dt = datetime.datetime.now()
        dtdata = [dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second]
        timestr = "{}-{}-{} {}:{}.{}".format(*dtdata)
        t0 = time.time()
        if latexify:
            plt.rc('text', usetex=True)
        else:
            plt.rc('text', usetex=False)
        plt.rc('font', family=font_family, size=9)
        n_cols = 3
        if n_objs < n_cols:
            f, axarr = plt.subplots(n_objs, squeeze=False)
        else:
            f, axarr = plt.subplots(n_cols,
                                    int(math.ceil(n_objs / n_cols)),
                                    squeeze=False)
        docset = self.c.find({'index': {'$exists': 1}})
        docs = [None] * docset.count()
        for i, doc in enumerate(docset):
            docs[i] = {'y': doc['y'], 'index': doc['index'], 'x': doc['x']}
        if n_objs > 1:
            all_y = np.asarray([doc['y'] for doc in docs])
            pareto_set = all_y[pareto(all_y, maximize=maximize)].tolist()
            pareto_graph = [(i + 1, doc['y']) for i, doc in enumerate(docs)
                            if doc['y'] in pareto_set]
            pareto_i = [i[0] for i in pareto_graph]

        print("Optimization Analysis:")
        print("Number of objectives: {}".format(n_objs))

        for obj in range(n_objs):
            ax = axarr[obj % n_cols, int(math.floor(obj / n_cols))]

            i = []
            fx = []
            best = []
            mean = []
            std = []
            n = self.c.find().count() - 2

            for doc in docs:
                fx.append(doc['y'] if n_objs == 1 else doc['y'][obj])
                i.append(doc['index'])
                best.append(opt(fx))
                mean.append(np.mean(fx))
                std.append(np.std(fx))

            if time.time() - t0 > 60:
                self.logger.warn(
                    "Gathering data from the db is taking a while. Ensure"
                    "the latency to your db is low and the bandwidth"
                    "is as high as possible!")

            mean = np.asarray(mean)
            std = np.asarray(std)

            ax.scatter(i, fx, color='blue', label=fxstr, s=10)
            ax.plot(i,
                    best,
                    color='orange',
                    label="best {} value found so far"
                    "".format(fxstr))
            if show_mean:
                ax.plot(i,
                        mean,
                        color='grey',
                        label="mean {} value (with std "
                        "dev.)".format(fxstr))
                ax.fill_between(i,
                                mean + std,
                                mean - std,
                                color='grey',
                                alpha=0.3)

            ax.set_xlabel("{} evaluation".format(fxstr))
            ax.set_ylabel("{} value".format(fxstr))
            best_val = opt(best)

            if show_best:

                if latexify:
                    best_label = "Best value: $f(x) = {}$" \
                                 "".format(latex_float(best_val))
                else:
                    best_label = "Best value: f(x) = {:.2E}".format(best_val)
                best = self.c.find({'y': best_val})

                if n_objs == 1:
                    print("\tNumber of optima: {}".format(best.count()))
                else:
                    print("\tNumber of optima for objective {}: {}"
                          "".format(obj + 1, best.count()))

                for b in best:
                    bl = None if n_objs > 1 else best_label
                    ax.scatter([b['index']], [best_val],
                               color='darkgreen',
                               s=50,
                               linewidth=3,
                               label=bl,
                               facecolors='none',
                               edgecolors='darkgreen')

                    artext = "$x = $ [" if latexify else "x = ["
                    for i, xi in enumerate(b['x']):
                        if i > 0:
                            artext += ". \mbox{~~~~~}" if latexify else "     "
                        if type(xi) in dtypes.floats:
                            if latexify:
                                artext += "${}$,\n".format(latex_float(xi))
                            else:
                                artext += "{:.2E},\n".format(xi)
                        else:
                            artext += str(xi) + ",\n"

                    artext = artext[:-2] + "]"
                    objstr = "objective {}".format(obj +
                                                   1) if n_objs > 1 else ""
                    if maximize:
                        print("\t\tmax(f(x)) {} is {} at x = {}"
                              "".format(objstr, best_val, b['x']))
                    else:
                        print("\t\tmin(f(x)) {} is {} at x = {}"
                              "".format(objstr, best_val, b['x']))
                    ax.annotate(artext,
                                xy=(b['index'] + 0.5, best_val),
                                xytext=(b['index'] + float(n) / 12.0,
                                        best_val),
                                arrowprops=dict(color='green'),
                                color='darkgreen',
                                bbox=dict(facecolor='white', alpha=1.0))
            else:
                best_label = ""
            if n_objs > 1:
                pareto_fx = [i[1][obj] for i in pareto_graph]
                ax.scatter(pareto_i,
                           pareto_fx,
                           color='red',
                           label="Pareto optimal",
                           s=20)
            if n_objs > 1:
                ax.set_title("Objective {}: {}".format(obj + 1, best_label))
            ax.set_yscale(scale)
        plt.gcf().set_size_inches(10, 10)
        if summarize:
            print(self.summarize())

        if print_pareto and n_objs > 1:
            print("Pareto Frontier: {} points, ranked by hypervolume".format(
                len(pareto_set)))
            pareto_y = [doc['y'] for doc in docs if doc['y'] in pareto_set]
            pareto_x = [doc['x'] for doc in docs if doc['y'] in pareto_set]

            # Order y by hypervolume
            hypervolumes = [np.prod(y) for y in pareto_y]
            pareto_y_ordered = [
                y for _, y in sorted(zip(hypervolumes, pareto_y), reverse=True)
            ]
            pareto_x_ordered = [
                x for _, x in sorted(zip(hypervolumes, pareto_x), reverse=True)
            ]
            hypervolumes_ordered = sorted(hypervolumes, reverse=True)

            for i, _ in enumerate(pareto_set):
                print("f(x) = {} @ x = {} with hypervolume {}".format(
                    pareto_y_ordered[i], pareto_x_ordered[i],
                    hypervolumes_ordered[i]))
        if n_objs % n_cols != 0 and n_objs > n_cols:
            for i in range(n_objs % n_cols, n_cols):
                plt.delaxes(axarr[i, -1])
        plt.legend()
        # plt.tight_layout(pad=0.01, w_pad=0.01, h_pad=0.01)
        plt.subplots_adjust(wspace=0.3, hspace=0.5)
        plt.suptitle("Rocketsled optimization results for {} - "
                     "{}".format(self.c.name, timestr),
                     y=0.99)
        return plt
Esempio n. 5
0
    def _predict(self, all_x, all_y, space, model, maximize, scaling):
        """
        Scikit-learn compatible model for stepwise optimization. It uses a
        regressive predictor evaluated on remaining points in a discrete space.

        Since sklearn modules cannot deal with categorical data, categorical
        data is preprocessed by _encode before being passed to _predict,
        and predicted x vectors are postprocessed by _decode to convert to
        the original categorical dimensions.

        Args:
            all_x ([list]): List of vectors containing input training data.
            all_y (list): List of scalars containing output training data. Can
                be a list of vectors if undergoing multiobjective optimization.
            space ([list]): List of vectors containing all unsearched inputs.
                Should be preprocessed.
            model (sklearn model): The regressor used for predicting the next
                best guess.
            maximize (bool): Makes predictor return the guess which maximizes
                the predicted objective function output. Else minmizes the
                predicted objective function output.
            scaling (bool): If True, scale data with StandardScaler (required
                for some optimizers, such as Gaussian processes).

        Returns:
            (list) A vector which is predicted to minimize (or maximize) the
                objective function.
        """

        # Scale data if all floats for dimensions in question
        if scaling:
            scaler = StandardScaler()
            train_set = np.vstack((all_x, space))
            scaler.fit(train_set)
            all_x_scaled = scaler.transform(all_x)
            space_scaled = scaler.transform(space)
        else:
            all_x_scaled = all_x
            space_scaled = space
        n_searched = len(all_x)
        n_unsearched = len(space)

        # If get_z defined, only use z features!
        if "get_z" in self:
            encoded_xlen = 0
            for t in self._xdim_types:
                if "int" in t or "float" in t:
                    encoded_xlen += 1
                else:
                    encoded_xlen += int(t[-1])
            all_x_scaled = np.asarray(all_x_scaled)[:, encoded_xlen:]
            space_scaled = np.asarray(space_scaled)[:, encoded_xlen:]
        all_y = np.asarray(all_y)
        if self.n_objs == 1:
            # Single objective
            if maximize:
                all_y = -1.0 * all_y
            if self.acq is None or n_searched < 10:
                model.fit(all_x_scaled, all_y)
                values = model.predict(space_scaled).tolist()
                evaluator = min
            else:
                # Use the acquistion function values
                values = acquire(
                    self.acq,
                    all_x_scaled,
                    all_y,
                    space_scaled,
                    model,
                    self.n_bootstraps,
                )
                evaluator = max
        else:
            evaluator = max
            # Multi-objective
            if self.acq is None or n_searched < 10:
                values = np.zeros((n_unsearched, self.n_objs))
                for i in range(self.n_objs):
                    yobj = [y[i] for y in all_y]
                    model.fit(all_x_scaled, yobj)
                    values[:, i] = model.predict(space_scaled)
                # In exploitative strategy, randomly weight pareto optimial
                # predictions!
                values = pareto(values, maximize=maximize) * np.random.uniform(
                    0, 1, n_unsearched)

            else:
                # Adapted from Multiobjective Optimization of Expensive Blackbox
                # Functions via Expected Maximin Improvement
                # by Joshua D. Svenson, Thomas J. Santner
                if maximize:
                    all_y = -1.0 * all_y

                if self.acq != "maximin":
                    raise ObjectiveError(
                        "Multiple objectives detected, but Maximin acquisition "
                        "function is not used. Please use a single objective "
                        "or change the acquisition function.")
                mu = np.zeros((n_unsearched, self.n_objs))
                values = np.zeros((n_unsearched, self.n_objs))
                for i in range(self.n_objs):
                    yobj = [y[i] for y in all_y]
                    values[:, i], mu[:, i] = acquire(
                        "pi",
                        all_x_scaled,
                        yobj,
                        space_scaled,
                        model,
                        self.n_bootstraps,
                        return_means=True,
                    )
                pf = all_y[pareto(all_y, maximize=maximize)]
                dmaximin = np.zeros(n_unsearched)
                for i, mui in enumerate(mu):
                    mins = np.zeros(len(pf))
                    for j, pfj in enumerate(pf):
                        # select max distance to pareto point (improvements
                        # are negative) among objectives
                        mins[j] = min(mui - pfj)
                    # minimum among all pareto points of the maximum improvement
                    # among objectives. Min/max are reversed bc. minimization
                    dmaximin[i] = max(mins)

                if len(dmaximin[dmaximin < 0.0]) != 0:
                    # Predicted pareto-optimal solutions are negative so far
                    # If we are here, it means there are still predicted pareto
                    # optimal solutions. This procedure is as shown in original
                    # EI paper.
                    dmaximin = dmaximin * -1.0
                    dmaximin = dmaximin.clip(min=0)
                else:
                    # Addition if there are no predicted pareto solutions.
                    # Without this, all dmaximin values are zero if no predicted
                    # pareto solutions. With this, dmaximin values are inverted
                    # to find the 'least bad' non-pareto optimal value.
                    # Only using the 'if' block above will result in pure
                    # exploration (random) if no pareto-optimal solutions
                    # are predicted.
                    dmaximin = 1.0 / dmaximin

                pi_product = np.prod(values, axis=1)
                values = pi_product * dmaximin
            values = values.tolist()
        prediction = evaluator(values)
        index = values.index(prediction)
        return space[index]
Esempio n. 6
0
    def _predict(self, all_x, all_y, space, model, maximize, scaling):
        """
        Scikit-learn compatible model for stepwise optimization. It uses a
        regressive predictor evaluated on remaining points in a discrete space.

        Since sklearn modules cannot deal with categorical data, categorical
        data is preprocessed by _encode before being passed to _predict,
        and predicted x vectors are postprocessed by _decode to convert to
        the original categorical dimensions.

        Args:
            all_x ([list]): List of vectors containing input training data.
            all_y (list): List of scalars containing output training data. Can
                be a list of vectors if undergoing multiobjective optimization.
            space ([list]): List of vectors containing all unsearched inputs.
                Should be preprocessed.
            model (sklearn model): The regressor used for predicting the next
                best guess.
            maximize (bool): Makes predictor return the guess which maximizes
                the predicted objective function output. Else minmizes the
                predicted objective function output.
            scaling (bool): If True, scale data with StandardScaler (required
                for some optimizers, such as Gaussian processes).

        Returns:
            (list) A vector which is predicted to minimize (or maximize) the
                objective function.
        """

        # Scale data if all floats for dimensions in question
        if scaling:
            scaler = StandardScaler()
            train_set = np.vstack((all_x, space))
            scaler.fit(train_set)
            all_x_scaled = scaler.transform(all_x)
            space_scaled = scaler.transform(space)
        else:
            all_x_scaled = all_x
            space_scaled = space
        n_searched = len(all_x)
        n_unsearched = len(space)

        # If get_z defined, only use z features!
        if 'get_z' in self:
            encoded_xlen = 0
            for t in self._xdim_types:
                if "int" in t or "float" in t:
                    encoded_xlen += 1
                else:
                    encoded_xlen += int(t[-1])
            all_x_scaled = np.asarray(all_x_scaled)[:, encoded_xlen:]
            space_scaled = np.asarray(space_scaled)[:, encoded_xlen:]
        all_y = np.asarray(all_y)
        if self.n_objs == 1:
            # Single objective
            if maximize:
                all_y = -1.0 * all_y
            if self.acq is None or n_searched < 10:
                model.fit(all_x_scaled, all_y)
                values = model.predict(space_scaled).tolist()
                evaluator = min
            else:
                # Use the acquistion function values
                values = acquire(self.acq, all_x_scaled, all_y, space_scaled,
                                 model, self.n_bootstraps)
                evaluator = max
        else:
            evaluator = max
            # Multi-objective
            if self.acq is None or n_searched < 10:
                values = np.zeros((n_unsearched, self.n_objs))
                for i in range(self.n_objs):
                    yobj = [y[i] for y in all_y]
                    model.fit(all_x_scaled, yobj)
                    values[:, i] = model.predict(space_scaled)
                # In exploitative strategy, randomly weight pareto optimial
                # predictions!
                values = pareto(values, maximize=maximize) * np.random.uniform(
                    0, 1, n_unsearched)

            else:
                # Adapted from Multiobjective Optimization of Expensive Blackbox
                # Functions via Expected Maximin Improvement
                # by Joshua D. Svenson, Thomas J. Santner
                if maximize:
                    all_y = -1.0 * all_y

                if self.acq != "maximin":
                    raise ObjectiveError(
                        "Multiple objectives detected, but Maximin acquisition "
                        "function is not used. Please use a single objective "
                        "or change the acquisition function.")
                mu = np.zeros((n_unsearched, self.n_objs))
                values = np.zeros((n_unsearched, self.n_objs))
                for i in range(self.n_objs):
                    yobj = [y[i] for y in all_y]
                    values[:, i], mu[:, i] = acquire("pi", all_x_scaled, yobj,
                                                     space_scaled, model,
                                                     self.n_bootstraps,
                                                     return_means=True)
                pf = all_y[pareto(all_y, maximize=maximize)]
                dmaximin = np.zeros(n_unsearched)
                for i, mui in enumerate(mu):
                    mins = np.zeros(len(pf))
                    for j, pfj in enumerate(pf):
                        # select max distance to pareto point (improvements
                        # are negative) among objectives
                        mins[j] = min(mui - pfj)
                    # minimum among all pareto points of the maximum improvement
                    # among objectives. Min/max are reversed bc. minimization
                    dmaximin[i] = max(mins)

                if len(dmaximin[dmaximin < 0.0]) != 0:
                    # Predicted pareto-optimal solutions are negative so far
                    # If we are here, it means there are still predicted pareto
                    # optimal solutions. This procedure is as shown in original
                    # EI paper.
                    dmaximin = dmaximin * -1.0
                    dmaximin = dmaximin.clip(min=0)
                else:
                    # Addition if there are no predicted pareto solutions.
                    # Without this, all dmaximin values are zero if no predicted
                    # pareto solutions. With this, dmaximin values are inverted
                    # to find the 'least bad' non-pareto optimal value.
                    # Only using the 'if' block above will result in pure
                    # exploration (random) if no pareto-optimal solutions
                    # are predicted.
                    dmaximin = 1.0 / dmaximin

                pi_product = np.prod(values, axis=1)
                values = pi_product * dmaximin
            values = values.tolist()
        prediction = evaluator(values)
        index = values.index(prediction)
        return space[index]
Esempio n. 7
0
    def plot(self, show_best=True, show_mean=True, latexify=False,
             font_family="serif", scale='linear', summarize=True,
             print_pareto=False):
        """
        Visualize the progress of an optimization.

        Args:
            show_best (bool): Point out the best point on legend and on plot. If
                more than one best point (i.e., multiple equal maxima), show
                them all. If multiobjective, shows best for each objective, and
                prints the best value and x for each objective.
            show_mean (bool): Show the mean and standard deviation for the
                guesses as the computations are carried out.
            latexify (bool): Use LaTeX for formatting.
            font_family (str): The font family to use for rendering. Choose from
                'serif', 'sans-serif', 'fantasy', 'monospace', or 'cursive'.
            scale (str): Whether to scale the plot's y axis according to log
                ('log') or 'linear' scale.
            summarize (bool): If True, stdouts summary from .summarize.
            print_pareto (bool): If True, display all Pareto-optimal objective
                values.

        Returns:
            A matplotlib plot object handle
        """
        if not self.is_configured:
            raise NotConfiguredError("Use MissionControl.configure to configure"
                                     "your optimization collection before "
                                     "plotting!")
        maximize = self.config["maximize"]
        fxstr = "$f(x)$" if latexify else "f(x)"
        opt = max if maximize else min
        objs = self.c.find_one({'index': {'$exists': 1}})['y']
        n_objs = len(objs) if isinstance(objs, (list, tuple)) else 1
        dt = datetime.datetime.now()
        dtdata = [dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second]
        timestr = "{}-{}-{} {}:{}.{}".format(*dtdata)
        t0 = time.time()
        if latexify:
            plt.rc('text', usetex=True)
        else:
            plt.rc('text', usetex=False)
        plt.rc('font', family=font_family, size=9)
        n_cols = 3
        if n_objs < n_cols:
            _, ax_arr = plt.subplots(n_objs, squeeze=False)
        else:
            _, ax_arr = plt.subplots(n_cols, int(math.ceil(n_objs / n_cols)),
                                    squeeze=False)
        docset = self.c.find({'index': {'$exists': 1}})
        docs = [None] * docset.count()
        for i, doc in enumerate(docset):
            docs[i] = {'y': doc['y'], 'index': doc['index'], 'x': doc['x']}
        if n_objs > 1:
            all_y = np.asarray([doc['y'] for doc in docs])
            pareto_set = all_y[pareto(all_y, maximize=maximize)].tolist()
            pareto_graph = [(i + 1, doc['y']) for i, doc in enumerate(docs)
                            if doc['y'] in pareto_set]
            pareto_i = [i[0] for i in pareto_graph]

        print("Optimization Analysis:")
        print("Number of objectives: {}".format(n_objs))

        for obj in range(n_objs):
            ax = ax_arr[obj % n_cols, int(math.floor(obj / n_cols))]

            i = []
            fx = []
            best = []
            mean = []
            std = []
            n = self.c.find().count() - 2

            for doc in docs:
                fx.append(doc['y'] if n_objs == 1 else doc['y'][obj])
                i.append(doc['index'])
                best.append(opt(fx))
                mean.append(np.mean(fx))
                std.append(np.std(fx))

            if time.time() - t0 > 60:
                self.logger.warn(
                    "Gathering data from the db is taking a while. Ensure"
                    "the latency to your db is low and the bandwidth"
                    "is as high as possible!")

            mean = np.asarray(mean)
            std = np.asarray(std)

            ax.scatter(i, fx, color='blue', label=fxstr, s=10)
            ax.plot(i, best, color='orange', label="best {} value found so far"
                                                   "".format(fxstr))
            if show_mean:
                ax.plot(i, mean, color='grey', label="mean {} value (with std "
                                                     "dev.)".format(fxstr))
                ax.fill_between(i, mean + std, mean - std, color='grey',
                                alpha=0.3)

            ax.set_xlabel("{} evaluation".format(fxstr))
            ax.set_ylabel("{} value".format(fxstr))
            best_val = opt(best)

            if show_best:

                if latexify:
                    best_label = "Best value: $f(x) = {}$" \
                                 "".format(latex_float(best_val))
                else:
                    best_label = "Best value: f(x) = {:.2E}".format(best_val)
                best = self.c.find({'y': best_val})

                if n_objs == 1:
                    print("\tNumber of optima: {}".format(best.count()))
                else:
                    print("\tNumber of optima for objective {}: {}"
                          "".format(obj + 1, best.count()))

                for b in best:
                    bl = None if n_objs > 1 else best_label
                    ax.scatter([b['index']], [best_val], color='darkgreen',
                               s=50,
                               linewidth=3, label=bl, facecolors='none',
                               edgecolors='darkgreen')

                    artext = "$x = $ [" if latexify else "x = ["
                    for i, xi in enumerate(b['x']):
                        if i > 0:
                            artext += ". \mbox{~~~~~}" if latexify else "     "
                        if type(xi) in dtypes.floats:
                            if latexify:
                                artext += "${}$,\n".format(latex_float(xi))
                            else:
                                artext += "{:.2E},\n".format(xi)
                        else:
                            artext += str(xi) + ",\n"

                    artext = artext[:-2] + "]"
                    objstr = "objective {}".format(
                        obj + 1) if n_objs > 1 else ""
                    if maximize:
                        print("\t\tmax(f(x)) {} is {} at x = {}"
                              "".format(objstr, best_val, b['x']))
                    else:
                        print("\t\tmin(f(x)) {} is {} at x = {}"
                              "".format(objstr, best_val, b['x']))
                    ax.annotate(artext,
                                xy=(b['index'] + 0.5, best_val),
                                xytext=(b['index'] + float(n) / 12.0, best_val),
                                arrowprops=dict(color='green'),
                                color='darkgreen',
                                bbox=dict(facecolor='white', alpha=1.0))
            else:
                best_label = ""
            if n_objs > 1:
                pareto_fx = [i[1][obj] for i in pareto_graph]
                ax.scatter(pareto_i, pareto_fx, color='red',
                           label="Pareto optimal", s=20)
            if n_objs > 1:
                ax.set_title("Objective {}: {}".format(obj + 1, best_label))
            ax.set_yscale(scale)
        plt.gcf().set_size_inches(10, 10)
        if summarize:
            print(self.summarize())

        if print_pareto and n_objs > 1:
            print("Pareto Frontier: {} points, ranked by hypervolume".format(
                len(pareto_set)))
            pareto_y = [doc['y'] for doc in docs if doc['y'] in pareto_set]
            pareto_x = [doc['x'] for doc in docs if doc['y'] in pareto_set]

            # Order y by hypervolume
            hypervolumes = [np.prod(y) for y in pareto_y]
            pareto_y_ordered = [y for _, y in
                                sorted(zip(hypervolumes, pareto_y),
                                       reverse=True)]
            pareto_x_ordered = [x for _, x in
                                sorted(zip(hypervolumes, pareto_x),
                                       reverse=True)]
            hypervolumes_ordered = sorted(hypervolumes, reverse=True)

            for i, _ in enumerate(pareto_set):
                print("f(x) = {} @ x = {} with hypervolume {}".format(
                    pareto_y_ordered[i], pareto_x_ordered[i],
                    hypervolumes_ordered[i]))
        if n_objs % n_cols != 0 and n_objs > n_cols:
            for i in range(n_objs % n_cols, n_cols):
                plt.delaxes(ax_arr[i, -1])
        plt.legend()
        # plt.tight_layout(pad=0.01, w_pad=0.01, h_pad=0.01)
        plt.subplots_adjust(wspace=0.3, hspace=0.5)
        plt.suptitle("Rocketsled optimization results for {} - "
                     "{}".format(self.c.name, timestr), y=0.99)
        return plt