def plot_optimization_trace(trial_list, name_list, times_list, optimum=0, title="",
                            log=True, save="", y_max=0, y_min=0, scale_std=1):
    markers = plot_util.get_plot_markers()
    colors = plot_util.get_plot_colors()
    linestyles = itertools.cycle(['-'])
    size = 1

    ratio = 5
    gs = matplotlib.gridspec.GridSpec(ratio, 1)
    fig = figure(1, dpi=100)
    fig.suptitle(title, fontsize=16)
    ax1 = subplot(gs[0:ratio, :])
    ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
    min_val = sys.maxint
    max_val = -sys.maxint
    max_trials = 0

    trial_list_means = list()
    trial_list_std = list()

    # One trialList represents all runs from one optimizer
    for i in range(len(trial_list)):
        if log:
            trial_list_means.append(np.log10(np.mean(np.array(trial_list[i]), axis=0)))
        else:
            trial_list_means.append(np.mean(np.array(trial_list[i]), axis=0))
        trial_list_std.append(np.std(np.array(trial_list[i]), axis=0)*scale_std)
        times_list[i] = np.array(times_list[i])

    fig.suptitle(title, fontsize=16)

    # Plot the average error and std
    for i in range(len(trial_list_means)):
        x = times_list[i]
        y = trial_list_means[i] - optimum
        # m = markers.next()
        c = colors.next()
        l = linestyles.next()
        std_up = y + trial_list_std[i]
        std_down = y - trial_list_std[i]

        ax1.fill_between(x, std_down, std_up,
                         facecolor=c, alpha=0.3, edgecolor=c)
        ax1.plot(x, y, color=c, linewidth=size*2,
                 label=name_list[i][0] + "(" + str(len(trial_list[i])) + ")",
                 linestyle=l, marker="")
        if min(std_down) < min_val:
            min_val = min(std_down)
        if max(y + std_up) > max_val:
            max_val = max(std_up)
        if max(times_list[i]) > max_trials:
            max_trials = max(times_list[i])

    # Maybe plot on logscale
    if scale_std != 1:
        ylabel = ", %s * std" % scale_std
    else:
        ylabel = ""

    if log:
        ax1.set_ylabel("log10(Minfunction value)" + ylabel)
    else:
        ax1.set_ylabel("Minfunction value" + ylabel)

    # Descript and label the stuff
    leg = ax1.legend(loc='best', fancybox=True)
    leg.get_frame().set_alpha(0.5)
    ax1.set_xlabel("Duration [sec] ")

    if y_max == y_min:
        # Set axes limit
        ax1.set_ylim([min_val-0.1*abs((max_val-min_val)), max_val+0.1*abs((max_val-min_val))])
    else:
        ax1.set_ylim([y_min, y_max])
    ax1.set_xlim([0, max_trials])

    tight_layout()
    subplots_adjust(top=0.85)
    if save != "":
        savefig(save, dpi=100, facecolor='w', edgecolor='w',
                orientation='portrait', papertype=None, format=None,
                transparent=False, bbox_inches="tight", pad_inches=0.1)
    else:
        show()
Exemple #2
0
def plot_time_trace(time_dict,
                    name_list,
                    title="",
                    log=True,
                    save="",
                    y_max=0,
                    y_min=0):
    colors = plot_util.get_plot_colors()
    markers = plot_util.get_plot_markers()
    linestyles = itertools.cycle(['-'])

    size = 5
    ratio = 5
    gs = matplotlib.gridspec.GridSpec(ratio, 1)
    fig = plt.figure(1, dpi=100)
    fig.suptitle(title, fontsize=16)
    ax1 = plt.subplot(gs[0:ratio, :])
    ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
    min_val = sys.maxint
    max_val = -sys.maxint
    max_trials = 0

    trial_list_means = list()
    trial_list_std = list()
    num_runs_list = list()

    # Get mean and std for all times and optimizers
    for entry in name_list:
        k = entry[0]
        trial_list_std.append(np.std(np.array(time_dict[k]), axis=0))
        if log:
            trial_list_means.append(
                np.log10(np.mean(np.array(time_dict[k]), axis=0)))
        else:
            trial_list_means.append(np.mean(np.array(time_dict[k]), axis=0))
        num_runs_list.append(len(time_dict[k]))

    for k in range(len(name_list)):
        # Plot mean and std for optimizer duration
        c = colors.next()
        m = markers.next()
        x = range(len(trial_list_means[k]))
        l = linestyles.next()
        ax1.fill_between(x,
                         trial_list_means[k] - trial_list_std[k],
                         trial_list_means[k] + trial_list_std[k],
                         facecolor=c,
                         alpha=0.3,
                         edgecolor=c)
        ax1.plot(x,
                 trial_list_means[k],
                 color=c,
                 linewidth=size,
                 label=name_list[k][0],
                 linestyle=l,
                 marker=m)
        # Plot number of func evals for this experiment

        if min(trial_list_means[k] - trial_list_std[k]) < min_val:
            min_val = min(trial_list_means[k] - trial_list_std[k])
        if max(trial_list_means[k] + trial_list_std[k]) > max_val:
            max_val = max(trial_list_means[k] + trial_list_std[k])
        if len(trial_list_means[k]) > max_trials:
            max_trials = len(trial_list_means[k])

    # Descript and label the stuff
    fig.suptitle(title, fontsize=16)
    leg = ax1.legend(loc='best', fancybox=True)
    leg.get_frame().set_alpha(0.5)
    if log:
        ax1.set_ylabel("log10(Optimizer time in [sec])")
    else:
        ax1.set_ylabel("Optimizer time in [sec]")
    if y_max == y_min:
        ax1.set_ylim([min_val - 2, max_val + 2])
    else:
        ax1.set_ylim([y_min, y_max])
    ax1.set_xlim([0, max_trials])

    plt.tight_layout()
    plt.subplots_adjust(top=0.85)
    if save != "":
        plt.savefig(save,
                    dpi=100,
                    facecolor='w',
                    edgecolor='w',
                    orientation='portrait',
                    papertype=None,
                    format=None,
                    transparent=False,
                    bbox_inches="tight",
                    pad_inches=0.1)
    else:
        plt.show()
def plot_optimization_trace(trial_list,
                            name_list,
                            optimum=0,
                            title="",
                            log=False,
                            save="",
                            y_min=0,
                            y_max=0,
                            cut=sys.maxint):
    markers = plot_util.get_plot_markers()
    colors = plot_util.get_plot_colors()
    linestyles = itertools.cycle(['-'])
    size = 1

    # get handles
    ratio = 5
    gs = matplotlib.gridspec.GridSpec(ratio, 1)
    fig = figure(1, dpi=100)
    fig.suptitle(title, fontsize=16)
    ax = subplot(gs[0:ratio, :])
    ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
    min_val = sys.maxint
    max_val = -sys.maxint
    max_trials = 0

    # This might not do what we actually want; Ideally it would only take the
    # ones that where actually run according to instance_order
    for i in range(len(name_list)):
        print cut, len(trial_list[i])
        num_plotted_trials = np.min([cut, len(trial_list[i])])
        print num_plotted_trials
        x = range(num_plotted_trials)
        y = np.zeros((num_plotted_trials))
        line = np.zeros((num_plotted_trials))
        for j, inst_res in enumerate(trial_list[i][:num_plotted_trials]):
            if j >= len(y):
                break
            if type(inst_res
                    ) == np.ndarray and not np.isfinite(inst_res).any():
                inst_res[np.isnan(inst_res)] = 1
            elif type(inst_res) != np.ndarray and np.isnan(inst_res):
                inst_res = 1
            tmp = sc.nanmean(np.array([inst_res,
                                       inst_res]).flat)  # Black Magic
            if log:
                y[j] = np.log(tmp - optimum)
                line[j] = np.min(y[:j + 1])
            else:
                y[j] = tmp - optimum
                line[j] = np.min(y[:j + 1])

        # Plot the stuff
        marker = markers.next()
        color = colors.next()
        l = linestyles.next()
        ax.scatter(np.argmin(line),
                   min(line),
                   facecolor="w",
                   edgecolor=color,
                   s=size * 10 * 15,
                   marker=marker)
        ax.scatter(x, y, color=color, marker=marker, s=size * 15)
        ax.plot(x,
                line,
                color=color,
                label=name_list[i][0],
                linestyle=l,
                linewidth=size)

        if min(y) < min_val:
            min_val = min(y)
        if max(y) > max_val:
            max_val = max(y)
        if num_plotted_trials > max_trials:
            max_trials = num_plotted_trials

    # Describe and label the stuff
    ax.set_xlabel("#Function evaluations")
    if log:
        ax.set_ylabel("log10(Minfunction value)")
    else:
        ax.set_ylabel("Minfunction value")

    if y_min == y_max:
        ax.set_ylim([min_val - 0.1, max_val + 0.1])
    else:
        ax.set_ylim([y_min, y_max])
    ax.set_xlim([0, max_trials])

    leg = ax.legend(loc='best', fancybox=True)
    leg.get_frame().set_alpha(0.5)

    tight_layout()
    subplots_adjust(top=0.85)

    if save != "":
        savefig(save,
                dpi=100,
                facecolor='w',
                edgecolor='w',
                orientation='portrait',
                papertype=None,
                format=None,
                transparent=False,
                bbox_inches="tight",
                pad_inches=0.1)
    else:
        show()
def plot_time_trace(time_dict, name_list, title="", log=True, save="", y_max=0, y_min=0):
    colors = plot_util.get_plot_colors()
    markers = plot_util.get_plot_markers()
    linestyles = itertools.cycle(['-'])

    size = 5
    ratio = 5
    gs = matplotlib.gridspec.GridSpec(ratio, 1)
    fig = plt.figure(1, dpi=100)
    fig.suptitle(title, fontsize=16)
    ax1 = plt.subplot(gs[0:ratio, :])
    ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
    min_val = sys.maxint
    max_val = -sys.maxint
    max_trials = 0

    trial_list_means = list()
    trial_list_std = list()
    num_runs_list = list()

    # Get mean and std for all times and optimizers
    for entry in name_list:
        k = entry[0]
        trial_list_std.append(np.std(np.array(time_dict[k]), axis=0))
        if log:
            trial_list_means.append(np.log10(np.mean(np.array(time_dict[k]), axis=0)))
        else:
            trial_list_means.append(np.mean(np.array(time_dict[k]), axis=0))
        num_runs_list.append(len(time_dict[k]))

    for k in range(len(name_list)):
        # Plot mean and std for optimizer duration
        c = colors.next()
        m = markers.next()
        x = range(len(trial_list_means[k]))
        l = linestyles.next()
        ax1.fill_between(x, trial_list_means[k] - trial_list_std[k],
                         trial_list_means[k] + trial_list_std[k],
                         facecolor=c, alpha=0.3, edgecolor=c)
        ax1.plot(x, trial_list_means[k], color=c, linewidth=size, label=name_list[k][0], linestyle=l, marker=m)
        # Plot number of func evals for this experiment

        if min(trial_list_means[k] - trial_list_std[k]) < min_val:
            min_val = min(trial_list_means[k] - trial_list_std[k])
        if max(trial_list_means[k] + trial_list_std[k]) > max_val:
            max_val = max(trial_list_means[k] + trial_list_std[k])
        if len(trial_list_means[k]) > max_trials:
            max_trials = len(trial_list_means[k])

    # Descript and label the stuff
    fig.suptitle(title, fontsize=16)
    leg = ax1.legend(loc='best', fancybox=True)
    leg.get_frame().set_alpha(0.5)
    if log:
        ax1.set_ylabel("log10(Optimizer time in [sec])")
    else:
        ax1.set_ylabel("Optimizer time in [sec]")
    if y_max == y_min:
        ax1.set_ylim([min_val-2, max_val+2])
    else:
        ax1.set_ylim([y_min, y_max])
    ax1.set_xlim([0, max_trials])

    plt.tight_layout()
    plt.subplots_adjust(top=0.85)
    if save != "":
        plt.savefig(save, dpi=100, facecolor='w', edgecolor='w',
                    orientation='portrait', papertype=None, format=None,
                    transparent=False, bbox_inches="tight", pad_inches=0.1)
    else:
        plt.show()
Exemple #5
0
def plot_optimization_trace_cv(trial_list, name_list, optimum=0, title="",
                               log=True, save="", y_max=0, y_min=0):
    markers =plot_util.get_plot_markers()
    colors = plot_util.get_plot_colors()
    linestyles = itertools.cycle(['-'])
    size = 1

    ratio = 5
    gs = matplotlib.gridspec.GridSpec(ratio, 1)
    fig = figure(1, dpi=100)
    fig.suptitle(title, fontsize=16)
    ax1 = subplot(gs[0:ratio, :])
    ax1.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
    min_val = sys.maxint
    max_val = -sys.maxint
    max_trials = 0

    fig.suptitle(title, fontsize=16)

    # Plot the average error and std
    for i in range(len(name_list)):
        m = markers.next()
        c = colors.next()
        l = linestyles.next()
        leg = False
        for tr in trial_list[i]:
            if log:
                tr = np.log10(tr)
            x = range(1, len(tr)+1)
            y = tr
            if not leg:
                ax1.plot(x, y, color=c, linewidth=size, linestyle=l, label=name_list[i][0])
                leg = True
            ax1.plot(x, y, color=c, linewidth=size, linestyle=l)
            min_val = min(min_val, min(tr))
            max_val = max(max_val, max(tr))
            max_trials = max(max_trials, len(tr))

    # Maybe plot on logscale
    ylabel = ""

    if log:
        ax1.set_ylabel("log10(Minfunction value)" + ylabel)
    else:
        ax1.set_ylabel("Minfunction value" + ylabel)

    # Descript and label the stuff
    leg = ax1.legend(loc='best', fancybox=True)
    leg.get_frame().set_alpha(0.5)
    ax1.set_xlabel("#Function evaluations")

    if y_max == y_min:
         # Set axes limits
        ax1.set_ylim([min_val-0.1*abs((max_val-min_val)), max_val+0.1*abs((max_val-min_val))])
    else:
        ax1.set_ylim([y_min, y_max])
    ax1.set_xlim([0, max_trials + 1])

    tight_layout()
    subplots_adjust(top=0.85)
    if save != "":
        savefig(save, dpi=100, facecolor='w', edgecolor='w',
                orientation='portrait', papertype=None, format=None,
                transparent=False, bbox_inches="tight", pad_inches=0.1)
    else:
        show()
Exemple #6
0
def plot_optimization_trace(trial_list, name_list, optimum=0, title="", log=False,
                            save="", y_min=0, y_max=0, cut=sys.maxint):
    markers = plot_util.get_plot_markers()
    colors = plot_util.get_plot_colors()
    linestyles = itertools.cycle(['-'])
    size = 1

    # get handles
    ratio = 5
    gs = matplotlib.gridspec.GridSpec(ratio, 1)
    fig = figure(1, dpi=100)
    fig.suptitle(title, fontsize=16)
    ax = subplot(gs[0:ratio, :])
    ax.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)
    min_val = sys.maxint
    max_val = -sys.maxint
    max_trials = 0

    # This might not do what we actually want; Ideally it would only take the
    # ones that where actually run according to instance_order
    for i in range(len(name_list)):
        print cut, len(trial_list[i])
        num_plotted_trials = np.min([cut, len(trial_list[i])])
        print num_plotted_trials
        x = range(num_plotted_trials)
        y = np.zeros((num_plotted_trials))
        line = np.zeros((num_plotted_trials))
        for j, inst_res in enumerate(trial_list[i][:num_plotted_trials]):
            if j >= len(y):
                break
            if type(inst_res) == np.ndarray and not np.isfinite(inst_res).any():
                inst_res[np.isnan(inst_res)] = 1
            elif type(inst_res) != np.ndarray and np.isnan(inst_res):
                inst_res = 1
            tmp = sc.nanmean(np.array([inst_res, inst_res]).flat)  # Black Magic
            if log:
                y[j] = np.log(tmp - optimum)
                line[j] = np.min(y[:j + 1])
            else:
                y[j] = tmp - optimum
                line[j] = np.min(y[:j + 1])

        # Plot the stuff
        marker = markers.next()
        color = colors.next()
        l = linestyles.next()
        ax.scatter(np.argmin(line), min(line), facecolor="w", edgecolor=color,
                   s=size*10*15, marker=marker)
        ax.scatter(x, y, color=color, marker=marker, s=size*15)
        ax.plot(x, line, color=color, label=name_list[i][0], linestyle=l, linewidth=size)

        if min(y) < min_val:
            min_val = min(y)
        if max(y) > max_val:
            max_val = max(y)
        if num_plotted_trials > max_trials:
            max_trials = num_plotted_trials

    # Describe and label the stuff
    ax.set_xlabel("#Function evaluations")
    if log:
        ax.set_ylabel("log10(Minfunction value)")
    else:
        ax.set_ylabel("Minfunction value")

    if y_min == y_max:
        ax.set_ylim([min_val - 0.1, max_val + 0.1])
    else:
        ax.set_ylim([y_min, y_max])
    ax.set_xlim([0, max_trials])

    leg = ax.legend(loc='best', fancybox=True)
    leg.get_frame().set_alpha(0.5)

    tight_layout()
    subplots_adjust(top=0.85)

    if save != "":
        savefig(save, dpi=100, facecolor='w', edgecolor='w',
                orientation='portrait', papertype=None, format=None,
                transparent=False, bbox_inches="tight", pad_inches=0.1)
    else:
        show()
def plot_optimization_trace(trial_list,
                            name_list,
                            optimum=0,
                            title="",
                            log=True,
                            save="",
                            y_max=0,
                            y_min=0,
                            scale_std=1,
                            xmax=100,
                            every=10):
    markers = plot_util.get_plot_markers()
    colors = plot_util.get_plot_colors()
    linestyles = itertools.cycle(['-'])
    size = 1

    ratio = 5
    gs = matplotlib.gridspec.GridSpec(ratio, 1)
    fig = figure(1, dpi=100)
    fig.suptitle(title, fontsize=16)
    ax1 = subplot(gs[0:ratio, :])
    ax1.grid(True,
             linestyle=':',
             which='major',
             color='grey',
             alpha=0.4,
             zorder=0)
    min_val = sys.maxint
    max_val = -sys.maxint
    max_trials = 0

    trial_list_means = list()
    trial_list_std = list()

    # One trialList represents all runs from one optimizer
    for i in range(len(trial_list)):
        if log:
            trial_list_means.append(
                np.log10(np.mean(np.array(trial_list[i]), axis=0)))
        else:
            trial_list_means.append(np.mean(np.array(trial_list[i]), axis=0))
        trial_list_std.append(
            np.std(np.array(trial_list[i]), axis=0) * scale_std)

    fig.suptitle(title, fontsize=16)

    # Plot the average error and std
    for i in range(len(trial_list_means)):
        x = range(1, len(trial_list_means[i]) + 1)
        y = trial_list_means[i] - optimum
        m = markers.next()
        c = colors.next()
        l = linestyles.next()
        std_up = y + trial_list_std[i]
        std_down = y - trial_list_std[i]
        ax1.fill_between(x,
                         std_down,
                         std_up,
                         facecolor=c,
                         alpha=0.3,
                         edgecolor=c)
        # ax1.plot(x, y, color=c, linewidth=size,
        #          label=name_list[i][0] + "(" + str(len(trial_list[i])) + ")",
        #          linestyle=l, marker=m)
        ax1.plot(x,
                 y,
                 color=c,
                 linewidth=size * 2,
                 label=name_list[i][0],
                 linestyle=l,
                 marker=m,
                 markevery=every)
        if min(std_down) < min_val:
            min_val = min(std_down)
        if max(std_up) > max_val:
            max_val = max(std_up)
        if len(trial_list_means[i]) > max_trials:
            max_trials = len(trial_list_means[i])

    # Maybe plot on logscale
    if scale_std != 1:
        ylabel = ", %s * std" % scale_std
    else:
        ylabel = ""

    ylabel = ''

    if log:
        ax1.set_ylabel("log10(Minfunction value)" + ylabel)
    else:
        ax1.set_ylabel("Test error rate (best so far)" + ylabel)

    # Descript and label the stuff
    leg = ax1.legend(loc='best', fancybox=True)
    leg.get_frame().set_alpha(0.5)
    ax1.set_xlabel("Number of pipeline runs")

    if y_max == y_min:
        # Set axes limits
        ax1.set_ylim([
            min_val - 0.1 * abs((max_val - min_val)), max_val + 0.1 * abs(
                (max_val - min_val))
        ])
    else:
        ax1.set_ylim([y_min, y_max])
    # ax1.set_xlim([0, max_trials + 1])
    ax1.set_xlim([0, xmax])

    tight_layout()
    subplots_adjust(top=0.85)
    if save != "":
        savefig(save,
                dpi=100,
                facecolor='w',
                edgecolor='w',
                orientation='portrait',
                papertype=None,
                format=None,
                transparent=False,
                bbox_inches="tight",
                pad_inches=0.1)
    else:
        show()