Example #1
0
     function evaluations
     (from right to left cycling cyan-magenta-black\dots) and final $\Df$-value (red),
     where \Df\ and \textsf{Df} denote the difference to the optimal function value.
     Light brown lines in the background show ECDFs for the most difficult target of all
     algorithms benchmarked during BBOB-2009."""
caption_single_fixed = caption_part_one + caption_left_fixed_targets + caption_wrap_up + caption_right
caption_single_rlbased = caption_part_one + caption_left_rlbased_targets + caption_wrap_up + caption_right

caption_two_part_one = r"""%
    Empirical cumulative distributions (ECDF)
    of run lengths and speed-up ratios in 5-D (left) and 20-D (right).
    Left sub-columns: ECDF of
    the number of function evaluations divided by dimension $D$
    (FEvals/D) """

symbAlgorithmA = r"{%s%s}" % (color_to_latex("k"), marker_to_latex(styles[0]["marker"]))
symbAlgorithmB = r"{%s%s}" % (color_to_latex("k"), marker_to_latex(styles[1]["marker"]))
caption_two_fixed_targets_part1 = r"""%
    to reach a target value $\fopt+\Df$ with $\Df=10^{k}$, where
    $k\in\{1, -1, -4, -8\}$ is given by the first value in the legend, for
    \algorithmA\ ("""
caption_two_fixed_targets_part2 = r""") and \algorithmB\ ("""
caption_two_fixed_targets_part3 = r""")%
    . Light beige lines show the ECDF of FEvals for target value $\Df=10^{-8}$
    of all algorithms benchmarked during BBOB-2009.
    Right sub-columns: 
    ECDF of FEval ratios of \algorithmA\ divided by \algorithmB for target
    function values $10^k$ with $k$ given in the legend; all
    trial pairs for each function. Pairs where both trials failed are disregarded,
    pairs where one trial failed are visible in the limits being $>0$ or $<1$. The
    legend also indicates, after the colon, the number of functions that were
Example #2
0
     (from right to left cycling cyan-magenta-black\dots) and final $\Df$-value (red),
     where \Df\ and \textsf{Df} denote the difference to the optimal function value.
     Light brown lines in the background show ECDFs for the most difficult target of all
     algorithms benchmarked during BBOB-2009."""
caption_single_fixed = caption_part_one + caption_left_fixed_targets + caption_wrap_up + caption_right
caption_single_rlbased = caption_part_one + caption_left_rlbased_targets + caption_wrap_up + caption_right

caption_two_part_one = r"""%
    Empirical cumulative distributions (ECDF)
    of run lengths and speed-up ratios in 5-D (left) and 20-D (right).
    Left sub-columns: ECDF of
    the number of function evaluations divided by dimension $D$
    (FEvals/D) """

symbAlgorithmA = r'{%s%s}' % (color_to_latex('k'),
    marker_to_latex(styles[0]['marker']))
symbAlgorithmB = r'{%s%s}' % (color_to_latex('k'),
    marker_to_latex(styles[1]['marker']))    
caption_two_fixed_targets_part1 = r"""%
    to reach a target value $\fopt+\Df$ with $\Df=10^{k}$, where
    $k\in\{1, -1, -4, -8\}$ is given by the first value in the legend, for
    \algorithmA\ ("""
caption_two_fixed_targets_part2 =  r""") and \algorithmB\ ("""
caption_two_fixed_targets_part3 = r""")%
    . Light beige lines show the ECDF of FEvals for target value $\Df=10^{-8}$
    of all algorithms benchmarked during BBOB-2009.
    Right sub-columns: 
    ECDF of FEval ratios of \algorithmA\ divided by \algorithmB for target
    function values $10^k$ with $k$ given in the legend; all
    trial pairs for each function. Pairs where both trials failed are disregarded,
    pairs where one trial failed are visible in the limits being $>0$ or $<1$. The
Example #3
0
     (from right to left cycling cyan-magenta-black\dots) and final $\Df$-value (red),
     where \Df\ and \textsf{Df} denote the difference to the optimal function value.
     Light brown lines in the background show ECDFs for the most difficult target of all
     algorithms benchmarked during BBOB-2009."""
caption_single_fixed = caption_part_one + caption_left_fixed_targets + caption_wrap_up + caption_right
caption_single_rlbased = caption_part_one + caption_left_rlbased_targets + caption_wrap_up + caption_right

caption_two_part_one = r"""%
    Empirical cumulative distributions (ECDF)
    of run lengths and speed-up ratios in 5-D (left) and 20-D (right).
    Left sub-columns: ECDF of
    the number of function evaluations divided by dimension $D$
    (FEvals/D) """

symbAlgorithmA = r'{%s%s}' % (color_to_latex('k'),
                              marker_to_latex(styles[0]['marker']))
symbAlgorithmB = r'{%s%s}' % (color_to_latex('k'),
                              marker_to_latex(styles[1]['marker']))
caption_two_fixed_targets_part1 = r"""%
    to reach a target value $\fopt+\Df$ with $\Df=10^{k}$, where
    $k\in\{1, -1, -4, -8\}$ is given by the first value in the legend, for
    \algorithmA\ ("""
caption_two_fixed_targets_part2 = r""") and \algorithmB\ ("""
caption_two_fixed_targets_part3 = r""")%
    . Light beige lines show the ECDF of FEvals for target value $\Df=10^{-8}$
    of all algorithms benchmarked during BBOB-2009.
    Right sub-columns: 
    ECDF of FEval ratios of \algorithmA\ divided by \algorithmB for target
    function values $10^k$ with $k$ given in the legend; all
    trial pairs for each function. Pairs where both trials failed are disregarded,
    pairs where one trial failed are visible in the limits being $>0$ or $<1$. The
Example #4
0
def main(dictAlg, sortedAlgs=None, target=ftarget_default, outputdir='ppdata', verbose=True):
    """From a DataSetList, returns figures showing the scaling: ERT/dim vs dim.
    
    One function and one target per figure.
    
    ``target`` can be a scalar, a list with one element or a 
    ``pproc.TargetValues`` instance with one target.
    
    ``sortedAlgs`` is a list of string-identifies (folder names)
    
    """
    # target becomes a TargetValues "list" with one element
    target = pproc.TargetValues.cast([target] if numpy.isscalar(target) else target)
    latex_commands_filename = os.path.join(outputdir, 'bbob_pproc_commands.tex')
    assert isinstance(target, pproc.TargetValues) 
    if len(target) != 1:
        raise ValueError('only a single target can be managed in ppfigs, ' + str(len(target)) + ' targets were given')
    
    dictFunc = pproc.dictAlgByFun(dictAlg)
    if sortedAlgs is None:
        sortedAlgs = sorted(dictAlg.keys())
    if not os.path.isdir(outputdir):
        os.mkdir(outputdir)
    for f in dictFunc:
        filename = os.path.join(outputdir,'ppfigs_f%03d' % (f))
        handles = []
        fix_styles(len(sortedAlgs))  # 
        for i, alg in enumerate(sortedAlgs):
            dictDim = dictFunc[f][alg].dictByDim()  # this does not look like the most obvious solution

            #Collect data
            dimert = []
            ert = []
            dimnbsucc = []
            ynbsucc = []
            nbsucc = []
            dimmaxevals = []
            maxevals = []
            dimmedian = []
            medianfes = []
            for dim in sorted(dictDim):
                assert len(dictDim[dim]) == 1
                entry = dictDim[dim][0]
                data = generateData(entry, target((f, dim))[0]) # TODO: here we might want a different target for each function
                if 1 < 3 or data[2] == 0: # No success
                    dimmaxevals.append(dim)
                    maxevals.append(float(data[3])/dim)
                if data[2] > 0:
                    dimmedian.append(dim)
                    medianfes.append(data[4]/dim)
                    dimert.append(dim)
                    ert.append(float(data[0])/dim)
                    if data[1] < 1.:
                        dimnbsucc.append(dim)
                        ynbsucc.append(float(data[0])/dim)
                        nbsucc.append('%d' % data[2])

            # Draw lines
            tmp = plt.plot(dimert, ert, **styles[i]) #label=alg, )
            plt.setp(tmp[0], markeredgecolor=plt.getp(tmp[0], 'color'))
            # For legend
            # tmp = plt.plot([], [], label=alg.replace('..' + os.sep, '').strip(os.sep), **styles[i])
            tmp = plt.plot([], [], label=alg.split(os.sep)[-1], **styles[i])
            plt.setp(tmp[0], markersize=12.,
                     markeredgecolor=plt.getp(tmp[0], 'color'))

            if dimmaxevals:
                tmp = plt.plot(dimmaxevals, maxevals, **styles[i])
                plt.setp(tmp[0], markersize=20, #label=alg,
                         markeredgecolor=plt.getp(tmp[0], 'color'),
                         markeredgewidth=1, 
                         markerfacecolor='None', linestyle='None')
                
            handles.append(tmp)
            #tmp2 = plt.plot(dimmedian, medianfes, ls='', marker='+',
            #               markersize=30, markeredgewidth=5,
            #               markeredgecolor=plt.getp(tmp, 'color'))[0]
            #for i, n in enumerate(nbsucc):
            #    plt.text(dimnbsucc[i], numpy.array(ynbsucc[i])*1.85, n,
            #             verticalalignment='bottom',
            #             horizontalalignment='center')

        if not bestalg.bestalgentries2009:
            bestalg.loadBBOB2009()

        bestalgdata = []
        dimbestalg = list(df[0] for df in bestalg.bestalgentries2009 if df[1] == f)
        dimbestalg.sort()
        dimbestalg2 = []
        for d in dimbestalg:
            entry = bestalg.bestalgentries2009[(d, f)]
            tmp = entry.detERT(target((f, d)))[0]
            if numpy.isfinite(tmp):
                bestalgdata.append(float(tmp)/d)
                dimbestalg2.append(d)

        tmp = plt.plot(dimbestalg2, bestalgdata, color=refcolor, linewidth=10,
                       marker='d', markersize=25, markeredgecolor=refcolor, zorder=-1
                       #label='best 2009', 
                       )
        handles.append(tmp)
        
        if show_significance: # plot significance-stars
            xstar, ystar = [], []
            dims = sorted(pproc.dictAlgByDim(dictFunc[f]))
            for i, dim in enumerate(dims):
                datasets = pproc.dictAlgByDim(dictFunc[f])[dim]
                assert all([len(datasets[ialg]) == 1 for ialg in sortedAlgs if datasets[ialg]])
                dsetlist =  [datasets[ialg][0] for ialg in sortedAlgs if datasets[ialg]]
                if len(dsetlist) > 1:
                    arzp, arialg = toolsstats.significance_all_best_vs_other(dsetlist, target((f, dim)))
                    if arzp[0][1] * len(dims) < show_significance:
                        ert = dsetlist[arialg[0]].detERT(target((f, dim)))[0]
                        if ert < numpy.inf: 
                            xstar.append(dim)
                            ystar.append(ert/dim)

            plt.plot(xstar, ystar, 'k*', markerfacecolor=None, markeredgewidth=2, markersize=0.5*styles[0]['markersize'])
        if funInfos:
            plt.gca().set_title(funInfos[f])

        isLegend = False
        if legend:
            plotLegend(handles)
        elif 1 < 3:
            if f in (1, 24, 101, 130) and len(sortedAlgs) < 6: # 6 elements at most in the boxed legend
                isLegend = True

        beautify(legend=isLegend, rightlegend=legend)

        plt.text(plt.xlim()[0], plt.ylim()[0], 'target ' + target.label_name() + ': ' + target.label(0))  # TODO: check

        saveFigure(filename, verbose=verbose)

        plt.close()

    # generate commands in tex file:
    try:
        abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
        alg_definitions = []
        for i in range(len(sortedAlgs)):
            symb = r'{%s%s}' % (color_to_latex(styles[i]['color']),
                                marker_to_latex(styles[i]['marker']))
            alg_definitions.append((', ' if i > 0 else '') + '%s:%s' % (symb, '\\algorithm' + abc[i % len(abc)]))
        toolsdivers.prepend_to_file(latex_commands_filename, 
                [#'\\providecommand{\\bbobppfigsftarget}{\\ensuremath{10^{%s}}}' 
                 #       % target.loglabel(0), # int(numpy.round(numpy.log10(target))),
                '\\providecommand{\\bbobppfigslegend}[1]{',
                scaling_figure_caption(target), 
                'Legend: '] + alg_definitions + ['}']
                )
        toolsdivers.prepend_to_file(latex_commands_filename, 
                ['\\providecommand{\\bbobECDFslegend}[1]{',
                ecdfs_figure_caption(target), '}']
                )


        if verbose:
            print 'Wrote commands and legend to %s' % filename

        # this is obsolete (however check templates)
        filename = os.path.join(outputdir,'ppfigs.tex') 
        f = open(filename, 'w')
        f.write('% Do not modify this file: calls to post-processing software'
                + ' will overwrite any modification.\n')
        f.write('Legend: ')
        
        for i in range(0, len(sortedAlgs)):
            symb = r'{%s%s}' % (color_to_latex(styles[i]['color']),
                                marker_to_latex(styles[i]['marker']))
            f.write((', ' if i > 0 else '') + '%s:%s' % (symb, writeLabels(sortedAlgs[i])))
        f.close()    
        if verbose:
            print '(obsolete) Wrote legend in %s' % filename
    except IOError:
        raise


        handles.append(tmp)

        if funInfos:
            plt.gca().set_title(funInfos[f])

        beautify(rightlegend=legend)

        if legend:
            plotLegend(handles)
        else:
            if f in (1, 24, 101, 130):
                plt.legend()

        saveFigure(filename, figFormat=genericsettings.fig_formats, verbose=verbose)

        plt.close()
Example #5
0
def main(dictAlg, sortedAlgs, target=1e-8, outputdir='ppdata', verbose=True):
    """From a DataSetList, returns figures showing the scaling: ERT/dim vs dim.
    
    One function and one target per figure.
    
    sortedAlgs is a list of string-identifies (folder names)
    
    """
    dictFunc = pproc.dictAlgByFun(dictAlg)

    for f in dictFunc:
        filename = os.path.join(outputdir,'ppfigs_f%03d' % (f))
        handles = []
        fix_styles(len(sortedAlgs))  # 
        for i, alg in enumerate(sortedAlgs):
            dictDim = dictFunc[f][alg].dictByDim()

            #Collect data
            dimert = []
            ert = []
            dimnbsucc = []
            ynbsucc = []
            nbsucc = []
            dimmaxevals = []
            maxevals = []
            dimmedian = []
            medianfes = []
            for dim in sorted(dictDim):
                assert len(dictDim[dim]) == 1
                entry = dictDim[dim][0]
                data = generateData(entry, target) # TODO: here we might want a different target for each function
                if 1 < 3 or data[2] == 0: # No success
                    dimmaxevals.append(dim)
                    maxevals.append(float(data[3])/dim)
                if data[2] > 0:
                    dimmedian.append(dim)
                    medianfes.append(data[4]/dim)
                    dimert.append(dim)
                    ert.append(float(data[0])/dim)
                    if data[1] < 1.:
                        dimnbsucc.append(dim)
                        ynbsucc.append(float(data[0])/dim)
                        nbsucc.append('%d' % data[2])

            # Draw lines
            tmp = plt.plot(dimert, ert, **styles[i]) #label=alg, )
            plt.setp(tmp[0], markeredgecolor=plt.getp(tmp[0], 'color'))
            # For legend
            # tmp = plt.plot([], [], label=alg.replace('..' + os.sep, '').strip(os.sep), **styles[i])
            tmp = plt.plot([], [], label=alg.split(os.sep)[-1], **styles[i])
            plt.setp(tmp[0], markersize=12.,
                     markeredgecolor=plt.getp(tmp[0], 'color'))

            if dimmaxevals:
                tmp = plt.plot(dimmaxevals, maxevals, **styles[i])
                plt.setp(tmp[0], markersize=20, #label=alg,
                         markeredgecolor=plt.getp(tmp[0], 'color'),
                         markeredgewidth=1, 
                         markerfacecolor='None', linestyle='None')
                
            handles.append(tmp)
            #tmp2 = plt.plot(dimmedian, medianfes, ls='', marker='+',
            #               markersize=30, markeredgewidth=5,
            #               markeredgecolor=plt.getp(tmp, 'color'))[0]
            #for i, n in enumerate(nbsucc):
            #    plt.text(dimnbsucc[i], numpy.array(ynbsucc[i])*1.85, n,
            #             verticalalignment='bottom',
            #             horizontalalignment='center')

        if not bestalg.bestalgentries2009:
            bestalg.loadBBOB2009()

        bestalgdata = []
        dimbestalg = list(df[0] for df in bestalg.bestalgentries2009 if df[1] == f)
        dimbestalg.sort()
        dimbestalg2 = []
        for d in dimbestalg:
            entry = bestalg.bestalgentries2009[(d, f)]
            tmp = entry.detERT([target])[0]
            if numpy.isfinite(tmp):
                bestalgdata.append(float(tmp)/d)
                dimbestalg2.append(d)

        tmp = plt.plot(dimbestalg2, bestalgdata, color=refcolor, linewidth=10,
                       marker='d', markersize=25, markeredgecolor=refcolor, zorder=-1
                       #label='best 2009', 
                       )
        handles.append(tmp)
        
        if show_significance: # plot significance-stars
            xstar, ystar = [], []
            dims = sorted(pproc.dictAlgByDim(dictFunc[f]))
            for i, dim in enumerate(dims):
                datasets = pproc.dictAlgByDim(dictFunc[f])[dim]
                assert all([len(datasets[ialg]) == 1 for ialg in sortedAlgs if datasets[ialg]])
                dsetlist =  [datasets[ialg][0] for ialg in sortedAlgs if datasets[ialg]]
                if len(dsetlist) > 1:
                    arzp, arialg = toolsstats.significance_all_best_vs_other(dsetlist, [target])
                    if arzp[0][1] * len(dims) < 0.05:
                        ert = dsetlist[arialg[0]].detERT([target])[0]
                        if ert < numpy.inf: 
                            xstar.append(dim)
                            ystar.append(ert/dim)

            plt.plot(xstar, ystar, 'k*', markerfacecolor=None, markeredgewidth=2, markersize=0.5*styles[0]['markersize'])
        if funInfos:
            plt.gca().set_title(funInfos[f])

        isLegend = False
        if legend:
            plotLegend(handles)
        elif 1 < 3:
            if f in (1, 24, 101, 130) and len(sortedAlgs) < 6: # 6 elements at most in the boxed legend
                isLegend = True

        beautify(legend=isLegend, rightlegend=legend)

        plt.text(plt.xlim()[0], plt.ylim()[0], 'ftarget=%.0e' % target)

        saveFigure(filename, verbose=verbose)

        plt.close()

    # generate commands in tex file:
    try:
        abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
        alg_definitions = []
        for i in range(len(sortedAlgs)):
            symb = r'{%s%s}' % (color_to_latex(styles[i]['color']),
                                marker_to_latex(styles[i]['marker']))
            alg_definitions.append((', ' if i > 0 else '') + '%s:%s' % (symb, '\\algorithm' + abc[i % len(abc)]))
        filename = os.path.join(outputdir, 'bbob_pproc_commands.tex')
        toolsdivers.prepend_to_file(filename, 
                ['\\providecommand{\\bbobppfigsftarget}{\\ensuremath{10^{%d}}}' 
                        % int(numpy.round(numpy.log10(target))),
                '\\providecommand{\\bbobppfigslegend}[1]{',
                scaling_figure_legend, 
                'Legend: '] + alg_definitions + ['}']
                )
        if verbose:
            print 'Wrote commands and legend to %s' % filename

        # this is obsolete (however check templates)
        filename = os.path.join(outputdir,'ppfigs.tex') 
        f = open(filename, 'w')
        f.write('% Do not modify this file: calls to post-processing software'
                + ' will overwrite any modification.\n')
        f.write('Legend: ')
        
        for i in range(0, len(sortedAlgs)):
            symb = r'{%s%s}' % (color_to_latex(styles[i]['color']),
                                marker_to_latex(styles[i]['marker']))
            f.write((', ' if i > 0 else '') + '%s:%s' % (symb, writeLabels(sortedAlgs[i])))
        f.close()    
        if verbose:
            print '(obsolete) Wrote legend in %s' % filename
    except IOError:
        raise


        handles.append(tmp)

        if funInfos:
            plt.gca().set_title(funInfos[f])

        beautify(rightlegend=legend)

        if legend:
            plotLegend(handles)
        else:
            if f in (1, 24, 101, 130):
                plt.legend()

        saveFigure(filename, figFormat=genericsettings.fig_formats, verbose=verbose)

        plt.close()