def tableonealg(dsList, allmintarget, allertbest, sortedAlgs=None,
                outputdir='.'):
    """Routine for the generation of a table for an algorithm."""

    header2 = ('evals/D', '\%trials', '\%inst', '\multicolumn{2}{c|}{fcts}', 'best', '10', '25', 'med', '75', '90')
    format2 = ('%.3g', '%d', '%d', '%d', '%d', '%1.1e', '%1.1e', '%1.1e', '%1.1e', '%1.1e', '%1.1e')
    ilines = [r'\begin{tabular}{cccc@{/}c|cccccc}',
              r'\multicolumn{5}{c|}{Solved} & \multicolumn{6}{|c}{ERT/ERT$_{\textrm{best}}$} \\',
              ' & '.join(header2)]

    dictDim = dsList.dictByDim()
    for d, dentries in dictDim.iteritems():
        dictAlg = dentries.dictByAlg()

        # one-alg table
        for alg in sortedAlgs:
            # Regroup entries by algorithm
            lines = ilines[:]
            algentries = DataSetList()
            for i in alg:
                if dictAlg.has_key(i):
                    algentries.extend(dictAlg[i])
            table = onealg(algentries, allmintarget, allertbest)
            for i in table:
                lines[-1] += r'\\'

                if numpy.isinf(i[0]):
                    tmpstr = r'$\infty$'
                else:
                    tmpstr = format2[0] % i[0]
                for j in range(1, 5):
                    tmpstr += (' & %s' % format2[j]) % i[j]
                for j in range(5, len(i)):
                    tmpstr += ' & %s' % writeFEvals(i[j])
                lines.append(tmpstr)

            lines.append(r'\end{tabular}')
            f = open(os.path.join(outputdir, 'pptable_%s_%02dD.tex' % (algShortInfos[alg[0]], d)), 'w')
            # any element of alg would convene.
            f.write('\n'.join(lines) + '\n')
            f.close()
def tablemanyalg(dsList, allmintarget, allertbest, sortedAlgs=None,
                 outputdir='.'):
    """Generate a table with the figures of multiple algorithms."""

    stargets = sorted(allmintarget.keys())
    dictDim = dsList.dictByDim()
    maxRank = 3

    for d, dentries in dictDim.iteritems():
        dictAlg = dentries.dictByAlg()
        # Multiple algorithms table.
        # Generate data
        table = []
        algnames = []

        for alg in sortedAlgs:
            # Regroup entries by algorithm
            algentries = DataSetList()
            for i in alg:
                if dictAlg.has_key(i):
                    algentries.extend(dictAlg[i])
            if not algentries:
                continue
            algnames.append(writeLabels(algPlotInfos[alg[0]]['label']))
            tmp = []
            for t in stargets:
                dictFunc = algentries.dictByFunc()
                erts = []
                for func, entry in dictFunc.iteritems():
                    try:
                        entry = entry[0]
                    except:
                        raise Usage('oops too many entries')

                    try:
                        if numpy.isnan(allmintarget[t][(func, d)]):
                            continue
                    except LookupError:
                        continue
                    # At this point the target exists.
                    try:
                        erts.append(entry.ert[entry.target<=allmintarget[t][(func, d)]][0]/allertbest[t][(func, d)])
                    except LookupError:
                        erts.append(numpy.inf)

                if numpy.isfinite(erts).any():
                    tmp += [numpy.median(erts), numpy.min(erts), numpy.sum(numpy.isfinite(erts))]
                else:
                    tmp += [numpy.inf, numpy.inf, 0]
            table.append(tmp)

        # Process over all data
        table = numpy.array(table)
        kept = [] # range(numpy.shape(table)[1])
        targetkept = []
        for i in range(1, (numpy.shape(table)[1])/3 + 1):
            if (table[:, 3*i - 1] != 0).any():
                kept.extend([3*i - 3, 3*i - 2 , 3*i - 1])
                targetkept.append(i-1)
        table = table[:, kept]
        #set_trace()
        dtype = []
        for i, t in enumerate(stargets):
            dtype.extend([('med%d' % i, 'f4'), ('min%d' % i, 'f4'),
                          ('nbsolved%d' % i, 'i1')])
        dtype = list(dtype[i] for i in kept)
        boldface = sortColumns(table, maxRank)

        idxsort = numpy.argsort(numpy.array(list(tuple(i) for i in table),
                                            dtype=dtype),
                                order=('med4', 'med2', 'med0', 'min0'))
        # Sorted successively by med(ERT) / ERTbest for fevals/D = 100, 10, 1
        # and then min(ERT) / ERTbest for fevals/D = 1

        # format the data
        lines = [r'\begin{tabular}{c' + 'c@{/}c@{(}c@{) }'*len(targetkept) + '}']
        tmpstr = 'evals/D'
        for t in list(stargets[i] for i in targetkept):
            nbsolved = sum(numpy.isfinite(list(allmintarget[t][i] for i in allmintarget[t] if i[1] == d)))
            #set_trace()
            tmpstr += (r' & \multicolumn{2}{c@{(}}{%s} & %d' % (writeFEvals(t), nbsolved))
        lines.append(tmpstr)

        for i in idxsort:
            line = table[i]

            lines[-1] += r'\\'
            curline = algnames[i]
            for j in range(len(table[i])):
                curline += ' & '
                if (j + 1) % 3 > 0: # the test may not be necessary
                    if numpy.isinf(line[j]):
                        tmpstr = '.'
                    else:
                        tmpstr = '%s' % (writeFEvals(line[j]))

                    if i in boldface[j]:
                        tmpstr = r'\textbf{' + tmpstr + '}'

                    curline += tmpstr
                else:
                    curline += '%d' % line[j] # nb solved.

            lines.append(curline)

        lines.append(r'\end{tabular}')

        f = open(os.path.join(outputdir, 'pptableall_%02dD.tex' % (d)), 'w')
        f.write('\n'.join(lines) + '\n')
        f.close()
예제 #3
0
            lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' +
                         str_to_latex(strip_pathname2(alg)) + '}')
        prepend_to_file(
            os.path.join(outputdir, 'bbob_pproc_commands.tex'), lines, 5000,
            'bbob_proc_commands.tex truncated, consider removing the file before the text run'
        )

        dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)

        if not dsList:
            sys.exit()

        for i in dictAlg:
            if isNoisy and not isNoiseFree:
                dictAlg[i] = dictAlg[i].dictByNoise().get(
                    'nzall', DataSetList())
            if isNoiseFree and not isNoisy:
                dictAlg[i] = dictAlg[i].dictByNoise().get(
                    'noiselessall', DataSetList())

        for i in dsList:
            if i.dim not in genericsettings.dimensions_to_display:
                continue

            if (dict((j, i.instancenumbers.count(j))
                     for j in set(i.instancenumbers)) <
                    inset.instancesOfInterest):
                warnings.warn('The data of %s do not list ' % (i) +
                              'the correct instances ' + 'of function F%d.' %
                              (i.funcId))
예제 #4
0
파일: rungeneric2.py 프로젝트: Oueee/SOS
def main(argv=None):
    r"""Routine for post-processing COCO data from two algorithms.

    Provided with some data, this routine outputs figure and TeX files
    in a folder needed for the compilation of latex document
    :file:`template2XXX.tex` or :file:`noisytemplate2XXX.tex`, where
    :file:`XXX` is either :file:`ecj` or :file:`generic`. The template
    file needs to be edited so that the command ``\bbobdatapath`` points
    to the output folder.

    These output files will contain performance tables, performance
    scaling figures and empirical cumulative distribution figures. On
    subsequent executions, new files will be added to the output folder,
    overwriting existing older files in the process.

    Keyword arguments:

    *argv* -- list of strings containing options and arguments. If not
    given, sys.argv is accessed.

    *argv* must list folders containing BBOB data files. Each of these
    folders should correspond to the data of ONE algorithm.

    Furthermore, argv can begin with, in any order, facultative option
    flags listed below.

        -h, --help
            displays this message.
        -v, --verbose
            verbose mode, prints out operations.
        -o OUTPUTDIR, --output-dir=OUTPUTDIR
            changes the default output directory (:file:`ppdata`) to
            :file:`OUTPUTDIR`
        --noise-free, --noisy
            processes only part of the data.
        --settings=SETTING
            changes the style of the output figures and tables. At the
            moment only the only differences are in the colors of the
            output figures. SETTING can be either "grayscale", "color"
            or "black-white". The default setting is "color".
        --fig-only, --rld-only, --tab-only, --sca-only
            these options can be used to output respectively the ERT
            graphs figures, run length distribution figures or the
            comparison tables scatter plot figures only. Any combination
            of these options results in no output.
        --conv 
            if this option is chosen, additionally convergence
            plots for each function and algorithm are generated.

    Exceptions raised:

    *Usage* -- Gives back a usage message.

    Examples:

    * Calling the rungeneric2.py interface from the command line::

        $ python bbob_pproc/rungeneric2.py -v Alg0-baseline Alg1-of-interest

      will post-process the data from folders :file:`Alg0-baseline` and
      :file:`Alg1-of-interest`, the former containing data for the
      reference algorithm (zero-th) and the latter data for the
      algorithm of concern (first). The results will be output in the
      default output folder. The ``-v`` option adds verbosity.

    * From the python interpreter (requires that the path to this
      package is in python search path)::

        >> import bbob_pproc as bb
        >> bb.rungeneric2.main('-o outputfolder PSO DEPSO'.split())

    This will execute the post-processing on the data found in folder
    :file:`PSO` and :file:`DEPSO`. The ``-o`` option changes the output
    folder from the default to :file:`outputfolder`.

    """

    if argv is None:
        argv = sys.argv[1:]
        # The zero-th input argument which is the name of the calling script is
        # disregarded.

    try:

        try:
            opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
        except getopt.error, msg:
             raise Usage(msg)

        if not (args):
            usage()
            sys.exit()

        isfigure = True
        isrldistr = True
        istable = True
        isscatter = True
        isscaleup = True
        isNoisy = False
        isNoiseFree = False
        verbose = False
        outputdir = 'ppdata'
        inputsettings = 'color'
        isConv= False

        #Process options
        for o, a in opts:
            if o in ("-v","--verbose"):
                verbose = True
            elif o in ("-h", "--help"):
                usage()
                sys.exit()
            elif o in ("-o", "--output-dir"):
                outputdir = a
            #elif o in ("-s", "--style"):
            #    inputsettings = a
            elif o == "--fig-only":
                isrldistr = False
                istable = False
                isscatter = False
            elif o == "--rld-only":
                isfigure = False
                istable = False
                isscatter = False
            elif o == "--tab-only":
                isfigure = False
                isrldistr = False
                isscatter = False
            elif o == "--sca-only":
                isfigure = False
                isrldistr = False
                istable = False
            elif o == "--noisy":
                isNoisy = True
            elif o == "--noise-free":
                isNoiseFree = True
            elif o == "--settings":
                inputsettings = a
            elif o == "--conv":
                isConv = True
            else:
                assert False, "unhandled option"

        # from bbob_pproc import bbob2010 as inset # input settings
        if inputsettings == "color":
            from bbob_pproc import config, genericsettings as inset # input settings
            config.config()
        elif inputsettings == "grayscale":
            from bbob_pproc import grayscalesettings as inset # input settings
        elif inputsettings == "black-white":
            from bbob_pproc import bwsettings as inset # input settings
        else:
            txt = ('Settings: %s is not an appropriate ' % inputsettings
                   + 'argument for input flag "--settings".')
            raise Usage(txt)

        if (not verbose):
            warnings.simplefilter('module')
            warnings.simplefilter('ignore')            

        print ("Post-processing will generate comparison " +
               "data in folder %s" % outputdir)
        print "  this might take several minutes."

        dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)

        if 1 < 3 and len(sortedAlgs) != 2:
            raise ValueError('rungeneric2.py needs exactly two algorithms to compare, found: ' 
                             + str(sortedAlgs)
                             + '\n use rungeneric.py (or rungenericmany.py) to compare more algorithms. ')
 
        if not dsList:
            sys.exit()

        for i in dictAlg:
            if isNoisy and not isNoiseFree:
                dictAlg[i] = dictAlg[i].dictByNoise().get('nzall', DataSetList())
            if isNoiseFree and not isNoisy:
                dictAlg[i] = dictAlg[i].dictByNoise().get('noiselessall', DataSetList())

        for i in dsList:
            if i.dim not in genericsettings.dimensions_to_display:
                continue

            if (dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) <
                inset.instancesOfInterest):
                warnings.warn('The data of %s do not list ' %(i) +
                              'the correct instances ' +
                              'of function F%d.' %(i.funcId))

        if len(sortedAlgs) < 2:
            raise Usage('Expect data from two different algorithms, could ' +
                        'only find one.')
        elif len(sortedAlgs) > 2:
            warnings.warn('Data from folders: %s ' % (sortedAlgs) +
                          'were found, the first two will be processed.')

        # Group by algorithm
        dsList0 = dictAlg[sortedAlgs[0]]
        if not dsList0:
            raise Usage('Could not find data for algorithm %s.' % (sortedAlgs[0]))

        dsList1 = dictAlg[sortedAlgs[1]]
        if not dsList1:
            raise Usage('Could not find data for algorithm %s.' % (sortedAlgs[0]))

        # get the name of each algorithm from the input arguments
        tmppath0, alg0name = os.path.split(sortedAlgs[0].rstrip(os.sep))
        tmppath1, alg1name = os.path.split(sortedAlgs[1].rstrip(os.sep))

        for i in dsList0:
            i.algId = alg0name
        for i in dsList1:
            i.algId = alg1name

        ######################### Post-processing #############################
        if isfigure or isrldistr or istable or isscatter:
            if not os.path.exists(outputdir):
                os.mkdir(outputdir)
                if verbose:
                    print 'Folder %s was created.' % (outputdir)
            
            # prepend the algorithm name command to the tex-command file
            abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
            lines = []
            for i, alg in enumerate(args):
                lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' + 
                        str_to_latex(strip_pathname(alg)) + '}')
            prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 
                         lines, 1000, 
                         'bbob_proc_commands.tex truncated, consider removing the file before the text run'
                         )

        # Check whether both input arguments list noisy and noise-free data
        dictFN0 = dsList0.dictByNoise()
        dictFN1 = dsList1.dictByNoise()
        k0 = set(dictFN0.keys())
        k1 = set(dictFN1.keys())
        symdiff = k1 ^ k0 # symmetric difference
        if symdiff:
            tmpdict = {}
            for i, noisegrp in enumerate(symdiff):
                if noisegrp == 'nzall':
                    tmp = 'noisy'
                elif noisegrp == 'noiselessall':
                    tmp = 'noiseless'

                if dictFN0.has_key(noisegrp):
                    tmp2 = sortedAlgs[0]
                elif dictFN1.has_key(noisegrp):
                    tmp2 = sortedAlgs[1]

                tmpdict.setdefault(tmp2, []).append(tmp)

            txt = []
            for i, j in tmpdict.iteritems():
                txt.append('Only input folder %s lists %s data.'
                            % (i, ' and '.join(j)))
            raise Usage('Data Mismatch: \n  ' + ' '.join(txt)
                        + '\nTry using --noise-free or --noisy flags.')

        if isfigure:
            plt.rc("axes", **inset.rcaxeslarger)
            plt.rc("xtick", **inset.rcticklarger)
            plt.rc("ytick", **inset.rcticklarger)
            plt.rc("font", **inset.rcfontlarger)
            plt.rc("legend", **inset.rclegendlarger)
            ppfig2.main(dsList0, dsList1, ftarget, outputdir, verbose)
            print "log ERT1/ERT0 vs target function values done."

        plt.rc("axes", **inset.rcaxes)
        plt.rc("xtick", **inset.rctick)
        plt.rc("ytick", **inset.rctick)
        plt.rc("font", **inset.rcfont)
        plt.rc("legend", **inset.rclegend)

        if isrldistr:
            if len(dictFN0) > 1 or len(dictFN1) > 1:
                warnings.warn('Data for functions from both the noisy and ' +
                              'non-noisy testbeds have been found. Their ' +
                              'results will be mixed in the "all functions" ' +
                              'ECDF figures.')
            dictDim0 = dsList0.dictByDim()
            dictDim1 = dsList1.dictByDim()

            # ECDFs of ERT ratios
            for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
                if dim in inset.rldDimsOfInterest:
                    # ECDF for all functions altogether
                    try:
                        pprldistr2.main(dictDim0[dim], dictDim1[dim],
                                        inset.rldValsOfInterest,
                                        outputdir, '%02dD_all' % dim, verbose)
                    except KeyError:
                        warnings.warn('Could not find some data in %d-D.'
                                      % (dim))
                        continue

                    # ECDFs per function groups
                    dictFG0 = dictDim0[dim].dictByFuncGroup()
                    dictFG1 = dictDim1[dim].dictByFuncGroup()

                    for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
                        pprldistr2.main(dictFG1[fGroup], dictFG0[fGroup],
                                        inset.rldValsOfInterest,
                                        outputdir, '%02dD_%s' % (dim, fGroup),
                                        verbose)

                    # ECDFs per noise groups
                    dictFN0 = dictDim0[dim].dictByNoise()
                    dictFN1 = dictDim1[dim].dictByNoise()

                    for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
                        pprldistr2.main(dictFN1[fGroup], dictFN0[fGroup],
                                        inset.rldValsOfInterest, outputdir,
                                        '%02dD_%s' % (dim, fGroup),
                                        verbose)
            print "ECDF runlength ratio graphs done."

            for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
                pprldistr.fmax = None #Resetting the max final value
                pprldistr.evalfmax = None #Resetting the max #fevalsfactor
                # ECDFs of all functions altogether
                if dim in inset.rldDimsOfInterest:
                    try:
                        pprldistr.comp(dictDim1[dim], dictDim0[dim],
                                       inset.rldValsOfInterest if isinstance(inset.rldValsOfInterest, TargetValues) 
                                                    else TargetValues(inset.rldValsOfInterest), 
                                       True,
                                       outputdir, 'all', verbose)
                    except KeyError:
                        warnings.warn('Could not find some data in %d-D.'
                                      % (dim))
                        continue

                    # ECDFs per function groups
                    dictFG0 = dictDim0[dim].dictByFuncGroup()
                    dictFG1 = dictDim1[dim].dictByFuncGroup()

                    for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
                        pprldistr.comp(dictFG1[fGroup], dictFG0[fGroup],
                                       inset.rldValsOfInterest if isinstance(inset.rldValsOfInterest, TargetValues) 
                                                    else TargetValues(inset.rldValsOfInterest), 
                                       True, outputdir,
                                       '%s' % fGroup, verbose)

                    # ECDFs per noise groups
                    dictFN0 = dictDim0[dim].dictByNoise()
                    dictFN1 = dictDim1[dim].dictByNoise()
                    for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
                        pprldistr.comp(dictFN1[fGroup], dictFN0[fGroup],
                                       inset.rldValsOfInterest if isinstance(inset.rldValsOfInterest, TargetValues) 
                                                    else TargetValues(inset.rldValsOfInterest), 
                                       True, outputdir,
                                       '%s' % fGroup, verbose)

            print "ECDF runlength graphs done."

        if isConv:
            ppconverrorbars.main(dictAlg,outputdir,verbose)

        if istable:
            dictNG0 = dsList0.dictByNoise()
            dictNG1 = dsList1.dictByNoise()

            for nGroup in set(dictNG0.keys()) & set(dictNG1.keys()):
                # split table in as many as necessary
                dictFunc0 = dictNG0[nGroup].dictByFunc()
                dictFunc1 = dictNG1[nGroup].dictByFunc()
                funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys()))
                if len(funcs) > 24:
                    funcs.sort()
                    nbgroups = int(numpy.ceil(len(funcs)/24.))
                    def split_seq(seq, nbgroups):
                        newseq = []
                        splitsize = 1.0/nbgroups*len(seq)
                        for i in range(nbgroups):
                            newseq.append(seq[int(round(i*splitsize)):int(round((i+1)*splitsize))])
                        return newseq

                    groups = split_seq(funcs, nbgroups)
                    # merge
                    group0 = []
                    group1 = []
                    for i, g in enumerate(groups):
                        tmp0 = DataSetList()
                        tmp1 = DataSetList()
                        for f in g:
                            tmp0.extend(dictFunc0[f])
                            tmp1.extend(dictFunc1[f])
                        group0.append(tmp0)
                        group1.append(tmp1)
                    for i, g in enumerate(zip(group0, group1)):
                        pptable2.main(g[0], g[1], inset.tabDimsOfInterest,
                                      outputdir, '%s%d' % (nGroup, i), verbose)
                else:
                    if 11 < 3:  # future handling: 
                        dictFunc0 = dsList0.dictByFunc()
                        dictFunc1 = dsList1.dictByFunc()
                        funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys()))
                        funcs.sort()
#                        nbgroups = int(numpy.ceil(len(funcs)/testbedsettings.numberOfFunctions))
#                        pptable2.main(dsList0, dsList1,
#                                      testbedsettings.tabDimsOfInterest, outputdir,
#                                      '%s' % (testbedsettings.testbedshortname), verbose)
                    else:
                        pptable2.main(dictNG0[nGroup], dictNG1[nGroup],
                                      inset.tabDimsOfInterest, outputdir,
                                      '%s' % (nGroup), verbose)
            prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 
                            ['\\providecommand{\\bbobpptablestwolegend}[1]{', 
                             pptable2.figure_legend, 
                             '}'
                            ])
            print "Tables done."

        if isscatter:
            ppscatter.main(dsList0, dsList1, outputdir, verbose=verbose)
            prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 
                            ['\\providecommand{\\bbobppscatterlegend}[1]{', 
                             ppscatter.figure_legend, 
                             '}'
                            ])
            print "Scatter plots done."

        if isscaleup:
            plt.rc("axes", labelsize=20, titlesize=24)
            plt.rc("xtick", labelsize=20)
            plt.rc("ytick", labelsize=20)
            plt.rc("font", size=20)
            plt.rc("legend", fontsize=20)
            ppfigs.main(dictAlg, sortedAlgs, ftarget, outputdir, verbose)
            plt.rcdefaults()
            print "Scaling figures done."

        if isfigure or isrldistr or istable or isscatter or isscaleup:
            print "Output data written to folder %s" % outputdir

        plt.rcdefaults()
예제 #5
0
def main(argv=None):
    r"""Post-processing COCO data of a single algorithm.

    Provided with some data, this routine outputs figure and TeX files
    in a folder needed for the compilation of latex document
    :file:`template1XXX.tex` or :file:`noisytemplate1XXX.tex`, where 
    :file:`XXX` is either :file:`ecj` or :file:`generic`. The template
    file needs to be edited so that the commands
    ``\bbobdatapath`` and ``\algfolder`` point to the output folder.

    These output files will contain performance tables, performance
    scaling figures and empirical cumulative distribution figures. On
    subsequent executions, new files will be added to the output folder,
    overwriting existing older files in the process.

    Keyword arguments:

    *argv* -- list of strings containing options and arguments. If not
    given, sys.argv is accessed.

    *argv* should list either names of :file:`info` files or folders
    containing :file:`info` files. argv can also contain post-processed
    :file:`pickle` files generated by this routine. Furthermore, *argv*
    can begin with, in any order, facultative option flags listed below.

        -h, --help
            displays this message.
        -v, --verbose
            verbose mode, prints out all operations.
        -p, --pickle
            generates pickle post processed data files.
        -o OUTPUTDIR, --output-dir=OUTPUTDIR
            changes the default output directory (:file:`ppdata`) to
            :file:`OUTPUTDIR`.
        --crafting-effort=VALUE
            sets the crafting effort to VALUE (float). Otherwise the
            default value of 0. will be used.
        --noise-free, --noisy
            processes only part of the data.
        --settings=SETTINGS
            changes the style of the output figures and tables. At the
            moment the only differences are  in the colors of the output
            figures. SETTINGS can be either "grayscale", "color" or
            "black-white". The default setting is "color".
        --tab-only, --fig-only, --rld-only, --los-only
            these options can be used to output respectively the TeX
            tables, convergence and ERTs graphs figures, run length
            distribution figures, ERT loss ratio figures only. A
            combination of any two of these options results in no
            output.
        --conv
            if this option is chosen, additionally convergence plots
            for each function and algorithm are generated.
        --expensive
            runlength-based f-target values and fixed display limits,
            useful with comparatively small budgets. By default the
            setting is based on the budget used in the data.
        --not-expensive
            expensive setting off. 
        --runlength-based
            runlength-based f-target values, such that the
            "level of difficulty" is similar for all functions. 

    Exceptions raised:

    *Usage* -- Gives back a usage message.

    Examples:

    * Calling the rungeneric1.py interface from the command line::

        $ python bbob_pproc/rungeneric1.py -v experiment1

      will post-process the folder experiment1 and all its containing
      data, base on the .info files found in the folder. The result will
      appear in the default output folder. The -v option adds verbosity. ::

        $ python bbob_pproc/rungeneric1.py -o exp2 experiment2/*.info

      This will execute the post-processing on the info files found in
      :file:`experiment2`. The result will be located in the alternative
      location :file:`exp2`.

    * Loading this package and calling the main from the command line
      (requires that the path to this package is in python search path)::

        $ python -m bbob_pproc.rungeneric1 -h

      This will print out this help message.

    * From the python interpreter (requires that the path to this
      package is in python search path)::

        >> import bbob_pproc as bb
        >> bb.rungeneric1.main('-o outputfolder folder1'.split())

      This will execute the post-processing on the index files found in
      :file:`folder1`. The ``-o`` option changes the output folder from
      the default to :file:`outputfolder`.

    """

    if argv is None:
        argv = sys.argv[1:]
        # The zero-th input argument which is the name of the calling script is
        # disregarded.

    if 1 < 3:
        opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
        if 11 < 3:
            try:
                opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
            except getopt.error, msg:
                raise Usage(msg)

        if not (args) and not '--help' in argv and not 'h' in argv:
            print 'not enough input arguments given'
            print 'cave: the following options also need an argument:'
            print[o for o in longoptlist if o[-1] == '=']
            print 'options given:'
            print opts
            print 'try --help for help'
            sys.exit()

        inputCrE = 0.
        isfigure = True
        istab = True
        isrldistr = True
        islogloss = True
        isPostProcessed = False
        isPickled = False
        verbose = False
        outputdir = 'ppdata'
        isNoisy = False
        isNoiseFree = False
        inputsettings = 'color'
        isConv = False
        isRLbased = None  # allows automatic choice
        isExpensive = None

        # Process options
        for o, a in opts:
            if o in ("-v", "--verbose"):
                verbose = True
            elif o in ("-h", "--help"):
                usage()
                sys.exit()
            elif o in ("-p", "--pickle"):
                isPickled = True
            elif o in ("-o", "--output-dir"):
                outputdir = a
            elif o == "--noisy":
                isNoisy = True
            elif o == "--noise-free":
                isNoiseFree = True
            # The next 4 are for testing purpose
            elif o == "--tab-only":
                isfigure = False
                isrldistr = False
                islogloss = False
            elif o == "--fig-only":
                istab = False
                isrldistr = False
                islogloss = False
            elif o == "--rld-only":
                istab = False
                isfigure = False
                islogloss = False
            elif o == "--los-only":
                istab = False
                isfigure = False
                isrldistr = False
            elif o == "--crafting-effort":
                try:
                    inputCrE = float(a)
                except ValueError:
                    raise Usage(
                        'Expect a valid float for flag crafting-effort.')
            elif o == "--settings":
                inputsettings = a
            elif o == "--conv":
                isConv = True
            elif o == "--runlength-based":
                isRLbased = True
            elif o == "--expensive":
                isExpensive = True  # comprises runlength-based
            elif o == "--not-expensive":
                isExpensive = False
            else:
                assert False, "unhandled option"

        # from bbob_pproc import bbob2010 as inset # input settings
        if inputsettings == "color":
            from bbob_pproc import genericsettings as inset  # input settings
        elif inputsettings == "grayscale":
            from bbob_pproc import grayscalesettings as inset  # input settings
        elif inputsettings == "black-white":
            from bbob_pproc import bwsettings as inset  # input settings
        else:
            txt = ('Settings: %s is not an appropriate ' % inputsettings +
                   'argument for input flag "--settings".')
            raise Usage(txt)

        if 11 < 3:
            from bbob_pproc import config  # input settings
            config.config()
            import imp
            # import testbedsettings as testbedsettings # input settings
            try:
                fp, pathname, description = imp.find_module("testbedsettings")
                testbedsettings = imp.load_module("testbedsettings", fp,
                                                  pathname, description)
            finally:
                fp.close()

        if (not verbose):
            warnings.simplefilter('module')
            # warnings.simplefilter('ignore')

        print("Post-processing (1): will generate output " +
              "data in folder %s" % outputdir)
        print "  this might take several minutes."

        filelist = list()
        for i in args:
            i = i.strip()
            if os.path.isdir(i):
                filelist.extend(findfiles.main(i, verbose))
            elif os.path.isfile(i):
                filelist.append(i)
            else:
                txt = 'Input file or folder %s could not be found.' % i
                print txt
                raise Usage(txt)
        dsList = DataSetList(filelist, verbose)

        if not dsList:
            raise Usage("Nothing to do: post-processing stopped.")

        if isNoisy and not isNoiseFree:
            dsList = dsList.dictByNoise().get('nzall', DataSetList())
        if isNoiseFree and not isNoisy:
            dsList = dsList.dictByNoise().get('noiselessall', DataSetList())

        # compute maxfuneval values
        dict_max_fun_evals = {}
        for ds in dsList:
            dict_max_fun_evals[ds.dim] = np.max(
                (dict_max_fun_evals.setdefault(ds.dim,
                                               0), float(np.max(ds.maxevals))))
        if isRLbased is not None:
            genericsettings.runlength_based_targets = isRLbased

        from bbob_pproc import config
        config.target_values(isExpensive, dict_max_fun_evals)
        config.config()

        if (verbose):
            for i in dsList:
                if (dict((j, i.instancenumbers.count(j))
                         for j in set(i.instancenumbers)) !=
                        inset.instancesOfInterest):
                    warnings.warn('The data of %s do not list ' % (i) +
                                  'the correct instances ' +
                                  'of function F%d.' % (i.funcId))

        dictAlg = dsList.dictByAlg()

        if len(dictAlg) > 1:
            warnings.warn('Data with multiple algId %s ' % (dictAlg) +
                          'will be processed together.')
            # TODO: in this case, all is well as long as for a given problem
            # (given dimension and function) there is a single instance of
            # DataSet associated. If there are more than one, the first one only
            # will be considered... which is probably not what one would expect.
            # TODO: put some errors where this case would be a problem.
            # raise Usage?

        if isfigure or istab or isrldistr or islogloss:
            if not os.path.exists(outputdir):
                os.makedirs(outputdir)
                if verbose:
                    print 'Folder %s was created.' % (outputdir)

        if isPickled:
            dsList.pickle(verbose=verbose)

        if isConv:
            ppconverrorbars.main(dictAlg, outputdir, verbose)

        if isfigure:
            print "Scaling figures...",
            sys.stdout.flush()
            # ERT/dim vs dim.
            plt.rc("axes", **inset.rcaxeslarger)
            plt.rc("xtick", **inset.rcticklarger)
            plt.rc("ytick", **inset.rcticklarger)
            plt.rc("font", **inset.rcfontlarger)
            plt.rc("legend", **inset.rclegendlarger)
            ppfigdim.main(dsList, ppfigdim.values_of_interest, outputdir,
                          verbose)
            plt.rcdefaults()
            print_done()

        plt.rc("axes", **inset.rcaxes)
        plt.rc("xtick", **inset.rctick)
        plt.rc("ytick", **inset.rctick)
        plt.rc("font", **inset.rcfont)
        plt.rc("legend", **inset.rclegend)

        if istab:
            print "TeX tables...",
            sys.stdout.flush()
            dictNoise = dsList.dictByNoise()
            for noise, sliceNoise in dictNoise.iteritems():
                pptable.main(sliceNoise, inset.tabDimsOfInterest, outputdir,
                             noise, verbose)
            print_done()

        if isrldistr:
            print "ECDF graphs...",
            sys.stdout.flush()
            dictNoise = dsList.dictByNoise()
            if len(dictNoise) > 1:
                warnings.warn('Data for functions from both the noisy and '
                              'non-noisy testbeds have been found. Their '
                              'results will be mixed in the "all functions" '
                              'ECDF figures.')
            dictDim = dsList.dictByDim()
            for dim in inset.rldDimsOfInterest:
                try:
                    sliceDim = dictDim[dim]
                except KeyError:
                    continue

                pprldistr.main(sliceDim, True, outputdir, 'all', verbose)
                dictNoise = sliceDim.dictByNoise()
                for noise, sliceNoise in dictNoise.iteritems():
                    pprldistr.main(sliceNoise, True, outputdir, '%s' % noise,
                                   verbose)
                dictFG = sliceDim.dictByFuncGroup()
                for fGroup, sliceFuncGroup in dictFG.items():
                    pprldistr.main(sliceFuncGroup, True, outputdir,
                                   '%s' % fGroup, verbose)

                pprldistr.fmax = None  # Resetting the max final value
                pprldistr.evalfmax = None  # Resetting the max #fevalsfactor

            print_done()

        if islogloss:
            print "ERT loss ratio figures and tables...",
            sys.stdout.flush()
            for ng, sliceNoise in dsList.dictByNoise().iteritems():
                if ng == 'noiselessall':
                    testbed = 'noiseless'
                elif ng == 'nzall':
                    testbed = 'noisy'
                txt = ("Please input crafting effort value " +
                       "for %s testbed:\n  CrE = " % testbed)
                CrE = inputCrE
                while CrE is None:
                    try:
                        CrE = float(raw_input(txt))
                    except (SyntaxError, NameError, ValueError):
                        print "Float value required."
                dictDim = sliceNoise.dictByDim()
                for d in inset.rldDimsOfInterest:
                    try:
                        sliceDim = dictDim[d]
                    except KeyError:
                        continue
                    info = '%s' % ng
                    pplogloss.main(sliceDim,
                                   CrE,
                                   True,
                                   outputdir,
                                   info,
                                   verbose=verbose)
                    pplogloss.generateTable(sliceDim,
                                            CrE,
                                            outputdir,
                                            info,
                                            verbose=verbose)
                    for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup(
                    ).iteritems():
                        info = '%s' % fGroup
                        pplogloss.main(sliceFuncGroup,
                                       CrE,
                                       True,
                                       outputdir,
                                       info,
                                       verbose=verbose)
                    pplogloss.evalfmax = None  # Resetting the max #fevalsfactor

            print_done()

        latex_commands_file = os.path.join(
            outputdir.split(os.sep)[0], 'bbob_pproc_commands.tex')
        prepend_to_file(latex_commands_file, [
            '\\providecommand{\\bbobloglosstablecaption}[1]{',
            pplogloss.table_caption, '}'
        ])
        prepend_to_file(latex_commands_file, [
            '\\providecommand{\\bbobloglossfigurecaption}[1]{',
            pplogloss.figure_caption, '}'
        ])
        prepend_to_file(
            latex_commands_file,
            [
                '\\providecommand{\\bbobpprldistrlegend}[1]{',
                pprldistr.caption_single(
                    np.max([
                        val / dim
                        for dim, val in dict_max_fun_evals.iteritems()
                    ])
                ),  # depends on the config setting, should depend on maxfevals
                '}'
            ])
        prepend_to_file(latex_commands_file, [
            '\\providecommand{\\bbobppfigdimlegend}[1]{',
            ppfigdim.scaling_figure_caption(), '}'
        ])
        prepend_to_file(latex_commands_file, [
            '\\providecommand{\\bbobpptablecaption}[1]{',
            pptable.table_caption, '}'
        ])
        prepend_to_file(latex_commands_file,
                        ['\\providecommand{\\algfolder}{}'
                         ])  # is overwritten in rungeneric.py
        prepend_to_file(latex_commands_file, [
            '\\providecommand{\\algname}{' +
            (str_to_latex(strip_pathname(args[0]))
             if len(args) == 1 else str_to_latex(dsList[0].algId)) + '{}}'
        ])
        if isfigure or istab or isrldistr or islogloss:
            print "Output data written to folder %s" % outputdir

        plt.rcdefaults()
예제 #6
0
def main(argv=None):
    """Generate python-formatted data from raw BBOB experimental data.

    The raw experimental data (files with the extension :file:`info`
    pointing to files with extension :file:`dat` and :file:`tdat`) are
    post-processed and stored in a more condensed way as files with the
    extension :file:`pickle`.
    Supposing the raw data are stored in folder :file:`mydata`, the new
    pickle files will be put in folder :file:`mydata-pickle`.

    :keyword list argv: strings containing options and arguments. If not
                        provided, sys.argv is accessed.

    *argv* should list either names of info files or folders containing
    info files.
    Furthermore, *argv* can begin with, in any order, facultative option
    flags listed below.

        -h, --help

            display this message

    :exception Usage: Gives back a usage message.

    Examples:

    * Calling the dataoutput.py interface from the command line::

        $ python bbob_pproc/dataoutput.py experiment2/*.info

    * Loading this package and calling the main from the command line
      (requires that the path to this package is in the search path)::

        $ python -m bbob_pproc.dataoutput -h

      This will print out this help message.

    * From the python interactive shell (requires that the path to this
      package is in python search path)::

        >> import bbob_pproc as bb
        >> bb.dataoutput.main('folder1')

    """
    if argv is None:
        argv = sys.argv[1:]

    try:
        try:
            opts, args = getopt.getopt(argv, "h", ["help"])
        except getopt.error, msg:
            raise Usage(msg)

        if not (args):
            usage()
            sys.exit()

        verbose = False

        #Process options
        for o, a in opts:
            if o in ("-h", "--help"):
                usage()
                sys.exit()
            else:
                assert False, "unhandled option"

        if (not verbose):
            warnings.simplefilter('ignore')

        dsList = DataSetList(args)
        outputPickle(dsList, verbose=True)
예제 #7
0
def main(argv=None):
    r"""Routine for post-processing COCO data from two algorithms.

    Provided with some data, this routine outputs figure and TeX files
    in a folder needed for the compilation of the provided LaTeX templates
    for comparing two algorithms (``*cmp.tex`` or ``*2*.tex``).
    
    The used template file needs to be edited so that the command
    ``\bbobdatapath`` points to the output folder created by this routine.

    The output files will contain performance tables, performance
    scaling figures and empirical cumulative distribution figures. On
    subsequent executions, new files will be added to the output folder,
    overwriting existing older files in the process.

    Keyword arguments:

    *argv* -- list of strings containing options and arguments. If not
    given, sys.argv is accessed.

    *argv* must list folders containing BBOB data files. Each of these
    folders should correspond to the data of ONE algorithm.

    Furthermore, argv can begin with, in any order, facultative option
    flags listed below.

        -h, --help
            displays this message.
        -v, --verbose
            verbose mode, prints out operations.
        -o OUTPUTDIR, --output-dir=OUTPUTDIR
            changes the default output directory (:file:`ppdata`) to
            :file:`OUTPUTDIR`
        --noise-free, --noisy
            processes only part of the data.
        --settings=SETTING
            changes the style of the output figures and tables. At the
            moment only the only differences are in the colors of the
            output figures. SETTING can be either "grayscale", "color"
            or "black-white". The default setting is "color".
        --fig-only, --rld-only, --tab-only, --sca-only
            these options can be used to output respectively the ERT
            graphs figures, run length distribution figures or the
            comparison tables scatter plot figures only. Any combination
            of these options results in no output.
        --conv 
            if this option is chosen, additionally convergence
            plots for each function and algorithm are generated.
        --rld-single-fcts
            generate also runlength distribution figures for each
            single function.
        --expensive
            runlength-based f-target values and fixed display limits,
            useful with comparatively small budgets. By default the
            setting is based on the budget used in the data.
        --not-expensive
            expensive setting off. 
        --svg
            generate also the svg figures which are used in html files 

    Exceptions raised:

    *Usage* -- Gives back a usage message.

    Examples:

    * Calling the rungeneric2.py interface from the command line::

        $ python bbob_pproc/rungeneric2.py -v Alg0-baseline Alg1-of-interest

      will post-process the data from folders :file:`Alg0-baseline` and
      :file:`Alg1-of-interest`, the former containing data for the
      reference algorithm (zero-th) and the latter data for the
      algorithm of concern (first). The results will be output in the
      default output folder. The ``-v`` option adds verbosity.

    * From the python interpreter (requires that the path to this
      package is in python search path)::

        >> import bbob_pproc as bb
        >> bb.rungeneric2.main('-o outputfolder PSO DEPSO'.split())

    This will execute the post-processing on the data found in folder
    :file:`PSO` and :file:`DEPSO`. The ``-o`` option changes the output
    folder from the default to :file:`outputfolder`.

    """

    if argv is None:
        argv = sys.argv[1:]
        # The zero-th input argument which is the name of the calling script is
        # disregarded.

    global ftarget
    try:

        try:
            opts, args = getopt.getopt(argv, genericsettings.shortoptlist, genericsettings.longoptlist)
        except getopt.error, msg:
             raise Usage(msg)

        if not (args):
            usage()
            sys.exit()

        #Process options
        outputdir = genericsettings.outputdir
        for o, a in opts:
            if o in ("-v", "--verbose"):
                genericsettings.verbose = True
            elif o in ("-h", "--help"):
                usage()
                sys.exit()
            elif o in ("-o", "--output-dir"):
                outputdir = a
            elif o == "--fig-only":
                genericsettings.isRLDistr = False
                genericsettings.isTab = False
                genericsettings.isScatter = False
            elif o == "--rld-only":
                genericsettings.isFig = False
                genericsettings.isTab = False
                genericsettings.isScatter = False
            elif o == "--tab-only":
                genericsettings.isFig = False
                genericsettings.isRLDistr = False
                genericsettings.isScatter = False
            elif o == "--sca-only":
                genericsettings.isFig = False
                genericsettings.isRLDistr = False
                genericsettings.isTab = False
            elif o == "--noisy":
                genericsettings.isNoisy = True
            elif o == "--noise-free":
                genericsettings.isNoiseFree = True
            elif o == "--settings":
                genericsettings.inputsettings = a
            elif o == "--conv":
                genericsettings.isConv = True
            elif o == "--rld-single-fcts":
                genericsettings.isRldOnSingleFcts = True
            elif o == "--runlength-based":
                genericsettings.runlength_based_targets = True
            elif o == "--expensive":
                genericsettings.isExpensive = True  # comprises runlength-based
            elif o == "--not-expensive":
                genericsettings.isExpensive = False  
            elif o == "--svg":
                genericsettings.generate_svg_files = True
            elif o == "--los-only":
                warnings.warn("option --los-only will have no effect with rungeneric2.py")
            elif o == "--crafting-effort=":
                warnings.warn("option --crafting-effort will have no effect with rungeneric2.py")
            elif o in ("-p", "--pickle"):
                warnings.warn("option --pickle will have no effect with rungeneric2.py")
            else:
                assert False, "unhandled option"

        # from bbob_pproc import bbob2010 as inset # input settings
        if genericsettings.inputsettings == "color":
            from bbob_pproc import genericsettings as inset # input settings
            config.config()
        elif genericsettings.inputsettings == "grayscale": # probably very much obsolete
            from bbob_pproc import grayscalesettings as inset # input settings
        elif genericsettings.inputsettings == "black-white": # probably very much obsolete
            from bbob_pproc import bwsettings as inset # input settings
        else:
            txt = ('Settings: %s is not an appropriate ' % genericsettings.inputsettings
                   + 'argument for input flag "--settings".')
            raise Usage(txt)

        if (not genericsettings.verbose):
            warnings.simplefilter('module')
            warnings.simplefilter('ignore')            

        print ("Post-processing will generate comparison " +
               "data in folder %s" % outputdir)
        print "  this might take several minutes."

        dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=genericsettings.verbose)

        if 1 < 3 and len(sortedAlgs) != 2:
            raise ValueError('rungeneric2.py needs exactly two algorithms to compare, found: ' 
                             + str(sortedAlgs)
                             + '\n use rungeneric.py (or rungenericmany.py) to compare more algorithms. ')
 
        if not dsList:
            sys.exit()

        for i in dictAlg:
            if genericsettings.isNoisy and not genericsettings.isNoiseFree:
                dictAlg[i] = dictAlg[i].dictByNoise().get('nzall', DataSetList())
            if genericsettings.isNoiseFree and not genericsettings.isNoisy:
                dictAlg[i] = dictAlg[i].dictByNoise().get('noiselessall', DataSetList())

        for i in dsList:
            if i.dim not in genericsettings.dimensions_to_display:
                continue

            if (dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) <
                inset.instancesOfInterest):
                warnings.warn('The data of %s do not list ' %(i) +
                              'the correct instances ' +
                              'of function F%d.' %(i.funcId))

        if len(sortedAlgs) < 2:
            raise Usage('Expect data from two different algorithms, could ' +
                        'only find one.')
        elif len(sortedAlgs) > 2:
            warnings.warn('Data from folders: %s ' % (sortedAlgs) +
                          'were found, the first two will be processed.')

        # Group by algorithm
        dsList0 = dictAlg[sortedAlgs[0]]
        if not dsList0:
            raise Usage('Could not find data for algorithm %s.' % (sortedAlgs[0]))

        dsList1 = dictAlg[sortedAlgs[1]]
        if not dsList1:
            raise Usage('Could not find data for algorithm %s.' % (sortedAlgs[0]))

        # get the name of each algorithm from the input arguments
        tmppath0, alg0name = os.path.split(sortedAlgs[0].rstrip(os.sep))
        tmppath1, alg1name = os.path.split(sortedAlgs[1].rstrip(os.sep))

        for i in dsList0:
            i.algId = alg0name
        for i in dsList1:
            i.algId = alg1name

        # compute maxfuneval values
        dict_max_fun_evals1 = {}
        dict_max_fun_evals2 = {}
        for ds in dsList0:
            dict_max_fun_evals1[ds.dim] = np.max((dict_max_fun_evals1.setdefault(ds.dim, 0), float(np.max(ds.maxevals))))
        for ds in dsList1:
            dict_max_fun_evals2[ds.dim] = np.max((dict_max_fun_evals2.setdefault(ds.dim, 0), float(np.max(ds.maxevals))))
        config.target_values(genericsettings.isExpensive, {1: min([max([val/dim for dim, val in dict_max_fun_evals1.iteritems()]), 
                                                   max([val/dim for dim, val in dict_max_fun_evals2.iteritems()])]
                                                  )})
        config.config()
        
        ######################### Post-processing #############################
        if genericsettings.isFig or genericsettings.isRLDistr or genericsettings.isTab or genericsettings.isScatter:
            if not os.path.exists(outputdir):
                os.mkdir(outputdir)
                if genericsettings.verbose:
                    print 'Folder %s was created.' % (outputdir)
            
            # prepend the algorithm name command to the tex-command file
            abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
            lines = []
            for i, alg in enumerate(args):
                lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' + 
                        str_to_latex(strip_pathname1(alg)) + '}')
            prepend_to_file(os.path.join(outputdir,
                         'bbob_pproc_commands.tex'), lines, 1000, 
                         'bbob_proc_commands.tex truncated, consider removing the file before the text run'
                         )

        # Check whether both input arguments list noisy and noise-free data
        dictFN0 = dsList0.dictByNoise()
        dictFN1 = dsList1.dictByNoise()
        k0 = set(dictFN0.keys())
        k1 = set(dictFN1.keys())
        symdiff = k1 ^ k0 # symmetric difference
        if symdiff:
            tmpdict = {}
            for i, noisegrp in enumerate(symdiff):
                if noisegrp == 'nzall':
                    tmp = 'noisy'
                elif noisegrp == 'noiselessall':
                    tmp = 'noiseless'

                if dictFN0.has_key(noisegrp):
                    tmp2 = sortedAlgs[0]
                elif dictFN1.has_key(noisegrp):
                    tmp2 = sortedAlgs[1]

                tmpdict.setdefault(tmp2, []).append(tmp)

            txt = []
            for i, j in tmpdict.iteritems():
                txt.append('Only input folder %s lists %s data.'
                            % (i, ' and '.join(j)))
            raise Usage('Data Mismatch: \n  ' + ' '.join(txt)
                        + '\nTry using --noise-free or --noisy flags.')

        if genericsettings.isFig:
            plt.rc("axes", **inset.rcaxeslarger)
            plt.rc("xtick", **inset.rcticklarger)
            plt.rc("ytick", **inset.rcticklarger)
            plt.rc("font", **inset.rcfontlarger)
            plt.rc("legend", **inset.rclegendlarger)
            plt.rc('pdf', fonttype = 42)
            ppfig2.main(dsList0, dsList1, ppfig2_ftarget,
                        outputdir, genericsettings.verbose)
            print "log ERT1/ERT0 vs target function values done."

        plt.rc("axes", **inset.rcaxes)
        plt.rc("xtick", **inset.rctick)
        plt.rc("ytick", **inset.rctick)
        plt.rc("font", **inset.rcfont)
        plt.rc("legend", **inset.rclegend)
        plt.rc('pdf', fonttype = 42)

        if genericsettings.isRLDistr:
            if len(dictFN0) > 1 or len(dictFN1) > 1:
                warnings.warn('Data for functions from both the noisy and ' +
                              'non-noisy testbeds have been found. Their ' +
                              'results will be mixed in the "all functions" ' +
                              'ECDF figures.')
            dictDim0 = dsList0.dictByDim()
            dictDim1 = dsList1.dictByDim()

            # ECDFs of ERT ratios
            for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
                if dim in inset.rldDimsOfInterest:
                    # ECDF for all functions altogether
                    try:
                        pprldistr2.main(dictDim0[dim], dictDim1[dim], dim,
                                        inset.rldValsOfInterest,
                                        outputdir,
                                        '%02dD_all' % dim,
                                        genericsettings.verbose)
                    except KeyError:
                        warnings.warn('Could not find some data in %d-D.'
                                      % (dim))
                        continue

                    # ECDFs per function groups
                    dictFG0 = dictDim0[dim].dictByFuncGroup()
                    dictFG1 = dictDim1[dim].dictByFuncGroup()

                    for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
                        pprldistr2.main(dictFG1[fGroup], dictFG0[fGroup], dim,
                                        inset.rldValsOfInterest,
                                        outputdir,
                                        '%02dD_%s' % (dim, fGroup),
                                        genericsettings.verbose)

                    # ECDFs per noise groups
                    dictFN0 = dictDim0[dim].dictByNoise()
                    dictFN1 = dictDim1[dim].dictByNoise()

                    for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
                        pprldistr2.main(dictFN1[fGroup], dictFN0[fGroup], dim,
                                        inset.rldValsOfInterest,
                                        outputdir,
                                        '%02dD_%s' % (dim, fGroup),
                                        genericsettings.verbose)
                                                
            prepend_to_file(os.path.join(outputdir,
                            'bbob_pproc_commands.tex'),
                            ['\\providecommand{\\bbobpprldistrlegendtwo}[1]{',
                             pprldistr.caption_two(),  # depends on the config setting, should depend on maxfevals
                             '}'
                            ])
            
            
            print "ECDF runlength ratio graphs done."

            for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
                pprldistr.fmax = None #Resetting the max final value
                pprldistr.evalfmax = None #Resetting the max #fevalsfactor
                # ECDFs of all functions altogether
                if dim in inset.rldDimsOfInterest:
                    try:
                        pprldistr.comp(dictDim1[dim], dictDim0[dim],
                                       inset.rldValsOfInterest, # TODO: let rldVals... possibly be RL-based targets
                                       True,
                                       outputdir, 'all', genericsettings.verbose)
                    except KeyError:
                        warnings.warn('Could not find some data in %d-D.'
                                      % (dim))
                        continue

                    # ECDFs per function groups
                    dictFG0 = dictDim0[dim].dictByFuncGroup()
                    dictFG1 = dictDim1[dim].dictByFuncGroup()

                    for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
                        pprldistr.comp(dictFG1[fGroup], dictFG0[fGroup],
                                       inset.rldValsOfInterest, True,
                                       outputdir,
                                       '%s' % fGroup, genericsettings.verbose)

                    # ECDFs per noise groups
                    dictFN0 = dictDim0[dim].dictByNoise()
                    dictFN1 = dictDim1[dim].dictByNoise()
                    for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
                        pprldistr.comp(dictFN1[fGroup], dictFN0[fGroup],
                                       inset.rldValsOfInterest, True,
                                       outputdir,
                                       '%s' % fGroup, genericsettings.verbose)

            if genericsettings.isRldOnSingleFcts: # copy-paste from above, here for each function instead of function groups
                # ECDFs for each function
                pprldmany.all_single_functions(dictAlg, sortedAlgs,
                        outputdir, genericsettings.verbose)
            print "ECDF runlength graphs done."

        if genericsettings.isConv:
            ppconverrorbars.main(dictAlg, outputdir, genericsettings.verbose)

        if genericsettings.isScatter:
            if genericsettings.runlength_based_targets:
                ppscatter.targets = ppscatter.runlength_based_targets
            ppscatter.main(dsList1, dsList0, outputdir,
                           verbose=genericsettings.verbose)
            prepend_to_file(os.path.join(outputdir,
                            'bbob_pproc_commands.tex'), 
                            ['\\providecommand{\\bbobppscatterlegend}[1]{', 
                             ppscatter.figure_caption(), 
                             '}'
                            ])
            
            replace_in_file(os.path.join(outputdir, genericsettings.two_algorithm_file_name + '.html'), '##bbobppscatterlegend##', ppscatter.figure_caption_html())
                            
            print "Scatter plots done."

        if genericsettings.isTab:
            dictNG0 = dsList0.dictByNoise()
            dictNG1 = dsList1.dictByNoise()

            for nGroup in set(dictNG0.keys()) & set(dictNG1.keys()):
                # split table in as many as necessary
                dictFunc0 = dictNG0[nGroup].dictByFunc()
                dictFunc1 = dictNG1[nGroup].dictByFunc()
                funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys()))
                if len(funcs) > 24:
                    funcs.sort()
                    nbgroups = int(numpy.ceil(len(funcs)/24.))
                    def split_seq(seq, nbgroups):
                        newseq = []
                        splitsize = 1.0/nbgroups*len(seq)
                        for i in range(nbgroups):
                            newseq.append(seq[int(round(i*splitsize)):int(round((i+1)*splitsize))])
                        return newseq

                    groups = split_seq(funcs, nbgroups)
                    # merge
                    group0 = []
                    group1 = []
                    for i, g in enumerate(groups):
                        tmp0 = DataSetList()
                        tmp1 = DataSetList()
                        for f in g:
                            tmp0.extend(dictFunc0[f])
                            tmp1.extend(dictFunc1[f])
                        group0.append(tmp0)
                        group1.append(tmp1)
                    for i, g in enumerate(zip(group0, group1)):
                        pptable2.main(g[0], g[1], inset.tabDimsOfInterest,
                                      outputdir,
                                      '%s%d' % (nGroup, i), genericsettings.verbose)
                else:
                    if 11 < 3:  # future handling: 
                        dictFunc0 = dsList0.dictByFunc()
                        dictFunc1 = dsList1.dictByFunc()
                        funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys()))
                        funcs.sort()
#                        nbgroups = int(numpy.ceil(len(funcs)/testbedsettings.numberOfFunctions))
#                        pptable2.main(dsList0, dsList1,
#                                      testbedsettings.tabDimsOfInterest, outputdir,
#                                      '%s' % (testbedsettings.testbedshortname), genericsettings.verbose)
                    else:
                        pptable2.main(dictNG0[nGroup], dictNG1[nGroup],
                                      inset.tabDimsOfInterest,
                                      outputdir,
                                      '%s' % (nGroup), genericsettings.verbose)

            if isinstance(pptable2.targetsOfInterest, pproc.RunlengthBasedTargetValues):
                prepend_to_file(os.path.join(outputdir,
                            'bbob_pproc_commands.tex'), 
                            ['\\providecommand{\\bbobpptablestwolegend}[1]{', 
                             pptable2.table_caption_expensive, 
                             '}'
                            ])
            else:
                prepend_to_file(os.path.join(outputdir,
                            'bbob_pproc_commands.tex'), 
                            ['\\providecommand{\\bbobpptablestwolegend}[1]{', 
                             pptable2.table_caption, 
                             '}'
                            ])
                            
            htmlFileName = os.path.join(outputdir, genericsettings.two_algorithm_file_name + '.html')            
            key =  '##bbobpptablestwolegendexpensive##' if isinstance(pptable2.targetsOfInterest, pproc.RunlengthBasedTargetValues) else '##bbobpptablestwolegend##'
            replace_in_file(htmlFileName, '##bbobpptablestwolegend##', htmldesc.getValue(key))
                        
            alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop().replace(genericsettings.extraction_folder_prefix, '')[0:3]
            alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop().replace(genericsettings.extraction_folder_prefix, '')[0:3]
            replace_in_file(htmlFileName, 'algorithmAshort', alg0)
            replace_in_file(htmlFileName, 'algorithmBshort', alg1)
            
            for i, alg in enumerate(args):
                replace_in_file(htmlFileName, 'algorithm' + abc[i], str_to_latex(strip_pathname1(alg)))

            print "Tables done."

        if genericsettings.isScaleUp:
            plt.rc("axes", labelsize=20, titlesize=24)
            plt.rc("xtick", labelsize=20)
            plt.rc("ytick", labelsize=20)
            plt.rc("font", size=20)
            plt.rc("legend", fontsize=20)
            plt.rc('pdf', fonttype = 42)
            if genericsettings.runlength_based_targets:
                ftarget = RunlengthBasedTargetValues([target_runlength])  # TODO: make this more variable but also consistent
            ppfigs.main(dictAlg, genericsettings.two_algorithm_file_name, sortedAlgs, ftarget,
                        outputdir, genericsettings.verbose)
            plt.rcdefaults()
            print "Scaling figures done."

        if genericsettings.isFig or genericsettings.isRLDistr or genericsettings.isTab or genericsettings.isScatter or genericsettings.isScaleUp:
            print "Output data written to folder %s" % outputdir

        plt.rcdefaults()
예제 #8
0
        for i, alg in enumerate(args):
            lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' + 
                    str_to_latex(strip_pathname2(alg)) + '}')
        prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 
                     lines, 5000, 
                     'bbob_proc_commands.tex truncated, consider removing the file before the text run'
                     )

        dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)

        if not dsList:
            sys.exit()

        for i in dictAlg:
            if isNoisy and not isNoiseFree:
                dictAlg[i] = dictAlg[i].dictByNoise().get('nzall', DataSetList())
            if isNoiseFree and not isNoisy:
                dictAlg[i] = dictAlg[i].dictByNoise().get('noiselessall', DataSetList())



        # compute maxfuneval values
        # TODO: we should rather take min_algorithm max_evals
        dict_max_fun_evals = {}
        for ds in dsList:
            dict_max_fun_evals[ds.dim] = numpy.max((dict_max_fun_evals.setdefault(ds.dim, 0), float(numpy.max(ds.maxevals))))
        if isRLbased is not None:
            genericsettings.runlength_based_targets = isRLbased
            
        # set target values
        from bbob_pproc import config
예제 #9
0
def main(argv=None):
    r"""Routine for post-processing COCO data from two algorithms.

    Provided with some data, this routine outputs figure and TeX files
    in a folder needed for the compilation of latex document
    :file:`template2XXX.tex` or :file:`noisytemplate2XXX.tex`, where
    :file:`XXX` is either :file:`ecj` or :file:`generic`. The template
    file needs to be edited so that the command ``\bbobdatapath`` points
    to the output folder.

    These output files will contain performance tables, performance
    scaling figures and empirical cumulative distribution figures. On
    subsequent executions, new files will be added to the output folder,
    overwriting existing older files in the process.

    Keyword arguments:

    *argv* -- list of strings containing options and arguments. If not
    given, sys.argv is accessed.

    *argv* must list folders containing BBOB data files. Each of these
    folders should correspond to the data of ONE algorithm.

    Furthermore, argv can begin with, in any order, facultative option
    flags listed below.

        -h, --help
            displays this message.
        -v, --verbose
            verbose mode, prints out operations.
        -o OUTPUTDIR, --output-dir=OUTPUTDIR
            changes the default output directory (:file:`ppdata`) to
            :file:`OUTPUTDIR`
        --noise-free, --noisy
            processes only part of the data.
        --settings=SETTING
            changes the style of the output figures and tables. At the
            moment only the only differences are in the colors of the
            output figures. SETTING can be either "grayscale", "color"
            or "black-white". The default setting is "color".
        --fig-only, --rld-only, --tab-only, --sca-only
            these options can be used to output respectively the ERT
            graphs figures, run length distribution figures or the
            comparison tables scatter plot figures only. Any combination
            of these options results in no output.
        --conv 
            if this option is chosen, additionally convergence
            plots for each function and algorithm are generated.

    Exceptions raised:

    *Usage* -- Gives back a usage message.

    Examples:

    * Calling the rungeneric2.py interface from the command line::

        $ python bbob_pproc/rungeneric2.py -v Alg0-baseline Alg1-of-interest

      will post-process the data from folders :file:`Alg0-baseline` and
      :file:`Alg1-of-interest`, the former containing data for the
      reference algorithm (zero-th) and the latter data for the
      algorithm of concern (first). The results will be output in the
      default output folder. The ``-v`` option adds verbosity.

    * From the python interpreter (requires that the path to this
      package is in python search path)::

        >> import bbob_pproc as bb
        >> bb.rungeneric2.main('-o outputfolder PSO DEPSO'.split())

    This will execute the post-processing on the data found in folder
    :file:`PSO` and :file:`DEPSO`. The ``-o`` option changes the output
    folder from the default to :file:`outputfolder`.

    """

    if argv is None:
        argv = sys.argv[1:]
        # The zero-th input argument which is the name of the calling script is
        # disregarded.

    try:

        try:
            opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
        except getopt.error, msg:
            raise Usage(msg)

        if not (args):
            usage()
            sys.exit()

        isfigure = True
        isrldistr = True
        istable = True
        isscatter = True
        isscaleup = True
        isNoisy = False
        isNoiseFree = False
        verbose = False
        outputdir = 'ppdata'
        inputsettings = 'color'
        isConv = False

        #Process options
        for o, a in opts:
            if o in ("-v", "--verbose"):
                verbose = True
            elif o in ("-h", "--help"):
                usage()
                sys.exit()
            elif o in ("-o", "--output-dir"):
                outputdir = a
            #elif o in ("-s", "--style"):
            #    inputsettings = a
            elif o == "--fig-only":
                isrldistr = False
                istable = False
                isscatter = False
            elif o == "--rld-only":
                isfigure = False
                istable = False
                isscatter = False
            elif o == "--tab-only":
                isfigure = False
                isrldistr = False
                isscatter = False
            elif o == "--sca-only":
                isfigure = False
                isrldistr = False
                istable = False
            elif o == "--noisy":
                isNoisy = True
            elif o == "--noise-free":
                isNoiseFree = True
            elif o == "--settings":
                inputsettings = a
            elif o == "--conv":
                isConv = True
            else:
                assert False, "unhandled option"

        # from bbob_pproc import bbob2010 as inset # input settings
        if inputsettings == "color":
            from bbob_pproc import config, genericsettings as inset  # input settings
            config.config()
        elif inputsettings == "grayscale":
            from bbob_pproc import grayscalesettings as inset  # input settings
        elif inputsettings == "black-white":
            from bbob_pproc import bwsettings as inset  # input settings
        else:
            txt = ('Settings: %s is not an appropriate ' % inputsettings +
                   'argument for input flag "--settings".')
            raise Usage(txt)

        if (not verbose):
            warnings.simplefilter('module')
            warnings.simplefilter('ignore')

        print("Post-processing will generate comparison " +
              "data in folder %s" % outputdir)
        print "  this might take several minutes."

        dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)

        if 1 < 3 and len(sortedAlgs) != 2:
            raise ValueError(
                'rungeneric2.py needs exactly two algorithms to compare, found: '
                + str(sortedAlgs) +
                '\n use rungeneric.py (or rungenericmany.py) to compare more algorithms. '
            )

        if not dsList:
            sys.exit()

        for i in dictAlg:
            if isNoisy and not isNoiseFree:
                dictAlg[i] = dictAlg[i].dictByNoise().get(
                    'nzall', DataSetList())
            if isNoiseFree and not isNoisy:
                dictAlg[i] = dictAlg[i].dictByNoise().get(
                    'noiselessall', DataSetList())

        for i in dsList:
            if i.dim not in genericsettings.dimensions_to_display:
                continue

            if (dict((j, i.instancenumbers.count(j))
                     for j in set(i.instancenumbers)) <
                    inset.instancesOfInterest):
                warnings.warn('The data of %s do not list ' % (i) +
                              'the correct instances ' + 'of function F%d.' %
                              (i.funcId))

        if len(sortedAlgs) < 2:
            raise Usage('Expect data from two different algorithms, could ' +
                        'only find one.')
        elif len(sortedAlgs) > 2:
            warnings.warn('Data from folders: %s ' % (sortedAlgs) +
                          'were found, the first two will be processed.')

        # Group by algorithm
        dsList0 = dictAlg[sortedAlgs[0]]
        if not dsList0:
            raise Usage('Could not find data for algorithm %s.' %
                        (sortedAlgs[0]))

        dsList1 = dictAlg[sortedAlgs[1]]
        if not dsList1:
            raise Usage('Could not find data for algorithm %s.' %
                        (sortedAlgs[0]))

        # get the name of each algorithm from the input arguments
        tmppath0, alg0name = os.path.split(sortedAlgs[0].rstrip(os.sep))
        tmppath1, alg1name = os.path.split(sortedAlgs[1].rstrip(os.sep))

        for i in dsList0:
            i.algId = alg0name
        for i in dsList1:
            i.algId = alg1name

        ######################### Post-processing #############################
        if isfigure or isrldistr or istable or isscatter:
            if not os.path.exists(outputdir):
                os.mkdir(outputdir)
                if verbose:
                    print 'Folder %s was created.' % (outputdir)

            # prepend the algorithm name command to the tex-command file
            abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
            lines = []
            for i, alg in enumerate(args):
                lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' +
                             str_to_latex(strip_pathname(alg)) + '}')
            prepend_to_file(
                os.path.join(outputdir,
                             'bbob_pproc_commands.tex'), lines, 1000,
                'bbob_proc_commands.tex truncated, consider removing the file before the text run'
            )

        # Check whether both input arguments list noisy and noise-free data
        dictFN0 = dsList0.dictByNoise()
        dictFN1 = dsList1.dictByNoise()
        k0 = set(dictFN0.keys())
        k1 = set(dictFN1.keys())
        symdiff = k1 ^ k0  # symmetric difference
        if symdiff:
            tmpdict = {}
            for i, noisegrp in enumerate(symdiff):
                if noisegrp == 'nzall':
                    tmp = 'noisy'
                elif noisegrp == 'noiselessall':
                    tmp = 'noiseless'

                if dictFN0.has_key(noisegrp):
                    tmp2 = sortedAlgs[0]
                elif dictFN1.has_key(noisegrp):
                    tmp2 = sortedAlgs[1]

                tmpdict.setdefault(tmp2, []).append(tmp)

            txt = []
            for i, j in tmpdict.iteritems():
                txt.append('Only input folder %s lists %s data.' %
                           (i, ' and '.join(j)))
            raise Usage('Data Mismatch: \n  ' + ' '.join(txt) +
                        '\nTry using --noise-free or --noisy flags.')

        if isfigure:
            plt.rc("axes", **inset.rcaxeslarger)
            plt.rc("xtick", **inset.rcticklarger)
            plt.rc("ytick", **inset.rcticklarger)
            plt.rc("font", **inset.rcfontlarger)
            plt.rc("legend", **inset.rclegendlarger)
            ppfig2.main(dsList0, dsList1, ftarget, outputdir, verbose)
            print "log ERT1/ERT0 vs target function values done."

        plt.rc("axes", **inset.rcaxes)
        plt.rc("xtick", **inset.rctick)
        plt.rc("ytick", **inset.rctick)
        plt.rc("font", **inset.rcfont)
        plt.rc("legend", **inset.rclegend)

        if isrldistr:
            if len(dictFN0) > 1 or len(dictFN1) > 1:
                warnings.warn('Data for functions from both the noisy and ' +
                              'non-noisy testbeds have been found. Their ' +
                              'results will be mixed in the "all functions" ' +
                              'ECDF figures.')
            dictDim0 = dsList0.dictByDim()
            dictDim1 = dsList1.dictByDim()

            # ECDFs of ERT ratios
            for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
                if dim in inset.rldDimsOfInterest:
                    # ECDF for all functions altogether
                    try:
                        pprldistr2.main(dictDim0[dim], dictDim1[dim],
                                        inset.rldValsOfInterest, outputdir,
                                        '%02dD_all' % dim, verbose)
                    except KeyError:
                        warnings.warn('Could not find some data in %d-D.' %
                                      (dim))
                        continue

                    # ECDFs per function groups
                    dictFG0 = dictDim0[dim].dictByFuncGroup()
                    dictFG1 = dictDim1[dim].dictByFuncGroup()

                    for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
                        pprldistr2.main(dictFG1[fGroup], dictFG0[fGroup],
                                        inset.rldValsOfInterest, outputdir,
                                        '%02dD_%s' % (dim, fGroup), verbose)

                    # ECDFs per noise groups
                    dictFN0 = dictDim0[dim].dictByNoise()
                    dictFN1 = dictDim1[dim].dictByNoise()

                    for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
                        pprldistr2.main(dictFN1[fGroup], dictFN0[fGroup],
                                        inset.rldValsOfInterest, outputdir,
                                        '%02dD_%s' % (dim, fGroup), verbose)
            print "ECDF runlength ratio graphs done."

            for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
                pprldistr.fmax = None  #Resetting the max final value
                pprldistr.evalfmax = None  #Resetting the max #fevalsfactor
                # ECDFs of all functions altogether
                if dim in inset.rldDimsOfInterest:
                    try:
                        pprldistr.comp(
                            dictDim1[dim], dictDim0[dim],
                            inset.rldValsOfInterest if isinstance(
                                inset.rldValsOfInterest, TargetValues) else
                            TargetValues(inset.rldValsOfInterest), True,
                            outputdir, 'all', verbose)
                    except KeyError:
                        warnings.warn('Could not find some data in %d-D.' %
                                      (dim))
                        continue

                    # ECDFs per function groups
                    dictFG0 = dictDim0[dim].dictByFuncGroup()
                    dictFG1 = dictDim1[dim].dictByFuncGroup()

                    for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
                        pprldistr.comp(
                            dictFG1[fGroup], dictFG0[fGroup],
                            inset.rldValsOfInterest if isinstance(
                                inset.rldValsOfInterest, TargetValues) else
                            TargetValues(inset.rldValsOfInterest), True,
                            outputdir, '%s' % fGroup, verbose)

                    # ECDFs per noise groups
                    dictFN0 = dictDim0[dim].dictByNoise()
                    dictFN1 = dictDim1[dim].dictByNoise()
                    for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
                        pprldistr.comp(
                            dictFN1[fGroup], dictFN0[fGroup],
                            inset.rldValsOfInterest if isinstance(
                                inset.rldValsOfInterest, TargetValues) else
                            TargetValues(inset.rldValsOfInterest), True,
                            outputdir, '%s' % fGroup, verbose)

            print "ECDF runlength graphs done."

        if isConv:
            ppconverrorbars.main(dictAlg, outputdir, verbose)

        if istable:
            dictNG0 = dsList0.dictByNoise()
            dictNG1 = dsList1.dictByNoise()

            for nGroup in set(dictNG0.keys()) & set(dictNG1.keys()):
                # split table in as many as necessary
                dictFunc0 = dictNG0[nGroup].dictByFunc()
                dictFunc1 = dictNG1[nGroup].dictByFunc()
                funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys()))
                if len(funcs) > 24:
                    funcs.sort()
                    nbgroups = int(numpy.ceil(len(funcs) / 24.))

                    def split_seq(seq, nbgroups):
                        newseq = []
                        splitsize = 1.0 / nbgroups * len(seq)
                        for i in range(nbgroups):
                            newseq.append(
                                seq[int(round(i * splitsize)
                                        ):int(round((i + 1) * splitsize))])
                        return newseq

                    groups = split_seq(funcs, nbgroups)
                    # merge
                    group0 = []
                    group1 = []
                    for i, g in enumerate(groups):
                        tmp0 = DataSetList()
                        tmp1 = DataSetList()
                        for f in g:
                            tmp0.extend(dictFunc0[f])
                            tmp1.extend(dictFunc1[f])
                        group0.append(tmp0)
                        group1.append(tmp1)
                    for i, g in enumerate(zip(group0, group1)):
                        pptable2.main(g[0], g[1], inset.tabDimsOfInterest,
                                      outputdir, '%s%d' % (nGroup, i), verbose)
                else:
                    if 11 < 3:  # future handling:
                        dictFunc0 = dsList0.dictByFunc()
                        dictFunc1 = dsList1.dictByFunc()
                        funcs = list(
                            set(dictFunc0.keys()) & set(dictFunc1.keys()))
                        funcs.sort()


#                        nbgroups = int(numpy.ceil(len(funcs)/testbedsettings.numberOfFunctions))
#                        pptable2.main(dsList0, dsList1,
#                                      testbedsettings.tabDimsOfInterest, outputdir,
#                                      '%s' % (testbedsettings.testbedshortname), verbose)
                    else:
                        pptable2.main(dictNG0[nGroup], dictNG1[nGroup],
                                      inset.tabDimsOfInterest, outputdir,
                                      '%s' % (nGroup), verbose)
            prepend_to_file(os.path.join(
                outputdir, 'bbob_pproc_commands.tex'), [
                    '\\providecommand{\\bbobpptablestwolegend}[1]{',
                    pptable2.figure_legend, '}'
                ])
            print "Tables done."

        if isscatter:
            ppscatter.main(dsList0, dsList1, outputdir, verbose=verbose)
            prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
                            [
                                '\\providecommand{\\bbobppscatterlegend}[1]{',
                                ppscatter.figure_legend, '}'
                            ])
            print "Scatter plots done."

        if isscaleup:
            plt.rc("axes", labelsize=20, titlesize=24)
            plt.rc("xtick", labelsize=20)
            plt.rc("ytick", labelsize=20)
            plt.rc("font", size=20)
            plt.rc("legend", fontsize=20)
            ppfigs.main(dictAlg, sortedAlgs, ftarget, outputdir, verbose)
            plt.rcdefaults()
            print "Scaling figures done."

        if isfigure or isrldistr or istable or isscatter or isscaleup:
            print "Output data written to folder %s" % outputdir

        plt.rcdefaults()
예제 #10
0
def main(argv=None):
    r"""Post-processing COCO data of a single algorithm.

    Provided with some data, this routine outputs figure and TeX files
    in a folder needed for the compilation of latex document
    :file:`template1XXX.tex` or :file:`noisytemplate1XXX.tex`, where 
    :file:`XXX` is either :file:`ecj` or :file:`generic`. The template
    file needs to be edited so that the commands
    ``\bbobdatapath`` and ``\algfolder`` point to the output folder.

    These output files will contain performance tables, performance
    scaling figures and empirical cumulative distribution figures. On
    subsequent executions, new files will be added to the output folder,
    overwriting existing older files in the process.

    Keyword arguments:

    *argv* -- list of strings containing options and arguments. If not
    given, sys.argv is accessed.

    *argv* should list either names of :file:`info` files or folders
    containing :file:`info` files. argv can also contain post-processed
    :file:`pickle` files generated by this routine. Furthermore, *argv*
    can begin with, in any order, facultative option flags listed below.

        -h, --help
            displays this message.
        -v, --verbose
            verbose mode, prints out all operations.
        -p, --pickle
            generates pickle post processed data files.
        -o OUTPUTDIR, --output-dir=OUTPUTDIR
            changes the default output directory (:file:`ppdata`) to
            :file:`OUTPUTDIR`.
        --crafting-effort=VALUE
            sets the crafting effort to VALUE (float). Otherwise the
            default value of 0. will be used.
        --noise-free, --noisy
            processes only part of the data.
        --settings=SETTINGS
            changes the style of the output figures and tables. At the
            moment the only differences are  in the colors of the output
            figures. SETTINGS can be either "grayscale", "color" or
            "black-white". The default setting is "color".
        --tab-only, --fig-only, --rld-only, --los-only
            these options can be used to output respectively the TeX
            tables, convergence and ERTs graphs figures, run length
            distribution figures, ERT loss ratio figures only. A
            combination of any two of these options results in no
            output.
        --conv
            if this option is chosen, additionally convergence plots
            for each function and algorithm are generated.
        --expensive
            runlength-based f-target values and fixed display limits,
            useful with comparatively small budgets. By default the
            setting is based on the budget used in the data.
        --not-expensive
            expensive setting off. 
        --runlength-based
            runlength-based f-target values, such that the
            "level of difficulty" is similar for all functions. 

    Exceptions raised:

    *Usage* -- Gives back a usage message.

    Examples:

    * Calling the rungeneric1.py interface from the command line::

        $ python bbob_pproc/rungeneric1.py -v experiment1

      will post-process the folder experiment1 and all its containing
      data, base on the .info files found in the folder. The result will
      appear in the default output folder. The -v option adds verbosity. ::

        $ python bbob_pproc/rungeneric1.py -o exp2 experiment2/*.info

      This will execute the post-processing on the info files found in
      :file:`experiment2`. The result will be located in the alternative
      location :file:`exp2`.

    * Loading this package and calling the main from the command line
      (requires that the path to this package is in python search path)::

        $ python -m bbob_pproc.rungeneric1 -h

      This will print out this help message.

    * From the python interpreter (requires that the path to this
      package is in python search path)::

        >> import bbob_pproc as bb
        >> bb.rungeneric1.main('-o outputfolder folder1'.split())

      This will execute the post-processing on the index files found in
      :file:`folder1`. The ``-o`` option changes the output folder from
      the default to :file:`outputfolder`.

    """

    if argv is None:
        argv = sys.argv[1:]
        # The zero-th input argument which is the name of the calling script is
        # disregarded.

    if 1 < 3:
        opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
        if 11 < 3:
            try:
                opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
            except getopt.error, msg:
                raise Usage(msg)

        if not (args) and not '--help' in argv and not 'h' in argv:
            print 'not enough input arguments given'
            print 'cave: the following options also need an argument:'
            print [o for o in longoptlist if o[-1] == '=']
            print 'options given:'
            print opts
            print 'try --help for help'
            sys.exit()

        inputCrE = 0.
        isfigure = True
        istab = True
        isrldistr = True
        islogloss = True
        isPostProcessed = False
        isPickled = False
        verbose = False
        outputdir = 'ppdata'
        isNoisy = False
        isNoiseFree = False
        inputsettings = 'color'
        isConv = False
        isRLbased = None  # allows automatic choice
        isExpensive = None 

        # Process options
        for o, a in opts:
            if o in ("-v", "--verbose"):
                verbose = True
            elif o in ("-h", "--help"):
                usage()
                sys.exit()
            elif o in ("-p", "--pickle"):
                isPickled = True
            elif o in ("-o", "--output-dir"):
                outputdir = a
            elif o == "--noisy":
                isNoisy = True
            elif o == "--noise-free":
                isNoiseFree = True
            # The next 4 are for testing purpose
            elif o == "--tab-only":
                isfigure = False
                isrldistr = False
                islogloss = False
            elif o == "--fig-only":
                istab = False
                isrldistr = False
                islogloss = False
            elif o == "--rld-only":
                istab = False
                isfigure = False
                islogloss = False
            elif o == "--los-only":
                istab = False
                isfigure = False
                isrldistr = False
            elif o == "--crafting-effort":
                try:
                    inputCrE = float(a)
                except ValueError:
                    raise Usage('Expect a valid float for flag crafting-effort.')
            elif o == "--settings":
                inputsettings = a
            elif o == "--conv":
                isConv = True
            elif o == "--runlength-based":
                isRLbased = True
            elif o == "--expensive":
                isExpensive = True  # comprises runlength-based
            elif o == "--not-expensive":
                isExpensive = False  
            else:
                assert False, "unhandled option"

        # from bbob_pproc import bbob2010 as inset # input settings
        if inputsettings == "color":
            from bbob_pproc import genericsettings as inset  # input settings
        elif inputsettings == "grayscale":
            from bbob_pproc import grayscalesettings as inset  # input settings
        elif inputsettings == "black-white":
            from bbob_pproc import bwsettings as inset  # input settings
        else:
            txt = ('Settings: %s is not an appropriate ' % inputsettings
                   + 'argument for input flag "--settings".')
            raise Usage(txt)
        
        if 11 < 3:
            from bbob_pproc import config  # input settings
            config.config()
            import imp
            # import testbedsettings as testbedsettings # input settings
            try:
                fp, pathname, description = imp.find_module("testbedsettings")
                testbedsettings = imp.load_module("testbedsettings", fp, pathname, description)
            finally:
                fp.close()

        if (not verbose):
            warnings.simplefilter('module')
            # warnings.simplefilter('ignore')            

        print ("Post-processing (1): will generate output " + 
               "data in folder %s" % outputdir)
        print "  this might take several minutes."

        filelist = list()
        for i in args:
            i = i.strip()
            if os.path.isdir(i):
                filelist.extend(findfiles.main(i, verbose))
            elif os.path.isfile(i):
                filelist.append(i)
            else:
                txt = 'Input file or folder %s could not be found.' % i
                print txt
                raise Usage(txt)
        dsList = DataSetList(filelist, verbose)
        
        if not dsList:
            raise Usage("Nothing to do: post-processing stopped.")

        if isNoisy and not isNoiseFree:
            dsList = dsList.dictByNoise().get('nzall', DataSetList())
        if isNoiseFree and not isNoisy:
            dsList = dsList.dictByNoise().get('noiselessall', DataSetList())

        # compute maxfuneval values
        dict_max_fun_evals = {}
        for ds in dsList:
            dict_max_fun_evals[ds.dim] = np.max((dict_max_fun_evals.setdefault(ds.dim, 0), float(np.max(ds.maxevals))))
        if isRLbased is not None:
            genericsettings.runlength_based_targets = isRLbased

        from bbob_pproc import config
        config.target_values(isExpensive, dict_max_fun_evals)
        config.config()

        if (verbose):
            for i in dsList:
                if (dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) != 
                    inset.instancesOfInterest):
                    warnings.warn('The data of %s do not list ' % (i) + 
                                  'the correct instances ' + 
                                  'of function F%d.' % (i.funcId))

        dictAlg = dsList.dictByAlg()

        if len(dictAlg) > 1:
            warnings.warn('Data with multiple algId %s ' % (dictAlg) + 
                          'will be processed together.')
            # TODO: in this case, all is well as long as for a given problem
            # (given dimension and function) there is a single instance of
            # DataSet associated. If there are more than one, the first one only
            # will be considered... which is probably not what one would expect.
            # TODO: put some errors where this case would be a problem.
            # raise Usage?

        if isfigure or istab or isrldistr or islogloss:
            if not os.path.exists(outputdir):
                os.makedirs(outputdir)
                if verbose:
                    print 'Folder %s was created.' % (outputdir)

        if isPickled:
            dsList.pickle(verbose=verbose)

        if isConv:
            ppconverrorbars.main(dictAlg, outputdir, verbose)

        if isfigure:
            print "Scaling figures...",
            sys.stdout.flush()
            # ERT/dim vs dim.
            plt.rc("axes", **inset.rcaxeslarger)
            plt.rc("xtick", **inset.rcticklarger)
            plt.rc("ytick", **inset.rcticklarger)
            plt.rc("font", **inset.rcfontlarger)
            plt.rc("legend", **inset.rclegendlarger)
            ppfigdim.main(dsList, ppfigdim.values_of_interest, outputdir, verbose)
            plt.rcdefaults()
            print_done()

        plt.rc("axes", **inset.rcaxes)
        plt.rc("xtick", **inset.rctick)
        plt.rc("ytick", **inset.rctick)
        plt.rc("font", **inset.rcfont)
        plt.rc("legend", **inset.rclegend)

        if istab:
            print "TeX tables...",
            sys.stdout.flush()
            dictNoise = dsList.dictByNoise()
            for noise, sliceNoise in dictNoise.iteritems():
                pptable.main(sliceNoise, inset.tabDimsOfInterest, outputdir,
                             noise, verbose)
            print_done()

        if isrldistr:
            print "ECDF graphs...",
            sys.stdout.flush()
            dictNoise = dsList.dictByNoise()
            if len(dictNoise) > 1:
                warnings.warn('Data for functions from both the noisy and '
                              'non-noisy testbeds have been found. Their '
                              'results will be mixed in the "all functions" '
                              'ECDF figures.')
            dictDim = dsList.dictByDim()
            for dim in inset.rldDimsOfInterest:
                try:
                    sliceDim = dictDim[dim]
                except KeyError:
                    continue

                pprldistr.main(sliceDim, True,
                               outputdir, 'all', verbose)
                dictNoise = sliceDim.dictByNoise()
                for noise, sliceNoise in dictNoise.iteritems():
                    pprldistr.main(sliceNoise, True,
                                   outputdir, '%s' % noise, verbose)
                dictFG = sliceDim.dictByFuncGroup()
                for fGroup, sliceFuncGroup in dictFG.items():
                    pprldistr.main(sliceFuncGroup,
                                   True, outputdir, '%s' % fGroup, verbose)

                pprldistr.fmax = None  # Resetting the max final value
                pprldistr.evalfmax = None  # Resetting the max #fevalsfactor

            print_done()

        if islogloss:
            print "ERT loss ratio figures and tables...",
            sys.stdout.flush()
            for ng, sliceNoise in dsList.dictByNoise().iteritems():
                if ng == 'noiselessall':
                    testbed = 'noiseless'
                elif ng == 'nzall':
                    testbed = 'noisy'
                txt = ("Please input crafting effort value "
                       + "for %s testbed:\n  CrE = " % testbed)
                CrE = inputCrE
                while CrE is None:
                    try:
                        CrE = float(raw_input(txt))
                    except (SyntaxError, NameError, ValueError):
                        print "Float value required."
                dictDim = sliceNoise.dictByDim()
                for d in inset.rldDimsOfInterest:
                    try:
                        sliceDim = dictDim[d]
                    except KeyError:
                        continue
                    info = '%s' % ng
                    pplogloss.main(sliceDim, CrE, True, outputdir, info,
                                   verbose=verbose)
                    pplogloss.generateTable(sliceDim, CrE, outputdir, info,
                                            verbose=verbose)
                    for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup().iteritems():
                        info = '%s' % fGroup
                        pplogloss.main(sliceFuncGroup, CrE, True, outputdir, info,
                                       verbose=verbose)
                    pplogloss.evalfmax = None  # Resetting the max #fevalsfactor

            print_done()

        latex_commands_file = os.path.join(outputdir.split(os.sep)[0], 'bbob_pproc_commands.tex')
        prepend_to_file(latex_commands_file,
                        ['\\providecommand{\\bbobloglosstablecaption}[1]{', 
                         pplogloss.table_caption, '}'])
        prepend_to_file(latex_commands_file,
                        ['\\providecommand{\\bbobloglossfigurecaption}[1]{', 
                         pplogloss.figure_caption, '}'])
        prepend_to_file(latex_commands_file,
                        ['\\providecommand{\\bbobpprldistrlegend}[1]{',
                         pprldistr.caption_single(np.max([ val / dim for dim, val in dict_max_fun_evals.iteritems()])),  # depends on the config setting, should depend on maxfevals
                         '}'])
        prepend_to_file(latex_commands_file,
                        ['\\providecommand{\\bbobppfigdimlegend}[1]{',
                         ppfigdim.scaling_figure_caption(),
                         '}'])
        prepend_to_file(latex_commands_file,
                        ['\\providecommand{\\bbobpptablecaption}[1]{',
                         pptable.table_caption,
                         '}'])
        prepend_to_file(latex_commands_file,
                        ['\\providecommand{\\algfolder}{}'])  # is overwritten in rungeneric.py
        prepend_to_file(latex_commands_file,
                        ['\\providecommand{\\algname}{' + 
                         (str_to_latex(strip_pathname(args[0])) if len(args) == 1 else str_to_latex(dsList[0].algId)) + '{}}'])
        if isfigure or istab or isrldistr or islogloss:
            print "Output data written to folder %s" % outputdir

        plt.rcdefaults()
예제 #11
0
def main(argv=None):
    """Generate python-formatted data from raw BBOB experimental data.

    The raw experimental data (files with the extension 'info' pointing to
    files with extension 'dat' and 'tdat') are post-processed and stored in a
    more condensed way as files with the extension 'pickle'. Supposing the
    raw data are stored in folder 'mydata', the new pickle files will be put in
    folder 'mydata-pickle'.

    Running this will also add an entry in file algorithmshortinfos.txt if it
    does not exist already.
    algorithmshortinfos.txt is a file which contain meta-information that are
    used by modules from the bbob_pproc.compall package.
    The new entry in algorithmshortinfos.txt is represented as a new line
    appended at the end of the file.
    The line in question will have 3 fields separated by colon (:) character.
    The 1st field must be the exact string used as algId in the info files in
    your data, the 2nd the exact string for the comment. The 3rd will be
    a python dictionary which will be used for the plotting.

    Keyword arguments:
    argv -- list of strings containing options and arguments. If not provided,
    sys.argv is accessed.

    argv should list either names of info files or folders containing info
    files or folders containing pickle files (preferred).
    Furthermore, argv can begin with, in any order, facultative option
    flags listed below.

        -h, --help

            display this message

        -v, --verbose
 
            verbose mode, prints out operations. When not in verbose mode, no
            output is to be expected, except for errors.

    Exceptions raised:
    Usage -- Gives back a usage message.

    Examples:

    * Calling the dataoutput.py interface from the command line:

        $ python bbob_pproc/dataoutput.py -v

        $ python bbob_pproc/dataoutput.py experiment2/*.info


    * Loading this package and calling the main from the command line
      (requires that the path to this package is in python search path):

        $ python -m bbob_pproc.dataoutput -h

    This will print out this help message.

    * From the python interactive shell (requires that the path to this
      package is in python search path):

        >>> from bbob_pproc import dataoutput
        >>> dataoutput.main('folder1')

    """

    if argv is None:
        argv = sys.argv[1:]

    try:
        try:
            opts, args = getopt.getopt(argv, "hv", ["help", "verbose"])
        except getopt.error, msg:
            raise Usage(msg)

        if not (args):
            usage()
            sys.exit()

        verbose = False

        #Process options
        for o, a in opts:
            if o in ("-v", "--verbose"):
                verbose = True
            elif o in ("-h", "--help"):
                usage()
                sys.exit()
            else:
                assert False, "unhandled option"

        if (not verbose):
            warnings.simplefilter('ignore')

        dsList = DataSetList(args)
        outputPickle(dsList, verbose=verbose)
        sys.exit()
예제 #12
0
def main(argv=None):
    """Main routine for post-processing the data of multiple algorithms.

    Keyword arguments:
    argv -- list of strings containing options and arguments. If not provided,
    sys.argv is accessed.

    argv must list folders containing BBOB data files. Each of these folders
    should correspond to the data of ONE algorithm and should be listed in
    algorithmshortinfos.txt, a file from the bbob_pproc.compall package listing
    the information of various algorithms treated using bbob_pproc.dataoutput

    Furthermore, argv can begin with, in any order, facultative option flags
    listed below.

        -h, --help

            display this message

        -v, --verbose
 
            verbose mode, prints out operations. When not in verbose mode, no
            output is to be expected, except for errors.

        -o, --output-dir OUTPUTDIR

            change the default output directory ('defaultoutputdirectory') to
            OUTPUTDIR

        --noise-free, --noisy

            restrain the post-processing to part of the data set only. Actually
            quicken the post-processing since it loads only part of the pickle
            files.

        --tab-only, --perfprof-only

            these options can be used to output respectively the comparison
            tex tables or the performance profiles only.
            A combination of any two of these options results in
            no output.

    Exceptions raised:
    Usage -- Gives back a usage message.

    Examples:

    * Calling the runcompall.py interface from the command line:

        $ python bbob_pproc/runcompall.py -v


    * Loading this package and calling the main from the command line
      (requires that the path to this package is in python search path):

        $ python -m bbob_pproc.runcompall -h

    This will print out this help message.

    * From the python interactive shell (requires that the path to this
      package is in python search path):

        >>> from bbob_pproc import runcompall
        >>> runcompall.main('-o outputfolder folder1 folder2'.split())

    This will execute the post-processing on the data found in folder1
    and folder2.
    The -o option changes the output folder from the default cmpalldata to
    outputfolder.

    * Generate post-processing data for some algorithms:

        $ python runcompall.py AMALGAM BFGS CMA-ES

    """

    if argv is None:
        argv = sys.argv[1:]

    try:
        try:
            opts, args = getopt.getopt(argv, "hvo:",
                                       ["help", "output-dir=", "noisy",
                                        "noise-free", "perfprof-only",
                                        "tab-only", "verbose"])
        except getopt.error, msg:
             raise Usage(msg)

        if not (args):
            usage()
            sys.exit()

        verbose = False
        outputdir = 'cmpalldata'
        isNoisy = False
        isNoiseFree = False

        isPer = True
        isTab = True

        #Process options
        for o, a in opts:
            if o in ("-v","--verbose"):
                verbose = True
            elif o in ("-h", "--help"):
                usage()
                sys.exit()
            elif o in ("-o", "--output-dir"):
                outputdir = a
            elif o == "--noisy":
                isNoisy = True
            elif o == "--noise-free":
                isNoiseFree = True
            elif o == "--tab-only":
                isPer = False
                isEff = False
            elif o == "--perfprof-only":
                isEff = False
                isTab = False
            else:
                assert False, "unhandled option"

        if (not verbose):
            warnings.simplefilter('ignore')

        print ("BBOB Post-processing: will generate comparison " +
               "data in folder %s" % outputdir)
        print "  this might take several minutes."

        dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)

        if not dsList:
            sys.exit()

        for i in dictAlg:
            if isNoisy and not isNoiseFree:
                dictAlg[i] = dictAlg[i].dictByNoise().get('nzall', DataSetList())
            elif isNoiseFree and not isNoisy:
                dictAlg[i] = dictAlg[i].dictByNoise().get('noiselessall', DataSetList())

            tmp = set((j.algId, j.comment) for j in dictAlg[i])
            for j in tmp:
                if not dataoutput.isListed(j):
                    dataoutput.updateAlgorithmInfo(j, verbose=verbose)

        for i in dsList:
            if not i.dim in (2, 3, 5, 10, 20):
                continue
            # Deterministic algorithms
            if i.algId in ('Original DIRECT', ):
                tmpInstancesOfInterest = instancesOfInterestDet
            else:
                tmpInstancesOfInterest = instancesOfInterest

            if ((dict((j, i.itrials.count(j)) for j in set(i.itrials)) <
                tmpInstancesOfInterest) and
                (dict((j, i.itrials.count(j)) for j in set(i.itrials)) <
                instancesOfInterest2010)):
                warnings.warn('The data of %s do not list ' %(i) +
                              'the correct instances ' +
                              'of function F%d or the ' %(i.funcId) +
                              'correct number of trials for each.')

        # group targets:
        dictTarget = {}
        for t in sorted(set(single_target_function_values + summarized_target_function_values)):
            tmpdict = dict.fromkeys(((f, d) for f in range(0, 25) + range(101, 131) for d in (2, 3, 5, 10, 20, 40)), t)
            stmp = 'E'
            if t == 1:
                stmp = 'E-'
            # dictTarget['_f' + stmp + '%2.1f' % numpy.log10(t)] = (tmpdict, )
            if t in single_target_function_values: 
                dictTarget['_f' + stmp + '%02d' % numpy.log10(t)] = (tmpdict, )
            if t in summarized_target_function_values: 
                dictTarget.setdefault('_allfs', []).append(tmpdict)

        if not os.path.exists(outputdir):
            os.mkdir(outputdir)
            if verbose:
                print 'Folder %s was created.' % (outputdir)

        # Performance profiles
        if isPer:
            dictNoi = pproc.dictAlgByNoi(dictAlg)
            for ng, tmpdictAlg in dictNoi.iteritems():
                dictDim = pproc.dictAlgByDim(tmpdictAlg)
                for d, entries in dictDim.iteritems():
                    for k, t in dictTarget.iteritems():
                        #set_trace()
                        ppperfprof.main(entries, target=t, order=sortedAlgs,
                                        plotArgs=algPlotInfos,
                                        outputdir=outputdir,
                                        info=('%02d%s_%s' % (d, k, ng)),
                                        verbose=verbose)
            organizeRTDpictures.do(outputdir)
            print "ECDFs of ERT figures done."

        if isTab:
            allmintarget, allertbest = detTarget(dsList)
            pptables.tablemanyalgonefunc(dictAlg, allmintarget, allertbest,
                                         sortedAlgs, outputdir)
            print "Comparison tables done."
예제 #13
0
def main(argv=None):
    """Generates from BBOB experiment data some outputs for a tex document.

    Provided with some index entries (found in files with the 'info' extension)
    this routine outputs figure and TeX files in the folder 'ppdata' needed for
    the compilation of  latex document templateBBOBarticle.tex. These output
    files will contain performance tables, performance scaling figures and
    empirical cumulative distribution figures. On subsequent executions, new
    files will be added to the output directory, overwriting existing older
    files in the process.

    Keyword arguments:
    argv -- list of strings containing options and arguments. If not given,
    sys.argv is accessed.

    argv should list either names of info files or folders containing info
    files. argv can also contain post-processed pickle files generated by this
    routine. Furthermore, argv can begin with, in any order, facultative option
    flags listed below.

        -h, --help

            display this message

        -v, --verbose

            verbose mode, prints out operations. When not in verbose mode, no
            output is to be expected, except for errors.

        -p, --pickle

            generates pickle post processed data files.

        -o, --output-dir OUTPUTDIR

            change the default output directory ('ppdata') to OUTPUTDIR

        --crafting-effort=VALUE

            sets the crafting effort to VALUE. Otherwise the user will be
            prompted. This flag is useful when running this script in batch.

        -f, --final

            lengthens the bootstrapping process used as dispersion measure in
            the tables generation. This might at least double the time of the
            whole post-processing. Please use this option when generating your
            final paper.

        --tab-only, --fig-only, --rld-only, --los-only

            these options can be used to output respectively the tex tables,
            convergence and ENFEs graphs figures, run length distribution
            figures, ERT loss ratio figures only. A combination of any two of
            these options results in no output.

    Exceptions raised:
    Usage -- Gives back a usage message.

    Examples:

    * Calling the run.py interface from the command line:

        $ python bbob_pproc/run.py -v experiment1

    will post-process the folder experiment1 and all its containing data,
    base on the found .info files in the folder. The result will appear
    in folder ppdata. The -v option adds verbosity.

        $ python bbob_pproc/run.py -o otherppdata experiment2/*.info

    This will execute the post-processing on the info files found in
    experiment2. The result will be located in the alternative location
    otherppdata.

    * Loading this package and calling the main from the command line
      (requires that the path to this package is in python search path):

        $ python -m bbob_pproc -h

    This will print out this help message.

    * From the python interactive shell (requires that the path to this
      package is in python search path):

        >>> import bbob_pproc
        >>> bbob_pproc.main('-o outputfolder folder1'.split())

    This will execute the post-processing on the index files found in folder1.
    The -o option changes the output folder from the default ppdata to
    outputfolder.

    """

    if argv is None:
        argv = sys.argv[1:]
        # The zero-th input argument which is the name of the calling script is
        # disregarded.

    try:

        try:
            opts, args = getopt.getopt(argv, "hvpfo:", [
                "help", "output-dir=", "tab-only", "fig-only", "rld-only",
                "los-only", "crafting-effort=", "pickle", "verbose", "final"
            ])
        except getopt.error, msg:
            raise Usage(msg)

        if not (args):
            usage()
            sys.exit()

        CrE = None
        isfigure = True
        istab = True
        isrldistr = True
        islogloss = True
        isPostProcessed = False
        isPickled = False
        isDraft = True
        verbose = False
        outputdir = 'ppdata'

        #Process options
        for o, a in opts:
            if o in ("-v", "--verbose"):
                verbose = True
            elif o in ("-h", "--help"):
                usage()
                sys.exit()
            elif o in ("-p", "--pickle"):
                isPickled = True
            elif o in ("-o", "--output-dir"):
                outputdir = a
            elif o in ("-f", "--final"):
                isDraft = False
            #The next 3 are for testing purpose
            elif o == "--tab-only":
                isfigure = False
                isrldistr = False
                islogloss = False
            elif o == "--fig-only":
                istab = False
                isrldistr = False
                islogloss = False
            elif o == "--rld-only":
                istab = False
                isfigure = False
                islogloss = False
            elif o == "--los-only":
                istab = False
                isfigure = False
                isrldistr = False
            elif o == "--crafting-effort":
                try:
                    CrE = float(a)
                except ValueError:
                    raise Usage(
                        'Expect a valid float for flag crafting-effort.')
            else:
                assert False, "unhandled option"

        if (not verbose):
            warnings.simplefilter('ignore')

        print("BBOB Post-processing: will generate post-processing " +
              "data in folder %s" % outputdir)
        print "  this might take several minutes."

        filelist = list()
        for i in args:
            if os.path.isdir(i):
                filelist.extend(findfiles.main(i, verbose))
            elif os.path.isfile(i):
                filelist.append(i)
            else:
                txt = 'Input file or folder %s could not be found.'
                raise Usage(txt)

        dsList = DataSetList(filelist, verbose)

        if not dsList:
            raise Usage("Nothing to do: post-processing stopped.")

        if (verbose):
            for i in dsList:
                if (dict((j, i.itrials.count(j))
                         for j in set(i.itrials)) != instancesOfInterest2010):
                    warnings.warn('The data of %s do not list ' % (i) +
                                  'the correct instances ' +
                                  'of function F%d.' % (i.funcId))

                # BBOB 2009 Checking
                #if ((dict((j, i.itrials.count(j)) for j in set(i.itrials)) !=
                #instancesOfInterest) and
                #(dict((j, i.itrials.count(j)) for j in set(i.itrials)) !=
                #instancesOfInterest2010)):
                #warnings.warn('The data of %s do not list ' %(i) +
                #'the correct instances ' +
                #'of function F%d or the ' %(i.funcId) +
                #'correct number of trials for each.')

        dictAlg = dsList.dictByAlg()
        if len(dictAlg) > 1:
            warnings.warn('Data with multiple algId %s ' % (dictAlg) +
                          'will be processed together.')
            #TODO: in this case, all is well as long as for a given problem
            #(given dimension and function) there is a single instance of
            #DataSet associated. If there are more than one, the first one only
            #will be considered... which is probably not what one would expect.
            #TODO: put some errors where this case would be a problem.

        if isfigure or istab or isrldistr or islogloss:
            if not os.path.exists(outputdir):
                os.mkdir(outputdir)
                if verbose:
                    print 'Folder %s was created.' % (outputdir)

        if isPickled:
            dsList.pickle(verbose=verbose)

        if isfigure:
            ppfigdim.main(dsList, figValsOfInterest, outputdir, verbose)
            #ERT/dim vs dim.
            #ppfigdim.main2(dsList, figValsOfInterest, outputdir,
            #verbose)
            print "Scaling figures done."

        if istab:
            dictFunc = dsList.dictByFunc()
            for fun, sliceFun in dictFunc.items():
                dictDim = sliceFun.dictByDim()
                tmp = []
                for dim in tabDimsOfInterest:
                    try:
                        if len(dictDim[dim]) > 1:
                            warnings.warn(
                                'Func: %d, DIM %d: ' % (fun, dim) +
                                'multiple index entries. Will only ' +
                                'process the first ' + '%s.' % dictDim[dim][0])
                        tmp.append(dictDim[dim][0])
                    except KeyError:
                        pass
                if tmp:
                    filename = os.path.join(outputdir, 'ppdata_f%d' % fun)
                    pptex.main(tmp, tabValsOfInterest, filename, isDraft,
                               verbose)
            print "TeX tables",
            if isDraft:
                print(
                    "(draft) done. To get final version tables, please "
                    "use the -f option with run.py")
            else:
                print "done."

        if isrldistr:
            dictNoise = dsList.dictByNoise()
            if len(dictNoise) > 1:
                warnings.warn('Data for functions from both the noisy and '
                              'non-noisy testbeds have been found. Their '
                              'results will be mixed in the "all functions" '
                              'ECDF figures.')
            dictDim = dsList.dictByDim()
            for dim in rldDimsOfInterest:
                try:
                    sliceDim = dictDim[dim]
                    pprldistr.main(sliceDim, rldValsOfInterest, True,
                                   outputdir, 'dim%02dall' % dim, verbose)
                    dictNoise = sliceDim.dictByNoise()
                    for noise, sliceNoise in dictNoise.iteritems():
                        pprldistr.main(sliceNoise, rldValsOfInterest, True,
                                       outputdir, 'dim%02d%s' % (dim, noise),
                                       verbose)
                    dictFG = sliceDim.dictByFuncGroup()
                    for fGroup, sliceFuncGroup in dictFG.items():
                        pprldistr.main(sliceFuncGroup, rldValsOfInterest, True,
                                       outputdir, 'dim%02d%s' % (dim, fGroup),
                                       verbose)

                    pprldistr.fmax = None  #Resetting the max final value
                    pprldistr.evalfmax = None  #Resetting the max #fevalsfactor
                except KeyError:
                    pass
            print "ECDF graphs done."

        if islogloss:
            for ng, sliceNoise in dsList.dictByNoise().iteritems():
                if ng == 'noiselessall':
                    testbed = 'noiseless'
                elif ng == 'nzall':
                    testbed = 'noisy'
                txt = ("Please input crafting effort value " +
                       "for %s testbed:\n  CrE = " % testbed)
                while CrE is None:
                    try:
                        CrE = float(input(txt))
                    except (SyntaxError, NameError, ValueError):
                        print "Float value required."
                dictDim = sliceNoise.dictByDim()
                for d in rldDimsOfInterest:
                    try:
                        sliceDim = dictDim[d]
                    except KeyError:
                        continue
                    info = 'dim%02d%s' % (d, ng)
                    pplogloss.main(sliceDim,
                                   CrE,
                                   True,
                                   outputdir,
                                   info,
                                   verbose=verbose)
                    pplogloss.generateTable(sliceDim,
                                            CrE,
                                            outputdir,
                                            info,
                                            verbose=verbose)
                    for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup(
                    ).iteritems():
                        info = 'dim%02d%s' % (d, fGroup)
                        pplogloss.main(sliceFuncGroup,
                                       CrE,
                                       True,
                                       outputdir,
                                       info,
                                       verbose=verbose)
                    pplogloss.evalfmax = None  #Resetting the max #fevalsfactor

            print "ERT loss ratio figures and tables done."

        if isfigure or istab or isrldistr or islogloss:
            print "Output data written to folder %s." % outputdir
예제 #14
0
def main(argv=None):
    r"""Routine for post-processing COCO data from two algorithms.

    Provided with some data, this routine outputs figure and TeX files
    in a folder needed for the compilation of the provided LaTeX templates
    for comparing two algorithms (``*cmp.tex`` or ``*2*.tex``).
    
    The used template file needs to be edited so that the command
    ``\bbobdatapath`` points to the output folder created by this routine.

    The output files will contain performance tables, performance
    scaling figures and empirical cumulative distribution figures. On
    subsequent executions, new files will be added to the output folder,
    overwriting existing older files in the process.

    Keyword arguments:

    *argv* -- list of strings containing options and arguments. If not
    given, sys.argv is accessed.

    *argv* must list folders containing BBOB data files. Each of these
    folders should correspond to the data of ONE algorithm.

    Furthermore, argv can begin with, in any order, facultative option
    flags listed below.

        -h, --help
            displays this message.
        -v, --verbose
            verbose mode, prints out operations.
        -o OUTPUTDIR, --output-dir=OUTPUTDIR
            changes the default output directory (:file:`ppdata`) to
            :file:`OUTPUTDIR`
        --noise-free, --noisy
            processes only part of the data.
        --settings=SETTING
            changes the style of the output figures and tables. At the
            moment only the only differences are in the colors of the
            output figures. SETTING can be either "grayscale", "color"
            or "black-white". The default setting is "color".
        --fig-only, --rld-only, --tab-only, --sca-only
            these options can be used to output respectively the ERT
            graphs figures, run length distribution figures or the
            comparison tables scatter plot figures only. Any combination
            of these options results in no output.
        --conv 
            if this option is chosen, additionally convergence
            plots for each function and algorithm are generated.
        --rld-single-fcts
            generate also runlength distribution figures for each
            single function.
        --expensive
            runlength-based f-target values and fixed display limits,
            useful with comparatively small budgets. By default the
            setting is based on the budget used in the data.
        --not-expensive
            expensive setting off. 
        --svg
            generate also the svg figures which are used in html files 

    Exceptions raised:

    *Usage* -- Gives back a usage message.

    Examples:

    * Calling the rungeneric2.py interface from the command line::

        $ python bbob_pproc/rungeneric2.py -v Alg0-baseline Alg1-of-interest

      will post-process the data from folders :file:`Alg0-baseline` and
      :file:`Alg1-of-interest`, the former containing data for the
      reference algorithm (zero-th) and the latter data for the
      algorithm of concern (first). The results will be output in the
      default output folder. The ``-v`` option adds verbosity.

    * From the python interpreter (requires that the path to this
      package is in python search path)::

        >> import bbob_pproc as bb
        >> bb.rungeneric2.main('-o outputfolder PSO DEPSO'.split())

    This will execute the post-processing on the data found in folder
    :file:`PSO` and :file:`DEPSO`. The ``-o`` option changes the output
    folder from the default to :file:`outputfolder`.

    """

    if argv is None:
        argv = sys.argv[1:]
        # The zero-th input argument which is the name of the calling script is
        # disregarded.

    global ftarget
    try:

        try:
            opts, args = getopt.getopt(argv, genericsettings.shortoptlist,
                                       genericsettings.longoptlist)
        except getopt.error, msg:
            raise Usage(msg)

        if not (args):
            usage()
            sys.exit()

        #Process options
        outputdir = genericsettings.outputdir
        for o, a in opts:
            if o in ("-v", "--verbose"):
                genericsettings.verbose = True
            elif o in ("-h", "--help"):
                usage()
                sys.exit()
            elif o in ("-o", "--output-dir"):
                outputdir = a
            elif o == "--fig-only":
                genericsettings.isRLDistr = False
                genericsettings.isTab = False
                genericsettings.isScatter = False
            elif o == "--rld-only":
                genericsettings.isFig = False
                genericsettings.isTab = False
                genericsettings.isScatter = False
            elif o == "--tab-only":
                genericsettings.isFig = False
                genericsettings.isRLDistr = False
                genericsettings.isScatter = False
            elif o == "--sca-only":
                genericsettings.isFig = False
                genericsettings.isRLDistr = False
                genericsettings.isTab = False
            elif o == "--noisy":
                genericsettings.isNoisy = True
            elif o == "--noise-free":
                genericsettings.isNoiseFree = True
            elif o == "--settings":
                genericsettings.inputsettings = a
            elif o == "--conv":
                genericsettings.isConv = True
            elif o == "--rld-single-fcts":
                genericsettings.isRldOnSingleFcts = True
            elif o == "--runlength-based":
                genericsettings.runlength_based_targets = True
            elif o == "--expensive":
                genericsettings.isExpensive = True  # comprises runlength-based
            elif o == "--not-expensive":
                genericsettings.isExpensive = False
            elif o == "--svg":
                genericsettings.generate_svg_files = True
            elif o == "--los-only":
                warnings.warn(
                    "option --los-only will have no effect with rungeneric2.py"
                )
            elif o == "--crafting-effort=":
                warnings.warn(
                    "option --crafting-effort will have no effect with rungeneric2.py"
                )
            elif o in ("-p", "--pickle"):
                warnings.warn(
                    "option --pickle will have no effect with rungeneric2.py")
            else:
                assert False, "unhandled option"

        # from bbob_pproc import bbob2010 as inset # input settings
        if genericsettings.inputsettings == "color":
            from bbob_pproc import genericsettings as inset  # input settings
            config.config()
        elif genericsettings.inputsettings == "grayscale":  # probably very much obsolete
            from bbob_pproc import grayscalesettings as inset  # input settings
        elif genericsettings.inputsettings == "black-white":  # probably very much obsolete
            from bbob_pproc import bwsettings as inset  # input settings
        else:
            txt = ('Settings: %s is not an appropriate ' %
                   genericsettings.inputsettings +
                   'argument for input flag "--settings".')
            raise Usage(txt)

        if (not genericsettings.verbose):
            warnings.simplefilter('module')
            warnings.simplefilter('ignore')

        print("Post-processing will generate comparison " +
              "data in folder %s" % outputdir)
        print "  this might take several minutes."

        dsList, sortedAlgs, dictAlg = processInputArgs(
            args, verbose=genericsettings.verbose)

        if 1 < 3 and len(sortedAlgs) != 2:
            raise ValueError(
                'rungeneric2.py needs exactly two algorithms to compare, found: '
                + str(sortedAlgs) +
                '\n use rungeneric.py (or rungenericmany.py) to compare more algorithms. '
            )

        if not dsList:
            sys.exit()

        for i in dictAlg:
            if genericsettings.isNoisy and not genericsettings.isNoiseFree:
                dictAlg[i] = dictAlg[i].dictByNoise().get(
                    'nzall', DataSetList())
            if genericsettings.isNoiseFree and not genericsettings.isNoisy:
                dictAlg[i] = dictAlg[i].dictByNoise().get(
                    'noiselessall', DataSetList())

        for i in dsList:
            if i.dim not in genericsettings.dimensions_to_display:
                continue

            if (dict((j, i.instancenumbers.count(j))
                     for j in set(i.instancenumbers)) <
                    inset.instancesOfInterest):
                warnings.warn('The data of %s do not list ' % (i) +
                              'the correct instances ' + 'of function F%d.' %
                              (i.funcId))

        if len(sortedAlgs) < 2:
            raise Usage('Expect data from two different algorithms, could ' +
                        'only find one.')
        elif len(sortedAlgs) > 2:
            warnings.warn('Data from folders: %s ' % (sortedAlgs) +
                          'were found, the first two will be processed.')

        # Group by algorithm
        dsList0 = dictAlg[sortedAlgs[0]]
        if not dsList0:
            raise Usage('Could not find data for algorithm %s.' %
                        (sortedAlgs[0]))

        dsList1 = dictAlg[sortedAlgs[1]]
        if not dsList1:
            raise Usage('Could not find data for algorithm %s.' %
                        (sortedAlgs[0]))

        # get the name of each algorithm from the input arguments
        tmppath0, alg0name = os.path.split(sortedAlgs[0].rstrip(os.sep))
        tmppath1, alg1name = os.path.split(sortedAlgs[1].rstrip(os.sep))

        for i in dsList0:
            i.algId = alg0name
        for i in dsList1:
            i.algId = alg1name

        # compute maxfuneval values
        dict_max_fun_evals1 = {}
        dict_max_fun_evals2 = {}
        for ds in dsList0:
            dict_max_fun_evals1[ds.dim] = np.max(
                (dict_max_fun_evals1.setdefault(ds.dim, 0),
                 float(np.max(ds.maxevals))))
        for ds in dsList1:
            dict_max_fun_evals2[ds.dim] = np.max(
                (dict_max_fun_evals2.setdefault(ds.dim, 0),
                 float(np.max(ds.maxevals))))
        config.target_values(
            genericsettings.isExpensive, {
                1:
                min([
                    max([
                        val / dim
                        for dim, val in dict_max_fun_evals1.iteritems()
                    ]),
                    max([
                        val / dim
                        for dim, val in dict_max_fun_evals2.iteritems()
                    ])
                ])
            })
        config.config()

        ######################### Post-processing #############################
        if genericsettings.isFig or genericsettings.isRLDistr or genericsettings.isTab or genericsettings.isScatter:
            if not os.path.exists(outputdir):
                os.mkdir(outputdir)
                if genericsettings.verbose:
                    print 'Folder %s was created.' % (outputdir)

            # prepend the algorithm name command to the tex-command file
            abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
            lines = []
            for i, alg in enumerate(args):
                lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' +
                             str_to_latex(strip_pathname1(alg)) + '}')
            prepend_to_file(
                os.path.join(outputdir,
                             'bbob_pproc_commands.tex'), lines, 1000,
                'bbob_proc_commands.tex truncated, consider removing the file before the text run'
            )

        # Check whether both input arguments list noisy and noise-free data
        dictFN0 = dsList0.dictByNoise()
        dictFN1 = dsList1.dictByNoise()
        k0 = set(dictFN0.keys())
        k1 = set(dictFN1.keys())
        symdiff = k1 ^ k0  # symmetric difference
        if symdiff:
            tmpdict = {}
            for i, noisegrp in enumerate(symdiff):
                if noisegrp == 'nzall':
                    tmp = 'noisy'
                elif noisegrp == 'noiselessall':
                    tmp = 'noiseless'

                if dictFN0.has_key(noisegrp):
                    tmp2 = sortedAlgs[0]
                elif dictFN1.has_key(noisegrp):
                    tmp2 = sortedAlgs[1]

                tmpdict.setdefault(tmp2, []).append(tmp)

            txt = []
            for i, j in tmpdict.iteritems():
                txt.append('Only input folder %s lists %s data.' %
                           (i, ' and '.join(j)))
            raise Usage('Data Mismatch: \n  ' + ' '.join(txt) +
                        '\nTry using --noise-free or --noisy flags.')

        if genericsettings.isFig:
            plt.rc("axes", **inset.rcaxeslarger)
            plt.rc("xtick", **inset.rcticklarger)
            plt.rc("ytick", **inset.rcticklarger)
            plt.rc("font", **inset.rcfontlarger)
            plt.rc("legend", **inset.rclegendlarger)
            plt.rc('pdf', fonttype=42)
            ppfig2.main(dsList0, dsList1, ppfig2_ftarget, outputdir,
                        genericsettings.verbose)
            print "log ERT1/ERT0 vs target function values done."

        plt.rc("axes", **inset.rcaxes)
        plt.rc("xtick", **inset.rctick)
        plt.rc("ytick", **inset.rctick)
        plt.rc("font", **inset.rcfont)
        plt.rc("legend", **inset.rclegend)
        plt.rc('pdf', fonttype=42)

        if genericsettings.isRLDistr:
            if len(dictFN0) > 1 or len(dictFN1) > 1:
                warnings.warn('Data for functions from both the noisy and ' +
                              'non-noisy testbeds have been found. Their ' +
                              'results will be mixed in the "all functions" ' +
                              'ECDF figures.')
            dictDim0 = dsList0.dictByDim()
            dictDim1 = dsList1.dictByDim()

            # ECDFs of ERT ratios
            for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
                if dim in inset.rldDimsOfInterest:
                    # ECDF for all functions altogether
                    try:
                        pprldistr2.main(dictDim0[dim], dictDim1[dim], dim,
                                        inset.rldValsOfInterest, outputdir,
                                        '%02dD_all' % dim,
                                        genericsettings.verbose)
                    except KeyError:
                        warnings.warn('Could not find some data in %d-D.' %
                                      (dim))
                        continue

                    # ECDFs per function groups
                    dictFG0 = dictDim0[dim].dictByFuncGroup()
                    dictFG1 = dictDim1[dim].dictByFuncGroup()

                    for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
                        pprldistr2.main(dictFG1[fGroup], dictFG0[fGroup], dim,
                                        inset.rldValsOfInterest, outputdir,
                                        '%02dD_%s' % (dim, fGroup),
                                        genericsettings.verbose)

                    # ECDFs per noise groups
                    dictFN0 = dictDim0[dim].dictByNoise()
                    dictFN1 = dictDim1[dim].dictByNoise()

                    for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
                        pprldistr2.main(dictFN1[fGroup], dictFN0[fGroup], dim,
                                        inset.rldValsOfInterest, outputdir,
                                        '%02dD_%s' % (dim, fGroup),
                                        genericsettings.verbose)

            prepend_to_file(
                os.path.join(outputdir, 'bbob_pproc_commands.tex'),
                [
                    '\\providecommand{\\bbobpprldistrlegendtwo}[1]{',
                    pprldistr.caption_two(
                    ),  # depends on the config setting, should depend on maxfevals
                    '}'
                ])

            print "ECDF runlength ratio graphs done."

            for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
                pprldistr.fmax = None  #Resetting the max final value
                pprldistr.evalfmax = None  #Resetting the max #fevalsfactor
                # ECDFs of all functions altogether
                if dim in inset.rldDimsOfInterest:
                    try:
                        pprldistr.comp(
                            dictDim1[dim],
                            dictDim0[dim],
                            inset.
                            rldValsOfInterest,  # TODO: let rldVals... possibly be RL-based targets
                            True,
                            outputdir,
                            'all',
                            genericsettings.verbose)
                    except KeyError:
                        warnings.warn('Could not find some data in %d-D.' %
                                      (dim))
                        continue

                    # ECDFs per function groups
                    dictFG0 = dictDim0[dim].dictByFuncGroup()
                    dictFG1 = dictDim1[dim].dictByFuncGroup()

                    for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
                        pprldistr.comp(dictFG1[fGroup], dictFG0[fGroup],
                                       inset.rldValsOfInterest, True,
                                       outputdir, '%s' % fGroup,
                                       genericsettings.verbose)

                    # ECDFs per noise groups
                    dictFN0 = dictDim0[dim].dictByNoise()
                    dictFN1 = dictDim1[dim].dictByNoise()
                    for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
                        pprldistr.comp(dictFN1[fGroup], dictFN0[fGroup],
                                       inset.rldValsOfInterest, True,
                                       outputdir, '%s' % fGroup,
                                       genericsettings.verbose)

            if genericsettings.isRldOnSingleFcts:  # copy-paste from above, here for each function instead of function groups
                # ECDFs for each function
                pprldmany.all_single_functions(dictAlg, sortedAlgs, outputdir,
                                               genericsettings.verbose)
            print "ECDF runlength graphs done."

        if genericsettings.isConv:
            ppconverrorbars.main(dictAlg, outputdir, genericsettings.verbose)

        if genericsettings.isScatter:
            if genericsettings.runlength_based_targets:
                ppscatter.targets = ppscatter.runlength_based_targets
            ppscatter.main(dsList1,
                           dsList0,
                           outputdir,
                           verbose=genericsettings.verbose)
            prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
                            [
                                '\\providecommand{\\bbobppscatterlegend}[1]{',
                                ppscatter.figure_caption(), '}'
                            ])

            replace_in_file(
                os.path.join(outputdir,
                             genericsettings.two_algorithm_file_name +
                             '.html'), '##bbobppscatterlegend##',
                ppscatter.figure_caption_html())

            print "Scatter plots done."

        if genericsettings.isTab:
            dictNG0 = dsList0.dictByNoise()
            dictNG1 = dsList1.dictByNoise()

            for nGroup in set(dictNG0.keys()) & set(dictNG1.keys()):
                # split table in as many as necessary
                dictFunc0 = dictNG0[nGroup].dictByFunc()
                dictFunc1 = dictNG1[nGroup].dictByFunc()
                funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys()))
                if len(funcs) > 24:
                    funcs.sort()
                    nbgroups = int(numpy.ceil(len(funcs) / 24.))

                    def split_seq(seq, nbgroups):
                        newseq = []
                        splitsize = 1.0 / nbgroups * len(seq)
                        for i in range(nbgroups):
                            newseq.append(
                                seq[int(round(i * splitsize)
                                        ):int(round((i + 1) * splitsize))])
                        return newseq

                    groups = split_seq(funcs, nbgroups)
                    # merge
                    group0 = []
                    group1 = []
                    for i, g in enumerate(groups):
                        tmp0 = DataSetList()
                        tmp1 = DataSetList()
                        for f in g:
                            tmp0.extend(dictFunc0[f])
                            tmp1.extend(dictFunc1[f])
                        group0.append(tmp0)
                        group1.append(tmp1)
                    for i, g in enumerate(zip(group0, group1)):
                        pptable2.main(g[0], g[1], inset.tabDimsOfInterest,
                                      outputdir, '%s%d' % (nGroup, i),
                                      genericsettings.verbose)
                else:
                    if 11 < 3:  # future handling:
                        dictFunc0 = dsList0.dictByFunc()
                        dictFunc1 = dsList1.dictByFunc()
                        funcs = list(
                            set(dictFunc0.keys()) & set(dictFunc1.keys()))
                        funcs.sort()


#                        nbgroups = int(numpy.ceil(len(funcs)/testbedsettings.numberOfFunctions))
#                        pptable2.main(dsList0, dsList1,
#                                      testbedsettings.tabDimsOfInterest, outputdir,
#                                      '%s' % (testbedsettings.testbedshortname), genericsettings.verbose)
                    else:
                        pptable2.main(dictNG0[nGroup], dictNG1[nGroup],
                                      inset.tabDimsOfInterest, outputdir,
                                      '%s' % (nGroup), genericsettings.verbose)

            if isinstance(pptable2.targetsOfInterest,
                          pproc.RunlengthBasedTargetValues):
                prepend_to_file(
                    os.path.join(outputdir, 'bbob_pproc_commands.tex'), [
                        '\\providecommand{\\bbobpptablestwolegend}[1]{',
                        pptable2.table_caption_expensive, '}'
                    ])
            else:
                prepend_to_file(
                    os.path.join(outputdir, 'bbob_pproc_commands.tex'), [
                        '\\providecommand{\\bbobpptablestwolegend}[1]{',
                        pptable2.table_caption, '}'
                    ])

            htmlFileName = os.path.join(
                outputdir, genericsettings.two_algorithm_file_name + '.html')
            key = '##bbobpptablestwolegendexpensive##' if isinstance(
                pptable2.targetsOfInterest, pproc.RunlengthBasedTargetValues
            ) else '##bbobpptablestwolegend##'
            replace_in_file(htmlFileName, '##bbobpptablestwolegend##',
                            htmldesc.getValue(key))

            alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop().replace(
                genericsettings.extraction_folder_prefix, '')[0:3]
            alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop().replace(
                genericsettings.extraction_folder_prefix, '')[0:3]
            replace_in_file(htmlFileName, 'algorithmAshort', alg0)
            replace_in_file(htmlFileName, 'algorithmBshort', alg1)

            for i, alg in enumerate(args):
                replace_in_file(htmlFileName, 'algorithm' + abc[i],
                                str_to_latex(strip_pathname1(alg)))

            print "Tables done."

        if genericsettings.isScaleUp:
            plt.rc("axes", labelsize=20, titlesize=24)
            plt.rc("xtick", labelsize=20)
            plt.rc("ytick", labelsize=20)
            plt.rc("font", size=20)
            plt.rc("legend", fontsize=20)
            plt.rc('pdf', fonttype=42)
            if genericsettings.runlength_based_targets:
                ftarget = RunlengthBasedTargetValues([
                    target_runlength
                ])  # TODO: make this more variable but also consistent
            ppfigs.main(dictAlg, genericsettings.two_algorithm_file_name,
                        sortedAlgs, ftarget, outputdir,
                        genericsettings.verbose)
            plt.rcdefaults()
            print "Scaling figures done."

        if genericsettings.isFig or genericsettings.isRLDistr or genericsettings.isTab or genericsettings.isScatter or genericsettings.isScaleUp:
            print "Output data written to folder %s" % outputdir

        plt.rcdefaults()
예제 #15
0
def main(argv=None):
    r"""Post-processing COCO data of a single algorithm.

    Provided with some data, this routine outputs figure and TeX files
    in a folder needed for the compilation of the provided LaTeX templates
    for one algorithm (``*article.tex`` or ``*1*.tex``).
    The used template file needs to be edited so that the commands
    ``\bbobdatapath`` and ``\algfolder`` point to the output folder created
    by this routine.

    These output files will contain performance tables, performance
    scaling figures and empirical cumulative distribution figures. On
    subsequent executions, new files will be added to the output folder,
    overwriting existing older files in the process.

    Keyword arguments:

    *argv* -- list of strings containing options and arguments. If not
    given, sys.argv is accessed.

    *argv* should list either names of :file:`info` files or folders
    containing :file:`info` files. argv can also contain post-processed
    :file:`pickle` files generated by this routine. Furthermore, *argv*
    can begin with, in any order, facultative option flags listed below.

        -h, --help
            displays this message.
        -v, --verbose
            verbose mode, prints out all operations.
        -p, --pickle
            generates pickle post processed data files.
        -o OUTPUTDIR, --output-dir=OUTPUTDIR
            changes the default output directory (:file:`ppdata`) to
            :file:`OUTPUTDIR`.
        --crafting-effort=VALUE
            sets the crafting effort to VALUE (float). Otherwise the
            default value of 0. will be used.
        --noise-free, --noisy
            processes only part of the data.
        --settings=SETTINGS
            changes the style of the output figures and tables. At the
            moment the only differences are  in the colors of the output
            figures. SETTINGS can be either "grayscale", "color" or
            "black-white". The default setting is "color".
        --tab-only, --fig-only, --rld-only, --los-only
            these options can be used to output respectively the TeX
            tables, convergence and ERTs graphs figures, run length
            distribution figures, ERT loss ratio figures only. A
            combination of any two of these options results in no
            output.
        --conv
            if this option is chosen, additionally convergence plots
            for each function and algorithm are generated.
        --rld-single-fcts
            generate also runlength distribution figures for each
            single function.
        --expensive
            runlength-based f-target values and fixed display limits,
            useful with comparatively small budgets. By default the
            setting is based on the budget used in the data.
        --not-expensive
            expensive setting off. 
        --svg
            generate also the svg figures which are used in html files 
        --runlength-based
            runlength-based f-target values, such that the
            "level of difficulty" is similar for all functions. 

    Exceptions raised:

    *Usage* -- Gives back a usage message.

    Examples:

    * Calling the rungeneric1.py interface from the command line::

        $ python bbob_pproc/rungeneric1.py -v experiment1

      will post-process the folder experiment1 and all its containing
      data, base on the .info files found in the folder. The result will
      appear in the default output folder. The -v option adds verbosity. ::

        $ python bbob_pproc/rungeneric1.py -o exp2 experiment2/*.info

      This will execute the post-processing on the info files found in
      :file:`experiment2`. The result will be located in the alternative
      location :file:`exp2`.

    * Loading this package and calling the main from the command line
      (requires that the path to this package is in python search path)::

        $ python -m bbob_pproc.rungeneric1 -h

      This will print out this help message.

    * From the python interpreter (requires that the path to this
      package is in python search path)::

        >> import bbob_pproc as bb
        >> bb.rungeneric1.main('-o outputfolder folder1'.split())

      This will execute the post-processing on the index files found in
      :file:`folder1`. The ``-o`` option changes the output folder from
      the default to :file:`outputfolder`.

    """

    if argv is None:
        argv = sys.argv[1:]
        # The zero-th input argument which is the name of the calling script is
        # disregarded.

    if 1 < 3:
        opts, args = getopt.getopt(argv, genericsettings.shortoptlist, genericsettings.longoptlist)
        if 11 < 3:
            try:
                opts, args = getopt.getopt(argv, genericsettings.shortoptlist, genericsettings.longoptlist)
            except getopt.error, msg:
                raise Usage(msg)

        if not (args) and not "--help" in argv and not "h" in argv:
            print "not enough input arguments given"
            print "cave: the following options also need an argument:"
            print [o for o in genericsettings.longoptlist if o[-1] == "="]
            print "options given:"
            print opts
            print "try --help for help"
            sys.exit()

        # Process options
        outputdir = genericsettings.outputdir
        for o, a in opts:
            if o in ("-v", "--verbose"):
                genericsettings.verbose = True
            elif o in ("-h", "--help"):
                usage()
                sys.exit()
            elif o in ("-p", "--pickle"):
                genericsettings.isPickled = True
            elif o in ("-o", "--output-dir"):
                outputdir = a
            elif o == "--noisy":
                genericsettings.isNoisy = True
            elif o == "--noise-free":
                genericsettings.isNoiseFree = True
            # The next 4 are for testing purpose
            elif o == "--tab-only":
                genericsettings.isFig = False
                genericsettings.isRLDistr = False
                genericsettings.isLogLoss = False
            elif o == "--fig-only":
                genericsettings.isTab = False
                genericsettings.isRLDistr = False
                genericsettings.isLogLoss = False
            elif o == "--rld-only":
                genericsettings.isTab = False
                genericsettings.isFig = False
                genericsettings.isLogLoss = False
            elif o == "--los-only":
                genericsettings.isTab = False
                genericsettings.isFig = False
                genericsettings.isRLDistr = False
            elif o == "--crafting-effort":
                try:
                    genericsettings.inputCrE = float(a)
                except ValueError:
                    raise Usage("Expect a valid float for flag crafting-effort.")
            elif o == "--settings":
                genericsettings.inputsettings = a
            elif o == "--conv":
                genericsettings.isConv = True
            elif o == "--rld-single-fcts":
                genericsettings.isRldOnSingleFcts = True
            elif o == "--runlength-based":
                genericsettings.runlength_based_targets = True
            elif o == "--expensive":
                genericsettings.isExpensive = True  # comprises runlength-based
            elif o == "--not-expensive":
                genericsettings.isExpensive = False
            elif o == "--svg":
                genericsettings.generate_svg_files = True
            elif o == "--sca-only":
                warnings.warn("option --sca-only will have no effect with rungeneric1.py")
            else:
                assert False, "unhandled option"

        # from bbob_pproc import bbob2010 as inset # input settings
        if genericsettings.inputsettings == "color":
            from bbob_pproc import genericsettings as inset  # input settings
        elif genericsettings.inputsettings == "grayscale":
            from bbob_pproc import grayscalesettings as inset  # input settings
        elif genericsettings.inputsettings == "black-white":
            from bbob_pproc import bwsettings as inset  # input settings
        else:
            txt = (
                "Settings: %s is not an appropriate " % genericsettings.inputsettings
                + 'argument for input flag "--settings".'
            )
            raise Usage(txt)

        if 11 < 3:
            from bbob_pproc import config  # input settings

            config.config(False)
            import imp

            # import testbedsettings as testbedsettings # input settings
            try:
                fp, pathname, description = imp.find_module("testbedsettings")
                testbedsettings = imp.load_module("testbedsettings", fp, pathname, description)
            finally:
                fp.close()

        if not genericsettings.verbose:
            warnings.simplefilter("module")
            # warnings.simplefilter('ignore')

        # get directory name if outputdir is a archive file
        algfolder = findfiles.get_output_directory_subfolder(args[0])
        outputdir = os.path.join(outputdir, algfolder)

        print ("Post-processing (1): will generate output " + "data in folder %s" % outputdir)
        print "  this might take several minutes."

        filelist = list()
        for i in args:
            i = i.strip()
            if os.path.isdir(i):
                filelist.extend(findfiles.main(i, genericsettings.verbose))
            elif os.path.isfile(i):
                filelist.append(i)
            else:
                txt = "Input file or folder %s could not be found." % i
                print txt
                raise Usage(txt)
        dsList = DataSetList(filelist, genericsettings.verbose)

        if not dsList:
            raise Usage("Nothing to do: post-processing stopped.")

        if genericsettings.isNoisy and not genericsettings.isNoiseFree:
            dsList = dsList.dictByNoise().get("nzall", DataSetList())
        if genericsettings.isNoiseFree and not genericsettings.isNoisy:
            dsList = dsList.dictByNoise().get("noiselessall", DataSetList())

        # compute maxfuneval values
        dict_max_fun_evals = {}
        for ds in dsList:
            dict_max_fun_evals[ds.dim] = np.max((dict_max_fun_evals.setdefault(ds.dim, 0), float(np.max(ds.maxevals))))

        from bbob_pproc import config

        config.target_values(genericsettings.isExpensive, dict_max_fun_evals)
        config.config(dsList.isBiobjective())

        if genericsettings.verbose:
            for i in dsList:
                if dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) != inset.instancesOfInterest:
                    warnings.warn(
                        "The data of %s do not list " % (i) + "the correct instances " + "of function F%d." % (i.funcId)
                    )

        dictAlg = dsList.dictByAlg()

        if len(dictAlg) > 1:
            warnings.warn("Data with multiple algId %s " % str(dictAlg.keys()) + "will be processed together.")
            # TODO: in this case, all is well as long as for a given problem
            # (given dimension and function) there is a single instance of
            # DataSet associated. If there are more than one, the first one only
            # will be considered... which is probably not what one would expect.
            # TODO: put some errors where this case would be a problem.
            # raise Usage?

        if genericsettings.isFig or genericsettings.isTab or genericsettings.isRLDistr or genericsettings.isLogLoss:
            if not os.path.exists(outputdir):
                os.makedirs(outputdir)
                if genericsettings.verbose:
                    print "Folder %s was created." % (outputdir)

        if genericsettings.isPickled:
            dsList.pickle(verbose=genericsettings.verbose)

        if genericsettings.isConv:
            ppconverrorbars.main(dictAlg, outputdir, genericsettings.verbose)

        if genericsettings.isFig:
            print "Scaling figures...",
            sys.stdout.flush()
            # ERT/dim vs dim.
            plt.rc("axes", **inset.rcaxeslarger)
            plt.rc("xtick", **inset.rcticklarger)
            plt.rc("ytick", **inset.rcticklarger)
            plt.rc("font", **inset.rcfontlarger)
            plt.rc("legend", **inset.rclegendlarger)
            plt.rc("pdf", fonttype=42)
            ppfigdim.main(dsList, ppfigdim.values_of_interest, outputdir, genericsettings.verbose)
            plt.rcdefaults()
            print_done()

        plt.rc("axes", **inset.rcaxes)
        plt.rc("xtick", **inset.rctick)
        plt.rc("ytick", **inset.rctick)
        plt.rc("font", **inset.rcfont)
        plt.rc("legend", **inset.rclegend)
        plt.rc("pdf", fonttype=42)

        if genericsettings.isTab:
            print "TeX tables...",
            sys.stdout.flush()
            dictNoise = dsList.dictByNoise()
            for noise, sliceNoise in dictNoise.iteritems():
                pptable.main(sliceNoise, inset.tabDimsOfInterest, outputdir, noise, genericsettings.verbose)
            print_done()

        if genericsettings.isRLDistr:
            print "ECDF graphs...",
            sys.stdout.flush()
            dictNoise = dsList.dictByNoise()
            if len(dictNoise) > 1:
                warnings.warn(
                    "Data for functions from both the noisy and "
                    "non-noisy testbeds have been found. Their "
                    'results will be mixed in the "all functions" '
                    "ECDF figures."
                )
            dictDim = dsList.dictByDim()
            for dim in inset.rldDimsOfInterest:
                try:
                    sliceDim = dictDim[dim]
                except KeyError:
                    continue

                pprldistr.main(sliceDim, True, outputdir, "all", genericsettings.verbose)
                dictNoise = sliceDim.dictByNoise()
                for noise, sliceNoise in dictNoise.iteritems():
                    pprldistr.main(sliceNoise, True, outputdir, "%s" % noise, genericsettings.verbose)
                dictFG = sliceDim.dictByFuncGroup()
                for fGroup, sliceFuncGroup in dictFG.items():
                    pprldistr.main(sliceFuncGroup, True, outputdir, "%s" % fGroup, genericsettings.verbose)

                pprldistr.fmax = None  # Resetting the max final value
                pprldistr.evalfmax = None  # Resetting the max #fevalsfactor

            if (
                genericsettings.isRldOnSingleFcts
            ):  # copy-paste from above, here for each function instead of function groups
                # ECDFs for each function
                pprldmany.all_single_functions(dictAlg, None, outputdir, genericsettings.verbose)
            print_done()

        if genericsettings.isLogLoss:
            print "ERT loss ratio figures and tables...",
            sys.stdout.flush()
            for ng, sliceNoise in dsList.dictByNoise().iteritems():
                if ng == "noiselessall":
                    testbed = "noiseless"
                elif ng == "nzall":
                    testbed = "noisy"
                txt = "Please input crafting effort value " + "for %s testbed:\n  CrE = " % testbed
                CrE = genericsettings.inputCrE
                while CrE is None:
                    try:
                        CrE = float(raw_input(txt))
                    except (SyntaxError, NameError, ValueError):
                        print "Float value required."
                dictDim = sliceNoise.dictByDim()
                for d in inset.rldDimsOfInterest:
                    try:
                        sliceDim = dictDim[d]
                    except KeyError:
                        continue
                    info = "%s" % ng
                    pplogloss.main(sliceDim, CrE, True, outputdir, info, verbose=genericsettings.verbose)
                    pplogloss.generateTable(sliceDim, CrE, outputdir, info, verbose=genericsettings.verbose)
                    for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup().iteritems():
                        info = "%s" % fGroup
                        pplogloss.main(sliceFuncGroup, CrE, True, outputdir, info, verbose=genericsettings.verbose)
                    pplogloss.evalfmax = None  # Resetting the max #fevalsfactor

            print_done()

        latex_commands_file = os.path.join(outputdir.split(os.sep)[0], "bbob_pproc_commands.tex")
        html_file = os.path.join(outputdir, genericsettings.single_algorithm_file_name + ".html")
        prepend_to_file(
            latex_commands_file, ["\\providecommand{\\bbobloglosstablecaption}[1]{", pplogloss.table_caption, "}"]
        )
        prepend_to_file(
            latex_commands_file, ["\\providecommand{\\bbobloglossfigurecaption}[1]{", pplogloss.figure_caption, "}"]
        )
        prepend_to_file(
            latex_commands_file,
            [
                "\\providecommand{\\bbobpprldistrlegend}[1]{",
                pprldistr.caption_single(
                    np.max([val / dim for dim, val in dict_max_fun_evals.iteritems()])
                ),  # depends on the config setting, should depend on maxfevals
                "}",
            ],
        )
        replace_in_file(
            html_file,
            r"TOBEREPLACED",
            "D, ".join([str(i) for i in pprldistr.single_runlength_factors[:6]]) + "D,&hellip;",
        )
        prepend_to_file(
            latex_commands_file, ["\\providecommand{\\bbobppfigdimlegend}[1]{", ppfigdim.scaling_figure_caption(), "}"]
        )
        prepend_to_file(latex_commands_file, ["\\providecommand{\\bbobpptablecaption}[1]{", pptable.table_caption, "}"])
        prepend_to_file(latex_commands_file, ["\\providecommand{\\algfolder}{" + algfolder + "/}"])
        prepend_to_file(
            latex_commands_file,
            [
                "\\providecommand{\\algname}{"
                + (str_to_latex(strip_pathname1(args[0])) if len(args) == 1 else str_to_latex(dsList[0].algId))
                + "{}}"
            ],
        )
        if genericsettings.isFig or genericsettings.isTab or genericsettings.isRLDistr or genericsettings.isLogLoss:
            print "Output data written to folder %s" % outputdir

        plt.rcdefaults()
예제 #16
0
def main(argv=None):
    """Generates some outputs from BBOB experiment data sets of two algorithms.

    Provided with some data, this routine outputs figure and TeX files in the
    folder 'cmp2data' needed for the compilation of the latex document
    templateBBOBcmparticle.tex. These output files will contain performance
    tables, performance scaling figures, scatter plot figures and empirical
    cumulative distribution figures. On subsequent executions, new files will
    be added to the output directory, overwriting existing files in the
    process.

    Keyword arguments:
    argv -- list of strings containing options and arguments. If not given,
    sys.argv is accessed.

    argv must list folders containing BBOB data files. Each of these folders
    should correspond to the data of ONE algorithm.

    Furthermore, argv can begin with, in any order, facultative option flags
    listed below.

        -h, --help

            display this message

        -v, --verbose

            verbose mode, prints out operations. When not in verbose mode, no
            output is to be expected, except for errors.

        -o, --output-dir OUTPUTDIR

            change the default output directory ('cmp2data') to OUTPUTDIR

        --noise-free, --noisy

            restrain the post-processing to part of the data set only. Actually
            quicken the post-processing since it loads only part of the pickle
            files.

        --fig-only, --rld-only, --tab-only, --sca-only

            these options can be used to output respectively the ERT graphs
            figures, run length distribution figures or the comparison tables
            scatter plot figures only. Any combination of these options results
            in no output.

    Exceptions raised:
    Usage -- Gives back a usage message.

    Examples:

    * Calling the runcomp2.py interface from the command line:

        $ python bbob_pproc/runcomp2.py -v Alg0-baseline Alg1-of-interest

    will post-process the data from folders Alg0-baseline and Alg1-of-interest,
    the former containing data for the reference algorithm (zero-th) and the
    latter data for the algorithm of concern (first). The results will be
    output in folder cmp2data. The -v option adds verbosity.

    * From the python interactive shell (requires that the path to this
      package is in python search path):

        >>> from bbob_pproc import runcomp2
        >>> runcomp2.main('-o outputfolder PSO DEPSO'.split())

    This will execute the post-processing on the data found in folder
    PSO and DEPSO. The -o option changes the output folder from the default
    cmp2data to outputfolder.

    """

    if argv is None:
        argv = sys.argv[1:]
        # The zero-th input argument which is the name of the calling script is
        # disregarded.

    try:

        try:
            opts, args = getopt.getopt(argv, "hvo:", [
                "help", "output-dir", "noisy", "noise-free", "fig-only",
                "rld-only", "tab-only", "sca-only", "verbose"
            ])
        except getopt.error, msg:
            raise Usage(msg)

        if not (args):
            usage()
            sys.exit()

        isfigure = True
        isrldistr = True
        istable = True
        isscatter = True
        isNoisy = False
        isNoiseFree = False  # Discern noisy and noisefree data?
        verbose = False
        outputdir = 'cmp2data'

        #Process options
        for o, a in opts:
            if o in ("-v", "--verbose"):
                verbose = True
            elif o in ("-h", "--help"):
                usage()
                sys.exit()
            elif o in ("-o", "--output-dir"):
                outputdir = a
            elif o == "--fig-only":
                isrldistr = False
                istable = False
                isscatter = False
            elif o == "--rld-only":
                isfigure = False
                istable = False
                isscatter = False
            elif o == "--tab-only":
                isfigure = False
                isrldistr = False
                isscatter = False
            elif o == "--sca-only":
                isfigure = False
                isrldistr = False
                istable = False
            elif o == "--noisy":
                isNoisy = True
            elif o == "--noise-free":
                isNoiseFree = True
            else:
                assert False, "unhandled option"

        if (not verbose):
            warnings.simplefilter('ignore')

        print("BBOB Post-processing: will generate comparison " +
              "data in folder %s" % outputdir)
        print "  this might take several minutes."

        dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)

        if not dsList:
            sys.exit()

        for i in dictAlg:
            if isNoisy and not isNoiseFree:
                dictAlg[i] = dictAlg[i].dictByNoise().get(
                    'nzall', DataSetList())
            if isNoiseFree and not isNoisy:
                dictAlg[i] = dictAlg[i].dictByNoise().get(
                    'noiselessall', DataSetList())

        for i in dsList:
            if not i.dim in (2, 3, 5, 10, 20):
                continue

            #### The following lines are BBOB 2009 checking.###################
            # Deterministic algorithms
            #if i.algId in ('Original DIRECT', ):
            #tmpInstancesOfInterest = instancesOfInterestDet
            #else:
            #tmpInstancesOfInterest = instancesOfInterest
            #if ((dict((j, i.itrials.count(j)) for j in set(i.itrials)) <
            #tmpInstancesOfInterest) and
            #(dict((j, i.itrials.count(j)) for j in set(i.itrials)) <
            #instancesOfInterest2010)):
            #warnings.warn('The data of %s do not list ' %(i) +
            #'the correct instances ' +
            #'of function F%d or the ' %(i.funcId) +
            #'correct number of trials for each.')
            ###################################################################

            if (dict((j, i.itrials.count(j))
                     for j in set(i.itrials)) < instancesOfInterest2010):
                warnings.warn('The data of %s do not list ' % (i) +
                              'the correct instances ' + 'of function F%d.' %
                              (i.funcId))

        if len(sortedAlgs) < 2:
            raise Usage('Expect data from two different algorithms, could ' +
                        'only find one.')
        elif len(sortedAlgs) > 2:
            #raise Usage('Expect data from two different algorithms, found ' +
            #            'more than two.')
            warnings.warn('Data from folders: %s ' % (sortedAlgs) +
                          'were found, the first two will be processed.')

        # Group by algorithm
        dsList0 = dictAlg[sortedAlgs[0]]
        if not dsList0:
            raise Usage('Could not find data for algorithm %s.' %
                        (sortedAlgs[0]))
        #set_trace()

        dsList1 = dictAlg[sortedAlgs[1]]
        if not dsList1:
            raise Usage('Could not find data for algorithm %s.' %
                        (sortedAlgs[0]))

        tmppath0, alg0name = os.path.split(sortedAlgs[0].rstrip(os.sep))
        tmppath1, alg1name = os.path.split(sortedAlgs[1].rstrip(os.sep))
        #Trick for having different algorithm names in the tables...
        #Does not really work.
        #while alg0name == alg1name:
        #    tmppath0, alg0name = os.path.split(tmppath0)
        #    tmppath1, alg1name = os.path.split(tmppath1)
        #
        #    if not tmppath0 and not tmppath1:
        #        break
        #    else:
        #        if not tmppath0:
        #            tmppath0 = alg0name
        #        if not tmppath1:
        #            tmppath1 = alg1name
        #assert alg0name != alg1name
        # should not be a problem, these are only used in the tables.
        for i in dsList0:
            i.algId = alg0name
        for i in dsList1:
            i.algId = alg1name

        #for i, entry in enumerate(sortedAlgs): #Nota: key is sortedAlgs
        #print "Alg%d is: %s" % (i, entry)

        if isfigure or isrldistr or istable:
            if not os.path.exists(outputdir):
                os.mkdir(outputdir)
                if verbose:
                    print 'Folder %s was created.' % (outputdir)

        dictFN0 = dsList0.dictByNoise()
        dictFN1 = dsList1.dictByNoise()
        k0 = set(dictFN0.keys())
        k1 = set(dictFN1.keys())
        symdiff = k1 ^ k0
        if symdiff:  # symmetric difference
            tmpdict = {}
            for i, noisegrp in enumerate(symdiff):
                if noisegrp == 'nzall':
                    tmp = 'noisy'
                elif noisegrp == 'noiselessall':
                    tmp = 'noiseless'

                if dictFN0.has_key(noisegrp):
                    tmp2 = sortedAlgs[0]
                elif dictFN1.has_key(noisegrp):
                    tmp2 = sortedAlgs[1]

                tmpdict.setdefault(tmp2, []).append(tmp)

            txt = []
            for i, j in tmpdict.iteritems():
                txt.append('Only input folder %s lists %s data.' %
                           (i, ' and '.join(j)))
            raise Usage('Data Mismatch: \n  ' + ' '.join(txt) +
                        '\nTry using --noise-free or --noisy flags.')

        if isfigure:
            ppfig2.main(dsList0, dsList1, 1e-8, outputdir, verbose)
            print "log ERT1/ERT0 vs target function values done."

        if isrldistr:
            if len(dictFN0) > 1 or len(dictFN1) > 1:
                warnings.warn('Data for functions from both the noisy and ' +
                              'non-noisy testbeds have been found. Their ' +
                              'results will be mixed in the "all functions" ' +
                              'ECDF figures.')

            dictDim0 = dsList0.dictByDim()
            dictDim1 = dsList1.dictByDim()

            for dim in set(dictDim0.keys()) | set(dictDim1.keys()):
                if dim in rldDimsOfInterest:
                    try:
                        pprldistr2.main2(dictDim0[dim], dictDim1[dim],
                                         rldValsOfInterest, outputdir,
                                         'dim%02dall' % dim, verbose)
                    except KeyError:
                        warnings.warn('Could not find some data in %d-D.' %
                                      (dim))
                        continue

                    dictFG0 = dictDim0[dim].dictByFuncGroup()
                    dictFG1 = dictDim1[dim].dictByFuncGroup()

                    for fGroup in set(dictFG0.keys()) | set(dictFG1.keys()):
                        pprldistr2.main2(dictFG0[fGroup], dictFG1[fGroup],
                                         rldValsOfInterest, outputdir,
                                         'dim%02d%s' % (dim, fGroup), verbose)

                    dictFN0 = dictDim0[dim].dictByNoise()
                    dictFN1 = dictDim1[dim].dictByNoise()

                    for fGroup in set(dictFN0.keys()) | set(dictFN1.keys()):
                        pprldistr2.main2(dictFN0[fGroup], dictFN1[fGroup],
                                         rldValsOfInterest, outputdir,
                                         'dim%02d%s' % (dim, fGroup), verbose)
            print "ECDF absolute target graphs done."

            #for dim in set(dictDim0.keys()) | set(dictDim1.keys()):
            #if dim in rldDimsOfInterest:
            #try:

            #pprldistr2.main(dictDim0[dim], dictDim1[dim], None,
            #True, outputdir, 'dim%02dall' % dim,
            #verbose)
            #except KeyError:
            #warnings.warn('Could not find some data in %d-D.'
            #% (dim))
            #continue

            #dictFG0 = dictDim0[dim].dictByFuncGroup()
            #dictFG1 = dictDim1[dim].dictByFuncGroup()

            #for fGroup in set(dictFG0.keys()) | set(dictFG1.keys()):
            #pprldistr2.main(dictFG0[fGroup], dictFG1[fGroup], None,
            #True, outputdir,
            #'dim%02d%s' % (dim, fGroup), verbose)

            #dictFN0 = dictDim0[dim].dictByNoise()
            #dictFN1 = dictDim1[dim].dictByNoise()
            #for fGroup in set(dictFN0.keys()) | set(dictFN1.keys()):
            #pprldistr2.main(dictFN0[fGroup], dictFN1[fGroup],
            #None, True, outputdir,
            #'dim%02d%s' % (dim, fGroup), verbose)

            #print "ECDF relative target graphs done."

            for dim in set(dictDim0.keys()) | set(dictDim1.keys()):
                pprldistr.fmax = None  #Resetting the max final value
                pprldistr.evalfmax = None  #Resetting the max #fevalsfactor
                if dim in rldDimsOfInterest:
                    try:
                        pprldistr.comp(dictDim0[dim], dictDim1[dim],
                                       rldValsOfInterest, True, outputdir,
                                       'dim%02dall' % dim, verbose)
                    except KeyError:
                        warnings.warn('Could not find some data in %d-D.' %
                                      (dim))
                        continue

                    dictFG0 = dictDim0[dim].dictByFuncGroup()
                    dictFG1 = dictDim1[dim].dictByFuncGroup()

                    for fGroup in set(dictFG0.keys()) | set(dictFG1.keys()):
                        pprldistr.comp(dictFG0[fGroup], dictFG1[fGroup],
                                       rldValsOfInterest, True, outputdir,
                                       'dim%02d%s' % (dim, fGroup), verbose)

                    dictFN0 = dictDim0[dim].dictByNoise()
                    dictFN1 = dictDim1[dim].dictByNoise()
                    for fGroup in set(dictFN0.keys()) | set(dictFN1.keys()):
                        pprldistr.comp(dictFN0[fGroup], dictFN1[fGroup],
                                       rldValsOfInterest, True, outputdir,
                                       'dim%02d%s' % (dim, fGroup), verbose)

            print "ECDF dashed-solid graphs done."

        if istable:
            dictFN0 = dsList0.dictByNoise()
            dictFN1 = dsList1.dictByNoise()

            for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
                pptable2.mainnew(dictFN0[fGroup], dictFN1[fGroup],
                                 tabDimsOfInterest, outputdir, '%s' % (fGroup),
                                 verbose)

            #pptable2.main2(dsList0, dsList1, tabDimsOfInterest, outputdir,
            #               verbose=verbose)

        if isscatter:
            ppscatter.main(dsList0, dsList1, outputdir, verbose=verbose)

        if isfigure or isrldistr or istable or isscatter:
            print "Output data written to folder %s." % outputdir