def main(argv=None):
r"""Routine for post-processing COCO data from two algorithms.
Provided with some data, this routine outputs figure and TeX files
in a folder needed for the compilation of latex document
:file:`template2XXX.tex` or :file:`noisytemplate2XXX.tex`, where
:file:`XXX` is either :file:`ecj` or :file:`generic`. The template
file needs to be edited so that the command ``\bbobdatapath`` points
to the output folder.
These output files will contain performance tables, performance
scaling figures and empirical cumulative distribution figures. On
subsequent executions, new files will be added to the output folder,
overwriting existing older files in the process.
Keyword arguments:
*argv* -- list of strings containing options and arguments. If not
given, sys.argv is accessed.
*argv* must list folders containing BBOB data files. Each of these
folders should correspond to the data of ONE algorithm.
Furthermore, argv can begin with, in any order, facultative option
flags listed below.
-h, --help
displays this message.
-v, --verbose
verbose mode, prints out operations.
-o OUTPUTDIR, --output-dir=OUTPUTDIR
changes the default output directory (:file:`ppdata`) to
:file:`OUTPUTDIR`
--settings=SETTING
changes the style of the output figures and tables. At the
moment only the only differences are in the colors of the
output figures. SETTING can be either "grayscale", "color"
or "black-white". The default setting is "color".
--fig-only, --rld-only, --tab-only, --sca-only
these options can be used to output respectively the ERT
graphs figures, run length distribution figures or the
comparison tables scatter plot figures only. Any combination
of these options results in no output.
--conv
if this option is chosen, additionally convergence
plots for each function and algorithm are generated.
--rld-single-fcts
generate also runlength distribution figures for each
single function.
--expensive
runlength-based f-target values and fixed display limits,
useful with comparatively small budgets. By default the
setting is based on the budget used in the data.
--not-expensive
expensive setting off.
Exceptions raised:
*Usage* -- Gives back a usage message.
Examples:
* Calling the rungeneric2.py interface from the command line::
$ python bbob_pproc/rungeneric2.py -v Alg0-baseline Alg1-of-interest
will post-process the data from folders :file:`Alg0-baseline` and
:file:`Alg1-of-interest`, the former containing data for the
reference algorithm (zero-th) and the latter data for the
algorithm of concern (first). The results will be output in the
default output folder. The ``-v`` option adds verbosity.
* From the python interpreter (requires that the path to this
package is in python search path)::
>> import bbob_pproc as bb
>> bb.rungeneric2.main('-o outputfolder PSO DEPSO'.split())
This will execute the post-processing on the data found in folder
:file:`PSO` and :file:`DEPSO`. The ``-o`` option changes the output
folder from the default to :file:`outputfolder`.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
global ftarget
try:
try:
opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not (args):
usage()
sys.exit()
isfigure = True
isrldistr = True
istable = True
isscatter = True
isscaleup = True
verbose = False
outputdir = 'ppdata'
inputsettings = 'color'
isConv= False
isRLbased = None # allows automatic choice
isExpensive = None
#Process options
for o, a in opts:
if o in ("-v","--verbose"):
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-o", "--output-dir"):
outputdir = a
#elif o in ("-s", "--style"):
# inputsettings = a
elif o == "--fig-only":
isrldistr = False
istable = False
isscatter = False
elif o == "--rld-only":
isfigure = False
istable = False
isscatter = False
elif o == "--tab-only":
isfigure = False
isrldistr = False
isscatter = False
elif o == "--sca-only":
isfigure = False
isrldistr = False
istable = False
elif o == "--settings":
inputsettings = a
elif o == "--conv":
isConv = True
elif o == "--runlength-based":
isRLbased = True
elif o == "--expensive":
isExpensive = True # comprises runlength-based
elif o == "--not-expensive":
isExpensive = False
else:
assert False, "unhandled option"
# from bbob_pproc import bbob2010 as inset # input settings
if inputsettings == "color":
from bbob_pproc import genericsettings as inset # input settings
config.config()
elif inputsettings == "grayscale": # probably very much obsolete
from bbob_pproc import grayscalesettings as inset # input settings
elif inputsettings == "black-white": # probably very much obsolete
from bbob_pproc import bwsettings as inset # input settings
else:
txt = ('Settings: %s is not an appropriate ' % inputsettings
+ 'argument for input flag "--settings".')
raise Usage(txt)
if (not verbose):
warnings.simplefilter('module')
warnings.simplefilter('ignore')
print ("Post-processing will generate comparison " +
"data in folder %s" % outputdir)
print " this might take several minutes."
dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)
if 1 < 3 and len(sortedAlgs) != 2:
raise ValueError('rungeneric2.py needs exactly two algorithms to compare, found: '
+ str(sortedAlgs)
+ '\n use rungeneric.py (or rungenericmany.py) to compare more algorithms. ')
if not dsList:
sys.exit()
for i in dictAlg:
dictAlg[i] = dictAlg[i].dictAll().get('All', DataSetList())
for i in dsList:
if i.dim not in genericsettings.dimensions_to_display:
continue
if (dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) <
inset.instancesOfInterest):
warnings.warn('The data of %s do not list ' %(i) +
'the correct instances ' +
'of function F%s.' %(i.funcId))
if len(sortedAlgs) < 2:
raise Usage('Expect data from two different algorithms, could ' +
'only find one.')
elif len(sortedAlgs) > 2:
warnings.warn('Data from folders: %s ' % (sortedAlgs) +
'were found, the first two will be processed.')
# Group by algorithm
dsList0 = dictAlg[sortedAlgs[0]]
if not dsList0:
raise Usage('Could not find data for algorithm %s.' % (sortedAlgs[0]))
dsList1 = dictAlg[sortedAlgs[1]]
if not dsList1:
raise Usage('Could not find data for algorithm %s.' % (sortedAlgs[0]))
# get the name of each algorithm from the input arguments
tmppath0, alg0name = os.path.split(sortedAlgs[0].rstrip(os.sep))
tmppath1, alg1name = os.path.split(sortedAlgs[1].rstrip(os.sep))
for i in dsList0:
i.algId = alg0name
for i in dsList1:
i.algId = alg1name
# compute maxfuneval values
dict_max_fun_evals1 = {}
dict_max_fun_evals2 = {}
for ds in dsList0:
dict_max_fun_evals1[ds.dim] = np.max((dict_max_fun_evals1.setdefault(ds.dim, 0), float(np.max(ds.maxevals))))
for ds in dsList1:
dict_max_fun_evals2[ds.dim] = np.max((dict_max_fun_evals2.setdefault(ds.dim, 0), float(np.max(ds.maxevals))))
if isRLbased is not None:
genericsettings.runlength_based_targets = isRLbased
config.target_values(isExpensive, {1: min([max([val/dim for dim, val in dict_max_fun_evals1.iteritems()]),
max([val/dim for dim, val in dict_max_fun_evals2.iteritems()])]
)})
config.config()
######################### Post-processing #############################
if isfigure or isrldistr or istable or isscatter:
if not os.path.exists(outputdir):
os.mkdir(outputdir)
if verbose:
print 'Folder %s was created.' % (outputdir)
# prepend the algorithm name command to the tex-command file
abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
lines = []
for i, alg in enumerate(args):
lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' +
str_to_latex(strip_pathname(alg)) + '}')
prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
lines, 1000,
'bbob_proc_commands.tex truncated, consider removing the file before the text run'
)
# # Check whether both input arguments list noisy and noise-free data
# dictFN0 = dsList0.dictAll()
# dictFN1 = dsList1.dictAll()
# k0 = set(dictFN0.keys())
# k1 = set(dictFN1.keys())
# symdiff = k1 ^ k0 # symmetric difference
# if symdiff:
# tmpdict = {}
# for i, noisegrp in enumerate(symdiff):
# if noisegrp == 'nzall':
# tmp = 'noisy'
# elif noisegrp == 'noiselessall':
# tmp = 'noiseless'
#
# if dictFN0.has_key(noisegrp):
# tmp2 = sortedAlgs[0]
# elif dictFN1.has_key(noisegrp):
# tmp2 = sortedAlgs[1]
#
# tmpdict.setdefault(tmp2, []).append(tmp)
#
# txt = []
# for i, j in tmpdict.iteritems():
# txt.append('Only input folder %s lists %s data.'
# % (i, ' and '.join(j)))
# raise Usage('Data Mismatch: \n ' + ' '.join(txt)
# + '\nTry using --noise-free or --noisy flags.')
if isfigure:
plt.rc("axes", **inset.rcaxeslarger)
plt.rc("xtick", **inset.rcticklarger)
plt.rc("ytick", **inset.rcticklarger)
plt.rc("font", **inset.rcfontlarger)
plt.rc("legend", **inset.rclegendlarger)
ppfig2.main(dsList0, dsList1, ppfig2_ftarget, outputdir, verbose)
print "log ERT1/ERT0 vs target function values done."
plt.rc("axes", **inset.rcaxes)
plt.rc("xtick", **inset.rctick)
plt.rc("ytick", **inset.rctick)
plt.rc("font", **inset.rcfont)
plt.rc("legend", **inset.rclegend)
if isrldistr:
# if len(dictFN0) > 1 or len(dictFN1) > 1:
# warnings.warn('Data for functions from both the noisy and ' +
# 'non-noisy testbeds have been found. Their ' +
# 'results will be mixed in the "all functions" ' +
# 'ECDF figures.')
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
# ECDFs of ERT ratios
for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
if dim in inset.rldDimsOfInterest:
# ECDF for all functions altogether
try:
pprldistr2.main(dictDim0[dim], dictDim1[dim], dim,
inset.rldValsOfInterest,
outputdir, '%02dD_all' % dim, verbose)
except KeyError:
warnings.warn('Could not find some data in %d-D.'
% (dim))
continue
# ECDFs per function groups
dictFG0 = dictDim0[dim].dictByFuncGroup()
dictFG1 = dictDim1[dim].dictByFuncGroup()
for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
pprldistr2.main(dictFG1[fGroup], dictFG0[fGroup], dim,
inset.rldValsOfInterest,
outputdir, '%02dD_%s' % (dim, fGroup),
verbose)
# ECDFs for all
dictFN0 = dictDim0[dim].dictAll()
dictFN1 = dictDim1[dim].dictAll()
for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
pprldistr2.main(dictFN1[fGroup], dictFN0[fGroup], dim,
inset.rldValsOfInterest,
outputdir, '%02dD_%s' % (dim, fGroup),
verbose)
prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
['\\providecommand{\\bbobpprldistrlegendtwo}[1]{',
pprldistr.caption_two(), # depends on the config setting, should depend on maxfevals
'}'
])
print "ECDF runlength ratio graphs done."
for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
pprldistr.fmax = None #Resetting the max final value
pprldistr.evalfmax = None #Resetting the max #fevalsfactor
# ECDFs of all functions altogether
if dim in inset.rldDimsOfInterest:
try:
pprldistr.comp(dictDim1[dim], dictDim0[dim],
inset.rldValsOfInterest, # TODO: let rldVals... possibly be RL-based targets
True,
outputdir, 'all', verbose)
except KeyError:
warnings.warn('Could not find some data in %d-D.'
% (dim))
continue
# ECDFs per function groups
dictFG0 = dictDim0[dim].dictByFuncGroup()
dictFG1 = dictDim1[dim].dictByFuncGroup()
for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
pprldistr.comp(dictFG1[fGroup], dictFG0[fGroup],
inset.rldValsOfInterest,
True, outputdir,
'%s' % fGroup, verbose)
# ECDFs for all
dictFN0 = dictDim0[dim].dictAll()
dictFN1 = dictDim1[dim].dictAll()
for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
pprldistr.comp(dictFN1[fGroup], dictFN0[fGroup],
inset.rldValsOfInterest,
True, outputdir,
'%s' % fGroup, verbose)
print "ECDF runlength graphs done."
if isConv:
ppconverrorbars.main(dictAlg,outputdir,verbose)
##TODO pptable
# if istable:
# dictNG0 = dsList0.dictAll()
# dictNG1 = dsList1.dictAll()
#
# for nGroup in set(dictNG0.keys()) & set(dictNG1.keys()):
# # split table in as many as necessary
# dictFunc0 = dictNG0[nGroup].dictByFunc()
# dictFunc1 = dictNG1[nGroup].dictByFunc()
# funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys()))
# if len(funcs) > 24:
# funcs.sort()
# nbgroups = int(numpy.ceil(len(funcs)/24.))
# def split_seq(seq, nbgroups):
# newseq = []
# splitsize = 1.0/nbgroups*len(seq)
# for i in range(nbgroups):
# newseq.append(seq[int(round(i*splitsize)):int(round((i+1)*splitsize))])
# return newseq
#
# groups = split_seq(funcs, nbgroups)
# # merge
# group0 = []
# group1 = []
# for i, g in enumerate(groups):
# tmp0 = DataSetList()
# tmp1 = DataSetList()
# for f in g:
# tmp0.extend(dictFunc0[f])
# tmp1.extend(dictFunc1[f])
# group0.append(tmp0)
# group1.append(tmp1)
# for i, g in enumerate(zip(group0, group1)):
# pptable2.main(g[0], g[1], inset.tabDimsOfInterest,
# outputdir, '%s%d' % (nGroup, i), verbose)
# else:
# if 11 < 3: # future handling:
# dictFunc0 = dsList0.dictByFunc()
# dictFunc1 = dsList1.dictByFunc()
# funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys()))
# funcs.sort()
# # nbgroups = int(numpy.ceil(len(funcs)/testbedsettings.numberOfFunctions))
# # pptable2.main(dsList0, dsList1,
# # testbedsettings.tabDimsOfInterest, outputdir,
# # '%s' % (testbedsettings.testbedshortname), verbose)
# else:
# pptable2.main(dictNG0[nGroup], dictNG1[nGroup],
# inset.tabDimsOfInterest, outputdir,
# '%s' % (nGroup), verbose)
#
# if isinstance(pptable2.targetsOfInterest, pproc.RunlengthBasedTargetValues):
# prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
# ['\\providecommand{\\bbobpptablestwolegend}[1]{',
# pptable2.table_caption_expensive,
# '}'
# ])
# else:
# prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
# ['\\providecommand{\\bbobpptablestwolegend}[1]{',
# pptable2.table_caption,
# '}'
# ])
# print "Tables done."
if isscatter:
if genericsettings.runlength_based_targets:
ppscatter.targets = ppscatter.runlength_based_targets
ppscatter.main(dsList1, dsList0, outputdir, verbose=verbose)
prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
['\\providecommand{\\bbobppscatterlegend}[1]{',
ppscatter.figure_caption(),
'}'
])
print "Scatter plots done."
if isscaleup:
plt.rc("axes", labelsize=20, titlesize=24)
plt.rc("xtick", labelsize=20)
plt.rc("ytick", labelsize=20)
plt.rc("font", size=20)
plt.rc("legend", fontsize=20)
if genericsettings.runlength_based_targets:
ftarget = RunlengthBasedTargetValues([target_runlength]) # TODO: make this more variable but also consistent
ppfigs.main(dictAlg, sortedAlgs, ftarget, outputdir, verbose)
plt.rcdefaults()
print "Scaling figures done."
if isfigure or isrldistr or istable or isscatter or isscaleup:
print "Output data written to folder %s" % outputdir
plt.rcdefaults()
def main(argv=None):
r"""Post-processing COCO data of a single algorithm.
Provided with some data, this routine outputs figure and TeX files
in a folder needed for the compilation of latex document
:file:`template1XXX.tex` or :file:`noisytemplate1XXX.tex`, where
:file:`XXX` is either :file:`ecj` or :file:`generic`. The template
file needs to be edited so that the commands
``\bbobdatapath`` and ``\algfolder`` point to the output folder.
These output files will contain performance tables, performance
scaling figures and empirical cumulative distribution figures. On
subsequent executions, new files will be added to the output folder,
overwriting existing older files in the process.
Keyword arguments:
*argv* -- list of strings containing options and arguments. If not
given, sys.argv is accessed.
*argv* should list either names of :file:`info` files or folders
containing :file:`info` files. argv can also contain post-processed
:file:`pickle` files generated by this routine. Furthermore, *argv*
can begin with, in any order, facultative option flags listed below.
-h, --help
displays this message.
-v, --verbose
verbose mode, prints out all operations.
-p, --pickle
generates pickle post processed data files.
-o OUTPUTDIR, --output-dir=OUTPUTDIR
changes the default output directory (:file:`ppdata`) to
:file:`OUTPUTDIR`.
--crafting-effort=VALUE
sets the crafting effort to VALUE (float). Otherwise the
default value of 0. will be used.
--noise-free, --noisy
processes only part of the data.
--settings=SETTINGS
changes the style of the output figures and tables. At the
moment the only differences are in the colors of the output
figures. SETTINGS can be either "grayscale", "color" or
"black-white". The default setting is "color".
--tab-only, --fig-only, --rld-only, --los-only
these options can be used to output respectively the TeX
tables, convergence and ERTs graphs figures, run length
distribution figures, ERT loss ratio figures only. A
combination of any two of these options results in no
output.
--conv
if this option is chosen, additionally convergence plots
for each function and algorithm are generated.
--expensive
runlength-based f-target values and fixed display limits,
useful with comparatively small budgets. By default the
setting is based on the budget used in the data.
--not-expensive
expensive setting off.
--runlength-based
runlength-based f-target values, such that the
"level of difficulty" is similar for all functions.
Exceptions raised:
*Usage* -- Gives back a usage message.
Examples:
* Calling the rungeneric1.py interface from the command line::
$ python bbob_pproc/rungeneric1.py -v experiment1
will post-process the folder experiment1 and all its containing
data, base on the .info files found in the folder. The result will
appear in the default output folder. The -v option adds verbosity. ::
$ python bbob_pproc/rungeneric1.py -o exp2 experiment2/*.info
This will execute the post-processing on the info files found in
:file:`experiment2`. The result will be located in the alternative
location :file:`exp2`.
* Loading this package and calling the main from the command line
(requires that the path to this package is in python search path)::
$ python -m bbob_pproc.rungeneric1 -h
This will print out this help message.
* From the python interpreter (requires that the path to this
package is in python search path)::
>> import bbob_pproc as bb
>> bb.rungeneric1.main('-o outputfolder folder1'.split())
This will execute the post-processing on the index files found in
:file:`folder1`. The ``-o`` option changes the output folder from
the default to :file:`outputfolder`.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
if 1 < 3:
opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
if 11 < 3:
try:
opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not (args) and not '--help' in argv and not 'h' in argv:
print 'not enough input arguments given'
print 'cave: the following options also need an argument:'
print[o for o in longoptlist if o[-1] == '=']
print 'options given:'
print opts
print 'try --help for help'
sys.exit()
inputCrE = 0.
isfigure = True
istab = True
isrldistr = True
islogloss = True
isPostProcessed = False
isPickled = False
verbose = False
outputdir = 'ppdata'
isNoisy = False
isNoiseFree = False
inputsettings = 'color'
isConv = False
isRLbased = None # allows automatic choice
isExpensive = None
# Process options
for o, a in opts:
if o in ("-v", "--verbose"):
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-p", "--pickle"):
isPickled = True
elif o in ("-o", "--output-dir"):
outputdir = a
elif o == "--noisy":
isNoisy = True
elif o == "--noise-free":
isNoiseFree = True
# The next 4 are for testing purpose
elif o == "--tab-only":
isfigure = False
isrldistr = False
islogloss = False
elif o == "--fig-only":
istab = False
isrldistr = False
islogloss = False
elif o == "--rld-only":
istab = False
isfigure = False
islogloss = False
elif o == "--los-only":
istab = False
isfigure = False
isrldistr = False
elif o == "--crafting-effort":
try:
inputCrE = float(a)
except ValueError:
raise Usage(
'Expect a valid float for flag crafting-effort.')
elif o == "--settings":
inputsettings = a
elif o == "--conv":
isConv = True
elif o == "--runlength-based":
isRLbased = True
elif o == "--expensive":
isExpensive = True # comprises runlength-based
elif o == "--not-expensive":
isExpensive = False
else:
assert False, "unhandled option"
# from bbob_pproc import bbob2010 as inset # input settings
if inputsettings == "color":
from bbob_pproc import genericsettings as inset # input settings
elif inputsettings == "grayscale":
from bbob_pproc import grayscalesettings as inset # input settings
elif inputsettings == "black-white":
from bbob_pproc import bwsettings as inset # input settings
else:
txt = ('Settings: %s is not an appropriate ' % inputsettings +
'argument for input flag "--settings".')
raise Usage(txt)
if 11 < 3:
from bbob_pproc import config # input settings
config.config()
import imp
# import testbedsettings as testbedsettings # input settings
try:
fp, pathname, description = imp.find_module("testbedsettings")
testbedsettings = imp.load_module("testbedsettings", fp,
pathname, description)
finally:
fp.close()
if (not verbose):
warnings.simplefilter('module')
# warnings.simplefilter('ignore')
print("Post-processing (1): will generate output " +
"data in folder %s" % outputdir)
print " this might take several minutes."
filelist = list()
for i in args:
i = i.strip()
if os.path.isdir(i):
filelist.extend(findfiles.main(i, verbose))
elif os.path.isfile(i):
filelist.append(i)
else:
txt = 'Input file or folder %s could not be found.' % i
print txt
raise Usage(txt)
dsList = DataSetList(filelist, verbose)
if not dsList:
raise Usage("Nothing to do: post-processing stopped.")
if isNoisy and not isNoiseFree:
dsList = dsList.dictByNoise().get('nzall', DataSetList())
if isNoiseFree and not isNoisy:
dsList = dsList.dictByNoise().get('noiselessall', DataSetList())
# compute maxfuneval values
dict_max_fun_evals = {}
for ds in dsList:
dict_max_fun_evals[ds.dim] = np.max(
(dict_max_fun_evals.setdefault(ds.dim,
0), float(np.max(ds.maxevals))))
if isRLbased is not None:
genericsettings.runlength_based_targets = isRLbased
from bbob_pproc import config
config.target_values(isExpensive, dict_max_fun_evals)
config.config()
if (verbose):
for i in dsList:
if (dict((j, i.instancenumbers.count(j))
for j in set(i.instancenumbers)) !=
inset.instancesOfInterest):
warnings.warn('The data of %s do not list ' % (i) +
'the correct instances ' +
'of function F%d.' % (i.funcId))
dictAlg = dsList.dictByAlg()
if len(dictAlg) > 1:
warnings.warn('Data with multiple algId %s ' % (dictAlg) +
'will be processed together.')
# TODO: in this case, all is well as long as for a given problem
# (given dimension and function) there is a single instance of
# DataSet associated. If there are more than one, the first one only
# will be considered... which is probably not what one would expect.
# TODO: put some errors where this case would be a problem.
# raise Usage?
if isfigure or istab or isrldistr or islogloss:
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if verbose:
print 'Folder %s was created.' % (outputdir)
if isPickled:
dsList.pickle(verbose=verbose)
if isConv:
ppconverrorbars.main(dictAlg, outputdir, verbose)
if isfigure:
print "Scaling figures...",
sys.stdout.flush()
# ERT/dim vs dim.
plt.rc("axes", **inset.rcaxeslarger)
plt.rc("xtick", **inset.rcticklarger)
plt.rc("ytick", **inset.rcticklarger)
plt.rc("font", **inset.rcfontlarger)
plt.rc("legend", **inset.rclegendlarger)
ppfigdim.main(dsList, ppfigdim.values_of_interest, outputdir,
verbose)
plt.rcdefaults()
print_done()
plt.rc("axes", **inset.rcaxes)
plt.rc("xtick", **inset.rctick)
plt.rc("ytick", **inset.rctick)
plt.rc("font", **inset.rcfont)
plt.rc("legend", **inset.rclegend)
if istab:
print "TeX tables...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pptable.main(sliceNoise, inset.tabDimsOfInterest, outputdir,
noise, verbose)
print_done()
if isrldistr:
print "ECDF graphs...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
if len(dictNoise) > 1:
warnings.warn('Data for functions from both the noisy and '
'non-noisy testbeds have been found. Their '
'results will be mixed in the "all functions" '
'ECDF figures.')
dictDim = dsList.dictByDim()
for dim in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[dim]
except KeyError:
continue
pprldistr.main(sliceDim, True, outputdir, 'all', verbose)
dictNoise = sliceDim.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pprldistr.main(sliceNoise, True, outputdir, '%s' % noise,
verbose)
dictFG = sliceDim.dictByFuncGroup()
for fGroup, sliceFuncGroup in dictFG.items():
pprldistr.main(sliceFuncGroup, True, outputdir,
'%s' % fGroup, verbose)
pprldistr.fmax = None # Resetting the max final value
pprldistr.evalfmax = None # Resetting the max #fevalsfactor
print_done()
if islogloss:
print "ERT loss ratio figures and tables...",
sys.stdout.flush()
for ng, sliceNoise in dsList.dictByNoise().iteritems():
if ng == 'noiselessall':
testbed = 'noiseless'
elif ng == 'nzall':
testbed = 'noisy'
txt = ("Please input crafting effort value " +
"for %s testbed:\n CrE = " % testbed)
CrE = inputCrE
while CrE is None:
try:
CrE = float(raw_input(txt))
except (SyntaxError, NameError, ValueError):
print "Float value required."
dictDim = sliceNoise.dictByDim()
for d in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[d]
except KeyError:
continue
info = '%s' % ng
pplogloss.main(sliceDim,
CrE,
True,
outputdir,
info,
verbose=verbose)
pplogloss.generateTable(sliceDim,
CrE,
outputdir,
info,
verbose=verbose)
for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup(
).iteritems():
info = '%s' % fGroup
pplogloss.main(sliceFuncGroup,
CrE,
True,
outputdir,
info,
verbose=verbose)
pplogloss.evalfmax = None # Resetting the max #fevalsfactor
print_done()
latex_commands_file = os.path.join(
outputdir.split(os.sep)[0], 'bbob_pproc_commands.tex')
prepend_to_file(latex_commands_file, [
'\\providecommand{\\bbobloglosstablecaption}[1]{',
pplogloss.table_caption, '}'
])
prepend_to_file(latex_commands_file, [
'\\providecommand{\\bbobloglossfigurecaption}[1]{',
pplogloss.figure_caption, '}'
])
prepend_to_file(
latex_commands_file,
[
'\\providecommand{\\bbobpprldistrlegend}[1]{',
pprldistr.caption_single(
np.max([
val / dim
for dim, val in dict_max_fun_evals.iteritems()
])
), # depends on the config setting, should depend on maxfevals
'}'
])
prepend_to_file(latex_commands_file, [
'\\providecommand{\\bbobppfigdimlegend}[1]{',
ppfigdim.scaling_figure_caption(), '}'
])
prepend_to_file(latex_commands_file, [
'\\providecommand{\\bbobpptablecaption}[1]{',
pptable.table_caption, '}'
])
prepend_to_file(latex_commands_file,
['\\providecommand{\\algfolder}{}'
]) # is overwritten in rungeneric.py
prepend_to_file(latex_commands_file, [
'\\providecommand{\\algname}{' +
(str_to_latex(strip_pathname(args[0]))
if len(args) == 1 else str_to_latex(dsList[0].algId)) + '{}}'
])
if isfigure or istab or isrldistr or islogloss:
print "Output data written to folder %s" % outputdir
plt.rcdefaults()
def plotLegend(handles, maxval):
"""Display right-side legend.
:param float maxval: rightmost x boundary
:returns: list of (ordered) labels and handles.
The figure is stopped at maxval (upper x-bound), and the graphs in
the figure are prolonged with straight lines to the right to connect
with labels of the graphs (uniformly spread out vertically). The
order of the graphs at the upper x-bound line give the order of the
labels, in case of ties, the best is the graph for which the x-value
of the first step (from the right) is smallest.
The annotation string is stripped from preceeding pathnames.
"""
reslabels = []
reshandles = []
ys = {}
lh = 0
for h in handles:
x2 = []
y2 = []
for i in h:
x2.append(plt.getp(i, "xdata"))
y2.append(plt.getp(i, "ydata"))
x2 = np.array(np.hstack(x2))
y2 = np.array(np.hstack(y2))
tmp = np.argsort(x2)
x2 = x2[tmp]
y2 = y2[tmp]
h = h[-1] # we expect the label to be in the last element of h
tmp = (x2 <= maxval)
try:
x2bis = x2[y2 < y2[tmp][-1]][-1]
except IndexError: # there is no data with a y smaller than max(y)
x2bis = 0.
ys.setdefault(y2[tmp][-1], {}).setdefault(x2bis, []).append(h)
lh += 1
if len(show_algorithms) > 0:
lh = min(lh, len(show_algorithms))
if lh <= 1:
lh = 2
fontsize = genericsettings.minmax_algorithm_fontsize[0] + np.min((1, np.exp(9-lh))) * (
genericsettings.minmax_algorithm_fontsize[-1] - genericsettings.minmax_algorithm_fontsize[0])
i = 0 # loop over the elements of ys
for j in sorted(ys.keys()):
for k in reversed(sorted(ys[j].keys())):
#enforce best ever comes last in case of equality
tmp = []
for h in ys[j][k]:
if plt.getp(h, 'label') == 'best 2009':
tmp.insert(0, h)
else:
tmp.append(h)
tmp.reverse()
ys[j][k] = tmp
for h in ys[j][k]:
if (not plt.getp(h, 'label').startswith('_line') and
(len(show_algorithms) == 0 or
plt.getp(h, 'label') in show_algorithms)):
y = 0.02 + i * 0.96/(lh-1)
tmp = {}
for attr in ('lw', 'ls', 'marker',
'markeredgewidth', 'markerfacecolor',
'markeredgecolor', 'markersize', 'zorder'):
tmp[attr] = plt.getp(h, attr)
legx = maxval**annotation_line_end_relative
if 'marker' in attr:
legx = maxval**annotation_line_end_relative
# reshandles.extend(plt_plot((maxval, legx), (j, y),
reshandles.extend(plt_plot((maxval, legx), (j, y),
color=plt.getp(h, 'markeredgecolor'), **tmp))
reshandles.append(
plt.text(maxval**(0.02 + annotation_line_end_relative), y,
toolsdivers.str_to_latex(toolsdivers.strip_pathname1(plt.getp(h, 'label'))),
horizontalalignment="left",
verticalalignment="center", size=fontsize))
reslabels.append(plt.getp(h, 'label'))
#set_trace()
i += 1
#plt.axvline(x=maxval, color='k') # Not as efficient?
reshandles.append(plt_plot((maxval, maxval), (0., 1.), color='k'))
reslabels.reverse()
plt.xlim(xmax=maxval**annotation_space_end_relative)
return reslabels, reshandles
def main(dictAlg, isBiobjective, order=None, outputdir='.', info='default',
dimension=None, verbose=True):
"""Generates a figure showing the performance of algorithms.
From a dictionary of :py:class:`DataSetList` sorted by algorithms,
generates the cumulative distribution function of the bootstrap
distribution of ERT for algorithms on multiple functions for
multiple targets altogether.
:param dict dictAlg: dictionary of :py:class:`DataSetList` instances
one instance is equivalent to one algorithm,
:param list targets: target function values
:param list order: sorted list of keys to dictAlg for plotting order
:param str outputdir: output directory
:param str info: output file name suffix
:param bool verbose: controls verbosity
"""
global x_limit # late assignment of default, because it can be set to None in config
global divide_by_dimension # not fully implemented/tested yet
if 'x_limit' not in globals() or x_limit is None:
x_limit = x_limit_default
tmp = pp.dictAlgByDim(dictAlg)
# tmp = pp.DictAlg(dictAlg).by_dim()
if len(tmp) != 1 and dimension is None:
raise ValueError('We never integrate over dimension.')
if dimension is not None:
if dimension not in tmp.keys():
raise ValueError('dimension %d not in dictAlg dimensions %s'
% (dimension, str(tmp.keys())))
tmp = {dimension: tmp[dimension]}
dim = tmp.keys()[0]
divisor = dim if divide_by_dimension else 1
algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []]
dictFunc = pp.dictAlgByFun(dictAlg)
# Collect data
# Crafting effort correction: should we consider any?
CrEperAlg = {}
for alg in algorithms_with_data:
CrE = 0.
if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL':
tmp = dictAlg[alg].dictByNoise()
assert len(tmp.keys()) == 1
if tmp.keys()[0] == 'noiselessall':
CrE = 0.5117
elif tmp.keys()[0] == 'nzall':
CrE = 0.6572
CrEperAlg[alg] = CrE
if CrE != 0.0:
print 'Crafting effort for', alg, 'is', CrE
dictData = {} # list of (ert per function) per algorithm
dictMaxEvals = {} # list of (maxevals per function) per algorithm
bestERT = [] # best ert per function
# funcsolved = [set()] * len(targets) # number of functions solved per target
xbest2009 = []
maxevalsbest2009 = []
for f, dictAlgperFunc in dictFunc.iteritems():
if function_IDs and f not in function_IDs:
continue
# print target_values((f, dim))
for j, t in enumerate(target_values((f, dim))):
# for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)):
# funcsolved[j].add(f)
for alg in algorithms_with_data:
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
try:
entry = dictAlgperFunc[alg][0] # one element per fun and per dim.
evals = entry.detEvals([t])[0]
assert entry.dim == dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
except (KeyError, IndexError):
#set_trace()
warntxt = ('Data for algorithm %s on function %d in %d-D '
% (alg, f, dim)
+ 'are missing.\n')
warnings.warn(warntxt)
dictData.setdefault(alg, []).extend(x)
dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc)
displaybest2009 = not isBiobjective #disabled until we find the bug
if displaybest2009:
#set_trace()
bestalgentries = bestalg.loadBestAlgorithm(isBiobjective)
bestalgentry = bestalgentries[(dim, f)]
bestalgevals = bestalgentry.detEvals(target_values((f, dim)))
# print bestalgevals
for j in range(len(bestalgevals[0])):
if bestalgevals[1][j]:
evals = bestalgevals[0][j]
#set_trace()
assert dim == bestalgentry.dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = bestalgentry.maxevals[bestalgevals[1][j]][np.isnan(evals)] / divisor
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
else:
x = perfprofsamplesize * [np.inf]
runlengthunsucc = []
xbest2009.extend(x)
maxevalsbest2009.extend(runlengthunsucc)
if order is None:
order = dictData.keys()
# Display data
lines = []
if displaybest2009:
args = {'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11.,
'markeredgewidth': 1.5, 'markerfacecolor': refcolor,
'markeredgecolor': refcolor, 'color': refcolor,
'label': 'best 2009', 'zorder': -1}
lines.append(plotdata(np.array(xbest2009), x_limit, maxevalsbest2009,
CrE = 0., **args))
def algname_to_label(algname, dirname=None):
"""to be extended to become generally useful"""
if isinstance(algname, (tuple, list)): # not sure this is needed
return ' '.join([str(name) for name in algname])
return str(algname)
for i, alg in enumerate(order):
try:
data = dictData[alg]
maxevals = dictMaxEvals[alg]
except KeyError:
continue
args = styles[(i) % len(styles)]
args['linewidth'] = 1.5
args['markersize'] = 12.
args['markeredgewidth'] = 1.5
args['markerfacecolor'] = 'None'
args['markeredgecolor'] = args['color']
args['label'] = algname_to_label(alg)
#args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib
#elif len(show_algorithms) > 0:
#args['color'] = 'wheat'
#args['ls'] = '-'
#args['zorder'] = -1
# plotdata calls pprldistr.plotECDF which calls ppfig.plotUnifLog... which does the work
lines.append(plotdata(np.array(data), x_limit, maxevals,
CrE=CrEperAlg[alg], **args))
labels, handles = plotLegend(lines, x_limit)
if True: # isLateXLeg:
fileName = os.path.join(outputdir,'pprldmany_%s.tex' % (info))
with open(fileName, 'w') as f:
f.write(r'\providecommand{\nperfprof}{7}')
algtocommand = {} # latex commands
for i, alg in enumerate(order):
tmp = r'\alg%sperfprof' % pptex.numtotext(i)
f.write(r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' %
(tmp, toolsdivers.str_to_latex(
toolsdivers.strip_pathname2(algname_to_label(alg)))))
algtocommand[algname_to_label(alg)] = tmp
if displaybest2009:
tmp = r'\algzeroperfprof'
f.write(r'\providecommand{%s}{best 2009}' % (tmp))
algtocommand['best 2009'] = tmp
commandnames = []
for label in labels:
commandnames.append(algtocommand[label])
# f.write(headleg)
if len(order) > 28: # latex sidepanel won't work well for more than 25 algorithms, but original labels are also clipped
f.write(r'\providecommand{\perfprofsidepanel}{\mbox{%s}\vfill\mbox{%s}}'
% (commandnames[0], commandnames[-1]))
else:
fontsize_command = r'\tiny{}' if len(order) > 19 else ''
f.write(r'\providecommand{\perfprofsidepanel}{{%s\mbox{%s}' %
(fontsize_command, commandnames[0])) # TODO: check len(labels) > 0
for i in range(1, len(labels)):
f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i])
f.write('}}\n')
# f.write(footleg)
if verbose:
print 'Wrote right-hand legend in %s' % fileName
figureName = os.path.join(outputdir,'pprldmany_%s' % (info))
#beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat)
beautify()
text = ppfig.consecutiveNumbers(sorted(dictFunc.keys()), 'f')
text += ',%d-D' % dim # TODO: this is strange when different dimensions are plotted
plt.text(0.01, 0.98, text, horizontalalignment="left",
verticalalignment="top", transform=plt.gca().transAxes)
if len(dictFunc) == 1:
plt.title(' '.join((str(dictFunc.keys()[0]),
genericsettings.current_testbed.short_names[dictFunc.keys()[0]])))
a = plt.gca()
plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative)
xticks, labels = plt.xticks()
tmp = []
for i in xticks:
tmp.append('%d' % round(np.log10(i)))
a.set_xticklabels(tmp)
if save_figure:
ppfig.saveFigure(figureName, verbose=verbose)
if len(dictFunc) == 1:
ppfig.save_single_functions_html(
os.path.join(outputdir, 'pprldmany'),
'', # algorithms names are clearly visible in the figure
add_to_names='_%02dD' %(dim),
algorithmCount=ppfig.AlgorithmCount.NON_SPECIFIED
)
if close_figure:
plt.close()
if 1 < 3:
print ("Post-processing: will generate output " +
"data in folder %s" % outputdir)
print " this might take several minutes."
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if genericsettings.verbose:
print 'Folder %s was created.' % (outputdir)
# prepend the algorithm name command to the tex-command file
lines = []
for i, alg in enumerate(args):
lines.append('\\providecommand{\\algorithm' + pptex.numtotext(i) +
'}{' + str_to_latex(strip_pathname1(alg)) + '}')
prepend_to_file(os.path.join(outputdir,
'bbob_pproc_commands.tex'),
lines, 5000,
'bbob_proc_commands.tex truncated, consider removing the file before the text run'
)
dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=genericsettings.verbose)
if not dsList:
sys.exit()
if (any(ds.isBiobjective() for ds in dsList) and any(not ds.isBiobjective() for ds in dsList)):
sys.exit()
for i in dictAlg:
def main(argv=None):
r"""Routine for post-processing COCO data from two algorithms.
Provided with some data, this routine outputs figure and TeX files
in a folder needed for the compilation of latex document
:file:`template2XXX.tex` or :file:`noisytemplate2XXX.tex`, where
:file:`XXX` is either :file:`ecj` or :file:`generic`. The template
file needs to be edited so that the command ``\bbobdatapath`` points
to the output folder.
These output files will contain performance tables, performance
scaling figures and empirical cumulative distribution figures. On
subsequent executions, new files will be added to the output folder,
overwriting existing older files in the process.
Keyword arguments:
*argv* -- list of strings containing options and arguments. If not
given, sys.argv is accessed.
*argv* must list folders containing BBOB data files. Each of these
folders should correspond to the data of ONE algorithm.
Furthermore, argv can begin with, in any order, facultative option
flags listed below.
-h, --help
displays this message.
-v, --verbose
verbose mode, prints out operations.
-o OUTPUTDIR, --output-dir=OUTPUTDIR
changes the default output directory (:file:`ppdata`) to
:file:`OUTPUTDIR`
--noise-free, --noisy
processes only part of the data.
--settings=SETTING
changes the style of the output figures and tables. At the
moment only the only differences are in the colors of the
output figures. SETTING can be either "grayscale", "color"
or "black-white". The default setting is "color".
--fig-only, --rld-only, --tab-only, --sca-only
these options can be used to output respectively the ERT
graphs figures, run length distribution figures or the
comparison tables scatter plot figures only. Any combination
of these options results in no output.
--conv
if this option is chosen, additionally convergence
plots for each function and algorithm are generated.
Exceptions raised:
*Usage* -- Gives back a usage message.
Examples:
* Calling the rungeneric2.py interface from the command line::
$ python bbob_pproc/rungeneric2.py -v Alg0-baseline Alg1-of-interest
will post-process the data from folders :file:`Alg0-baseline` and
:file:`Alg1-of-interest`, the former containing data for the
reference algorithm (zero-th) and the latter data for the
algorithm of concern (first). The results will be output in the
default output folder. The ``-v`` option adds verbosity.
* From the python interpreter (requires that the path to this
package is in python search path)::
>> import bbob_pproc as bb
>> bb.rungeneric2.main('-o outputfolder PSO DEPSO'.split())
This will execute the post-processing on the data found in folder
:file:`PSO` and :file:`DEPSO`. The ``-o`` option changes the output
folder from the default to :file:`outputfolder`.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
try:
try:
opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not (args):
usage()
sys.exit()
isfigure = True
isrldistr = True
istable = True
isscatter = True
isscaleup = True
isNoisy = False
isNoiseFree = False
verbose = False
outputdir = 'ppdata'
inputsettings = 'color'
isConv = False
#Process options
for o, a in opts:
if o in ("-v", "--verbose"):
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-o", "--output-dir"):
outputdir = a
#elif o in ("-s", "--style"):
# inputsettings = a
elif o == "--fig-only":
isrldistr = False
istable = False
isscatter = False
elif o == "--rld-only":
isfigure = False
istable = False
isscatter = False
elif o == "--tab-only":
isfigure = False
isrldistr = False
isscatter = False
elif o == "--sca-only":
isfigure = False
isrldistr = False
istable = False
elif o == "--noisy":
isNoisy = True
elif o == "--noise-free":
isNoiseFree = True
elif o == "--settings":
inputsettings = a
elif o == "--conv":
isConv = True
else:
assert False, "unhandled option"
# from bbob_pproc import bbob2010 as inset # input settings
if inputsettings == "color":
from bbob_pproc import config, genericsettings as inset # input settings
config.config()
elif inputsettings == "grayscale":
from bbob_pproc import grayscalesettings as inset # input settings
elif inputsettings == "black-white":
from bbob_pproc import bwsettings as inset # input settings
else:
txt = ('Settings: %s is not an appropriate ' % inputsettings +
'argument for input flag "--settings".')
raise Usage(txt)
if (not verbose):
warnings.simplefilter('module')
warnings.simplefilter('ignore')
print("Post-processing will generate comparison " +
"data in folder %s" % outputdir)
print " this might take several minutes."
dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)
if 1 < 3 and len(sortedAlgs) != 2:
raise ValueError(
'rungeneric2.py needs exactly two algorithms to compare, found: '
+ str(sortedAlgs) +
'\n use rungeneric.py (or rungenericmany.py) to compare more algorithms. '
)
if not dsList:
sys.exit()
for i in dictAlg:
if isNoisy and not isNoiseFree:
dictAlg[i] = dictAlg[i].dictByNoise().get(
'nzall', DataSetList())
if isNoiseFree and not isNoisy:
dictAlg[i] = dictAlg[i].dictByNoise().get(
'noiselessall', DataSetList())
for i in dsList:
if i.dim not in genericsettings.dimensions_to_display:
continue
if (dict((j, i.instancenumbers.count(j))
for j in set(i.instancenumbers)) <
inset.instancesOfInterest):
warnings.warn('The data of %s do not list ' % (i) +
'the correct instances ' + 'of function F%d.' %
(i.funcId))
if len(sortedAlgs) < 2:
raise Usage('Expect data from two different algorithms, could ' +
'only find one.')
elif len(sortedAlgs) > 2:
warnings.warn('Data from folders: %s ' % (sortedAlgs) +
'were found, the first two will be processed.')
# Group by algorithm
dsList0 = dictAlg[sortedAlgs[0]]
if not dsList0:
raise Usage('Could not find data for algorithm %s.' %
(sortedAlgs[0]))
dsList1 = dictAlg[sortedAlgs[1]]
if not dsList1:
raise Usage('Could not find data for algorithm %s.' %
(sortedAlgs[0]))
# get the name of each algorithm from the input arguments
tmppath0, alg0name = os.path.split(sortedAlgs[0].rstrip(os.sep))
tmppath1, alg1name = os.path.split(sortedAlgs[1].rstrip(os.sep))
for i in dsList0:
i.algId = alg0name
for i in dsList1:
i.algId = alg1name
######################### Post-processing #############################
if isfigure or isrldistr or istable or isscatter:
if not os.path.exists(outputdir):
os.mkdir(outputdir)
if verbose:
print 'Folder %s was created.' % (outputdir)
# prepend the algorithm name command to the tex-command file
abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
lines = []
for i, alg in enumerate(args):
lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' +
str_to_latex(strip_pathname(alg)) + '}')
prepend_to_file(
os.path.join(outputdir,
'bbob_pproc_commands.tex'), lines, 1000,
'bbob_proc_commands.tex truncated, consider removing the file before the text run'
)
# Check whether both input arguments list noisy and noise-free data
dictFN0 = dsList0.dictByNoise()
dictFN1 = dsList1.dictByNoise()
k0 = set(dictFN0.keys())
k1 = set(dictFN1.keys())
symdiff = k1 ^ k0 # symmetric difference
if symdiff:
tmpdict = {}
for i, noisegrp in enumerate(symdiff):
if noisegrp == 'nzall':
tmp = 'noisy'
elif noisegrp == 'noiselessall':
tmp = 'noiseless'
if dictFN0.has_key(noisegrp):
tmp2 = sortedAlgs[0]
elif dictFN1.has_key(noisegrp):
tmp2 = sortedAlgs[1]
tmpdict.setdefault(tmp2, []).append(tmp)
txt = []
for i, j in tmpdict.iteritems():
txt.append('Only input folder %s lists %s data.' %
(i, ' and '.join(j)))
raise Usage('Data Mismatch: \n ' + ' '.join(txt) +
'\nTry using --noise-free or --noisy flags.')
if isfigure:
plt.rc("axes", **inset.rcaxeslarger)
plt.rc("xtick", **inset.rcticklarger)
plt.rc("ytick", **inset.rcticklarger)
plt.rc("font", **inset.rcfontlarger)
plt.rc("legend", **inset.rclegendlarger)
ppfig2.main(dsList0, dsList1, ftarget, outputdir, verbose)
print "log ERT1/ERT0 vs target function values done."
plt.rc("axes", **inset.rcaxes)
plt.rc("xtick", **inset.rctick)
plt.rc("ytick", **inset.rctick)
plt.rc("font", **inset.rcfont)
plt.rc("legend", **inset.rclegend)
if isrldistr:
if len(dictFN0) > 1 or len(dictFN1) > 1:
warnings.warn('Data for functions from both the noisy and ' +
'non-noisy testbeds have been found. Their ' +
'results will be mixed in the "all functions" ' +
'ECDF figures.')
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
# ECDFs of ERT ratios
for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
if dim in inset.rldDimsOfInterest:
# ECDF for all functions altogether
try:
pprldistr2.main(dictDim0[dim], dictDim1[dim],
inset.rldValsOfInterest, outputdir,
'%02dD_all' % dim, verbose)
except KeyError:
warnings.warn('Could not find some data in %d-D.' %
(dim))
continue
# ECDFs per function groups
dictFG0 = dictDim0[dim].dictByFuncGroup()
dictFG1 = dictDim1[dim].dictByFuncGroup()
for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
pprldistr2.main(dictFG1[fGroup], dictFG0[fGroup],
inset.rldValsOfInterest, outputdir,
'%02dD_%s' % (dim, fGroup), verbose)
# ECDFs per noise groups
dictFN0 = dictDim0[dim].dictByNoise()
dictFN1 = dictDim1[dim].dictByNoise()
for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
pprldistr2.main(dictFN1[fGroup], dictFN0[fGroup],
inset.rldValsOfInterest, outputdir,
'%02dD_%s' % (dim, fGroup), verbose)
print "ECDF runlength ratio graphs done."
for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
pprldistr.fmax = None #Resetting the max final value
pprldistr.evalfmax = None #Resetting the max #fevalsfactor
# ECDFs of all functions altogether
if dim in inset.rldDimsOfInterest:
try:
pprldistr.comp(
dictDim1[dim], dictDim0[dim],
inset.rldValsOfInterest if isinstance(
inset.rldValsOfInterest, TargetValues) else
TargetValues(inset.rldValsOfInterest), True,
outputdir, 'all', verbose)
except KeyError:
warnings.warn('Could not find some data in %d-D.' %
(dim))
continue
# ECDFs per function groups
dictFG0 = dictDim0[dim].dictByFuncGroup()
dictFG1 = dictDim1[dim].dictByFuncGroup()
for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
pprldistr.comp(
dictFG1[fGroup], dictFG0[fGroup],
inset.rldValsOfInterest if isinstance(
inset.rldValsOfInterest, TargetValues) else
TargetValues(inset.rldValsOfInterest), True,
outputdir, '%s' % fGroup, verbose)
# ECDFs per noise groups
dictFN0 = dictDim0[dim].dictByNoise()
dictFN1 = dictDim1[dim].dictByNoise()
for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
pprldistr.comp(
dictFN1[fGroup], dictFN0[fGroup],
inset.rldValsOfInterest if isinstance(
inset.rldValsOfInterest, TargetValues) else
TargetValues(inset.rldValsOfInterest), True,
outputdir, '%s' % fGroup, verbose)
print "ECDF runlength graphs done."
if isConv:
ppconverrorbars.main(dictAlg, outputdir, verbose)
if istable:
dictNG0 = dsList0.dictByNoise()
dictNG1 = dsList1.dictByNoise()
for nGroup in set(dictNG0.keys()) & set(dictNG1.keys()):
# split table in as many as necessary
dictFunc0 = dictNG0[nGroup].dictByFunc()
dictFunc1 = dictNG1[nGroup].dictByFunc()
funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys()))
if len(funcs) > 24:
funcs.sort()
nbgroups = int(numpy.ceil(len(funcs) / 24.))
def split_seq(seq, nbgroups):
newseq = []
splitsize = 1.0 / nbgroups * len(seq)
for i in range(nbgroups):
newseq.append(
seq[int(round(i * splitsize)
):int(round((i + 1) * splitsize))])
return newseq
groups = split_seq(funcs, nbgroups)
# merge
group0 = []
group1 = []
for i, g in enumerate(groups):
tmp0 = DataSetList()
tmp1 = DataSetList()
for f in g:
tmp0.extend(dictFunc0[f])
tmp1.extend(dictFunc1[f])
group0.append(tmp0)
group1.append(tmp1)
for i, g in enumerate(zip(group0, group1)):
pptable2.main(g[0], g[1], inset.tabDimsOfInterest,
outputdir, '%s%d' % (nGroup, i), verbose)
else:
if 11 < 3: # future handling:
dictFunc0 = dsList0.dictByFunc()
dictFunc1 = dsList1.dictByFunc()
funcs = list(
set(dictFunc0.keys()) & set(dictFunc1.keys()))
funcs.sort()
# nbgroups = int(numpy.ceil(len(funcs)/testbedsettings.numberOfFunctions))
# pptable2.main(dsList0, dsList1,
# testbedsettings.tabDimsOfInterest, outputdir,
# '%s' % (testbedsettings.testbedshortname), verbose)
else:
pptable2.main(dictNG0[nGroup], dictNG1[nGroup],
inset.tabDimsOfInterest, outputdir,
'%s' % (nGroup), verbose)
prepend_to_file(os.path.join(
outputdir, 'bbob_pproc_commands.tex'), [
'\\providecommand{\\bbobpptablestwolegend}[1]{',
pptable2.figure_legend, '}'
])
print "Tables done."
if isscatter:
ppscatter.main(dsList0, dsList1, outputdir, verbose=verbose)
prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
[
'\\providecommand{\\bbobppscatterlegend}[1]{',
ppscatter.figure_legend, '}'
])
print "Scatter plots done."
if isscaleup:
plt.rc("axes", labelsize=20, titlesize=24)
plt.rc("xtick", labelsize=20)
plt.rc("ytick", labelsize=20)
plt.rc("font", size=20)
plt.rc("legend", fontsize=20)
ppfigs.main(dictAlg, sortedAlgs, ftarget, outputdir, verbose)
plt.rcdefaults()
print "Scaling figures done."
if isfigure or isrldistr or istable or isscatter or isscaleup:
print "Output data written to folder %s" % outputdir
plt.rcdefaults()
if 1 < 3:
print("Post-processing: will generate output " +
"data in folder %s" % outputdir)
print " this might take several minutes."
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if genericsettings.verbose:
print 'Folder %s was created.' % (outputdir)
# prepend the algorithm name command to the tex-command file
lines = []
for i, alg in enumerate(args):
lines.append('\\providecommand{\\algorithm' + pptex.numtotext(i) +
'}{' + str_to_latex(strip_pathname1(alg)) + '}')
prepend_to_file(
os.path.join(outputdir, 'bbob_pproc_commands.tex'), lines, 5000,
'bbob_proc_commands.tex truncated, consider removing the file before the text run'
)
dsList, sortedAlgs, dictAlg = processInputArgs(
args, verbose=genericsettings.verbose)
if not dsList:
sys.exit()
for i in dictAlg:
if genericsettings.isNoisy and not genericsettings.isNoiseFree:
dictAlg[i] = dictAlg[i].dictByNoise().get(
'nzall', DataSetList())
def plotLegend(handles, maxval):
"""Display right-side legend.
:param float maxval: rightmost x boundary
:returns: list of (ordered) labels and handles.
The figure is stopped at maxval (upper x-bound), and the graphs in
the figure are prolonged with straight lines to the right to connect
with labels of the graphs (uniformly spread out vertically). The
order of the graphs at the upper x-bound line give the order of the
labels, in case of ties, the best is the graph for which the x-value
of the first step (from the right) is smallest.
The annotation string is stripped from preceeding pathnames.
"""
reslabels = []
reshandles = []
ys = {}
lh = 0
for h in handles:
x2 = []
y2 = []
for i in h:
x2.append(plt.getp(i, "xdata"))
y2.append(plt.getp(i, "ydata"))
x2 = np.array(np.hstack(x2))
y2 = np.array(np.hstack(y2))
tmp = np.argsort(x2)
x2 = x2[tmp]
y2 = y2[tmp]
h = h[-1] # we expect the label to be in the last element of h
tmp = (x2 <= maxval)
try:
x2bis = x2[y2 < y2[tmp][-1]][-1]
except IndexError: # there is no data with a y smaller than max(y)
x2bis = 0.
ys.setdefault(y2[tmp][-1], {}).setdefault(x2bis, []).append(h)
lh += 1
if len(show_algorithms) > 0:
lh = min(lh, len(show_algorithms))
if lh <= 1:
lh = 2
fontsize = genericsettings.minmax_algorithm_fontsize[0] + np.min(
(1, np.exp(9 - lh))) * (genericsettings.minmax_algorithm_fontsize[-1] -
genericsettings.minmax_algorithm_fontsize[0])
i = 0 # loop over the elements of ys
for j in sorted(ys.keys()):
for k in reversed(sorted(ys[j].keys())):
#enforce best ever comes last in case of equality
tmp = []
for h in ys[j][k]:
if plt.getp(h, 'label') == 'best 2009':
tmp.insert(0, h)
else:
tmp.append(h)
tmp.reverse()
ys[j][k] = tmp
for h in ys[j][k]:
if (not plt.getp(h, 'label').startswith('_line')
and (len(show_algorithms) == 0
or plt.getp(h, 'label') in show_algorithms)):
y = 0.02 + i * 0.96 / (lh - 1)
tmp = {}
for attr in ('lw', 'ls', 'marker', 'markeredgewidth',
'markerfacecolor', 'markeredgecolor',
'markersize', 'zorder'):
tmp[attr] = plt.getp(h, attr)
legx = maxval**annotation_line_end_relative
if 'marker' in attr:
legx = maxval**annotation_line_end_relative
# reshandles.extend(plt_plot((maxval, legx), (j, y),
reshandles.extend(
plt_plot((maxval, legx), (j, y),
color=plt.getp(h, 'markeredgecolor'),
**tmp))
reshandles.append(
plt.text(maxval**(0.02 + annotation_line_end_relative),
y,
toolsdivers.str_to_latex(
toolsdivers.strip_pathname1(
plt.getp(h, 'label'))),
horizontalalignment="left",
verticalalignment="center",
size=fontsize))
reslabels.append(plt.getp(h, 'label'))
#set_trace()
i += 1
#plt.axvline(x=maxval, color='k') # Not as efficient?
reshandles.append(plt_plot((maxval, maxval), (0., 1.), color='k'))
reslabels.reverse()
plt.xlim(xmax=maxval**annotation_space_end_relative)
return reslabels, reshandles
def main(argv=None):
r"""Post-processing COCO data of a single algorithm.
Provided with some data, this routine outputs figure and TeX files
in a folder needed for the compilation of latex document
:file:`template1XXX.tex` or :file:`noisytemplate1XXX.tex`, where
:file:`XXX` is either :file:`ecj` or :file:`generic`. The template
file needs to be edited so that the commands
``\bbobdatapath`` and ``\algfolder`` point to the output folder.
These output files will contain performance tables, performance
scaling figures and empirical cumulative distribution figures. On
subsequent executions, new files will be added to the output folder,
overwriting existing older files in the process.
Keyword arguments:
*argv* -- list of strings containing options and arguments. If not
given, sys.argv is accessed.
*argv* should list either names of :file:`info` files or folders
containing :file:`info` files. argv can also contain post-processed
:file:`pickle` files generated by this routine. Furthermore, *argv*
can begin with, in any order, facultative option flags listed below.
-h, --help
displays this message.
-v, --verbose
verbose mode, prints out all operations.
-p, --pickle
generates pickle post processed data files.
-o OUTPUTDIR, --output-dir=OUTPUTDIR
changes the default output directory (:file:`ppdata`) to
:file:`OUTPUTDIR`.
--crafting-effort=VALUE
sets the crafting effort to VALUE (float). Otherwise the
default value of 0. will be used.
--noise-free, --noisy
processes only part of the data.
--settings=SETTINGS
changes the style of the output figures and tables. At the
moment the only differences are in the colors of the output
figures. SETTINGS can be either "grayscale", "color" or
"black-white". The default setting is "color".
--tab-only, --fig-only, --rld-only, --los-only
these options can be used to output respectively the TeX
tables, convergence and ERTs graphs figures, run length
distribution figures, ERT loss ratio figures only. A
combination of any two of these options results in no
output.
--conv
if this option is chosen, additionally convergence plots
for each function and algorithm are generated.
--expensive
runlength-based f-target values and fixed display limits,
useful with comparatively small budgets. By default the
setting is based on the budget used in the data.
--not-expensive
expensive setting off.
--runlength-based
runlength-based f-target values, such that the
"level of difficulty" is similar for all functions.
Exceptions raised:
*Usage* -- Gives back a usage message.
Examples:
* Calling the rungeneric1.py interface from the command line::
$ python bbob_pproc/rungeneric1.py -v experiment1
will post-process the folder experiment1 and all its containing
data, base on the .info files found in the folder. The result will
appear in the default output folder. The -v option adds verbosity. ::
$ python bbob_pproc/rungeneric1.py -o exp2 experiment2/*.info
This will execute the post-processing on the info files found in
:file:`experiment2`. The result will be located in the alternative
location :file:`exp2`.
* Loading this package and calling the main from the command line
(requires that the path to this package is in python search path)::
$ python -m bbob_pproc.rungeneric1 -h
This will print out this help message.
* From the python interpreter (requires that the path to this
package is in python search path)::
>> import bbob_pproc as bb
>> bb.rungeneric1.main('-o outputfolder folder1'.split())
This will execute the post-processing on the index files found in
:file:`folder1`. The ``-o`` option changes the output folder from
the default to :file:`outputfolder`.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
if 1 < 3:
opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
if 11 < 3:
try:
opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not (args) and not '--help' in argv and not 'h' in argv:
print 'not enough input arguments given'
print 'cave: the following options also need an argument:'
print [o for o in longoptlist if o[-1] == '=']
print 'options given:'
print opts
print 'try --help for help'
sys.exit()
inputCrE = 0.
isfigure = True
istab = True
isrldistr = True
islogloss = True
isPostProcessed = False
isPickled = False
verbose = False
outputdir = 'ppdata'
isNoisy = False
isNoiseFree = False
inputsettings = 'color'
isConv = False
isRLbased = None # allows automatic choice
isExpensive = None
# Process options
for o, a in opts:
if o in ("-v", "--verbose"):
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-p", "--pickle"):
isPickled = True
elif o in ("-o", "--output-dir"):
outputdir = a
elif o == "--noisy":
isNoisy = True
elif o == "--noise-free":
isNoiseFree = True
# The next 4 are for testing purpose
elif o == "--tab-only":
isfigure = False
isrldistr = False
islogloss = False
elif o == "--fig-only":
istab = False
isrldistr = False
islogloss = False
elif o == "--rld-only":
istab = False
isfigure = False
islogloss = False
elif o == "--los-only":
istab = False
isfigure = False
isrldistr = False
elif o == "--crafting-effort":
try:
inputCrE = float(a)
except ValueError:
raise Usage('Expect a valid float for flag crafting-effort.')
elif o == "--settings":
inputsettings = a
elif o == "--conv":
isConv = True
elif o == "--runlength-based":
isRLbased = True
elif o == "--expensive":
isExpensive = True # comprises runlength-based
elif o == "--not-expensive":
isExpensive = False
else:
assert False, "unhandled option"
# from bbob_pproc import bbob2010 as inset # input settings
if inputsettings == "color":
from bbob_pproc import genericsettings as inset # input settings
elif inputsettings == "grayscale":
from bbob_pproc import grayscalesettings as inset # input settings
elif inputsettings == "black-white":
from bbob_pproc import bwsettings as inset # input settings
else:
txt = ('Settings: %s is not an appropriate ' % inputsettings
+ 'argument for input flag "--settings".')
raise Usage(txt)
if 11 < 3:
from bbob_pproc import config # input settings
config.config()
import imp
# import testbedsettings as testbedsettings # input settings
try:
fp, pathname, description = imp.find_module("testbedsettings")
testbedsettings = imp.load_module("testbedsettings", fp, pathname, description)
finally:
fp.close()
if (not verbose):
warnings.simplefilter('module')
# warnings.simplefilter('ignore')
print ("Post-processing (1): will generate output " +
"data in folder %s" % outputdir)
print " this might take several minutes."
filelist = list()
for i in args:
i = i.strip()
if os.path.isdir(i):
filelist.extend(findfiles.main(i, verbose))
elif os.path.isfile(i):
filelist.append(i)
else:
txt = 'Input file or folder %s could not be found.' % i
print txt
raise Usage(txt)
dsList = DataSetList(filelist, verbose)
if not dsList:
raise Usage("Nothing to do: post-processing stopped.")
if isNoisy and not isNoiseFree:
dsList = dsList.dictByNoise().get('nzall', DataSetList())
if isNoiseFree and not isNoisy:
dsList = dsList.dictByNoise().get('noiselessall', DataSetList())
# compute maxfuneval values
dict_max_fun_evals = {}
for ds in dsList:
dict_max_fun_evals[ds.dim] = np.max((dict_max_fun_evals.setdefault(ds.dim, 0), float(np.max(ds.maxevals))))
if isRLbased is not None:
genericsettings.runlength_based_targets = isRLbased
from bbob_pproc import config
config.target_values(isExpensive, dict_max_fun_evals)
config.config()
if (verbose):
for i in dsList:
if (dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) !=
inset.instancesOfInterest):
warnings.warn('The data of %s do not list ' % (i) +
'the correct instances ' +
'of function F%d.' % (i.funcId))
dictAlg = dsList.dictByAlg()
if len(dictAlg) > 1:
warnings.warn('Data with multiple algId %s ' % (dictAlg) +
'will be processed together.')
# TODO: in this case, all is well as long as for a given problem
# (given dimension and function) there is a single instance of
# DataSet associated. If there are more than one, the first one only
# will be considered... which is probably not what one would expect.
# TODO: put some errors where this case would be a problem.
# raise Usage?
if isfigure or istab or isrldistr or islogloss:
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if verbose:
print 'Folder %s was created.' % (outputdir)
if isPickled:
dsList.pickle(verbose=verbose)
if isConv:
ppconverrorbars.main(dictAlg, outputdir, verbose)
if isfigure:
print "Scaling figures...",
sys.stdout.flush()
# ERT/dim vs dim.
plt.rc("axes", **inset.rcaxeslarger)
plt.rc("xtick", **inset.rcticklarger)
plt.rc("ytick", **inset.rcticklarger)
plt.rc("font", **inset.rcfontlarger)
plt.rc("legend", **inset.rclegendlarger)
ppfigdim.main(dsList, ppfigdim.values_of_interest, outputdir, verbose)
plt.rcdefaults()
print_done()
plt.rc("axes", **inset.rcaxes)
plt.rc("xtick", **inset.rctick)
plt.rc("ytick", **inset.rctick)
plt.rc("font", **inset.rcfont)
plt.rc("legend", **inset.rclegend)
if istab:
print "TeX tables...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pptable.main(sliceNoise, inset.tabDimsOfInterest, outputdir,
noise, verbose)
print_done()
if isrldistr:
print "ECDF graphs...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
if len(dictNoise) > 1:
warnings.warn('Data for functions from both the noisy and '
'non-noisy testbeds have been found. Their '
'results will be mixed in the "all functions" '
'ECDF figures.')
dictDim = dsList.dictByDim()
for dim in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[dim]
except KeyError:
continue
pprldistr.main(sliceDim, True,
outputdir, 'all', verbose)
dictNoise = sliceDim.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pprldistr.main(sliceNoise, True,
outputdir, '%s' % noise, verbose)
dictFG = sliceDim.dictByFuncGroup()
for fGroup, sliceFuncGroup in dictFG.items():
pprldistr.main(sliceFuncGroup,
True, outputdir, '%s' % fGroup, verbose)
pprldistr.fmax = None # Resetting the max final value
pprldistr.evalfmax = None # Resetting the max #fevalsfactor
print_done()
if islogloss:
print "ERT loss ratio figures and tables...",
sys.stdout.flush()
for ng, sliceNoise in dsList.dictByNoise().iteritems():
if ng == 'noiselessall':
testbed = 'noiseless'
elif ng == 'nzall':
testbed = 'noisy'
txt = ("Please input crafting effort value "
+ "for %s testbed:\n CrE = " % testbed)
CrE = inputCrE
while CrE is None:
try:
CrE = float(raw_input(txt))
except (SyntaxError, NameError, ValueError):
print "Float value required."
dictDim = sliceNoise.dictByDim()
for d in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[d]
except KeyError:
continue
info = '%s' % ng
pplogloss.main(sliceDim, CrE, True, outputdir, info,
verbose=verbose)
pplogloss.generateTable(sliceDim, CrE, outputdir, info,
verbose=verbose)
for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup().iteritems():
info = '%s' % fGroup
pplogloss.main(sliceFuncGroup, CrE, True, outputdir, info,
verbose=verbose)
pplogloss.evalfmax = None # Resetting the max #fevalsfactor
print_done()
latex_commands_file = os.path.join(outputdir.split(os.sep)[0], 'bbob_pproc_commands.tex')
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobloglosstablecaption}[1]{',
pplogloss.table_caption, '}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobloglossfigurecaption}[1]{',
pplogloss.figure_caption, '}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobpprldistrlegend}[1]{',
pprldistr.caption_single(np.max([ val / dim for dim, val in dict_max_fun_evals.iteritems()])), # depends on the config setting, should depend on maxfevals
'}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobppfigdimlegend}[1]{',
ppfigdim.scaling_figure_caption(),
'}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobpptablecaption}[1]{',
pptable.table_caption,
'}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\algfolder}{}']) # is overwritten in rungeneric.py
prepend_to_file(latex_commands_file,
['\\providecommand{\\algname}{' +
(str_to_latex(strip_pathname(args[0])) if len(args) == 1 else str_to_latex(dsList[0].algId)) + '{}}'])
if isfigure or istab or isrldistr or islogloss:
print "Output data written to folder %s" % outputdir
plt.rcdefaults()
return 2
if 1 < 3:
print ("Post-processing: will generate output " + "data in folder %s" % outputdir)
print " this might take several minutes."
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if genericsettings.verbose:
print "Folder %s was created." % (outputdir)
# prepend the algorithm name command to the tex-command file
lines = []
for i, alg in enumerate(args):
lines.append(
"\\providecommand{\\algorithm" + pptex.numtotext(i) + "}{" + str_to_latex(strip_pathname1(alg)) + "}"
)
prepend_to_file(
os.path.join(outputdir, "bbob_pproc_commands.tex"),
lines,
5000,
"bbob_proc_commands.tex truncated, consider removing the file before the text run",
)
dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=genericsettings.verbose)
if not dsList:
sys.exit()
for i in dictAlg:
if genericsettings.isNoisy and not genericsettings.isNoiseFree:
def main(dictAlg,
sortedAlgs,
targets,
outputdir='.',
verbose=True,
function_targets_line=True): # [1, 13, 101]
"""Generate one table per func with results of multiple algorithms."""
"""Difference with the first version:
* numbers aligned using the decimal separator
* premices for dispersion measure
* significance test against best algorithm
* table width...
"""
# TODO: method is long, terrible to read, split if possible
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
# Sort data per dimension and function
dictData = {}
dsListperAlg = list(dictAlg[i] for i in sortedAlgs)
for n, entries in enumerate(dsListperAlg):
tmpdictdim = entries.dictByDim()
for d in tmpdictdim:
tmpdictfun = tmpdictdim[d].dictByFunc()
for f in tmpdictfun:
dictData.setdefault((d, f), {})[n] = tmpdictfun[f]
nbtests = len(dictData)
for df in dictData:
# Generate one table per df
# best 2009
refalgentry = bestalg.bestalgentries2009[df]
refalgert = refalgentry.detERT(targets)
refalgevals = (refalgentry.detEvals((targetf, ))[0][0])
refalgnbruns = len(refalgevals)
refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False)
# Process the data
# The following variables will be lists of elements each corresponding
# to an algorithm
algnames = []
#algdata = []
algerts = []
algevals = []
algdisp = []
algnbsucc = []
algnbruns = []
algmedmaxevals = []
algmedfinalfunvals = []
algtestres = []
algentries = []
for n in sorted(dictData[df].keys()):
entries = dictData[df][n]
# the number of datasets for a given dimension and function (df)
# should be strictly 1. TODO: find a way to warn
# TODO: do this checking before... why wasn't it triggered by ppperprof?
if len(entries) > 1:
txt = ("There is more than a single entry associated with "
"folder %s on %d-D f%d." %
(sortedAlgs[n], df[0], df[1]))
raise Exception(txt)
entry = entries[0]
algentries.append(entry)
algnames.append(sortedAlgs[n])
evals = entry.detEvals(targets)
#tmpdata = []
tmpdisp = []
tmpert = []
for i, e in enumerate(evals):
succ = (numpy.isnan(e) == False)
e[succ == False] = entry.maxevals[succ == False]
ert = toolsstats.sp(e, issuccessful=succ)[0]
#tmpdata.append(ert/refalgert[i])
if succ.any():
tmp = toolsstats.drawSP(e[succ],
entry.maxevals[succ == False],
[10, 50, 90],
samplesize=samplesize)[0]
tmpdisp.append((tmp[-1] - tmp[0]) / 2.)
else:
tmpdisp.append(numpy.nan)
tmpert.append(ert)
algerts.append(tmpert)
algevals.append(evals)
#algdata.append(tmpdata)
algdisp.append(tmpdisp)
algmedmaxevals.append(numpy.median(entry.maxevals))
algmedfinalfunvals.append(numpy.median(entry.finalfunvals))
#algmedmaxevals.append(numpy.median(entry.maxevals)/df[0])
#algmedfinalfunvals.append(numpy.median(entry.finalfunvals))
algtestres.append(significancetest(refalgentry, entry, targets))
# determine success probability for Df = 1e-8
e = entry.detEvals((targetf, ))[0]
algnbsucc.append(numpy.sum(numpy.isnan(e) == False))
algnbruns.append(len(e))
# Process over all data
# find best values...
nalgs = len(dictData[df])
maxRank = 1 + numpy.floor(
0.14 * nalgs) # number of algs to be displayed in bold
isBoldArray = [] # Point out the best values
algfinaldata = [
] # Store median function values/median number of function evaluations
tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank)
for i, erts in enumerate(algerts):
tmp = []
for j, ert in enumerate(erts): # algi targetj
tmp.append(i in tmptop[j] or
(nalgs > 7 and algerts[i][j] <= 3. * refalgert[j]))
isBoldArray.append(tmp)
algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i]))
# significance test of best given algorithm against all others
best_alg_idx = numpy.array(algerts).argsort(0)[
0, :] # indexed by target index
significance_versus_others = significance_all_best_vs_other(
algentries, targets, best_alg_idx)[0]
# Create the table
table = []
spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % (
len(targets)
) # in case StrLeft not working: replaced c@{} with l@{ }
spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % (
len(targets)
) # in case StrLeft not working: replaced c@{} with l@{ }
extraeol = []
# Generate header lines
if with_table_heading:
header = funInfos[df[1]] if funInfos else 'f%d' % df[1]
table.append([
r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}' %
(2 * len(targets) + 2, header)
])
extraeol.append('')
if function_targets_line is True or (function_targets_line and df[1]
in function_targets_line):
curline = [r'$\Delta f_\mathrm{opt}$']
for t in targets[0:-1]:
curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}' %
writeFEvals2(t, precision=1, isscientific=True))
curline.append(
r'\multicolumn{2}{@{\,}X@{}|}{%s}' %
writeFEvals2(targets[-1], precision=1, isscientific=True))
curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}')
table.append(curline)
extraeol.append(r'\hline')
# extraeol.append(r'\hline\arrayrulecolor{tableShade}')
curline = [r'ERT$_{\text{best}}$'
] if with_table_heading else [r'\textbf{f%d}' % df[1]]
for i in refalgert[0:-1]:
curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}' %
writeFEvalsMaxPrec(float(i), 2))
curline.append(r'\multicolumn{2}{@{\,}X@{\,}|}{%s}' %
writeFEvalsMaxPrec(float(refalgert[-1]), 2))
curline.append('%d' % refalgnbsucc)
if refalgnbsucc:
curline.append('/%d' % refalgnbruns)
#curline.append(curline[0])
table.append(curline)
extraeol.append('')
#for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))):
#print i, gna, gno
#set_trace()
# Format data
#if df == (5, 17):
#set_trace()
header = r'\providecommand{\ntables}{7}'
for i, alg in enumerate(algnames):
#algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames,
#data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres):
commandname = r'\alg%stables' % numtotext(i)
# header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg)))
header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % (
commandname, str_to_latex(strip_pathname(alg)))
curline = [
commandname + r'\hspace*{\fill}'
] # each list element becomes a &-separated table entry?
for j, tmp in enumerate(
zip(
algerts[i],
algdisp[i], # j is target index
isBoldArray[i],
algtestres[i])):
ert, dispersion, isBold, testres = tmp
alignment = '@{\,}X@{\,}'
if j == len(algerts[i]) - 1:
alignment = '@{\,}X@{\,}|'
data = ert / refalgert[j]
# write star for significance against all other algorithms
str_significance_subsup = ''
if (len(best_alg_idx) > 0
and len(significance_versus_others) > 0
and i == best_alg_idx[j]
and nbtests * significance_versus_others[j][1] < 0.05):
logp = -numpy.ceil(
numpy.log10(
nbtests * significance_versus_others[j][1]))
str_significance_subsup = r"^{%s%s}" % (
significance_vs_others_symbol,
str(int(logp)) if logp > 1 else '')
# moved out of the above else: this was a bug!?
z, p = testres
if (nbtests * p) < 0.05 and data < 1. and z < 0.:
if not numpy.isinf(refalgert[j]):
tmpevals = algevals[i][j].copy()
tmpevals[numpy.isnan(tmpevals)] = algentries[
i].maxevals[numpy.isnan(tmpevals)]
bestevals = refalgentry.detEvals([targets[j]])
bestevals, bestalgalg = (bestevals[0][0],
bestevals[1][0])
bestevals[numpy.isnan(
bestevals)] = refalgentry.maxevals[bestalgalg][
numpy.isnan(bestevals)]
tmpevals = numpy.array(sorted(
tmpevals))[0:min(len(tmpevals), len(bestevals))]
bestevals = numpy.array(sorted(
bestevals))[0:min(len(tmpevals), len(bestevals))]
#The conditions are now that ERT < ERT_best and
# all(sorted(FEvals_best) > sorted(FEvals_current)).
if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
# tmp2[-1] += r'$^{%s}$' % superscript
str_significance_subsup += r'_{%s%s}' % (
significance_vs_ref_symbol,
str(int(nbstars)) if nbstars > 1 else '')
if str_significance_subsup:
str_significance_subsup = '$%s$' % str_significance_subsup
# format number in variable data
if numpy.isnan(data):
curline.append(r'\multicolumn{2}{%s}{.}' % alignment)
else:
if numpy.isinf(refalgert[j]):
curline.append(
r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}'
% (alignment, writeFEvalsMaxPrec(algerts[i][j], 2),
writeFEvalsMaxPrec(dispersion, precdispersion),
str_significance_subsup))
continue
tmp = writeFEvalsMaxPrec(data,
precfloat,
maxfloatrepr=maxfloatrepr)
if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation
if numpy.isinf(data) and j == len(algerts[i]) - 1:
tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec(
algfinaldata[i][1],
0,
maxfloatrepr=maxfloatrepr)
else:
tmp = writeFEvalsMaxPrec(data,
precscien,
maxfloatrepr=data)
if isBold:
tmp = r'\textbf{%s}' % tmp
if not numpy.isnan(dispersion):
tmpdisp = dispersion / refalgert[j]
if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=tmpdisp)
else:
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=maxfloatrepr)
tmp += r'\mbox{\tiny (%s)}' % tmpdisp
curline.append(
r'\multicolumn{2}{%s}{%s%s}' %
(alignment, tmp, str_significance_subsup))
else:
tmp2 = tmp.split('.', 1)
if len(tmp2) < 2:
tmp2.append('')
else:
tmp2[-1] = '.' + tmp2[-1]
if isBold:
tmp3 = []
for k in tmp2:
tmp3.append(r'\textbf{%s}' % k)
tmp2 = tmp3
if not numpy.isnan(dispersion):
tmpdisp = dispersion / refalgert[j]
if tmpdisp >= maxfloatrepr or tmpdisp < 0.01:
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=tmpdisp)
else:
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=maxfloatrepr)
tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp))
tmp2[-1] += str_significance_subsup
curline.extend(tmp2)
curline.append('%d' % algnbsucc[i])
curline.append('/%d' % algnbruns[i])
table.append(curline)
extraeol.append('')
# Write table
res = tableXLaTeX(table, spec=spec, extraeol=extraeol)
try:
filename = os.path.join(
outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0]))
f = open(filename, 'w')
f.write(header + '\n')
f.write(res)
if verbose:
print 'Wrote table in %s' % filename
except:
raise
else:
f.close()
def main(dictAlg,
sortedAlgs,
outputdir='.',
verbose=True,
function_targets_line=True): # [1, 13, 101]
"""Generate one table per func with results of multiple algorithms."""
"""Difference with the first version:
* numbers aligned using the decimal separator
* premices for dispersion measure
* significance test against best algorithm
* table width...
Takes ``targetsOfInterest`` from this file as "input argument" to compute
the desired target values. ``targetsOfInterest`` might be configured via
config.
"""
# TODO: method is long, terrible to read, split if possible
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
# Sort data per dimension and function
dictData = {}
dsListperAlg = list(dictAlg[i] for i in sortedAlgs)
for n, entries in enumerate(dsListperAlg):
tmpdictdim = entries.dictByDim()
for d in tmpdictdim:
tmpdictfun = tmpdictdim[d].dictByFunc()
for f in tmpdictfun:
dictData.setdefault((d, f), {})[n] = tmpdictfun[f]
nbtests = len(dictData)
for df in dictData:
# Generate one table per df
# first update targets for each dimension-function pair if needed:
targets = targetsOfInterest((df[1], df[0]))
targetf = targets[-1]
# best 2009
refalgentry = bestalg.bestalgentries2009[df]
refalgert = refalgentry.detERT(targets)
refalgevals = (refalgentry.detEvals((targetf, ))[0][0])
refalgnbruns = len(refalgevals)
refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False)
# Process the data
# The following variables will be lists of elements each corresponding
# to an algorithm
algnames = []
#algdata = []
algerts = []
algevals = []
algdisp = []
algnbsucc = []
algnbruns = []
algmedmaxevals = []
algmedfinalfunvals = []
algtestres = []
algentries = []
for n in sorted(dictData[df].keys()):
entries = dictData[df][n]
# the number of datasets for a given dimension and function (df)
# should be strictly 1. TODO: find a way to warn
# TODO: do this checking before... why wasn't it triggered by ppperprof?
if len(entries) > 1:
print entries
txt = ("There is more than a single entry associated with "
"folder %s on %d-D f%d." %
(sortedAlgs[n], df[0], df[1]))
raise Exception(txt)
entry = entries[0]
algentries.append(entry)
algnames.append(sortedAlgs[n])
evals = entry.detEvals(targets)
#tmpdata = []
tmpdisp = []
tmpert = []
for i, e in enumerate(evals):
succ = (numpy.isnan(e) == False)
ec = e.copy(
) # note: here was the previous bug (changes made in e also appeared in evals !)
ec[succ == False] = entry.maxevals[succ == False]
ert = toolsstats.sp(ec, issuccessful=succ)[0]
#tmpdata.append(ert/refalgert[i])
if succ.any():
tmp = toolsstats.drawSP(ec[succ],
entry.maxevals[succ == False],
[10, 50, 90],
samplesize=samplesize)[0]
tmpdisp.append((tmp[-1] - tmp[0]) / 2.)
else:
tmpdisp.append(numpy.nan)
tmpert.append(ert)
algerts.append(tmpert)
algevals.append(evals)
#algdata.append(tmpdata)
algdisp.append(tmpdisp)
algmedmaxevals.append(numpy.median(entry.maxevals))
algmedfinalfunvals.append(numpy.median(entry.finalfunvals))
#algmedmaxevals.append(numpy.median(entry.maxevals)/df[0])
#algmedfinalfunvals.append(numpy.median(entry.finalfunvals))
algtestres.append(significancetest(refalgentry, entry, targets))
# determine success probability for Df = 1e-8
e = entry.detEvals((targetf, ))[0]
algnbsucc.append(numpy.sum(numpy.isnan(e) == False))
algnbruns.append(len(e))
# Process over all data
# find best values...
nalgs = len(dictData[df])
maxRank = 1 + numpy.floor(
0.14 * nalgs) # number of algs to be displayed in bold
isBoldArray = [] # Point out the best values
algfinaldata = [
] # Store median function values/median number of function evaluations
tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank)
for i, erts in enumerate(algerts):
tmp = []
for j, ert in enumerate(erts): # algi targetj
tmp.append(i in tmptop[j] or
(nalgs > 7 and algerts[i][j] <= 3. * refalgert[j]))
isBoldArray.append(tmp)
algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i]))
# significance test of best given algorithm against all others
best_alg_idx = numpy.array(algerts).argsort(0)[
0, :] # indexed by target index
significance_versus_others = significance_all_best_vs_other(
algentries, targets, best_alg_idx)[0]
# Create the table
table = []
tableHtml = []
spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % (
len(targets)
) # in case StrLeft not working: replaced c@{} with l@{ }
spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % (
len(targets)
) # in case StrLeft not working: replaced c@{} with l@{ }
extraeol = []
# Generate header lines
if with_table_heading:
header = funInfos[df[1]] if funInfos else 'f%d' % df[1]
table.append([
r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}' %
(2 * len(targets) + 2, header)
])
extraeol.append('')
if function_targets_line is True or (function_targets_line and df[1]
in function_targets_line):
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
curline = [r'\#FEs/D']
curlineHtml = ['<thead>\n<tr>\n<th>#FEs/D<br>REPLACEH</th>\n']
counter = 1
for i in targetsOfInterest.labels():
curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' % i)
curlineHtml.append('<td>%s<br>REPLACE%d</td>\n' %
(i, counter))
counter += 1
else:
curline = [r'$\Delta f_\mathrm{opt}$']
curlineHtml = [
'<thead>\n<tr>\n<th>Δ f<sub>opt</sub><br>REPLACEH</th>\n'
]
counter = 1
for t in targets:
curline.append(
r'\multicolumn{2}{@{\,}X@{\,}}{%s}' %
writeFEvals2(t, precision=1, isscientific=True))
curlineHtml.append(
'<td>%s<br>REPLACE%d</td>\n' % (writeFEvals2(
t, precision=1, isscientific=True), counter))
counter += 1
# curline.append(r'\multicolumn{2}{@{\,}X@{}|}{%s}'
# % writeFEvals2(targets[-1], precision=1, isscientific=True))
curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}')
curlineHtml.append('<td>#succ<br>REPLACEF</td>\n</tr>\n</thead>\n')
table.append(curline)
extraeol.append(r'\hline')
# extraeol.append(r'\hline\arrayrulecolor{tableShade}')
curline = [r'ERT$_{\text{best}}$'
] if with_table_heading else [r'\textbf{f%d}' % df[1]]
replaceValue = 'ERT<sub>best</sub>' if with_table_heading else (
'<b>f%d</b>' % df[1])
curlineHtml = [
item.replace('REPLACEH', replaceValue) for item in curlineHtml
]
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
# write ftarget:fevals
counter = 1
for i in xrange(len(refalgert[:-1])):
temp = "%.1e" % targetsOfInterest((df[1], df[0]))[i]
if temp[-2] == "0":
temp = temp[:-2] + temp[-1]
curline.append(
r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' %
(temp, writeFEvalsMaxPrec(refalgert[i], 2)))
replaceValue = '<i>%s</i>:%s' % (
temp, writeFEvalsMaxPrec(refalgert[i], 2))
curlineHtml = [
item.replace('REPLACE%d' % counter, replaceValue)
for item in curlineHtml
]
counter += 1
temp = "%.1e" % targetsOfInterest((df[1], df[0]))[-1]
if temp[-2] == "0":
temp = temp[:-2] + temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' %
(temp, writeFEvalsMaxPrec(refalgert[-1], 2)))
replaceValue = '<i>%s</i>:%s' % (
temp, writeFEvalsMaxPrec(refalgert[-1], 2))
curlineHtml = [
item.replace('REPLACE%d' % counter, replaceValue)
for item in curlineHtml
]
else:
# write #fevals of the reference alg
counter = 1
for i in refalgert[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' %
writeFEvalsMaxPrec(i, 2))
curlineHtml = [
item.replace('REPLACE%d' % counter,
writeFEvalsMaxPrec(i, 2))
for item in curlineHtml
]
counter += 1
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' %
writeFEvalsMaxPrec(refalgert[-1], 2))
curlineHtml = [
item.replace('REPLACE%d' % counter,
writeFEvalsMaxPrec(refalgert[-1], 2))
for item in curlineHtml
]
# write the success ratio for the reference alg
tmp2 = numpy.sum(
numpy.isnan(refalgevals) == False) # count the nb of success
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(refalgevals))
replaceValue = '%d/%d' % (tmp2, len(refalgevals))
else:
replaceValue = '%d' % tmp2
curlineHtml = [
item.replace('REPLACEF', replaceValue) for item in curlineHtml
]
table.append(curline[:])
tableHtml.extend(curlineHtml[:])
tableHtml.append('<tbody>\n')
extraeol.append('')
#for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))):
#print i, gna, gno
#set_trace()
# Format data
#if df == (5, 17):
#set_trace()
header = r'\providecommand{\ntables}{7}'
for i, alg in enumerate(algnames):
tableHtml.append('<tr>\n')
#algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames,
#data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres):
commandname = r'\alg%stables' % numtotext(i)
# header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg)))
header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % (
commandname, str_to_latex(strip_pathname1(alg)))
curline = [
commandname + r'\hspace*{\fill}'
] # each list element becomes a &-separated table entry?
curlineHtml = [
'<th>%s</th>\n' % str_to_latex(strip_pathname1(alg))
]
for j, tmp in enumerate(
zip(
algerts[i],
algdisp[i], # j is target index
isBoldArray[i],
algtestres[i])):
ert, dispersion, isBold, testres = tmp
alignment = '@{\,}X@{\,}'
if j == len(algerts[i]) - 1:
alignment = '@{\,}X@{\,}|'
data = ert / refalgert[j]
# write star for significance against all other algorithms
str_significance_subsup = ''
str_significance_subsup_html = ''
if (len(best_alg_idx) > 0
and len(significance_versus_others) > 0
and i == best_alg_idx[j]
and nbtests * significance_versus_others[j][1] < 0.05):
logp = -numpy.ceil(
numpy.log10(
nbtests * significance_versus_others[j][1]))
logp = numpy.min(
(9,
logp)) # not messing up the format and handling inf
str_significance_subsup = r"^{%s%s}" % (
significance_vs_others_symbol,
str(int(logp)) if logp > 1 else '')
str_significance_subsup_html = '<sup>%s%s</sup>' % (
significance_vs_others_symbol_html,
str(int(logp)) if logp > 1 else '')
# moved out of the above else: this was a bug!?
z, p = testres
if (nbtests * p) < 0.05 and data < 1. and z < 0.:
if not numpy.isinf(refalgert[j]):
tmpevals = algevals[i][j].copy()
tmpevals[numpy.isnan(tmpevals)] = algentries[
i].maxevals[numpy.isnan(tmpevals)]
bestevals = refalgentry.detEvals(targets)
bestevals, bestalgalg = (bestevals[0][0],
bestevals[1][0])
bestevals[numpy.isnan(
bestevals)] = refalgentry.maxevals[bestalgalg][
numpy.isnan(bestevals)]
tmpevals = numpy.array(sorted(
tmpevals))[0:min(len(tmpevals), len(bestevals))]
bestevals = numpy.array(sorted(
bestevals))[0:min(len(tmpevals), len(bestevals))]
#The conditions are now that ERT < ERT_best and
# all(sorted(FEvals_best) > sorted(FEvals_current)).
if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
# tmp2[-1] += r'$^{%s}$' % superscript
str_significance_subsup += r'_{%s%s}' % (
significance_vs_ref_symbol,
str(int(nbstars)) if nbstars > 1 else '')
str_significance_subsup_html = '<sub>%s%s</sub>' % (
significance_vs_ref_symbol_html,
str(int(nbstars)) if nbstars > 1 else '')
if str_significance_subsup:
str_significance_subsup = '$%s$' % str_significance_subsup
# format number in variable data
if numpy.isnan(data):
curline.append(r'\multicolumn{2}{%s}{.}' % alignment)
else:
if numpy.isinf(refalgert[j]):
curline.append(
r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}'
% (alignment, writeFEvalsMaxPrec(algerts[i][j], 2),
writeFEvalsMaxPrec(dispersion, precdispersion),
str_significance_subsup))
curlineHtml.append(
'<td sorttable_customkey=\"%f\"><b>%s</b> (%s)%s</td>\n'
% (algerts[i][j],
writeFEvalsMaxPrec(algerts[i][j], 2),
writeFEvalsMaxPrec(dispersion, precdispersion),
str_significance_subsup_html))
continue
tmp = writeFEvalsMaxPrec(data,
precfloat,
maxfloatrepr=maxfloatrepr)
tmpHtml = writeFEvalsMaxPrec(data,
precfloat,
maxfloatrepr=maxfloatrepr)
sortKey = data
if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation
if numpy.isinf(data) and j == len(algerts[i]) - 1:
tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec(
algfinaldata[i][1],
0,
maxfloatrepr=maxfloatrepr)
tmpHtml += '<i>%s</i>' % writeFEvalsMaxPrec(
algfinaldata[i][1],
0,
maxfloatrepr=maxfloatrepr)
sortKey = algfinaldata[i][1]
else:
tmp = writeFEvalsMaxPrec(data,
precscien,
maxfloatrepr=data)
if isBold:
tmpHtml = '<b>%s</b>' % tmp
tmp = r'\textbf{%s}' % tmp
if not numpy.isnan(dispersion):
tmpdisp = dispersion / refalgert[j]
if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=tmpdisp)
else:
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=maxfloatrepr)
tmp += r'\mbox{\tiny (%s)}' % tmpdisp
tmpHtml += ' (%s)' % tmpdisp
curline.append(
r'\multicolumn{2}{%s}{%s%s}' %
(alignment, tmp, str_significance_subsup))
tmpHtml = tmpHtml.replace('$\infty$', '∞')
if (numpy.isinf(sortKey)):
sortKey = sys.maxint
curlineHtml.append(
'<td sorttable_customkey=\"%f\">%s%s</td>' %
(sortKey, tmpHtml, str_significance_subsup_html))
else:
tmp2 = tmp.split('.', 1)
if len(tmp2) < 2:
tmp2.append('')
else:
tmp2[-1] = '.' + tmp2[-1]
if isBold:
tmp3 = []
tmp3html = []
for k in tmp2:
tmp3.append(r'\textbf{%s}' % k)
tmp3html.append('<b>%s</b>' % k)
tmp2 = tmp3
tmp2html = tmp3html
else:
tmp2html = []
tmp2html.extend(tmp2)
if not numpy.isnan(dispersion):
tmpdisp = dispersion / refalgert[j]
if tmpdisp >= maxfloatrepr or tmpdisp < 0.01:
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=tmpdisp)
else:
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=maxfloatrepr)
tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp))
tmp2html[-1] += ' (%s)' % tmpdisp
tmp2[-1] += str_significance_subsup
tmp2html[-1] += str_significance_subsup_html
curline.extend(tmp2)
tmp2html = ("").join(str(item) for item in tmp2html)
tmp2html = tmp2html.replace('$\infty$', '∞')
curlineHtml.append(
'<td sorttable_customkey=\"%f\">%s</td>' %
(data, tmp2html))
curline.append('%d' % algnbsucc[i])
curline.append('/%d' % algnbruns[i])
table.append(curline)
curlineHtml.append('<td sorttable_customkey=\"%d\">%d/%d</td>\n' %
(algnbsucc[i], algnbsucc[i], algnbruns[i]))
tableHtml.extend(curlineHtml[:])
extraeol.append('')
# Write table
res = tableXLaTeX(table, spec=spec, extraeol=extraeol)
try:
filename = os.path.join(
outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0]))
f = open(filename, 'w')
f.write(header + '\n')
f.write(res)
res = ("").join(str(item) for item in tableHtml)
res = '\n<table class=\"sortable\" style=\"width:800px \">\n%s</table>\n<p/>\n' % res
if df[0] in (5, 20):
filename = os.path.join(
outputdir,
genericsettings.many_algorithm_file_name + '.html')
lines = []
with open(filename) as infile:
for line in infile:
if '<!--' + 'pptablesf%03d%02dDHtml' % (
df[1], df[0]) + '-->' in line:
lines.append(res)
lines.append(line)
with open(filename, 'w') as outfile:
for line in lines:
outfile.write(line)
if verbose:
print 'Wrote table in %s' % filename
except:
raise
else:
f.close()
def main(argv=None):
r"""Routine for post-processing COCO data from two algorithms.
Provided with some data, this routine outputs figure and TeX files
in a folder needed for the compilation of the provided LaTeX templates
for comparing two algorithms (``*cmp.tex`` or ``*2*.tex``).
The used template file needs to be edited so that the command
``\bbobdatapath`` points to the output folder created by this routine.
The output files will contain performance tables, performance
scaling figures and empirical cumulative distribution figures. On
subsequent executions, new files will be added to the output folder,
overwriting existing older files in the process.
Keyword arguments:
*argv* -- list of strings containing options and arguments. If not
given, sys.argv is accessed.
*argv* must list folders containing BBOB data files. Each of these
folders should correspond to the data of ONE algorithm.
Furthermore, argv can begin with, in any order, facultative option
flags listed below.
-h, --help
displays this message.
-v, --verbose
verbose mode, prints out operations.
-o OUTPUTDIR, --output-dir=OUTPUTDIR
changes the default output directory (:file:`ppdata`) to
:file:`OUTPUTDIR`
--noise-free, --noisy
processes only part of the data.
--settings=SETTING
changes the style of the output figures and tables. At the
moment only the only differences are in the colors of the
output figures. SETTING can be either "grayscale", "color"
or "black-white". The default setting is "color".
--fig-only, --rld-only, --tab-only, --sca-only
these options can be used to output respectively the ERT
graphs figures, run length distribution figures or the
comparison tables scatter plot figures only. Any combination
of these options results in no output.
--conv
if this option is chosen, additionally convergence
plots for each function and algorithm are generated.
--rld-single-fcts
generate also runlength distribution figures for each
single function.
--expensive
runlength-based f-target values and fixed display limits,
useful with comparatively small budgets. By default the
setting is based on the budget used in the data.
--not-expensive
expensive setting off.
--svg
generate also the svg figures which are used in html files
Exceptions raised:
*Usage* -- Gives back a usage message.
Examples:
* Calling the rungeneric2.py interface from the command line::
$ python bbob_pproc/rungeneric2.py -v Alg0-baseline Alg1-of-interest
will post-process the data from folders :file:`Alg0-baseline` and
:file:`Alg1-of-interest`, the former containing data for the
reference algorithm (zero-th) and the latter data for the
algorithm of concern (first). The results will be output in the
default output folder. The ``-v`` option adds verbosity.
* From the python interpreter (requires that the path to this
package is in python search path)::
>> import bbob_pproc as bb
>> bb.rungeneric2.main('-o outputfolder PSO DEPSO'.split())
This will execute the post-processing on the data found in folder
:file:`PSO` and :file:`DEPSO`. The ``-o`` option changes the output
folder from the default to :file:`outputfolder`.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
global ftarget
try:
try:
opts, args = getopt.getopt(argv, genericsettings.shortoptlist,
genericsettings.longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not (args):
usage()
sys.exit()
#Process options
outputdir = genericsettings.outputdir
for o, a in opts:
if o in ("-v", "--verbose"):
genericsettings.verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-o", "--output-dir"):
outputdir = a
elif o == "--fig-only":
genericsettings.isRLDistr = False
genericsettings.isTab = False
genericsettings.isScatter = False
elif o == "--rld-only":
genericsettings.isFig = False
genericsettings.isTab = False
genericsettings.isScatter = False
elif o == "--tab-only":
genericsettings.isFig = False
genericsettings.isRLDistr = False
genericsettings.isScatter = False
elif o == "--sca-only":
genericsettings.isFig = False
genericsettings.isRLDistr = False
genericsettings.isTab = False
elif o == "--noisy":
genericsettings.isNoisy = True
elif o == "--noise-free":
genericsettings.isNoiseFree = True
elif o == "--settings":
genericsettings.inputsettings = a
elif o == "--conv":
genericsettings.isConv = True
elif o == "--rld-single-fcts":
genericsettings.isRldOnSingleFcts = True
elif o == "--runlength-based":
genericsettings.runlength_based_targets = True
elif o == "--expensive":
genericsettings.isExpensive = True # comprises runlength-based
elif o == "--not-expensive":
genericsettings.isExpensive = False
elif o == "--svg":
genericsettings.generate_svg_files = True
elif o == "--los-only":
warnings.warn(
"option --los-only will have no effect with rungeneric2.py"
)
elif o == "--crafting-effort=":
warnings.warn(
"option --crafting-effort will have no effect with rungeneric2.py"
)
elif o in ("-p", "--pickle"):
warnings.warn(
"option --pickle will have no effect with rungeneric2.py")
else:
assert False, "unhandled option"
# from bbob_pproc import bbob2010 as inset # input settings
if genericsettings.inputsettings == "color":
from bbob_pproc import genericsettings as inset # input settings
config.config()
elif genericsettings.inputsettings == "grayscale": # probably very much obsolete
from bbob_pproc import grayscalesettings as inset # input settings
elif genericsettings.inputsettings == "black-white": # probably very much obsolete
from bbob_pproc import bwsettings as inset # input settings
else:
txt = ('Settings: %s is not an appropriate ' %
genericsettings.inputsettings +
'argument for input flag "--settings".')
raise Usage(txt)
if (not genericsettings.verbose):
warnings.simplefilter('module')
warnings.simplefilter('ignore')
print("Post-processing will generate comparison " +
"data in folder %s" % outputdir)
print " this might take several minutes."
dsList, sortedAlgs, dictAlg = processInputArgs(
args, verbose=genericsettings.verbose)
if 1 < 3 and len(sortedAlgs) != 2:
raise ValueError(
'rungeneric2.py needs exactly two algorithms to compare, found: '
+ str(sortedAlgs) +
'\n use rungeneric.py (or rungenericmany.py) to compare more algorithms. '
)
if not dsList:
sys.exit()
for i in dictAlg:
if genericsettings.isNoisy and not genericsettings.isNoiseFree:
dictAlg[i] = dictAlg[i].dictByNoise().get(
'nzall', DataSetList())
if genericsettings.isNoiseFree and not genericsettings.isNoisy:
dictAlg[i] = dictAlg[i].dictByNoise().get(
'noiselessall', DataSetList())
for i in dsList:
if i.dim not in genericsettings.dimensions_to_display:
continue
if (dict((j, i.instancenumbers.count(j))
for j in set(i.instancenumbers)) <
inset.instancesOfInterest):
warnings.warn('The data of %s do not list ' % (i) +
'the correct instances ' + 'of function F%d.' %
(i.funcId))
if len(sortedAlgs) < 2:
raise Usage('Expect data from two different algorithms, could ' +
'only find one.')
elif len(sortedAlgs) > 2:
warnings.warn('Data from folders: %s ' % (sortedAlgs) +
'were found, the first two will be processed.')
# Group by algorithm
dsList0 = dictAlg[sortedAlgs[0]]
if not dsList0:
raise Usage('Could not find data for algorithm %s.' %
(sortedAlgs[0]))
dsList1 = dictAlg[sortedAlgs[1]]
if not dsList1:
raise Usage('Could not find data for algorithm %s.' %
(sortedAlgs[0]))
# get the name of each algorithm from the input arguments
tmppath0, alg0name = os.path.split(sortedAlgs[0].rstrip(os.sep))
tmppath1, alg1name = os.path.split(sortedAlgs[1].rstrip(os.sep))
for i in dsList0:
i.algId = alg0name
for i in dsList1:
i.algId = alg1name
# compute maxfuneval values
dict_max_fun_evals1 = {}
dict_max_fun_evals2 = {}
for ds in dsList0:
dict_max_fun_evals1[ds.dim] = np.max(
(dict_max_fun_evals1.setdefault(ds.dim, 0),
float(np.max(ds.maxevals))))
for ds in dsList1:
dict_max_fun_evals2[ds.dim] = np.max(
(dict_max_fun_evals2.setdefault(ds.dim, 0),
float(np.max(ds.maxevals))))
config.target_values(
genericsettings.isExpensive, {
1:
min([
max([
val / dim
for dim, val in dict_max_fun_evals1.iteritems()
]),
max([
val / dim
for dim, val in dict_max_fun_evals2.iteritems()
])
])
})
config.config()
######################### Post-processing #############################
if genericsettings.isFig or genericsettings.isRLDistr or genericsettings.isTab or genericsettings.isScatter:
if not os.path.exists(outputdir):
os.mkdir(outputdir)
if genericsettings.verbose:
print 'Folder %s was created.' % (outputdir)
# prepend the algorithm name command to the tex-command file
abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
lines = []
for i, alg in enumerate(args):
lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' +
str_to_latex(strip_pathname1(alg)) + '}')
prepend_to_file(
os.path.join(outputdir,
'bbob_pproc_commands.tex'), lines, 1000,
'bbob_proc_commands.tex truncated, consider removing the file before the text run'
)
# Check whether both input arguments list noisy and noise-free data
dictFN0 = dsList0.dictByNoise()
dictFN1 = dsList1.dictByNoise()
k0 = set(dictFN0.keys())
k1 = set(dictFN1.keys())
symdiff = k1 ^ k0 # symmetric difference
if symdiff:
tmpdict = {}
for i, noisegrp in enumerate(symdiff):
if noisegrp == 'nzall':
tmp = 'noisy'
elif noisegrp == 'noiselessall':
tmp = 'noiseless'
if dictFN0.has_key(noisegrp):
tmp2 = sortedAlgs[0]
elif dictFN1.has_key(noisegrp):
tmp2 = sortedAlgs[1]
tmpdict.setdefault(tmp2, []).append(tmp)
txt = []
for i, j in tmpdict.iteritems():
txt.append('Only input folder %s lists %s data.' %
(i, ' and '.join(j)))
raise Usage('Data Mismatch: \n ' + ' '.join(txt) +
'\nTry using --noise-free or --noisy flags.')
if genericsettings.isFig:
plt.rc("axes", **inset.rcaxeslarger)
plt.rc("xtick", **inset.rcticklarger)
plt.rc("ytick", **inset.rcticklarger)
plt.rc("font", **inset.rcfontlarger)
plt.rc("legend", **inset.rclegendlarger)
plt.rc('pdf', fonttype=42)
ppfig2.main(dsList0, dsList1, ppfig2_ftarget, outputdir,
genericsettings.verbose)
print "log ERT1/ERT0 vs target function values done."
plt.rc("axes", **inset.rcaxes)
plt.rc("xtick", **inset.rctick)
plt.rc("ytick", **inset.rctick)
plt.rc("font", **inset.rcfont)
plt.rc("legend", **inset.rclegend)
plt.rc('pdf', fonttype=42)
if genericsettings.isRLDistr:
if len(dictFN0) > 1 or len(dictFN1) > 1:
warnings.warn('Data for functions from both the noisy and ' +
'non-noisy testbeds have been found. Their ' +
'results will be mixed in the "all functions" ' +
'ECDF figures.')
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
# ECDFs of ERT ratios
for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
if dim in inset.rldDimsOfInterest:
# ECDF for all functions altogether
try:
pprldistr2.main(dictDim0[dim], dictDim1[dim], dim,
inset.rldValsOfInterest, outputdir,
'%02dD_all' % dim,
genericsettings.verbose)
except KeyError:
warnings.warn('Could not find some data in %d-D.' %
(dim))
continue
# ECDFs per function groups
dictFG0 = dictDim0[dim].dictByFuncGroup()
dictFG1 = dictDim1[dim].dictByFuncGroup()
for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
pprldistr2.main(dictFG1[fGroup], dictFG0[fGroup], dim,
inset.rldValsOfInterest, outputdir,
'%02dD_%s' % (dim, fGroup),
genericsettings.verbose)
# ECDFs per noise groups
dictFN0 = dictDim0[dim].dictByNoise()
dictFN1 = dictDim1[dim].dictByNoise()
for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
pprldistr2.main(dictFN1[fGroup], dictFN0[fGroup], dim,
inset.rldValsOfInterest, outputdir,
'%02dD_%s' % (dim, fGroup),
genericsettings.verbose)
prepend_to_file(
os.path.join(outputdir, 'bbob_pproc_commands.tex'),
[
'\\providecommand{\\bbobpprldistrlegendtwo}[1]{',
pprldistr.caption_two(
), # depends on the config setting, should depend on maxfevals
'}'
])
print "ECDF runlength ratio graphs done."
for dim in set(dictDim0.keys()) & set(dictDim1.keys()):
pprldistr.fmax = None #Resetting the max final value
pprldistr.evalfmax = None #Resetting the max #fevalsfactor
# ECDFs of all functions altogether
if dim in inset.rldDimsOfInterest:
try:
pprldistr.comp(
dictDim1[dim],
dictDim0[dim],
inset.
rldValsOfInterest, # TODO: let rldVals... possibly be RL-based targets
True,
outputdir,
'all',
genericsettings.verbose)
except KeyError:
warnings.warn('Could not find some data in %d-D.' %
(dim))
continue
# ECDFs per function groups
dictFG0 = dictDim0[dim].dictByFuncGroup()
dictFG1 = dictDim1[dim].dictByFuncGroup()
for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()):
pprldistr.comp(dictFG1[fGroup], dictFG0[fGroup],
inset.rldValsOfInterest, True,
outputdir, '%s' % fGroup,
genericsettings.verbose)
# ECDFs per noise groups
dictFN0 = dictDim0[dim].dictByNoise()
dictFN1 = dictDim1[dim].dictByNoise()
for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()):
pprldistr.comp(dictFN1[fGroup], dictFN0[fGroup],
inset.rldValsOfInterest, True,
outputdir, '%s' % fGroup,
genericsettings.verbose)
if genericsettings.isRldOnSingleFcts: # copy-paste from above, here for each function instead of function groups
# ECDFs for each function
pprldmany.all_single_functions(dictAlg, sortedAlgs, outputdir,
genericsettings.verbose)
print "ECDF runlength graphs done."
if genericsettings.isConv:
ppconverrorbars.main(dictAlg, outputdir, genericsettings.verbose)
if genericsettings.isScatter:
if genericsettings.runlength_based_targets:
ppscatter.targets = ppscatter.runlength_based_targets
ppscatter.main(dsList1,
dsList0,
outputdir,
verbose=genericsettings.verbose)
prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
[
'\\providecommand{\\bbobppscatterlegend}[1]{',
ppscatter.figure_caption(), '}'
])
replace_in_file(
os.path.join(outputdir,
genericsettings.two_algorithm_file_name +
'.html'), '##bbobppscatterlegend##',
ppscatter.figure_caption_html())
print "Scatter plots done."
if genericsettings.isTab:
dictNG0 = dsList0.dictByNoise()
dictNG1 = dsList1.dictByNoise()
for nGroup in set(dictNG0.keys()) & set(dictNG1.keys()):
# split table in as many as necessary
dictFunc0 = dictNG0[nGroup].dictByFunc()
dictFunc1 = dictNG1[nGroup].dictByFunc()
funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys()))
if len(funcs) > 24:
funcs.sort()
nbgroups = int(numpy.ceil(len(funcs) / 24.))
def split_seq(seq, nbgroups):
newseq = []
splitsize = 1.0 / nbgroups * len(seq)
for i in range(nbgroups):
newseq.append(
seq[int(round(i * splitsize)
):int(round((i + 1) * splitsize))])
return newseq
groups = split_seq(funcs, nbgroups)
# merge
group0 = []
group1 = []
for i, g in enumerate(groups):
tmp0 = DataSetList()
tmp1 = DataSetList()
for f in g:
tmp0.extend(dictFunc0[f])
tmp1.extend(dictFunc1[f])
group0.append(tmp0)
group1.append(tmp1)
for i, g in enumerate(zip(group0, group1)):
pptable2.main(g[0], g[1], inset.tabDimsOfInterest,
outputdir, '%s%d' % (nGroup, i),
genericsettings.verbose)
else:
if 11 < 3: # future handling:
dictFunc0 = dsList0.dictByFunc()
dictFunc1 = dsList1.dictByFunc()
funcs = list(
set(dictFunc0.keys()) & set(dictFunc1.keys()))
funcs.sort()
# nbgroups = int(numpy.ceil(len(funcs)/testbedsettings.numberOfFunctions))
# pptable2.main(dsList0, dsList1,
# testbedsettings.tabDimsOfInterest, outputdir,
# '%s' % (testbedsettings.testbedshortname), genericsettings.verbose)
else:
pptable2.main(dictNG0[nGroup], dictNG1[nGroup],
inset.tabDimsOfInterest, outputdir,
'%s' % (nGroup), genericsettings.verbose)
if isinstance(pptable2.targetsOfInterest,
pproc.RunlengthBasedTargetValues):
prepend_to_file(
os.path.join(outputdir, 'bbob_pproc_commands.tex'), [
'\\providecommand{\\bbobpptablestwolegend}[1]{',
pptable2.table_caption_expensive, '}'
])
else:
prepend_to_file(
os.path.join(outputdir, 'bbob_pproc_commands.tex'), [
'\\providecommand{\\bbobpptablestwolegend}[1]{',
pptable2.table_caption, '}'
])
htmlFileName = os.path.join(
outputdir, genericsettings.two_algorithm_file_name + '.html')
key = '##bbobpptablestwolegendexpensive##' if isinstance(
pptable2.targetsOfInterest, pproc.RunlengthBasedTargetValues
) else '##bbobpptablestwolegend##'
replace_in_file(htmlFileName, '##bbobpptablestwolegend##',
htmldesc.getValue(key))
alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop().replace(
genericsettings.extraction_folder_prefix, '')[0:3]
alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop().replace(
genericsettings.extraction_folder_prefix, '')[0:3]
replace_in_file(htmlFileName, 'algorithmAshort', alg0)
replace_in_file(htmlFileName, 'algorithmBshort', alg1)
for i, alg in enumerate(args):
replace_in_file(htmlFileName, 'algorithm' + abc[i],
str_to_latex(strip_pathname1(alg)))
print "Tables done."
if genericsettings.isScaleUp:
plt.rc("axes", labelsize=20, titlesize=24)
plt.rc("xtick", labelsize=20)
plt.rc("ytick", labelsize=20)
plt.rc("font", size=20)
plt.rc("legend", fontsize=20)
plt.rc('pdf', fonttype=42)
if genericsettings.runlength_based_targets:
ftarget = RunlengthBasedTargetValues([
target_runlength
]) # TODO: make this more variable but also consistent
ppfigs.main(dictAlg, genericsettings.two_algorithm_file_name,
sortedAlgs, ftarget, outputdir,
genericsettings.verbose)
plt.rcdefaults()
print "Scaling figures done."
if genericsettings.isFig or genericsettings.isRLDistr or genericsettings.isTab or genericsettings.isScatter or genericsettings.isScaleUp:
print "Output data written to folder %s" % outputdir
plt.rcdefaults()
if 1 < 3:
print("Post-processing: will generate output " +
"data in folder %s" % outputdir)
print " this might take several minutes."
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if verbose:
print 'Folder %s was created.' % (outputdir)
# prepend the algorithm name command to the tex-command file
abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
lines = []
for i, alg in enumerate(args):
lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' +
str_to_latex(strip_pathname2(alg)) + '}')
prepend_to_file(
os.path.join(outputdir, 'bbob_pproc_commands.tex'), lines, 5000,
'bbob_proc_commands.tex truncated, consider removing the file before the text run'
)
dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)
if not dsList:
sys.exit()
for i in dictAlg:
if isNoisy and not isNoiseFree:
dictAlg[i] = dictAlg[i].dictByNoise().get(
'nzall', DataSetList())
if isNoiseFree and not isNoisy:
def main(dictAlg, order=None, outputdir='.', info='default',
verbose=True):
"""Generates a figure showing the performance of algorithms.
From a dictionary of :py:class:`DataSetList` sorted by algorithms,
generates the cumulative distribution function of the bootstrap
distribution of ERT for algorithms on multiple functions for
multiple targets altogether.
:param dict dictAlg: dictionary of :py:class:`DataSetList` instances
one instance is equivalent to one algorithm,
:param list targets: target function values
:param list order: sorted list of keys to dictAlg for plotting order
:param str outputdir: output directory
:param str info: output file name suffix
:param bool verbose: controls verbosity
"""
global x_limit # late assignment of default, because it can be set to None in config
if 'x_limit' not in globals() or x_limit is None:
x_limit = x_limit_default
tmp = pp.dictAlgByDim(dictAlg)
# tmp = pp.DictAlg(dictAlg).by_dim()
if len(tmp) != 1:
raise Exception('We never integrate over dimension.')
dim = tmp.keys()[0]
algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []]
dictFunc = pp.dictAlgByFun(dictAlg)
# Collect data
# Crafting effort correction: should we consider any?
CrEperAlg = {}
for alg in algorithms_with_data:
CrE = 0.
if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL':
tmp = dictAlg[alg].dictByNoise()
assert len(tmp.keys()) == 1
if tmp.keys()[0] == 'noiselessall':
CrE = 0.5117
elif tmp.keys()[0] == 'nzall':
CrE = 0.6572
CrEperAlg[alg] = CrE
if CrE != 0.0:
print 'Crafting effort for', alg, 'is', CrE
dictData = {} # list of (ert per function) per algorithm
dictMaxEvals = {} # list of (maxevals per function) per algorithm
bestERT = [] # best ert per function
# funcsolved = [set()] * len(targets) # number of functions solved per target
xbest2009 = []
maxevalsbest2009 = []
for f, dictAlgperFunc in dictFunc.iteritems():
if function_IDs and f not in function_IDs:
continue
# print target_values((f, dim))
for j, t in enumerate(target_values((f, dim))):
# for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)):
# funcsolved[j].add(f)
for alg in algorithms_with_data:
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
try:
entry = dictAlgperFunc[alg][0] # one element per fun and per dim.
evals = entry.detEvals([t])[0]
runlengthsucc = evals[np.isnan(evals) == False] / entry.dim
runlengthunsucc = entry.maxevals[np.isnan(evals)] / entry.dim
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
except (KeyError, IndexError):
#set_trace()
warntxt = ('Data for algorithm %s on function %d in %d-D '
% (alg, f, dim)
+ 'are missing.\n')
warnings.warn(warntxt)
dictData.setdefault(alg, []).extend(x)
dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc)
if displaybest2009:
#set_trace()
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgentry = bestalg.bestalgentries2009[(dim, f)]
bestalgevals = bestalgentry.detEvals(target_values((f, dim)))
# print bestalgevals
for j in range(len(bestalgevals[0])):
if bestalgevals[1][j]:
evals = bestalgevals[0][j]
#set_trace()
runlengthsucc = evals[np.isnan(evals) == False] / bestalgentry.dim
runlengthunsucc = bestalgentry.maxevals[bestalgevals[1][j]][np.isnan(evals)] / bestalgentry.dim
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
else:
x = perfprofsamplesize * [np.inf]
runlengthunsucc = []
xbest2009.extend(x)
maxevalsbest2009.extend(runlengthunsucc)
if order is None:
order = dictData.keys()
# Display data
lines = []
if displaybest2009:
args = {'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11.,
'markeredgewidth': 1.5, 'markerfacecolor': refcolor,
'markeredgecolor': refcolor, 'color': refcolor,
'label': 'best 2009', 'zorder': -1}
lines.append(plotdata(np.array(xbest2009), x_limit, maxevalsbest2009,
CrE = 0., **args))
for i, alg in enumerate(order):
try:
data = dictData[alg]
maxevals = dictMaxEvals[alg]
except KeyError:
continue
args = styles[(i) % len(styles)]
args['linewidth'] = 1.5
args['markersize'] = 12.
args['markeredgewidth'] = 1.5
args['markerfacecolor'] = 'None'
args['markeredgecolor'] = args['color']
args['label'] = alg
#args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib
#elif len(show_algorithms) > 0:
#args['color'] = 'wheat'
#args['ls'] = '-'
#args['zorder'] = -1
lines.append(plotdata(np.array(data), x_limit, maxevals,
CrE=CrEperAlg[alg], **args))
labels, handles = plotLegend(lines, x_limit)
if True: #isLateXLeg:
fileName = os.path.join(outputdir,'pprldmany_%s.tex' % (info))
try:
f = open(fileName, 'w')
f.write(r'\providecommand{\nperfprof}{7}')
algtocommand = {}
for i, alg in enumerate(order):
tmp = r'\alg%sperfprof' % pptex.numtotext(i)
f.write(r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' % (tmp, toolsdivers.str_to_latex(toolsdivers.strip_pathname2(alg))))
algtocommand[alg] = tmp
commandnames = []
if displaybest2009:
tmp = r'\algzeroperfprof'
f.write(r'\providecommand{%s}{best 2009}' % (tmp))
algtocommand['best 2009'] = tmp
for l in labels:
commandnames.append(algtocommand[l])
# f.write(headleg)
f.write(r'\providecommand{\perfprofsidepanel}{\mbox{%s}' % commandnames[0]) # TODO: check len(labels) > 0
for i in range(1, len(labels)):
f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i])
f.write('}\n')
# f.write(footleg)
if verbose:
print 'Wrote right-hand legend in %s' % fileName
except:
raise # TODO: Does this make sense?
else:
f.close()
figureName = os.path.join(outputdir,'pprldmany_%s' % (info))
#beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat)
beautify()
text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys())))
text += ',%d-D' % dim
plt.text(0.01, 0.98, text, horizontalalignment="left",
verticalalignment="top", transform=plt.gca().transAxes)
a = plt.gca()
plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative)
xticks, labels = plt.xticks()
tmp = []
for i in xticks:
tmp.append('%d' % round(np.log10(i)))
a.set_xticklabels(tmp)
ppfig.saveFigure(figureName, verbose=verbose)
plt.close()
if 1 < 3:
print ("Post-processing: will generate output " +
"data in folder %s" % outputdir)
print " this might take several minutes."
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if verbose:
print 'Folder %s was created.' % (outputdir)
# prepend the algorithm name command to the tex-command file
abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
lines = []
for i, alg in enumerate(args):
lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' +
str_to_latex(strip_pathname2(alg)) + '}')
prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'),
lines, 5000,
'bbob_proc_commands.tex truncated, consider removing the file before the text run'
)
dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)
if not dsList:
sys.exit()
for i in dictAlg:
if isNoisy and not isNoiseFree:
dictAlg[i] = dictAlg[i].dictByNoise().get('nzall', DataSetList())
if isNoiseFree and not isNoisy:
dictAlg[i] = dictAlg[i].dictByNoise().get('noiselessall', DataSetList())
def main(dictAlg, sortedAlgs, outputdir='.', verbose=True, function_targets_line=True): # [1, 13, 101]
"""Generate one table per func with results of multiple algorithms."""
"""Difference with the first version:
* numbers aligned using the decimal separator
* premices for dispersion measure
* significance test against best algorithm
* table width...
Takes ``targetsOfInterest`` from this file as "input argument" to compute
the desired target values. ``targetsOfInterest`` might be configured via
config.
"""
# TODO: method is long, terrible to read, split if possible
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
# Sort data per dimension and function
dictData = {}
dsListperAlg = list(dictAlg[i] for i in sortedAlgs)
for n, entries in enumerate(dsListperAlg):
tmpdictdim = entries.dictByDim()
for d in tmpdictdim:
tmpdictfun = tmpdictdim[d].dictByFunc()
for f in tmpdictfun:
dictData.setdefault((d, f), {})[n] = tmpdictfun[f]
nbtests = len(dictData)
for df in dictData:
# Generate one table per df
# first update targets for each dimension-function pair if needed:
targets = targetsOfInterest((df[1], df[0]))
targetf = targets[-1]
# best 2009
refalgentry = bestalg.bestalgentries2009[df]
refalgert = refalgentry.detERT(targets)
refalgevals = (refalgentry.detEvals((targetf, ))[0][0])
refalgnbruns = len(refalgevals)
refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False)
# Process the data
# The following variables will be lists of elements each corresponding
# to an algorithm
algnames = []
#algdata = []
algerts = []
algevals = []
algdisp = []
algnbsucc = []
algnbruns = []
algmedmaxevals = []
algmedfinalfunvals = []
algtestres = []
algentries = []
for n in sorted(dictData[df].keys()):
entries = dictData[df][n]
# the number of datasets for a given dimension and function (df)
# should be strictly 1. TODO: find a way to warn
# TODO: do this checking before... why wasn't it triggered by ppperprof?
if len(entries) > 1:
txt = ("There is more than a single entry associated with "
"folder %s on %d-D f%d." % (sortedAlgs[n], df[0], df[1]))
raise Exception(txt)
entry = entries[0]
algentries.append(entry)
algnames.append(sortedAlgs[n])
evals = entry.detEvals(targets)
#tmpdata = []
tmpdisp = []
tmpert = []
for i, e in enumerate(evals):
succ = (numpy.isnan(e) == False)
ec = e.copy() # note: here was the previous bug (changes made in e also appeared in evals !)
ec[succ == False] = entry.maxevals[succ == False]
ert = toolsstats.sp(ec, issuccessful=succ)[0]
#tmpdata.append(ert/refalgert[i])
if succ.any():
tmp = toolsstats.drawSP(ec[succ], entry.maxevals[succ == False],
[10, 50, 90], samplesize=samplesize)[0]
tmpdisp.append((tmp[-1] - tmp[0])/2.)
else:
tmpdisp.append(numpy.nan)
tmpert.append(ert)
algerts.append(tmpert)
algevals.append(evals)
#algdata.append(tmpdata)
algdisp.append(tmpdisp)
algmedmaxevals.append(numpy.median(entry.maxevals))
algmedfinalfunvals.append(numpy.median(entry.finalfunvals))
#algmedmaxevals.append(numpy.median(entry.maxevals)/df[0])
#algmedfinalfunvals.append(numpy.median(entry.finalfunvals))
algtestres.append(significancetest(refalgentry, entry, targets))
# determine success probability for Df = 1e-8
e = entry.detEvals((targetf ,))[0]
algnbsucc.append(numpy.sum(numpy.isnan(e) == False))
algnbruns.append(len(e))
# Process over all data
# find best values...
nalgs = len(dictData[df])
maxRank = 1 + numpy.floor(0.14 * nalgs) # number of algs to be displayed in bold
isBoldArray = [] # Point out the best values
algfinaldata = [] # Store median function values/median number of function evaluations
tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank)
for i, erts in enumerate(algerts):
tmp = []
for j, ert in enumerate(erts): # algi targetj
tmp.append(i in tmptop[j] or (nalgs > 7 and algerts[i][j] <= 3. * refalgert[j]))
isBoldArray.append(tmp)
algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i]))
# significance test of best given algorithm against all others
best_alg_idx = numpy.array(algerts).argsort(0)[0, :] # indexed by target index
significance_versus_others = significance_all_best_vs_other(algentries, targets, best_alg_idx)[0]
# Create the table
table = []
spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % (len(targets)) # in case StrLeft not working: replaced c@{} with l@{ }
spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % (len(targets)) # in case StrLeft not working: replaced c@{} with l@{ }
extraeol = []
# Generate header lines
if with_table_heading:
header = funInfos[df[1]] if funInfos else 'f%d' % df[1]
table.append([r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}'
% (2 * len(targets) + 2, header)])
extraeol.append('')
if function_targets_line is True or (function_targets_line and df[1] in function_targets_line):
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
curline = [r'\#FEs/D']
for i in targetsOfInterest.labels():
curline.append(r'\multicolumn{2}{@{}c@{}}{%s}'
% i)
else:
curline = [r'$\Delta f_\mathrm{opt}$']
for t in targets:
curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}'
% writeFEvals2(t, precision=1, isscientific=True))
# curline.append(r'\multicolumn{2}{@{\,}X@{}|}{%s}'
# % writeFEvals2(targets[-1], precision=1, isscientific=True))
curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}')
table.append(curline)
extraeol.append(r'\hline')
# extraeol.append(r'\hline\arrayrulecolor{tableShade}')
curline = [r'ERT$_{\text{best}}$'] if with_table_heading else [r'\textbf{f%d}' % df[1]]
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
# write ftarget:fevals
for i in xrange(len(refalgert[:-1])):
temp="%.1e" %targetsOfInterest((df[1], df[0]))[i]
if temp[-2]=="0":
temp=temp[:-2]+temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}'
% (temp,writeFEvalsMaxPrec(refalgert[i], 2)))
temp="%.1e" %targetsOfInterest((df[1], df[0]))[-1]
if temp[-2]=="0":
temp=temp[:-2]+temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }'
% (temp,writeFEvalsMaxPrec(refalgert[-1], 2)))
else:
# write #fevals of the reference alg
for i in refalgert[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}'
% writeFEvalsMaxPrec(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}'
% writeFEvalsMaxPrec(refalgert[-1], 2))
# write the success ratio for the reference alg
tmp2 = numpy.sum(numpy.isnan(refalgevals) == False) # count the nb of success
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(refalgevals))
table.append(curline[:])
extraeol.append('')
#for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))):
#print i, gna, gno
#set_trace()
# Format data
#if df == (5, 17):
#set_trace()
header = r'\providecommand{\ntables}{7}'
for i, alg in enumerate(algnames):
#algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames,
#data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres):
commandname = r'\alg%stables' % numtotext(i)
# header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg)))
header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % (commandname, str_to_latex(strip_pathname2(alg)))
curline = [commandname + r'\hspace*{\fill}'] # each list element becomes a &-separated table entry?
for j, tmp in enumerate(zip(algerts[i], algdisp[i], # j is target index
isBoldArray[i], algtestres[i])):
ert, dispersion, isBold, testres = tmp
alignment = '@{\,}X@{\,}'
if j == len(algerts[i]) - 1:
alignment = '@{\,}X@{\,}|'
data = ert/refalgert[j]
# write star for significance against all other algorithms
str_significance_subsup = ''
if (len(best_alg_idx) > 0 and len(significance_versus_others) > 0 and
i == best_alg_idx[j] and nbtests * significance_versus_others[j][1] < 0.05):
logp = -numpy.ceil(numpy.log10(nbtests * significance_versus_others[j][1]))
str_significance_subsup = r"^{%s%s}" % (significance_vs_others_symbol, str(int(logp)) if logp > 1 else '')
# moved out of the above else: this was a bug!?
z, p = testres
if (nbtests * p) < 0.05 and data < 1. and z < 0.:
if not numpy.isinf(refalgert[j]):
tmpevals = algevals[i][j].copy()
tmpevals[numpy.isnan(tmpevals)] = algentries[i].maxevals[numpy.isnan(tmpevals)]
bestevals = refalgentry.detEvals(targets)
bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0])
bestevals[numpy.isnan(bestevals)] = refalgentry.maxevals[bestalgalg][numpy.isnan(bestevals)]
tmpevals = numpy.array(sorted(tmpevals))[0:min(len(tmpevals), len(bestevals))]
bestevals = numpy.array(sorted(bestevals))[0:min(len(tmpevals), len(bestevals))]
#The conditions are now that ERT < ERT_best and
# all(sorted(FEvals_best) > sorted(FEvals_current)).
if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
# tmp2[-1] += r'$^{%s}$' % superscript
str_significance_subsup += r'_{%s%s}' % (significance_vs_ref_symbol,
str(int(nbstars)) if nbstars > 1 else '')
if str_significance_subsup:
str_significance_subsup = '$%s$' % str_significance_subsup
# format number in variable data
if numpy.isnan(data):
curline.append(r'\multicolumn{2}{%s}{.}' % alignment)
else:
if numpy.isinf(refalgert[j]):
curline.append(r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}'
% (alignment,
writeFEvalsMaxPrec(algerts[i][j], 2),
writeFEvalsMaxPrec(dispersion, precdispersion),
str_significance_subsup))
continue
tmp = writeFEvalsMaxPrec(data, precfloat, maxfloatrepr=maxfloatrepr)
if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation
if numpy.isinf(data) and j == len(algerts[i]) - 1:
tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec(algfinaldata[i][1], 0, maxfloatrepr=maxfloatrepr)
else:
tmp = writeFEvalsMaxPrec(data, precscien, maxfloatrepr=data)
if isBold:
tmp = r'\textbf{%s}' % tmp
if not numpy.isnan(dispersion):
tmpdisp = dispersion/refalgert[j]
if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack
tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=tmpdisp)
else:
tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr)
tmp += r'\mbox{\tiny (%s)}' % tmpdisp
curline.append(r'\multicolumn{2}{%s}{%s%s}' % (alignment, tmp, str_significance_subsup))
else:
tmp2 = tmp.split('.', 1)
if len(tmp2) < 2:
tmp2.append('')
else:
tmp2[-1] = '.' + tmp2[-1]
if isBold:
tmp3 = []
for k in tmp2:
tmp3.append(r'\textbf{%s}' % k)
tmp2 = tmp3
if not numpy.isnan(dispersion):
tmpdisp = dispersion/refalgert[j]
if tmpdisp >= maxfloatrepr or tmpdisp < 0.01:
tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=tmpdisp)
else:
tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr)
tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp))
tmp2[-1] += str_significance_subsup
curline.extend(tmp2)
curline.append('%d' % algnbsucc[i])
curline.append('/%d' % algnbruns[i])
table.append(curline)
extraeol.append('')
# Write table
res = tableXLaTeX(table, spec=spec, extraeol=extraeol)
try:
filename = os.path.join(outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0]))
f = open(filename, 'w')
f.write(header + '\n')
f.write(res)
if verbose:
print 'Wrote table in %s' % filename
except:
raise
else:
f.close()
def main(dictAlg,
order=None,
outputdir='.',
info='default',
dimension=None,
verbose=True):
"""Generates a figure showing the performance of algorithms.
From a dictionary of :py:class:`DataSetList` sorted by algorithms,
generates the cumulative distribution function of the bootstrap
distribution of ERT for algorithms on multiple functions for
multiple targets altogether.
:param dict dictAlg: dictionary of :py:class:`DataSetList` instances
one instance is equivalent to one algorithm,
:param list targets: target function values
:param list order: sorted list of keys to dictAlg for plotting order
:param str outputdir: output directory
:param str info: output file name suffix
:param bool verbose: controls verbosity
"""
global x_limit # late assignment of default, because it can be set to None in config
global divide_by_dimension # not fully implemented/tested yet
if 'x_limit' not in globals() or x_limit is None:
x_limit = x_limit_default
tmp = pp.dictAlgByDim(dictAlg)
# tmp = pp.DictAlg(dictAlg).by_dim()
if len(tmp) != 1 and dimension is None:
raise ValueError('We never integrate over dimension.')
if dimension is not None:
if dimension not in tmp.keys():
raise ValueError('dimension %d not in dictAlg dimensions %s' %
(dimension, str(tmp.keys())))
tmp = {dimension: tmp[dimension]}
dim = tmp.keys()[0]
divisor = dim if divide_by_dimension else 1
algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []]
dictFunc = pp.dictAlgByFun(dictAlg)
# Collect data
# Crafting effort correction: should we consider any?
CrEperAlg = {}
for alg in algorithms_with_data:
CrE = 0.
if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL':
tmp = dictAlg[alg].dictByNoise()
assert len(tmp.keys()) == 1
if tmp.keys()[0] == 'noiselessall':
CrE = 0.5117
elif tmp.keys()[0] == 'nzall':
CrE = 0.6572
CrEperAlg[alg] = CrE
if CrE != 0.0:
print 'Crafting effort for', alg, 'is', CrE
dictData = {} # list of (ert per function) per algorithm
dictMaxEvals = {} # list of (maxevals per function) per algorithm
bestERT = [] # best ert per function
# funcsolved = [set()] * len(targets) # number of functions solved per target
xbest2009 = []
maxevalsbest2009 = []
for f, dictAlgperFunc in dictFunc.iteritems():
if function_IDs and f not in function_IDs:
continue
# print target_values((f, dim))
for j, t in enumerate(target_values((f, dim))):
# for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)):
# funcsolved[j].add(f)
for alg in algorithms_with_data:
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
try:
entry = dictAlgperFunc[alg][
0] # one element per fun and per dim.
evals = entry.detEvals([t])[0]
assert entry.dim == dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(runlengthsucc,
runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
except (KeyError, IndexError):
#set_trace()
warntxt = (
'Data for algorithm %s on function %d in %d-D ' %
(alg, f, dim) + 'are missing.\n')
warnings.warn(warntxt)
dictData.setdefault(alg, []).extend(x)
dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc)
if displaybest2009:
#set_trace()
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgentry = bestalg.bestalgentries2009[(dim, f)]
bestalgevals = bestalgentry.detEvals(target_values((f, dim)))
# print bestalgevals
for j in range(len(bestalgevals[0])):
if bestalgevals[1][j]:
evals = bestalgevals[0][j]
#set_trace()
assert dim == bestalgentry.dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = bestalgentry.maxevals[
bestalgevals[1][j]][np.isnan(evals)] / divisor
x = toolsstats.drawSP(runlengthsucc,
runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
else:
x = perfprofsamplesize * [np.inf]
runlengthunsucc = []
xbest2009.extend(x)
maxevalsbest2009.extend(runlengthunsucc)
if order is None:
order = dictData.keys()
# Display data
lines = []
if displaybest2009:
args = {
'ls': '-',
'linewidth': 6,
'marker': 'D',
'markersize': 11.,
'markeredgewidth': 1.5,
'markerfacecolor': refcolor,
'markeredgecolor': refcolor,
'color': refcolor,
'label': 'best 2009',
'zorder': -1
}
lines.append(
plotdata(np.array(xbest2009),
x_limit,
maxevalsbest2009,
CrE=0.,
**args))
def algname_to_label(algname, dirname=None):
"""to be extended to become generally useful"""
if isinstance(algname, (tuple, list)): # not sure this is needed
return ' '.join([str(name) for name in algname])
return str(algname)
for i, alg in enumerate(order):
try:
data = dictData[alg]
maxevals = dictMaxEvals[alg]
except KeyError:
continue
args = styles[(i) % len(styles)]
args['linewidth'] = 1.5
args['markersize'] = 12.
args['markeredgewidth'] = 1.5
args['markerfacecolor'] = 'None'
args['markeredgecolor'] = args['color']
args['label'] = algname_to_label(alg)
#args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib
#elif len(show_algorithms) > 0:
#args['color'] = 'wheat'
#args['ls'] = '-'
#args['zorder'] = -1
# plotdata calls pprldistr.plotECDF which calls ppfig.plotUnifLog... which does the work
lines.append(
plotdata(np.array(data),
x_limit,
maxevals,
CrE=CrEperAlg[alg],
**args))
labels, handles = plotLegend(lines, x_limit)
if True: # isLateXLeg:
fileName = os.path.join(outputdir, 'pprldmany_%s.tex' % (info))
with open(fileName, 'w') as f:
f.write(r'\providecommand{\nperfprof}{7}')
algtocommand = {} # latex commands
for i, alg in enumerate(order):
tmp = r'\alg%sperfprof' % pptex.numtotext(i)
f.write(
r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' %
(tmp,
toolsdivers.str_to_latex(
toolsdivers.strip_pathname2(algname_to_label(alg)))))
algtocommand[algname_to_label(alg)] = tmp
if displaybest2009:
tmp = r'\algzeroperfprof'
f.write(r'\providecommand{%s}{best 2009}' % (tmp))
algtocommand['best 2009'] = tmp
commandnames = []
for label in labels:
commandnames.append(algtocommand[label])
# f.write(headleg)
if len(
order
) > 28: # latex sidepanel won't work well for more than 25 algorithms, but original labels are also clipped
f.write(
r'\providecommand{\perfprofsidepanel}{\mbox{%s}\vfill\mbox{%s}}'
% (commandnames[0], commandnames[-1]))
else:
fontsize_command = r'\tiny{}' if len(order) > 19 else ''
f.write(r'\providecommand{\perfprofsidepanel}{{%s\mbox{%s}' %
(fontsize_command,
commandnames[0])) # TODO: check len(labels) > 0
for i in range(1, len(labels)):
f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i])
f.write('}}\n')
# f.write(footleg)
if verbose:
print 'Wrote right-hand legend in %s' % fileName
figureName = os.path.join(outputdir, 'pprldmany_%s' % (info))
#beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat)
beautify()
text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys())))
text += ',%d-D' % dim # TODO: this is strange when different dimensions are plotted
plt.text(0.01,
0.98,
text,
horizontalalignment="left",
verticalalignment="top",
transform=plt.gca().transAxes)
if len(dictFunc) == 1:
plt.title(' '.join(
(str(dictFunc.keys()[0]),
genericsettings.current_testbed.short_names[dictFunc.keys()[0]])))
a = plt.gca()
plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative)
xticks, labels = plt.xticks()
tmp = []
for i in xticks:
tmp.append('%d' % round(np.log10(i)))
a.set_xticklabels(tmp)
if save_figure:
ppfig.saveFigure(figureName, verbose=verbose)
if len(dictFunc) == 1:
ppfig.save_single_functions_html(
os.path.join(outputdir, 'pprldmany'),
'', # algorithms names are clearly visible in the figure
add_to_names='_%02dD' % (dim),
algorithmCount=ppfig.AlgorithmCount.NON_SPECIFIED)
if close_figure:
plt.close()
def main(argv=None):
r"""Post-processing COCO data of a single algorithm.
Provided with some data, this routine outputs figure and TeX files
in a folder needed for the compilation of the provided LaTeX templates
for one algorithm (``*article.tex`` or ``*1*.tex``).
The used template file needs to be edited so that the commands
``\bbobdatapath`` and ``\algfolder`` point to the output folder created
by this routine.
These output files will contain performance tables, performance
scaling figures and empirical cumulative distribution figures. On
subsequent executions, new files will be added to the output folder,
overwriting existing older files in the process.
Keyword arguments:
*argv* -- list of strings containing options and arguments. If not
given, sys.argv is accessed.
*argv* should list either names of :file:`info` files or folders
containing :file:`info` files. argv can also contain post-processed
:file:`pickle` files generated by this routine. Furthermore, *argv*
can begin with, in any order, facultative option flags listed below.
-h, --help
displays this message.
-v, --verbose
verbose mode, prints out all operations.
-p, --pickle
generates pickle post processed data files.
-o OUTPUTDIR, --output-dir=OUTPUTDIR
changes the default output directory (:file:`ppdata`) to
:file:`OUTPUTDIR`.
--crafting-effort=VALUE
sets the crafting effort to VALUE (float). Otherwise the
default value of 0. will be used.
--noise-free, --noisy
processes only part of the data.
--settings=SETTINGS
changes the style of the output figures and tables. At the
moment the only differences are in the colors of the output
figures. SETTINGS can be either "grayscale", "color" or
"black-white". The default setting is "color".
--tab-only, --fig-only, --rld-only, --los-only
these options can be used to output respectively the TeX
tables, convergence and ERTs graphs figures, run length
distribution figures, ERT loss ratio figures only. A
combination of any two of these options results in no
output.
--conv
if this option is chosen, additionally convergence plots
for each function and algorithm are generated.
--rld-single-fcts
generate also runlength distribution figures for each
single function.
--expensive
runlength-based f-target values and fixed display limits,
useful with comparatively small budgets. By default the
setting is based on the budget used in the data.
--not-expensive
expensive setting off.
--svg
generate also the svg figures which are used in html files
--runlength-based
runlength-based f-target values, such that the
"level of difficulty" is similar for all functions.
Exceptions raised:
*Usage* -- Gives back a usage message.
Examples:
* Calling the rungeneric1.py interface from the command line::
$ python bbob_pproc/rungeneric1.py -v experiment1
will post-process the folder experiment1 and all its containing
data, base on the .info files found in the folder. The result will
appear in the default output folder. The -v option adds verbosity. ::
$ python bbob_pproc/rungeneric1.py -o exp2 experiment2/*.info
This will execute the post-processing on the info files found in
:file:`experiment2`. The result will be located in the alternative
location :file:`exp2`.
* Loading this package and calling the main from the command line
(requires that the path to this package is in python search path)::
$ python -m bbob_pproc.rungeneric1 -h
This will print out this help message.
* From the python interpreter (requires that the path to this
package is in python search path)::
>> import bbob_pproc as bb
>> bb.rungeneric1.main('-o outputfolder folder1'.split())
This will execute the post-processing on the index files found in
:file:`folder1`. The ``-o`` option changes the output folder from
the default to :file:`outputfolder`.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
if 1 < 3:
opts, args = getopt.getopt(argv, genericsettings.shortoptlist, genericsettings.longoptlist)
if 11 < 3:
try:
opts, args = getopt.getopt(argv, genericsettings.shortoptlist, genericsettings.longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not (args) and not "--help" in argv and not "h" in argv:
print "not enough input arguments given"
print "cave: the following options also need an argument:"
print [o for o in genericsettings.longoptlist if o[-1] == "="]
print "options given:"
print opts
print "try --help for help"
sys.exit()
# Process options
outputdir = genericsettings.outputdir
for o, a in opts:
if o in ("-v", "--verbose"):
genericsettings.verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-p", "--pickle"):
genericsettings.isPickled = True
elif o in ("-o", "--output-dir"):
outputdir = a
elif o == "--noisy":
genericsettings.isNoisy = True
elif o == "--noise-free":
genericsettings.isNoiseFree = True
# The next 4 are for testing purpose
elif o == "--tab-only":
genericsettings.isFig = False
genericsettings.isRLDistr = False
genericsettings.isLogLoss = False
elif o == "--fig-only":
genericsettings.isTab = False
genericsettings.isRLDistr = False
genericsettings.isLogLoss = False
elif o == "--rld-only":
genericsettings.isTab = False
genericsettings.isFig = False
genericsettings.isLogLoss = False
elif o == "--los-only":
genericsettings.isTab = False
genericsettings.isFig = False
genericsettings.isRLDistr = False
elif o == "--crafting-effort":
try:
genericsettings.inputCrE = float(a)
except ValueError:
raise Usage("Expect a valid float for flag crafting-effort.")
elif o == "--settings":
genericsettings.inputsettings = a
elif o == "--conv":
genericsettings.isConv = True
elif o == "--rld-single-fcts":
genericsettings.isRldOnSingleFcts = True
elif o == "--runlength-based":
genericsettings.runlength_based_targets = True
elif o == "--expensive":
genericsettings.isExpensive = True # comprises runlength-based
elif o == "--not-expensive":
genericsettings.isExpensive = False
elif o == "--svg":
genericsettings.generate_svg_files = True
elif o == "--sca-only":
warnings.warn("option --sca-only will have no effect with rungeneric1.py")
else:
assert False, "unhandled option"
# from bbob_pproc import bbob2010 as inset # input settings
if genericsettings.inputsettings == "color":
from bbob_pproc import genericsettings as inset # input settings
elif genericsettings.inputsettings == "grayscale":
from bbob_pproc import grayscalesettings as inset # input settings
elif genericsettings.inputsettings == "black-white":
from bbob_pproc import bwsettings as inset # input settings
else:
txt = (
"Settings: %s is not an appropriate " % genericsettings.inputsettings
+ 'argument for input flag "--settings".'
)
raise Usage(txt)
if 11 < 3:
from bbob_pproc import config # input settings
config.config(False)
import imp
# import testbedsettings as testbedsettings # input settings
try:
fp, pathname, description = imp.find_module("testbedsettings")
testbedsettings = imp.load_module("testbedsettings", fp, pathname, description)
finally:
fp.close()
if not genericsettings.verbose:
warnings.simplefilter("module")
# warnings.simplefilter('ignore')
# get directory name if outputdir is a archive file
algfolder = findfiles.get_output_directory_subfolder(args[0])
outputdir = os.path.join(outputdir, algfolder)
print ("Post-processing (1): will generate output " + "data in folder %s" % outputdir)
print " this might take several minutes."
filelist = list()
for i in args:
i = i.strip()
if os.path.isdir(i):
filelist.extend(findfiles.main(i, genericsettings.verbose))
elif os.path.isfile(i):
filelist.append(i)
else:
txt = "Input file or folder %s could not be found." % i
print txt
raise Usage(txt)
dsList = DataSetList(filelist, genericsettings.verbose)
if not dsList:
raise Usage("Nothing to do: post-processing stopped.")
if genericsettings.isNoisy and not genericsettings.isNoiseFree:
dsList = dsList.dictByNoise().get("nzall", DataSetList())
if genericsettings.isNoiseFree and not genericsettings.isNoisy:
dsList = dsList.dictByNoise().get("noiselessall", DataSetList())
# compute maxfuneval values
dict_max_fun_evals = {}
for ds in dsList:
dict_max_fun_evals[ds.dim] = np.max((dict_max_fun_evals.setdefault(ds.dim, 0), float(np.max(ds.maxevals))))
from bbob_pproc import config
config.target_values(genericsettings.isExpensive, dict_max_fun_evals)
config.config(dsList.isBiobjective())
if genericsettings.verbose:
for i in dsList:
if dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) != inset.instancesOfInterest:
warnings.warn(
"The data of %s do not list " % (i) + "the correct instances " + "of function F%d." % (i.funcId)
)
dictAlg = dsList.dictByAlg()
if len(dictAlg) > 1:
warnings.warn("Data with multiple algId %s " % str(dictAlg.keys()) + "will be processed together.")
# TODO: in this case, all is well as long as for a given problem
# (given dimension and function) there is a single instance of
# DataSet associated. If there are more than one, the first one only
# will be considered... which is probably not what one would expect.
# TODO: put some errors where this case would be a problem.
# raise Usage?
if genericsettings.isFig or genericsettings.isTab or genericsettings.isRLDistr or genericsettings.isLogLoss:
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if genericsettings.verbose:
print "Folder %s was created." % (outputdir)
if genericsettings.isPickled:
dsList.pickle(verbose=genericsettings.verbose)
if genericsettings.isConv:
ppconverrorbars.main(dictAlg, outputdir, genericsettings.verbose)
if genericsettings.isFig:
print "Scaling figures...",
sys.stdout.flush()
# ERT/dim vs dim.
plt.rc("axes", **inset.rcaxeslarger)
plt.rc("xtick", **inset.rcticklarger)
plt.rc("ytick", **inset.rcticklarger)
plt.rc("font", **inset.rcfontlarger)
plt.rc("legend", **inset.rclegendlarger)
plt.rc("pdf", fonttype=42)
ppfigdim.main(dsList, ppfigdim.values_of_interest, outputdir, genericsettings.verbose)
plt.rcdefaults()
print_done()
plt.rc("axes", **inset.rcaxes)
plt.rc("xtick", **inset.rctick)
plt.rc("ytick", **inset.rctick)
plt.rc("font", **inset.rcfont)
plt.rc("legend", **inset.rclegend)
plt.rc("pdf", fonttype=42)
if genericsettings.isTab:
print "TeX tables...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pptable.main(sliceNoise, inset.tabDimsOfInterest, outputdir, noise, genericsettings.verbose)
print_done()
if genericsettings.isRLDistr:
print "ECDF graphs...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
if len(dictNoise) > 1:
warnings.warn(
"Data for functions from both the noisy and "
"non-noisy testbeds have been found. Their "
'results will be mixed in the "all functions" '
"ECDF figures."
)
dictDim = dsList.dictByDim()
for dim in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[dim]
except KeyError:
continue
pprldistr.main(sliceDim, True, outputdir, "all", genericsettings.verbose)
dictNoise = sliceDim.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pprldistr.main(sliceNoise, True, outputdir, "%s" % noise, genericsettings.verbose)
dictFG = sliceDim.dictByFuncGroup()
for fGroup, sliceFuncGroup in dictFG.items():
pprldistr.main(sliceFuncGroup, True, outputdir, "%s" % fGroup, genericsettings.verbose)
pprldistr.fmax = None # Resetting the max final value
pprldistr.evalfmax = None # Resetting the max #fevalsfactor
if (
genericsettings.isRldOnSingleFcts
): # copy-paste from above, here for each function instead of function groups
# ECDFs for each function
pprldmany.all_single_functions(dictAlg, None, outputdir, genericsettings.verbose)
print_done()
if genericsettings.isLogLoss:
print "ERT loss ratio figures and tables...",
sys.stdout.flush()
for ng, sliceNoise in dsList.dictByNoise().iteritems():
if ng == "noiselessall":
testbed = "noiseless"
elif ng == "nzall":
testbed = "noisy"
txt = "Please input crafting effort value " + "for %s testbed:\n CrE = " % testbed
CrE = genericsettings.inputCrE
while CrE is None:
try:
CrE = float(raw_input(txt))
except (SyntaxError, NameError, ValueError):
print "Float value required."
dictDim = sliceNoise.dictByDim()
for d in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[d]
except KeyError:
continue
info = "%s" % ng
pplogloss.main(sliceDim, CrE, True, outputdir, info, verbose=genericsettings.verbose)
pplogloss.generateTable(sliceDim, CrE, outputdir, info, verbose=genericsettings.verbose)
for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup().iteritems():
info = "%s" % fGroup
pplogloss.main(sliceFuncGroup, CrE, True, outputdir, info, verbose=genericsettings.verbose)
pplogloss.evalfmax = None # Resetting the max #fevalsfactor
print_done()
latex_commands_file = os.path.join(outputdir.split(os.sep)[0], "bbob_pproc_commands.tex")
html_file = os.path.join(outputdir, genericsettings.single_algorithm_file_name + ".html")
prepend_to_file(
latex_commands_file, ["\\providecommand{\\bbobloglosstablecaption}[1]{", pplogloss.table_caption, "}"]
)
prepend_to_file(
latex_commands_file, ["\\providecommand{\\bbobloglossfigurecaption}[1]{", pplogloss.figure_caption, "}"]
)
prepend_to_file(
latex_commands_file,
[
"\\providecommand{\\bbobpprldistrlegend}[1]{",
pprldistr.caption_single(
np.max([val / dim for dim, val in dict_max_fun_evals.iteritems()])
), # depends on the config setting, should depend on maxfevals
"}",
],
)
replace_in_file(
html_file,
r"TOBEREPLACED",
"D, ".join([str(i) for i in pprldistr.single_runlength_factors[:6]]) + "D,…",
)
prepend_to_file(
latex_commands_file, ["\\providecommand{\\bbobppfigdimlegend}[1]{", ppfigdim.scaling_figure_caption(), "}"]
)
prepend_to_file(latex_commands_file, ["\\providecommand{\\bbobpptablecaption}[1]{", pptable.table_caption, "}"])
prepend_to_file(latex_commands_file, ["\\providecommand{\\algfolder}{" + algfolder + "/}"])
prepend_to_file(
latex_commands_file,
[
"\\providecommand{\\algname}{"
+ (str_to_latex(strip_pathname1(args[0])) if len(args) == 1 else str_to_latex(dsList[0].algId))
+ "{}}"
],
)
if genericsettings.isFig or genericsettings.isTab or genericsettings.isRLDistr or genericsettings.isLogLoss:
print "Output data written to folder %s" % outputdir
plt.rcdefaults()
