def tablemanyalg(dsList, allmintarget, allertbest, sortedAlgs=None,
outputdir='.'):
"""Generate a table with the figures of multiple algorithms."""
stargets = sorted(allmintarget.keys())
dictDim = dsList.dictByDim()
maxRank = 3
for d, dentries in dictDim.iteritems():
dictAlg = dentries.dictByAlg()
# Multiple algorithms table.
# Generate data
table = []
algnames = []
for alg in sortedAlgs:
# Regroup entries by algorithm
algentries = DataSetList()
for i in alg:
if dictAlg.has_key(i):
algentries.extend(dictAlg[i])
if not algentries:
continue
algnames.append(writeLabels(algPlotInfos[alg[0]]['label']))
tmp = []
for t in stargets:
dictFunc = algentries.dictByFunc()
erts = []
for func, entry in dictFunc.iteritems():
try:
entry = entry[0]
except:
raise Usage('oops too many entries')
try:
if numpy.isnan(allmintarget[t][(func, d)]):
continue
except LookupError:
continue
# At this point the target exists.
try:
erts.append(entry.ert[entry.target<=allmintarget[t][(func, d)]][0]/allertbest[t][(func, d)])
except LookupError:
erts.append(numpy.inf)
if numpy.isfinite(erts).any():
tmp += [numpy.median(erts), numpy.min(erts), numpy.sum(numpy.isfinite(erts))]
else:
tmp += [numpy.inf, numpy.inf, 0]
table.append(tmp)
# Process over all data
table = numpy.array(table)
kept = [] # range(numpy.shape(table)[1])
targetkept = []
for i in range(1, (numpy.shape(table)[1])/3 + 1):
if (table[:, 3*i - 1] != 0).any():
kept.extend([3*i - 3, 3*i - 2 , 3*i - 1])
targetkept.append(i-1)
table = table[:, kept]
#set_trace()
dtype = []
for i, t in enumerate(stargets):
dtype.extend([('med%d' % i, 'f4'), ('min%d' % i, 'f4'),
('nbsolved%d' % i, 'i1')])
dtype = list(dtype[i] for i in kept)
boldface = sortColumns(table, maxRank)
idxsort = numpy.argsort(numpy.array(list(tuple(i) for i in table),
dtype=dtype),
order=('med4', 'med2', 'med0', 'min0'))
# Sorted successively by med(ERT) / ERTbest for fevals/D = 100, 10, 1
# and then min(ERT) / ERTbest for fevals/D = 1
# format the data
lines = [r'\begin{tabular}{c' + 'c@{/}c@{(}c@{) }'*len(targetkept) + '}']
tmpstr = 'evals/D'
for t in list(stargets[i] for i in targetkept):
nbsolved = sum(numpy.isfinite(list(allmintarget[t][i] for i in allmintarget[t] if i[1] == d)))
#set_trace()
tmpstr += (r' & \multicolumn{2}{c@{(}}{%s} & %d' % (writeFEvals(t), nbsolved))
lines.append(tmpstr)
for i in idxsort:
line = table[i]
lines[-1] += r'\\'
curline = algnames[i]
for j in range(len(table[i])):
curline += ' & '
if (j + 1) % 3 > 0: # the test may not be necessary
if numpy.isinf(line[j]):
tmpstr = '.'
else:
tmpstr = '%s' % (writeFEvals(line[j]))
if i in boldface[j]:
tmpstr = r'\textbf{' + tmpstr + '}'
curline += tmpstr
else:
curline += '%d' % line[j] # nb solved.
lines.append(curline)
lines.append(r'\end{tabular}')
f = open(os.path.join(outputdir, 'pptableall_%02dD.tex' % (d)), 'w')
f.write('\n'.join(lines) + '\n')
f.close()
def main(argv=None):
"""Generates from BBOB experiment data some outputs for a tex document.
Provided with some index entries (found in files with the 'info' extension)
this routine outputs figure and TeX files in the folder 'ppdata' needed for
the compilation of latex document templateBBOBarticle.tex. These output
files will contain performance tables, performance scaling figures and
empirical cumulative distribution figures. On subsequent executions, new
files will be added to the output directory, overwriting existing older
files in the process.
Keyword arguments:
argv -- list of strings containing options and arguments. If not given,
sys.argv is accessed.
argv should list either names of info files or folders containing info
files. argv can also contain post-processed pickle files generated by this
routine. Furthermore, argv can begin with, in any order, facultative option
flags listed below.
-h, --help
display this message
-v, --verbose
verbose mode, prints out operations. When not in verbose mode, no
output is to be expected, except for errors.
-p, --pickle
generates pickle post processed data files.
-o, --output-dir OUTPUTDIR
change the default output directory ('ppdata') to OUTPUTDIR
--crafting-effort=VALUE
sets the crafting effort to VALUE. Otherwise the user will be
prompted. This flag is useful when running this script in batch.
-f, --final
lengthens the bootstrapping process used as dispersion measure in
the tables generation. This might at least double the time of the
whole post-processing. Please use this option when generating your
final paper.
--tab-only, --fig-only, --rld-only, --los-only
these options can be used to output respectively the tex tables,
convergence and ENFEs graphs figures, run length distribution
figures, ERT loss ratio figures only. A combination of any two of
these options results in no output.
Exceptions raised:
Usage -- Gives back a usage message.
Examples:
* Calling the run.py interface from the command line:
$ python bbob_pproc/run.py -v experiment1
will post-process the folder experiment1 and all its containing data,
base on the found .info files in the folder. The result will appear
in folder ppdata. The -v option adds verbosity.
$ python bbob_pproc/run.py -o otherppdata experiment2/*.info
This will execute the post-processing on the info files found in
experiment2. The result will be located in the alternative location
otherppdata.
* Loading this package and calling the main from the command line
(requires that the path to this package is in python search path):
$ python -m bbob_pproc -h
This will print out this help message.
* From the python interactive shell (requires that the path to this
package is in python search path):
>>> import bbob_pproc
>>> bbob_pproc.main('-o outputfolder folder1'.split())
This will execute the post-processing on the index files found in folder1.
The -o option changes the output folder from the default ppdata to
outputfolder.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
try:
try:
opts, args = getopt.getopt(argv, "hvpfo:", [
"help", "output-dir=", "tab-only", "fig-only", "rld-only",
"los-only", "crafting-effort=", "pickle", "verbose", "final"
])
except getopt.error, msg:
raise Usage(msg)
if not (args):
usage()
sys.exit()
CrE = None
isfigure = True
istab = True
isrldistr = True
islogloss = True
isPostProcessed = False
isPickled = False
isDraft = True
verbose = False
outputdir = 'ppdata'
#Process options
for o, a in opts:
if o in ("-v", "--verbose"):
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-p", "--pickle"):
isPickled = True
elif o in ("-o", "--output-dir"):
outputdir = a
elif o in ("-f", "--final"):
isDraft = False
#The next 3 are for testing purpose
elif o == "--tab-only":
isfigure = False
isrldistr = False
islogloss = False
elif o == "--fig-only":
istab = False
isrldistr = False
islogloss = False
elif o == "--rld-only":
istab = False
isfigure = False
islogloss = False
elif o == "--los-only":
istab = False
isfigure = False
isrldistr = False
elif o == "--crafting-effort":
try:
CrE = float(a)
except ValueError:
raise Usage(
'Expect a valid float for flag crafting-effort.')
else:
assert False, "unhandled option"
if (not verbose):
warnings.simplefilter('ignore')
print("BBOB Post-processing: will generate post-processing " +
"data in folder %s" % outputdir)
print " this might take several minutes."
filelist = list()
for i in args:
if os.path.isdir(i):
filelist.extend(findfiles.main(i, verbose))
elif os.path.isfile(i):
filelist.append(i)
else:
txt = 'Input file or folder %s could not be found.'
raise Usage(txt)
dsList = DataSetList(filelist, verbose)
if not dsList:
raise Usage("Nothing to do: post-processing stopped.")
if (verbose):
for i in dsList:
if (dict((j, i.itrials.count(j))
for j in set(i.itrials)) != instancesOfInterest2010):
warnings.warn('The data of %s do not list ' % (i) +
'the correct instances ' +
'of function F%d.' % (i.funcId))
# BBOB 2009 Checking
#if ((dict((j, i.itrials.count(j)) for j in set(i.itrials)) !=
#instancesOfInterest) and
#(dict((j, i.itrials.count(j)) for j in set(i.itrials)) !=
#instancesOfInterest2010)):
#warnings.warn('The data of %s do not list ' %(i) +
#'the correct instances ' +
#'of function F%d or the ' %(i.funcId) +
#'correct number of trials for each.')
dictAlg = dsList.dictByAlg()
if len(dictAlg) > 1:
warnings.warn('Data with multiple algId %s ' % (dictAlg) +
'will be processed together.')
#TODO: in this case, all is well as long as for a given problem
#(given dimension and function) there is a single instance of
#DataSet associated. If there are more than one, the first one only
#will be considered... which is probably not what one would expect.
#TODO: put some errors where this case would be a problem.
if isfigure or istab or isrldistr or islogloss:
if not os.path.exists(outputdir):
os.mkdir(outputdir)
if verbose:
print 'Folder %s was created.' % (outputdir)
if isPickled:
dsList.pickle(verbose=verbose)
if isfigure:
ppfigdim.main(dsList, figValsOfInterest, outputdir, verbose)
#ERT/dim vs dim.
#ppfigdim.main2(dsList, figValsOfInterest, outputdir,
#verbose)
print "Scaling figures done."
if istab:
dictFunc = dsList.dictByFunc()
for fun, sliceFun in dictFunc.items():
dictDim = sliceFun.dictByDim()
tmp = []
for dim in tabDimsOfInterest:
try:
if len(dictDim[dim]) > 1:
warnings.warn(
'Func: %d, DIM %d: ' % (fun, dim) +
'multiple index entries. Will only ' +
'process the first ' + '%s.' % dictDim[dim][0])
tmp.append(dictDim[dim][0])
except KeyError:
pass
if tmp:
filename = os.path.join(outputdir, 'ppdata_f%d' % fun)
pptex.main(tmp, tabValsOfInterest, filename, isDraft,
verbose)
print "TeX tables",
if isDraft:
print(
"(draft) done. To get final version tables, please "
"use the -f option with run.py")
else:
print "done."
if isrldistr:
dictNoise = dsList.dictByNoise()
if len(dictNoise) > 1:
warnings.warn('Data for functions from both the noisy and '
'non-noisy testbeds have been found. Their '
'results will be mixed in the "all functions" '
'ECDF figures.')
dictDim = dsList.dictByDim()
for dim in rldDimsOfInterest:
try:
sliceDim = dictDim[dim]
pprldistr.main(sliceDim, rldValsOfInterest, True,
outputdir, 'dim%02dall' % dim, verbose)
dictNoise = sliceDim.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pprldistr.main(sliceNoise, rldValsOfInterest, True,
outputdir, 'dim%02d%s' % (dim, noise),
verbose)
dictFG = sliceDim.dictByFuncGroup()
for fGroup, sliceFuncGroup in dictFG.items():
pprldistr.main(sliceFuncGroup, rldValsOfInterest, True,
outputdir, 'dim%02d%s' % (dim, fGroup),
verbose)
pprldistr.fmax = None #Resetting the max final value
pprldistr.evalfmax = None #Resetting the max #fevalsfactor
except KeyError:
pass
print "ECDF graphs done."
if islogloss:
for ng, sliceNoise in dsList.dictByNoise().iteritems():
if ng == 'noiselessall':
testbed = 'noiseless'
elif ng == 'nzall':
testbed = 'noisy'
txt = ("Please input crafting effort value " +
"for %s testbed:\n CrE = " % testbed)
while CrE is None:
try:
CrE = float(input(txt))
except (SyntaxError, NameError, ValueError):
print "Float value required."
dictDim = sliceNoise.dictByDim()
for d in rldDimsOfInterest:
try:
sliceDim = dictDim[d]
except KeyError:
continue
info = 'dim%02d%s' % (d, ng)
pplogloss.main(sliceDim,
CrE,
True,
outputdir,
info,
verbose=verbose)
pplogloss.generateTable(sliceDim,
CrE,
outputdir,
info,
verbose=verbose)
for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup(
).iteritems():
info = 'dim%02d%s' % (d, fGroup)
pplogloss.main(sliceFuncGroup,
CrE,
True,
outputdir,
info,
verbose=verbose)
pplogloss.evalfmax = None #Resetting the max #fevalsfactor
print "ERT loss ratio figures and tables done."
if isfigure or istab or isrldistr or islogloss:
print "Output data written to folder %s." % outputdir
