def main(argv=None):
"""Main routine for COCO data checking procedure.
The routine will stop at the first problem encountered.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
try:
try:
opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not (args):
usage()
sys.exit()
#Process options
verbose = False
for o, a in opts:
if o in ('-v', '--verbose'):
verbose = True
print 'COCO Checking procedure: This may take a couple of minutes.'
filelist = list()
for i in args:
if os.path.isdir(i):
filelist.extend(findfiles.main(i, verbose))
elif os.path.isfile(i):
filelist.append(i)
else:
txt = 'Input file or folder %s could not be found.' % i
raise Usage(txt)
for i in filelist:
try:
if verbose:
print 'Checking %s.' % i
extension = os.path.splitext(i)[1]
if extension == '.info':
checkinfofile(i, verbose)
elif extension == '.pickle':
# cocofy(i)
with open(i) as f:
ds = pickle.load(f)
if not is_correct_instances(ds.instancenumbers):
msg = ('File %s: The instances listed do not respect '
'the specifications BBOB-2009 or BBOB-2010.' %
i)
raise Usage(msg)
except Usage, err:
print >> sys.stderr, err.msg
continue
def processInputArgs(args, verbose=True):
"""Process command line arguments into data useable by bbob_pproc scripts.
Returns an instance of DataSetList, a list of algorithms from
a list of strings representing file and folder names.
This command will operate folder-wise: one folder will correspond to an
algorithm.
It is recommended that if a folder listed in args contain both info files
and the associated pickle files, they be kept in different locations for
efficiency reasons.
Keyword arguments:
args -- list of string arguments for folder names
verbose -- bool controlling verbosity
Returns: (dsList, sortedAlgs, dictAlg), where
dsList is a list containing all DataSet instances, this is to prevent the
regrouping done in instances of DataSetList
dictAlg is a dictionary which associates algorithms to an instance of
DataSetList,
sortedAlgs is the sorted list of keys of dictAlg, the sorting is given
by the input argument args.
"""
dsList = list()
sortedAlgs = list()
dictAlg = {}
for i in args:
if os.path.isfile(i):
txt = ('The post-processing cannot operate on the single file' +
' %s.' % i)
warnings.warn(txt)
continue
elif os.path.isdir(i):
filelist = findfiles.main(i, verbose)
#Do here any sorting or filtering necessary.
#filelist = list(i for i in filelist if i.count('ppdata_f005'))
tmpDsList = DataSetList(filelist, verbose)
#Nota: findfiles will find all info AND pickle files in folder i.
#No problem should arise if the info and pickle files have
#redundant information. Only, the process could be more efficient
#if pickle files were in a whole other location.
dsList.extend(tmpDsList)
#alg = os.path.split(i.rstrip(os.sep))[1] # trailing slash or backslash
#if alg == '':
# alg = os.path.split(os.path.split(i)[0])[1]
alg = i
print ' using:', alg
if all(i != alg for i in sortedAlgs):
sortedAlgs.append(
alg) # TODO: watch out there could be duplicates.
dictAlg[alg] = tmpDsList
else:
txt = 'Input folder %s could not be found.' % i
#raise Usage(txt) #TODO how to call Usage?
warnings.warn(txt)
return dsList, sortedAlgs, dictAlg
def main(argv=None):
"""Main routine for COCO data checking procedure.
The routine will stop at the first problem encountered.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
try:
try:
opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not (args):
usage()
sys.exit()
#Process options
verbose = False
for o, a in opts:
if o in ('-v', '--verbose'):
verbose = True
print 'COCO Checking procedure: This may take a couple of minutes.'
filelist = list()
for i in args:
if os.path.isdir(i):
filelist.extend(findfiles.main(i, verbose))
elif os.path.isfile(i):
filelist.append(i)
else:
txt = 'Input file or folder %s could not be found.' % i
raise Usage(txt)
for i in filelist:
try:
if verbose:
print 'Checking %s.' % i
extension = os.path.splitext(i)[1]
if extension == '.info':
checkinfofile(i, verbose)
elif extension == '.pickle':
# cocofy(i)
with open(i) as f:
ds = pickle.load(f)
if not is_correct_instances(ds.instancenumbers):
msg = ('File %s: The instances listed do not respect '
'the specifications BBOB-2009 or BBOB-2010.' % i)
raise Usage(msg)
except Usage, err:
print >>sys.stderr, err.msg
continue
def __init__(self, args=[], verbose=True):
"""Instantiate self from a list of inputs.
Keyword arguments:
args -- list of strings being either info file names, folder containing
info files or pickled data files.
verbose -- controls verbosity.
Exception:
Warning -- Unexpected user input.
pickle.UnpicklingError
"""
#Nota: part of the input argument processing is now in
#pproc.processInputArguments.
if not args:
super(DataSetList, self).__init__()
return
if isinstance(args, basestring):
args = [args]
tmp = []
for i in args:
if os.path.isdir(i):
tmp.extend(findfiles.main(i, verbose))
else:
tmp.append(i)
for i in tmp:
if i.endswith('.info'):
self.processIndexFile(i, verbose)
elif i.endswith('.pickle'):
try:
f = open(i, 'r')
try:
entry = pickle.load(f)
except pickle.UnpicklingError:
print '%s could not be unpickled.' % (i)
f.close()
if verbose:
print 'Unpickled %s.' % (i)
self.append(entry)
#set_trace()
except IOError, (errno, strerror):
print "I/O error(%s): %s" % (errno, strerror)
else:
warnings.warn('File or folder ' + i + ' not found. ' +
'Expecting as input argument either .info ' +
'file(s), .pickle file(s) or a folder ' +
'containing .info file(s).')
def main(argv=None):
r"""Post-processing COCO data of a single algorithm.
Provided with some data, this routine outputs figure and TeX files
in a folder needed for the compilation of latex document
:file:`template1XXX.tex` or :file:`noisytemplate1XXX.tex`, where
:file:`XXX` is either :file:`ecj` or :file:`generic`. The template
file needs to be edited so that the commands
``\bbobdatapath`` and ``\algfolder`` point to the output folder.
These output files will contain performance tables, performance
scaling figures and empirical cumulative distribution figures. On
subsequent executions, new files will be added to the output folder,
overwriting existing older files in the process.
Keyword arguments:
*argv* -- list of strings containing options and arguments. If not
given, sys.argv is accessed.
*argv* should list either names of :file:`info` files or folders
containing :file:`info` files. argv can also contain post-processed
:file:`pickle` files generated by this routine. Furthermore, *argv*
can begin with, in any order, facultative option flags listed below.
-h, --help
displays this message.
-v, --verbose
verbose mode, prints out all operations.
-p, --pickle
generates pickle post processed data files.
-o OUTPUTDIR, --output-dir=OUTPUTDIR
changes the default output directory (:file:`ppdata`) to
:file:`OUTPUTDIR`.
--crafting-effort=VALUE
sets the crafting effort to VALUE (float). Otherwise the
default value of 0. will be used.
--noise-free, --noisy
processes only part of the data.
--settings=SETTINGS
changes the style of the output figures and tables. At the
moment the only differences are in the colors of the output
figures. SETTINGS can be either "grayscale", "color" or
"black-white". The default setting is "color".
--tab-only, --fig-only, --rld-only, --los-only
these options can be used to output respectively the TeX
tables, convergence and ERTs graphs figures, run length
distribution figures, ERT loss ratio figures only. A
combination of any two of these options results in no
output.
--conv
if this option is chosen, additionally convergence plots
for each function and algorithm are generated.
--expensive
runlength-based f-target values and fixed display limits,
useful with comparatively small budgets. By default the
setting is based on the budget used in the data.
--not-expensive
expensive setting off.
--runlength-based
runlength-based f-target values, such that the
"level of difficulty" is similar for all functions.
Exceptions raised:
*Usage* -- Gives back a usage message.
Examples:
* Calling the rungeneric1.py interface from the command line::
$ python bbob_pproc/rungeneric1.py -v experiment1
will post-process the folder experiment1 and all its containing
data, base on the .info files found in the folder. The result will
appear in the default output folder. The -v option adds verbosity. ::
$ python bbob_pproc/rungeneric1.py -o exp2 experiment2/*.info
This will execute the post-processing on the info files found in
:file:`experiment2`. The result will be located in the alternative
location :file:`exp2`.
* Loading this package and calling the main from the command line
(requires that the path to this package is in python search path)::
$ python -m bbob_pproc.rungeneric1 -h
This will print out this help message.
* From the python interpreter (requires that the path to this
package is in python search path)::
>> import bbob_pproc as bb
>> bb.rungeneric1.main('-o outputfolder folder1'.split())
This will execute the post-processing on the index files found in
:file:`folder1`. The ``-o`` option changes the output folder from
the default to :file:`outputfolder`.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
if 1 < 3:
opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
if 11 < 3:
try:
opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not (args) and not '--help' in argv and not 'h' in argv:
print 'not enough input arguments given'
print 'cave: the following options also need an argument:'
print[o for o in longoptlist if o[-1] == '=']
print 'options given:'
print opts
print 'try --help for help'
sys.exit()
inputCrE = 0.
isfigure = True
istab = True
isrldistr = True
islogloss = True
isPostProcessed = False
isPickled = False
verbose = False
outputdir = 'ppdata'
isNoisy = False
isNoiseFree = False
inputsettings = 'color'
isConv = False
isRLbased = None # allows automatic choice
isExpensive = None
# Process options
for o, a in opts:
if o in ("-v", "--verbose"):
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-p", "--pickle"):
isPickled = True
elif o in ("-o", "--output-dir"):
outputdir = a
elif o == "--noisy":
isNoisy = True
elif o == "--noise-free":
isNoiseFree = True
# The next 4 are for testing purpose
elif o == "--tab-only":
isfigure = False
isrldistr = False
islogloss = False
elif o == "--fig-only":
istab = False
isrldistr = False
islogloss = False
elif o == "--rld-only":
istab = False
isfigure = False
islogloss = False
elif o == "--los-only":
istab = False
isfigure = False
isrldistr = False
elif o == "--crafting-effort":
try:
inputCrE = float(a)
except ValueError:
raise Usage(
'Expect a valid float for flag crafting-effort.')
elif o == "--settings":
inputsettings = a
elif o == "--conv":
isConv = True
elif o == "--runlength-based":
isRLbased = True
elif o == "--expensive":
isExpensive = True # comprises runlength-based
elif o == "--not-expensive":
isExpensive = False
else:
assert False, "unhandled option"
# from bbob_pproc import bbob2010 as inset # input settings
if inputsettings == "color":
from bbob_pproc import genericsettings as inset # input settings
elif inputsettings == "grayscale":
from bbob_pproc import grayscalesettings as inset # input settings
elif inputsettings == "black-white":
from bbob_pproc import bwsettings as inset # input settings
else:
txt = ('Settings: %s is not an appropriate ' % inputsettings +
'argument for input flag "--settings".')
raise Usage(txt)
if 11 < 3:
from bbob_pproc import config # input settings
config.config()
import imp
# import testbedsettings as testbedsettings # input settings
try:
fp, pathname, description = imp.find_module("testbedsettings")
testbedsettings = imp.load_module("testbedsettings", fp,
pathname, description)
finally:
fp.close()
if (not verbose):
warnings.simplefilter('module')
# warnings.simplefilter('ignore')
print("Post-processing (1): will generate output " +
"data in folder %s" % outputdir)
print " this might take several minutes."
filelist = list()
for i in args:
i = i.strip()
if os.path.isdir(i):
filelist.extend(findfiles.main(i, verbose))
elif os.path.isfile(i):
filelist.append(i)
else:
txt = 'Input file or folder %s could not be found.' % i
print txt
raise Usage(txt)
dsList = DataSetList(filelist, verbose)
if not dsList:
raise Usage("Nothing to do: post-processing stopped.")
if isNoisy and not isNoiseFree:
dsList = dsList.dictByNoise().get('nzall', DataSetList())
if isNoiseFree and not isNoisy:
dsList = dsList.dictByNoise().get('noiselessall', DataSetList())
# compute maxfuneval values
dict_max_fun_evals = {}
for ds in dsList:
dict_max_fun_evals[ds.dim] = np.max(
(dict_max_fun_evals.setdefault(ds.dim,
0), float(np.max(ds.maxevals))))
if isRLbased is not None:
genericsettings.runlength_based_targets = isRLbased
from bbob_pproc import config
config.target_values(isExpensive, dict_max_fun_evals)
config.config()
if (verbose):
for i in dsList:
if (dict((j, i.instancenumbers.count(j))
for j in set(i.instancenumbers)) !=
inset.instancesOfInterest):
warnings.warn('The data of %s do not list ' % (i) +
'the correct instances ' +
'of function F%d.' % (i.funcId))
dictAlg = dsList.dictByAlg()
if len(dictAlg) > 1:
warnings.warn('Data with multiple algId %s ' % (dictAlg) +
'will be processed together.')
# TODO: in this case, all is well as long as for a given problem
# (given dimension and function) there is a single instance of
# DataSet associated. If there are more than one, the first one only
# will be considered... which is probably not what one would expect.
# TODO: put some errors where this case would be a problem.
# raise Usage?
if isfigure or istab or isrldistr or islogloss:
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if verbose:
print 'Folder %s was created.' % (outputdir)
if isPickled:
dsList.pickle(verbose=verbose)
if isConv:
ppconverrorbars.main(dictAlg, outputdir, verbose)
if isfigure:
print "Scaling figures...",
sys.stdout.flush()
# ERT/dim vs dim.
plt.rc("axes", **inset.rcaxeslarger)
plt.rc("xtick", **inset.rcticklarger)
plt.rc("ytick", **inset.rcticklarger)
plt.rc("font", **inset.rcfontlarger)
plt.rc("legend", **inset.rclegendlarger)
ppfigdim.main(dsList, ppfigdim.values_of_interest, outputdir,
verbose)
plt.rcdefaults()
print_done()
plt.rc("axes", **inset.rcaxes)
plt.rc("xtick", **inset.rctick)
plt.rc("ytick", **inset.rctick)
plt.rc("font", **inset.rcfont)
plt.rc("legend", **inset.rclegend)
if istab:
print "TeX tables...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pptable.main(sliceNoise, inset.tabDimsOfInterest, outputdir,
noise, verbose)
print_done()
if isrldistr:
print "ECDF graphs...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
if len(dictNoise) > 1:
warnings.warn('Data for functions from both the noisy and '
'non-noisy testbeds have been found. Their '
'results will be mixed in the "all functions" '
'ECDF figures.')
dictDim = dsList.dictByDim()
for dim in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[dim]
except KeyError:
continue
pprldistr.main(sliceDim, True, outputdir, 'all', verbose)
dictNoise = sliceDim.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pprldistr.main(sliceNoise, True, outputdir, '%s' % noise,
verbose)
dictFG = sliceDim.dictByFuncGroup()
for fGroup, sliceFuncGroup in dictFG.items():
pprldistr.main(sliceFuncGroup, True, outputdir,
'%s' % fGroup, verbose)
pprldistr.fmax = None # Resetting the max final value
pprldistr.evalfmax = None # Resetting the max #fevalsfactor
print_done()
if islogloss:
print "ERT loss ratio figures and tables...",
sys.stdout.flush()
for ng, sliceNoise in dsList.dictByNoise().iteritems():
if ng == 'noiselessall':
testbed = 'noiseless'
elif ng == 'nzall':
testbed = 'noisy'
txt = ("Please input crafting effort value " +
"for %s testbed:\n CrE = " % testbed)
CrE = inputCrE
while CrE is None:
try:
CrE = float(raw_input(txt))
except (SyntaxError, NameError, ValueError):
print "Float value required."
dictDim = sliceNoise.dictByDim()
for d in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[d]
except KeyError:
continue
info = '%s' % ng
pplogloss.main(sliceDim,
CrE,
True,
outputdir,
info,
verbose=verbose)
pplogloss.generateTable(sliceDim,
CrE,
outputdir,
info,
verbose=verbose)
for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup(
).iteritems():
info = '%s' % fGroup
pplogloss.main(sliceFuncGroup,
CrE,
True,
outputdir,
info,
verbose=verbose)
pplogloss.evalfmax = None # Resetting the max #fevalsfactor
print_done()
latex_commands_file = os.path.join(
outputdir.split(os.sep)[0], 'bbob_pproc_commands.tex')
prepend_to_file(latex_commands_file, [
'\\providecommand{\\bbobloglosstablecaption}[1]{',
pplogloss.table_caption, '}'
])
prepend_to_file(latex_commands_file, [
'\\providecommand{\\bbobloglossfigurecaption}[1]{',
pplogloss.figure_caption, '}'
])
prepend_to_file(
latex_commands_file,
[
'\\providecommand{\\bbobpprldistrlegend}[1]{',
pprldistr.caption_single(
np.max([
val / dim
for dim, val in dict_max_fun_evals.iteritems()
])
), # depends on the config setting, should depend on maxfevals
'}'
])
prepend_to_file(latex_commands_file, [
'\\providecommand{\\bbobppfigdimlegend}[1]{',
ppfigdim.scaling_figure_caption(), '}'
])
prepend_to_file(latex_commands_file, [
'\\providecommand{\\bbobpptablecaption}[1]{',
pptable.table_caption, '}'
])
prepend_to_file(latex_commands_file,
['\\providecommand{\\algfolder}{}'
]) # is overwritten in rungeneric.py
prepend_to_file(latex_commands_file, [
'\\providecommand{\\algname}{' +
(str_to_latex(strip_pathname(args[0]))
if len(args) == 1 else str_to_latex(dsList[0].algId)) + '{}}'
])
if isfigure or istab or isrldistr or islogloss:
print "Output data written to folder %s" % outputdir
plt.rcdefaults()
def main(argv=None):
r"""Post-processing COCO data of a single algorithm.
Provided with some data, this routine outputs figure and TeX files
in a folder needed for the compilation of latex document
:file:`template1XXX.tex` or :file:`noisytemplate1XXX.tex`, where
:file:`XXX` is either :file:`ecj` or :file:`generic`. The template
file needs to be edited so that the commands
``\bbobdatapath`` and ``\algfolder`` point to the output folder.
These output files will contain performance tables, performance
scaling figures and empirical cumulative distribution figures. On
subsequent executions, new files will be added to the output folder,
overwriting existing older files in the process.
Keyword arguments:
*argv* -- list of strings containing options and arguments. If not
given, sys.argv is accessed.
*argv* should list either names of :file:`info` files or folders
containing :file:`info` files. argv can also contain post-processed
:file:`pickle` files generated by this routine. Furthermore, *argv*
can begin with, in any order, facultative option flags listed below.
-h, --help
displays this message.
-v, --verbose
verbose mode, prints out all operations.
-p, --pickle
generates pickle post processed data files.
-o OUTPUTDIR, --output-dir=OUTPUTDIR
changes the default output directory (:file:`ppdata`) to
:file:`OUTPUTDIR`.
--crafting-effort=VALUE
sets the crafting effort to VALUE (float). Otherwise the
default value of 0. will be used.
--noise-free, --noisy
processes only part of the data.
--settings=SETTINGS
changes the style of the output figures and tables. At the
moment the only differences are in the colors of the output
figures. SETTINGS can be either "grayscale", "color" or
"black-white". The default setting is "color".
--tab-only, --fig-only, --rld-only, --los-only
these options can be used to output respectively the TeX
tables, convergence and ERTs graphs figures, run length
distribution figures, ERT loss ratio figures only. A
combination of any two of these options results in no
output.
--conv
if this option is chosen, additionally convergence plots
for each function and algorithm are generated.
--expensive
runlength-based f-target values and fixed display limits,
useful with comparatively small budgets. By default the
setting is based on the budget used in the data.
--not-expensive
expensive setting off.
--runlength-based
runlength-based f-target values, such that the
"level of difficulty" is similar for all functions.
Exceptions raised:
*Usage* -- Gives back a usage message.
Examples:
* Calling the rungeneric1.py interface from the command line::
$ python bbob_pproc/rungeneric1.py -v experiment1
will post-process the folder experiment1 and all its containing
data, base on the .info files found in the folder. The result will
appear in the default output folder. The -v option adds verbosity. ::
$ python bbob_pproc/rungeneric1.py -o exp2 experiment2/*.info
This will execute the post-processing on the info files found in
:file:`experiment2`. The result will be located in the alternative
location :file:`exp2`.
* Loading this package and calling the main from the command line
(requires that the path to this package is in python search path)::
$ python -m bbob_pproc.rungeneric1 -h
This will print out this help message.
* From the python interpreter (requires that the path to this
package is in python search path)::
>> import bbob_pproc as bb
>> bb.rungeneric1.main('-o outputfolder folder1'.split())
This will execute the post-processing on the index files found in
:file:`folder1`. The ``-o`` option changes the output folder from
the default to :file:`outputfolder`.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
if 1 < 3:
opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
if 11 < 3:
try:
opts, args = getopt.getopt(argv, shortoptlist, longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not (args) and not '--help' in argv and not 'h' in argv:
print 'not enough input arguments given'
print 'cave: the following options also need an argument:'
print [o for o in longoptlist if o[-1] == '=']
print 'options given:'
print opts
print 'try --help for help'
sys.exit()
inputCrE = 0.
isfigure = True
istab = True
isrldistr = True
islogloss = True
isPostProcessed = False
isPickled = False
verbose = False
outputdir = 'ppdata'
isNoisy = False
isNoiseFree = False
inputsettings = 'color'
isConv = False
isRLbased = None # allows automatic choice
isExpensive = None
# Process options
for o, a in opts:
if o in ("-v", "--verbose"):
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-p", "--pickle"):
isPickled = True
elif o in ("-o", "--output-dir"):
outputdir = a
elif o == "--noisy":
isNoisy = True
elif o == "--noise-free":
isNoiseFree = True
# The next 4 are for testing purpose
elif o == "--tab-only":
isfigure = False
isrldistr = False
islogloss = False
elif o == "--fig-only":
istab = False
isrldistr = False
islogloss = False
elif o == "--rld-only":
istab = False
isfigure = False
islogloss = False
elif o == "--los-only":
istab = False
isfigure = False
isrldistr = False
elif o == "--crafting-effort":
try:
inputCrE = float(a)
except ValueError:
raise Usage('Expect a valid float for flag crafting-effort.')
elif o == "--settings":
inputsettings = a
elif o == "--conv":
isConv = True
elif o == "--runlength-based":
isRLbased = True
elif o == "--expensive":
isExpensive = True # comprises runlength-based
elif o == "--not-expensive":
isExpensive = False
else:
assert False, "unhandled option"
# from bbob_pproc import bbob2010 as inset # input settings
if inputsettings == "color":
from bbob_pproc import genericsettings as inset # input settings
elif inputsettings == "grayscale":
from bbob_pproc import grayscalesettings as inset # input settings
elif inputsettings == "black-white":
from bbob_pproc import bwsettings as inset # input settings
else:
txt = ('Settings: %s is not an appropriate ' % inputsettings
+ 'argument for input flag "--settings".')
raise Usage(txt)
if 11 < 3:
from bbob_pproc import config # input settings
config.config()
import imp
# import testbedsettings as testbedsettings # input settings
try:
fp, pathname, description = imp.find_module("testbedsettings")
testbedsettings = imp.load_module("testbedsettings", fp, pathname, description)
finally:
fp.close()
if (not verbose):
warnings.simplefilter('module')
# warnings.simplefilter('ignore')
print ("Post-processing (1): will generate output " +
"data in folder %s" % outputdir)
print " this might take several minutes."
filelist = list()
for i in args:
i = i.strip()
if os.path.isdir(i):
filelist.extend(findfiles.main(i, verbose))
elif os.path.isfile(i):
filelist.append(i)
else:
txt = 'Input file or folder %s could not be found.' % i
print txt
raise Usage(txt)
dsList = DataSetList(filelist, verbose)
if not dsList:
raise Usage("Nothing to do: post-processing stopped.")
if isNoisy and not isNoiseFree:
dsList = dsList.dictByNoise().get('nzall', DataSetList())
if isNoiseFree and not isNoisy:
dsList = dsList.dictByNoise().get('noiselessall', DataSetList())
# compute maxfuneval values
dict_max_fun_evals = {}
for ds in dsList:
dict_max_fun_evals[ds.dim] = np.max((dict_max_fun_evals.setdefault(ds.dim, 0), float(np.max(ds.maxevals))))
if isRLbased is not None:
genericsettings.runlength_based_targets = isRLbased
from bbob_pproc import config
config.target_values(isExpensive, dict_max_fun_evals)
config.config()
if (verbose):
for i in dsList:
if (dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) !=
inset.instancesOfInterest):
warnings.warn('The data of %s do not list ' % (i) +
'the correct instances ' +
'of function F%d.' % (i.funcId))
dictAlg = dsList.dictByAlg()
if len(dictAlg) > 1:
warnings.warn('Data with multiple algId %s ' % (dictAlg) +
'will be processed together.')
# TODO: in this case, all is well as long as for a given problem
# (given dimension and function) there is a single instance of
# DataSet associated. If there are more than one, the first one only
# will be considered... which is probably not what one would expect.
# TODO: put some errors where this case would be a problem.
# raise Usage?
if isfigure or istab or isrldistr or islogloss:
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if verbose:
print 'Folder %s was created.' % (outputdir)
if isPickled:
dsList.pickle(verbose=verbose)
if isConv:
ppconverrorbars.main(dictAlg, outputdir, verbose)
if isfigure:
print "Scaling figures...",
sys.stdout.flush()
# ERT/dim vs dim.
plt.rc("axes", **inset.rcaxeslarger)
plt.rc("xtick", **inset.rcticklarger)
plt.rc("ytick", **inset.rcticklarger)
plt.rc("font", **inset.rcfontlarger)
plt.rc("legend", **inset.rclegendlarger)
ppfigdim.main(dsList, ppfigdim.values_of_interest, outputdir, verbose)
plt.rcdefaults()
print_done()
plt.rc("axes", **inset.rcaxes)
plt.rc("xtick", **inset.rctick)
plt.rc("ytick", **inset.rctick)
plt.rc("font", **inset.rcfont)
plt.rc("legend", **inset.rclegend)
if istab:
print "TeX tables...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pptable.main(sliceNoise, inset.tabDimsOfInterest, outputdir,
noise, verbose)
print_done()
if isrldistr:
print "ECDF graphs...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
if len(dictNoise) > 1:
warnings.warn('Data for functions from both the noisy and '
'non-noisy testbeds have been found. Their '
'results will be mixed in the "all functions" '
'ECDF figures.')
dictDim = dsList.dictByDim()
for dim in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[dim]
except KeyError:
continue
pprldistr.main(sliceDim, True,
outputdir, 'all', verbose)
dictNoise = sliceDim.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pprldistr.main(sliceNoise, True,
outputdir, '%s' % noise, verbose)
dictFG = sliceDim.dictByFuncGroup()
for fGroup, sliceFuncGroup in dictFG.items():
pprldistr.main(sliceFuncGroup,
True, outputdir, '%s' % fGroup, verbose)
pprldistr.fmax = None # Resetting the max final value
pprldistr.evalfmax = None # Resetting the max #fevalsfactor
print_done()
if islogloss:
print "ERT loss ratio figures and tables...",
sys.stdout.flush()
for ng, sliceNoise in dsList.dictByNoise().iteritems():
if ng == 'noiselessall':
testbed = 'noiseless'
elif ng == 'nzall':
testbed = 'noisy'
txt = ("Please input crafting effort value "
+ "for %s testbed:\n CrE = " % testbed)
CrE = inputCrE
while CrE is None:
try:
CrE = float(raw_input(txt))
except (SyntaxError, NameError, ValueError):
print "Float value required."
dictDim = sliceNoise.dictByDim()
for d in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[d]
except KeyError:
continue
info = '%s' % ng
pplogloss.main(sliceDim, CrE, True, outputdir, info,
verbose=verbose)
pplogloss.generateTable(sliceDim, CrE, outputdir, info,
verbose=verbose)
for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup().iteritems():
info = '%s' % fGroup
pplogloss.main(sliceFuncGroup, CrE, True, outputdir, info,
verbose=verbose)
pplogloss.evalfmax = None # Resetting the max #fevalsfactor
print_done()
latex_commands_file = os.path.join(outputdir.split(os.sep)[0], 'bbob_pproc_commands.tex')
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobloglosstablecaption}[1]{',
pplogloss.table_caption, '}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobloglossfigurecaption}[1]{',
pplogloss.figure_caption, '}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobpprldistrlegend}[1]{',
pprldistr.caption_single(np.max([ val / dim for dim, val in dict_max_fun_evals.iteritems()])), # depends on the config setting, should depend on maxfevals
'}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobppfigdimlegend}[1]{',
ppfigdim.scaling_figure_caption(),
'}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\bbobpptablecaption}[1]{',
pptable.table_caption,
'}'])
prepend_to_file(latex_commands_file,
['\\providecommand{\\algfolder}{}']) # is overwritten in rungeneric.py
prepend_to_file(latex_commands_file,
['\\providecommand{\\algname}{' +
(str_to_latex(strip_pathname(args[0])) if len(args) == 1 else str_to_latex(dsList[0].algId)) + '{}}'])
if isfigure or istab or isrldistr or islogloss:
print "Output data written to folder %s" % outputdir
plt.rcdefaults()
f.write("function = %s dim = %s %s%s\n" %
(key[0], key[1], key[2], instancedict[key]))
if __name__ == '__main__':
"""Merges lines of .info files that contain different instances but the
same filename.
Reconstructs the .info files within a given folder and writes everything
into a clean new folder with the given output folder name.
"""
if not len(sys.argv) == 3:
print(
r'Usage:\n python merge_lines_in_info_files.py FOLDERNAME OUTPUTFOLDERNAME'
)
else:
inputfolder = sys.argv[1]
outputfolder = sys.argv[2]
try:
shutil.copytree(inputfolder, outputfolder)
except:
print("Problem while copying folder %s to %s" %
(inputfolder, outputfolder))
e = sys.exc_info()[0]
print(" Error: %s" % e)
filelist = findfiles.main(sys.argv[1])
for f in filelist:
print("Processing %s..." % f)
merge_lines_in(f, inputfolder, outputfolder)
def main(argv=None):
"""Generates from BBOB experiment data some outputs for a tex document.
Provided with some index entries (found in files with the 'info' extension)
this routine outputs figure and TeX files in the folder 'ppdata' needed for
the compilation of latex document templateBBOBarticle.tex. These output
files will contain performance tables, performance scaling figures and
empirical cumulative distribution figures. On subsequent executions, new
files will be added to the output directory, overwriting existing older
files in the process.
Keyword arguments:
argv -- list of strings containing options and arguments. If not given,
sys.argv is accessed.
argv should list either names of info files or folders containing info
files. argv can also contain post-processed pickle files generated by this
routine. Furthermore, argv can begin with, in any order, facultative option
flags listed below.
-h, --help
display this message
-v, --verbose
verbose mode, prints out operations. When not in verbose mode, no
output is to be expected, except for errors.
-p, --pickle
generates pickle post processed data files.
-o, --output-dir OUTPUTDIR
change the default output directory ('ppdata') to OUTPUTDIR
--crafting-effort=VALUE
sets the crafting effort to VALUE. Otherwise the user will be
prompted. This flag is useful when running this script in batch.
-f, --final
lengthens the bootstrapping process used as dispersion measure in
the tables generation. This might at least double the time of the
whole post-processing. Please use this option when generating your
final paper.
--tab-only, --fig-only, --rld-only, --los-only
these options can be used to output respectively the tex tables,
convergence and ENFEs graphs figures, run length distribution
figures, ERT loss ratio figures only. A combination of any two of
these options results in no output.
Exceptions raised:
Usage -- Gives back a usage message.
Examples:
* Calling the run.py interface from the command line:
$ python bbob_pproc/run.py -v experiment1
will post-process the folder experiment1 and all its containing data,
base on the found .info files in the folder. The result will appear
in folder ppdata. The -v option adds verbosity.
$ python bbob_pproc/run.py -o otherppdata experiment2/*.info
This will execute the post-processing on the info files found in
experiment2. The result will be located in the alternative location
otherppdata.
* Loading this package and calling the main from the command line
(requires that the path to this package is in python search path):
$ python -m bbob_pproc -h
This will print out this help message.
* From the python interactive shell (requires that the path to this
package is in python search path):
>>> import bbob_pproc
>>> bbob_pproc.main('-o outputfolder folder1'.split())
This will execute the post-processing on the index files found in folder1.
The -o option changes the output folder from the default ppdata to
outputfolder.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
try:
try:
opts, args = getopt.getopt(argv, "hvpfo:", [
"help", "output-dir=", "tab-only", "fig-only", "rld-only",
"los-only", "crafting-effort=", "pickle", "verbose", "final"
])
except getopt.error, msg:
raise Usage(msg)
if not (args):
usage()
sys.exit()
CrE = None
isfigure = True
istab = True
isrldistr = True
islogloss = True
isPostProcessed = False
isPickled = False
isDraft = True
verbose = False
outputdir = 'ppdata'
#Process options
for o, a in opts:
if o in ("-v", "--verbose"):
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-p", "--pickle"):
isPickled = True
elif o in ("-o", "--output-dir"):
outputdir = a
elif o in ("-f", "--final"):
isDraft = False
#The next 3 are for testing purpose
elif o == "--tab-only":
isfigure = False
isrldistr = False
islogloss = False
elif o == "--fig-only":
istab = False
isrldistr = False
islogloss = False
elif o == "--rld-only":
istab = False
isfigure = False
islogloss = False
elif o == "--los-only":
istab = False
isfigure = False
isrldistr = False
elif o == "--crafting-effort":
try:
CrE = float(a)
except ValueError:
raise Usage(
'Expect a valid float for flag crafting-effort.')
else:
assert False, "unhandled option"
if (not verbose):
warnings.simplefilter('ignore')
print("BBOB Post-processing: will generate post-processing " +
"data in folder %s" % outputdir)
print " this might take several minutes."
filelist = list()
for i in args:
if os.path.isdir(i):
filelist.extend(findfiles.main(i, verbose))
elif os.path.isfile(i):
filelist.append(i)
else:
txt = 'Input file or folder %s could not be found.'
raise Usage(txt)
dsList = DataSetList(filelist, verbose)
if not dsList:
raise Usage("Nothing to do: post-processing stopped.")
if (verbose):
for i in dsList:
if (dict((j, i.itrials.count(j))
for j in set(i.itrials)) != instancesOfInterest2010):
warnings.warn('The data of %s do not list ' % (i) +
'the correct instances ' +
'of function F%d.' % (i.funcId))
# BBOB 2009 Checking
#if ((dict((j, i.itrials.count(j)) for j in set(i.itrials)) !=
#instancesOfInterest) and
#(dict((j, i.itrials.count(j)) for j in set(i.itrials)) !=
#instancesOfInterest2010)):
#warnings.warn('The data of %s do not list ' %(i) +
#'the correct instances ' +
#'of function F%d or the ' %(i.funcId) +
#'correct number of trials for each.')
dictAlg = dsList.dictByAlg()
if len(dictAlg) > 1:
warnings.warn('Data with multiple algId %s ' % (dictAlg) +
'will be processed together.')
#TODO: in this case, all is well as long as for a given problem
#(given dimension and function) there is a single instance of
#DataSet associated. If there are more than one, the first one only
#will be considered... which is probably not what one would expect.
#TODO: put some errors where this case would be a problem.
if isfigure or istab or isrldistr or islogloss:
if not os.path.exists(outputdir):
os.mkdir(outputdir)
if verbose:
print 'Folder %s was created.' % (outputdir)
if isPickled:
dsList.pickle(verbose=verbose)
if isfigure:
ppfigdim.main(dsList, figValsOfInterest, outputdir, verbose)
#ERT/dim vs dim.
#ppfigdim.main2(dsList, figValsOfInterest, outputdir,
#verbose)
print "Scaling figures done."
if istab:
dictFunc = dsList.dictByFunc()
for fun, sliceFun in dictFunc.items():
dictDim = sliceFun.dictByDim()
tmp = []
for dim in tabDimsOfInterest:
try:
if len(dictDim[dim]) > 1:
warnings.warn(
'Func: %d, DIM %d: ' % (fun, dim) +
'multiple index entries. Will only ' +
'process the first ' + '%s.' % dictDim[dim][0])
tmp.append(dictDim[dim][0])
except KeyError:
pass
if tmp:
filename = os.path.join(outputdir, 'ppdata_f%d' % fun)
pptex.main(tmp, tabValsOfInterest, filename, isDraft,
verbose)
print "TeX tables",
if isDraft:
print(
"(draft) done. To get final version tables, please "
"use the -f option with run.py")
else:
print "done."
if isrldistr:
dictNoise = dsList.dictByNoise()
if len(dictNoise) > 1:
warnings.warn('Data for functions from both the noisy and '
'non-noisy testbeds have been found. Their '
'results will be mixed in the "all functions" '
'ECDF figures.')
dictDim = dsList.dictByDim()
for dim in rldDimsOfInterest:
try:
sliceDim = dictDim[dim]
pprldistr.main(sliceDim, rldValsOfInterest, True,
outputdir, 'dim%02dall' % dim, verbose)
dictNoise = sliceDim.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pprldistr.main(sliceNoise, rldValsOfInterest, True,
outputdir, 'dim%02d%s' % (dim, noise),
verbose)
dictFG = sliceDim.dictByFuncGroup()
for fGroup, sliceFuncGroup in dictFG.items():
pprldistr.main(sliceFuncGroup, rldValsOfInterest, True,
outputdir, 'dim%02d%s' % (dim, fGroup),
verbose)
pprldistr.fmax = None #Resetting the max final value
pprldistr.evalfmax = None #Resetting the max #fevalsfactor
except KeyError:
pass
print "ECDF graphs done."
if islogloss:
for ng, sliceNoise in dsList.dictByNoise().iteritems():
if ng == 'noiselessall':
testbed = 'noiseless'
elif ng == 'nzall':
testbed = 'noisy'
txt = ("Please input crafting effort value " +
"for %s testbed:\n CrE = " % testbed)
while CrE is None:
try:
CrE = float(input(txt))
except (SyntaxError, NameError, ValueError):
print "Float value required."
dictDim = sliceNoise.dictByDim()
for d in rldDimsOfInterest:
try:
sliceDim = dictDim[d]
except KeyError:
continue
info = 'dim%02d%s' % (d, ng)
pplogloss.main(sliceDim,
CrE,
True,
outputdir,
info,
verbose=verbose)
pplogloss.generateTable(sliceDim,
CrE,
outputdir,
info,
verbose=verbose)
for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup(
).iteritems():
info = 'dim%02d%s' % (d, fGroup)
pplogloss.main(sliceFuncGroup,
CrE,
True,
outputdir,
info,
verbose=verbose)
pplogloss.evalfmax = None #Resetting the max #fevalsfactor
print "ERT loss ratio figures and tables done."
if isfigure or istab or isrldistr or islogloss:
print "Output data written to folder %s." % outputdir
def main(argv=None):
r"""Post-processing COCO data of a single algorithm.
Provided with some data, this routine outputs figure and TeX files
in a folder needed for the compilation of the provided LaTeX templates
for one algorithm (``*article.tex`` or ``*1*.tex``).
The used template file needs to be edited so that the commands
``\bbobdatapath`` and ``\algfolder`` point to the output folder created
by this routine.
These output files will contain performance tables, performance
scaling figures and empirical cumulative distribution figures. On
subsequent executions, new files will be added to the output folder,
overwriting existing older files in the process.
Keyword arguments:
*argv* -- list of strings containing options and arguments. If not
given, sys.argv is accessed.
*argv* should list either names of :file:`info` files or folders
containing :file:`info` files. argv can also contain post-processed
:file:`pickle` files generated by this routine. Furthermore, *argv*
can begin with, in any order, facultative option flags listed below.
-h, --help
displays this message.
-v, --verbose
verbose mode, prints out all operations.
-p, --pickle
generates pickle post processed data files.
-o OUTPUTDIR, --output-dir=OUTPUTDIR
changes the default output directory (:file:`ppdata`) to
:file:`OUTPUTDIR`.
--crafting-effort=VALUE
sets the crafting effort to VALUE (float). Otherwise the
default value of 0. will be used.
--noise-free, --noisy
processes only part of the data.
--settings=SETTINGS
changes the style of the output figures and tables. At the
moment the only differences are in the colors of the output
figures. SETTINGS can be either "grayscale", "color" or
"black-white". The default setting is "color".
--tab-only, --fig-only, --rld-only, --los-only
these options can be used to output respectively the TeX
tables, convergence and ERTs graphs figures, run length
distribution figures, ERT loss ratio figures only. A
combination of any two of these options results in no
output.
--conv
if this option is chosen, additionally convergence plots
for each function and algorithm are generated.
--rld-single-fcts
generate also runlength distribution figures for each
single function.
--expensive
runlength-based f-target values and fixed display limits,
useful with comparatively small budgets. By default the
setting is based on the budget used in the data.
--not-expensive
expensive setting off.
--svg
generate also the svg figures which are used in html files
--runlength-based
runlength-based f-target values, such that the
"level of difficulty" is similar for all functions.
Exceptions raised:
*Usage* -- Gives back a usage message.
Examples:
* Calling the rungeneric1.py interface from the command line::
$ python bbob_pproc/rungeneric1.py -v experiment1
will post-process the folder experiment1 and all its containing
data, base on the .info files found in the folder. The result will
appear in the default output folder. The -v option adds verbosity. ::
$ python bbob_pproc/rungeneric1.py -o exp2 experiment2/*.info
This will execute the post-processing on the info files found in
:file:`experiment2`. The result will be located in the alternative
location :file:`exp2`.
* Loading this package and calling the main from the command line
(requires that the path to this package is in python search path)::
$ python -m bbob_pproc.rungeneric1 -h
This will print out this help message.
* From the python interpreter (requires that the path to this
package is in python search path)::
>> import bbob_pproc as bb
>> bb.rungeneric1.main('-o outputfolder folder1'.split())
This will execute the post-processing on the index files found in
:file:`folder1`. The ``-o`` option changes the output folder from
the default to :file:`outputfolder`.
"""
if argv is None:
argv = sys.argv[1:]
# The zero-th input argument which is the name of the calling script is
# disregarded.
if 1 < 3:
opts, args = getopt.getopt(argv, genericsettings.shortoptlist, genericsettings.longoptlist)
if 11 < 3:
try:
opts, args = getopt.getopt(argv, genericsettings.shortoptlist, genericsettings.longoptlist)
except getopt.error, msg:
raise Usage(msg)
if not (args) and not "--help" in argv and not "h" in argv:
print "not enough input arguments given"
print "cave: the following options also need an argument:"
print [o for o in genericsettings.longoptlist if o[-1] == "="]
print "options given:"
print opts
print "try --help for help"
sys.exit()
# Process options
outputdir = genericsettings.outputdir
for o, a in opts:
if o in ("-v", "--verbose"):
genericsettings.verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-p", "--pickle"):
genericsettings.isPickled = True
elif o in ("-o", "--output-dir"):
outputdir = a
elif o == "--noisy":
genericsettings.isNoisy = True
elif o == "--noise-free":
genericsettings.isNoiseFree = True
# The next 4 are for testing purpose
elif o == "--tab-only":
genericsettings.isFig = False
genericsettings.isRLDistr = False
genericsettings.isLogLoss = False
elif o == "--fig-only":
genericsettings.isTab = False
genericsettings.isRLDistr = False
genericsettings.isLogLoss = False
elif o == "--rld-only":
genericsettings.isTab = False
genericsettings.isFig = False
genericsettings.isLogLoss = False
elif o == "--los-only":
genericsettings.isTab = False
genericsettings.isFig = False
genericsettings.isRLDistr = False
elif o == "--crafting-effort":
try:
genericsettings.inputCrE = float(a)
except ValueError:
raise Usage("Expect a valid float for flag crafting-effort.")
elif o == "--settings":
genericsettings.inputsettings = a
elif o == "--conv":
genericsettings.isConv = True
elif o == "--rld-single-fcts":
genericsettings.isRldOnSingleFcts = True
elif o == "--runlength-based":
genericsettings.runlength_based_targets = True
elif o == "--expensive":
genericsettings.isExpensive = True # comprises runlength-based
elif o == "--not-expensive":
genericsettings.isExpensive = False
elif o == "--svg":
genericsettings.generate_svg_files = True
elif o == "--sca-only":
warnings.warn("option --sca-only will have no effect with rungeneric1.py")
else:
assert False, "unhandled option"
# from bbob_pproc import bbob2010 as inset # input settings
if genericsettings.inputsettings == "color":
from bbob_pproc import genericsettings as inset # input settings
elif genericsettings.inputsettings == "grayscale":
from bbob_pproc import grayscalesettings as inset # input settings
elif genericsettings.inputsettings == "black-white":
from bbob_pproc import bwsettings as inset # input settings
else:
txt = (
"Settings: %s is not an appropriate " % genericsettings.inputsettings
+ 'argument for input flag "--settings".'
)
raise Usage(txt)
if 11 < 3:
from bbob_pproc import config # input settings
config.config(False)
import imp
# import testbedsettings as testbedsettings # input settings
try:
fp, pathname, description = imp.find_module("testbedsettings")
testbedsettings = imp.load_module("testbedsettings", fp, pathname, description)
finally:
fp.close()
if not genericsettings.verbose:
warnings.simplefilter("module")
# warnings.simplefilter('ignore')
# get directory name if outputdir is a archive file
algfolder = findfiles.get_output_directory_subfolder(args[0])
outputdir = os.path.join(outputdir, algfolder)
print ("Post-processing (1): will generate output " + "data in folder %s" % outputdir)
print " this might take several minutes."
filelist = list()
for i in args:
i = i.strip()
if os.path.isdir(i):
filelist.extend(findfiles.main(i, genericsettings.verbose))
elif os.path.isfile(i):
filelist.append(i)
else:
txt = "Input file or folder %s could not be found." % i
print txt
raise Usage(txt)
dsList = DataSetList(filelist, genericsettings.verbose)
if not dsList:
raise Usage("Nothing to do: post-processing stopped.")
if genericsettings.isNoisy and not genericsettings.isNoiseFree:
dsList = dsList.dictByNoise().get("nzall", DataSetList())
if genericsettings.isNoiseFree and not genericsettings.isNoisy:
dsList = dsList.dictByNoise().get("noiselessall", DataSetList())
# compute maxfuneval values
dict_max_fun_evals = {}
for ds in dsList:
dict_max_fun_evals[ds.dim] = np.max((dict_max_fun_evals.setdefault(ds.dim, 0), float(np.max(ds.maxevals))))
from bbob_pproc import config
config.target_values(genericsettings.isExpensive, dict_max_fun_evals)
config.config(dsList.isBiobjective())
if genericsettings.verbose:
for i in dsList:
if dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) != inset.instancesOfInterest:
warnings.warn(
"The data of %s do not list " % (i) + "the correct instances " + "of function F%d." % (i.funcId)
)
dictAlg = dsList.dictByAlg()
if len(dictAlg) > 1:
warnings.warn("Data with multiple algId %s " % str(dictAlg.keys()) + "will be processed together.")
# TODO: in this case, all is well as long as for a given problem
# (given dimension and function) there is a single instance of
# DataSet associated. If there are more than one, the first one only
# will be considered... which is probably not what one would expect.
# TODO: put some errors where this case would be a problem.
# raise Usage?
if genericsettings.isFig or genericsettings.isTab or genericsettings.isRLDistr or genericsettings.isLogLoss:
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if genericsettings.verbose:
print "Folder %s was created." % (outputdir)
if genericsettings.isPickled:
dsList.pickle(verbose=genericsettings.verbose)
if genericsettings.isConv:
ppconverrorbars.main(dictAlg, outputdir, genericsettings.verbose)
if genericsettings.isFig:
print "Scaling figures...",
sys.stdout.flush()
# ERT/dim vs dim.
plt.rc("axes", **inset.rcaxeslarger)
plt.rc("xtick", **inset.rcticklarger)
plt.rc("ytick", **inset.rcticklarger)
plt.rc("font", **inset.rcfontlarger)
plt.rc("legend", **inset.rclegendlarger)
plt.rc("pdf", fonttype=42)
ppfigdim.main(dsList, ppfigdim.values_of_interest, outputdir, genericsettings.verbose)
plt.rcdefaults()
print_done()
plt.rc("axes", **inset.rcaxes)
plt.rc("xtick", **inset.rctick)
plt.rc("ytick", **inset.rctick)
plt.rc("font", **inset.rcfont)
plt.rc("legend", **inset.rclegend)
plt.rc("pdf", fonttype=42)
if genericsettings.isTab:
print "TeX tables...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pptable.main(sliceNoise, inset.tabDimsOfInterest, outputdir, noise, genericsettings.verbose)
print_done()
if genericsettings.isRLDistr:
print "ECDF graphs...",
sys.stdout.flush()
dictNoise = dsList.dictByNoise()
if len(dictNoise) > 1:
warnings.warn(
"Data for functions from both the noisy and "
"non-noisy testbeds have been found. Their "
'results will be mixed in the "all functions" '
"ECDF figures."
)
dictDim = dsList.dictByDim()
for dim in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[dim]
except KeyError:
continue
pprldistr.main(sliceDim, True, outputdir, "all", genericsettings.verbose)
dictNoise = sliceDim.dictByNoise()
for noise, sliceNoise in dictNoise.iteritems():
pprldistr.main(sliceNoise, True, outputdir, "%s" % noise, genericsettings.verbose)
dictFG = sliceDim.dictByFuncGroup()
for fGroup, sliceFuncGroup in dictFG.items():
pprldistr.main(sliceFuncGroup, True, outputdir, "%s" % fGroup, genericsettings.verbose)
pprldistr.fmax = None # Resetting the max final value
pprldistr.evalfmax = None # Resetting the max #fevalsfactor
if (
genericsettings.isRldOnSingleFcts
): # copy-paste from above, here for each function instead of function groups
# ECDFs for each function
pprldmany.all_single_functions(dictAlg, None, outputdir, genericsettings.verbose)
print_done()
if genericsettings.isLogLoss:
print "ERT loss ratio figures and tables...",
sys.stdout.flush()
for ng, sliceNoise in dsList.dictByNoise().iteritems():
if ng == "noiselessall":
testbed = "noiseless"
elif ng == "nzall":
testbed = "noisy"
txt = "Please input crafting effort value " + "for %s testbed:\n CrE = " % testbed
CrE = genericsettings.inputCrE
while CrE is None:
try:
CrE = float(raw_input(txt))
except (SyntaxError, NameError, ValueError):
print "Float value required."
dictDim = sliceNoise.dictByDim()
for d in inset.rldDimsOfInterest:
try:
sliceDim = dictDim[d]
except KeyError:
continue
info = "%s" % ng
pplogloss.main(sliceDim, CrE, True, outputdir, info, verbose=genericsettings.verbose)
pplogloss.generateTable(sliceDim, CrE, outputdir, info, verbose=genericsettings.verbose)
for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup().iteritems():
info = "%s" % fGroup
pplogloss.main(sliceFuncGroup, CrE, True, outputdir, info, verbose=genericsettings.verbose)
pplogloss.evalfmax = None # Resetting the max #fevalsfactor
print_done()
latex_commands_file = os.path.join(outputdir.split(os.sep)[0], "bbob_pproc_commands.tex")
html_file = os.path.join(outputdir, genericsettings.single_algorithm_file_name + ".html")
prepend_to_file(
latex_commands_file, ["\\providecommand{\\bbobloglosstablecaption}[1]{", pplogloss.table_caption, "}"]
)
prepend_to_file(
latex_commands_file, ["\\providecommand{\\bbobloglossfigurecaption}[1]{", pplogloss.figure_caption, "}"]
)
prepend_to_file(
latex_commands_file,
[
"\\providecommand{\\bbobpprldistrlegend}[1]{",
pprldistr.caption_single(
np.max([val / dim for dim, val in dict_max_fun_evals.iteritems()])
), # depends on the config setting, should depend on maxfevals
"}",
],
)
replace_in_file(
html_file,
r"TOBEREPLACED",
"D, ".join([str(i) for i in pprldistr.single_runlength_factors[:6]]) + "D,…",
)
prepend_to_file(
latex_commands_file, ["\\providecommand{\\bbobppfigdimlegend}[1]{", ppfigdim.scaling_figure_caption(), "}"]
)
prepend_to_file(latex_commands_file, ["\\providecommand{\\bbobpptablecaption}[1]{", pptable.table_caption, "}"])
prepend_to_file(latex_commands_file, ["\\providecommand{\\algfolder}{" + algfolder + "/}"])
prepend_to_file(
latex_commands_file,
[
"\\providecommand{\\algname}{"
+ (str_to_latex(strip_pathname1(args[0])) if len(args) == 1 else str_to_latex(dsList[0].algId))
+ "{}}"
],
)
if genericsettings.isFig or genericsettings.isTab or genericsettings.isRLDistr or genericsettings.isLogLoss:
print "Output data written to folder %s" % outputdir
plt.rcdefaults()
with open(outputfilename, 'w') as f:
f.write(towrite)
for idx, key in enumerate(instancedict):
f.write("function = %s dim = %s %s%s\n" % (key[0], key[1], key[2], instancedict[key]))
if __name__ == '__main__':
"""Merges lines of .info files that contain different instances but the
same filename.
Reconstructs the .info files within a given folder and writes everything
into a clean new folder with the given output folder name.
"""
if not len(sys.argv) == 3:
print(r'Usage:\n python merge_lines_in_info_files.py FOLDERNAME OUTPUTFOLDERNAME')
else:
inputfolder = sys.argv[1]
outputfolder = sys.argv[2]
try:
shutil.copytree(inputfolder, outputfolder)
except:
print("Problem while copying folder %s to %s" % (inputfolder, outputfolder))
e = sys.exc_info()[0]
print(" Error: %s" % e)
filelist = findfiles.main(sys.argv[1])
for f in filelist:
print("Processing %s..." % f)
merge_lines_in(f, inputfolder, outputfolder)
