def main(argv=None): r"""Post-processing COCO data of a single algorithm. Provided with some data, this routine outputs figure and TeX files in a folder needed for the compilation of latex document :file:`template1XXX.tex` or :file:`noisytemplate1XXX.tex`, where :file:`XXX` is either :file:`ecj` or :file:`generic`. The template file needs to be edited so that the commands ``\bbobdatapath`` and ``\algfolder`` point to the output folder. These output files will contain performance tables, performance scaling figures and empirical cumulative distribution figures. On subsequent executions, new files will be added to the output folder, overwriting existing older files in the process. Keyword arguments: *argv* -- list of strings containing options and arguments. If not given, sys.argv is accessed. *argv* should list either names of :file:`info` files or folders containing :file:`info` files. argv can also contain post-processed :file:`pickle` files generated by this routine. Furthermore, *argv* can begin with, in any order, facultative option flags listed below. -h, --help displays this message. -v, --verbose verbose mode, prints out all operations. -p, --pickle generates pickle post processed data files. -o OUTPUTDIR, --output-dir=OUTPUTDIR changes the default output directory (:file:`ppdata`) to :file:`OUTPUTDIR`. --crafting-effort=VALUE sets the crafting effort to VALUE (float). Otherwise the default value of 0. will be used. --noise-free, --noisy processes only part of the data. --settings=SETTINGS changes the style of the output figures and tables. At the moment the only differences are in the colors of the output figures. SETTINGS can be either "grayscale", "color" or "black-white". The default setting is "color". --tab-only, --fig-only, --rld-only, --los-only these options can be used to output respectively the TeX tables, convergence and ERTs graphs figures, run length distribution figures, ERT loss ratio figures only. A combination of any two of these options results in no output. --conv if this option is chosen, additionally convergence plots for each function and algorithm are generated. --expensive runlength-based f-target values and fixed display limits, useful with comparatively small budgets. By default the setting is based on the budget used in the data. --not-expensive expensive setting off. --runlength-based runlength-based f-target values, such that the "level of difficulty" is similar for all functions. Exceptions raised: *Usage* -- Gives back a usage message. Examples: * Calling the rungeneric1.py interface from the command line:: $ python bbob_pproc/rungeneric1.py -v experiment1 will post-process the folder experiment1 and all its containing data, base on the .info files found in the folder. The result will appear in the default output folder. The -v option adds verbosity. :: $ python bbob_pproc/rungeneric1.py -o exp2 experiment2/*.info This will execute the post-processing on the info files found in :file:`experiment2`. The result will be located in the alternative location :file:`exp2`. * Loading this package and calling the main from the command line (requires that the path to this package is in python search path):: $ python -m bbob_pproc.rungeneric1 -h This will print out this help message. * From the python interpreter (requires that the path to this package is in python search path):: >> import bbob_pproc as bb >> bb.rungeneric1.main('-o outputfolder folder1'.split()) This will execute the post-processing on the index files found in :file:`folder1`. The ``-o`` option changes the output folder from the default to :file:`outputfolder`. """ if argv is None: argv = sys.argv[1:] # The zero-th input argument which is the name of the calling script is # disregarded. if 1 < 3: opts, args = getopt.getopt(argv, shortoptlist, longoptlist) if 11 < 3: try: opts, args = getopt.getopt(argv, shortoptlist, longoptlist) except getopt.error, msg: raise Usage(msg) if not (args) and not '--help' in argv and not 'h' in argv: print 'not enough input arguments given' print 'cave: the following options also need an argument:' print[o for o in longoptlist if o[-1] == '='] print 'options given:' print opts print 'try --help for help' sys.exit() inputCrE = 0. isfigure = True istab = True isrldistr = True islogloss = True isPostProcessed = False isPickled = False verbose = False outputdir = 'ppdata' isNoisy = False isNoiseFree = False inputsettings = 'color' isConv = False isRLbased = None # allows automatic choice isExpensive = None # Process options for o, a in opts: if o in ("-v", "--verbose"): verbose = True elif o in ("-h", "--help"): usage() sys.exit() elif o in ("-p", "--pickle"): isPickled = True elif o in ("-o", "--output-dir"): outputdir = a elif o == "--noisy": isNoisy = True elif o == "--noise-free": isNoiseFree = True # The next 4 are for testing purpose elif o == "--tab-only": isfigure = False isrldistr = False islogloss = False elif o == "--fig-only": istab = False isrldistr = False islogloss = False elif o == "--rld-only": istab = False isfigure = False islogloss = False elif o == "--los-only": istab = False isfigure = False isrldistr = False elif o == "--crafting-effort": try: inputCrE = float(a) except ValueError: raise Usage( 'Expect a valid float for flag crafting-effort.') elif o == "--settings": inputsettings = a elif o == "--conv": isConv = True elif o == "--runlength-based": isRLbased = True elif o == "--expensive": isExpensive = True # comprises runlength-based elif o == "--not-expensive": isExpensive = False else: assert False, "unhandled option" # from bbob_pproc import bbob2010 as inset # input settings if inputsettings == "color": from bbob_pproc import genericsettings as inset # input settings elif inputsettings == "grayscale": from bbob_pproc import grayscalesettings as inset # input settings elif inputsettings == "black-white": from bbob_pproc import bwsettings as inset # input settings else: txt = ('Settings: %s is not an appropriate ' % inputsettings + 'argument for input flag "--settings".') raise Usage(txt) if 11 < 3: from bbob_pproc import config # input settings config.config() import imp # import testbedsettings as testbedsettings # input settings try: fp, pathname, description = imp.find_module("testbedsettings") testbedsettings = imp.load_module("testbedsettings", fp, pathname, description) finally: fp.close() if (not verbose): warnings.simplefilter('module') # warnings.simplefilter('ignore') print("Post-processing (1): will generate output " + "data in folder %s" % outputdir) print " this might take several minutes." filelist = list() for i in args: i = i.strip() if os.path.isdir(i): filelist.extend(findfiles.main(i, verbose)) elif os.path.isfile(i): filelist.append(i) else: txt = 'Input file or folder %s could not be found.' % i print txt raise Usage(txt) dsList = DataSetList(filelist, verbose) if not dsList: raise Usage("Nothing to do: post-processing stopped.") if isNoisy and not isNoiseFree: dsList = dsList.dictByNoise().get('nzall', DataSetList()) if isNoiseFree and not isNoisy: dsList = dsList.dictByNoise().get('noiselessall', DataSetList()) # compute maxfuneval values dict_max_fun_evals = {} for ds in dsList: dict_max_fun_evals[ds.dim] = np.max( (dict_max_fun_evals.setdefault(ds.dim, 0), float(np.max(ds.maxevals)))) if isRLbased is not None: genericsettings.runlength_based_targets = isRLbased from bbob_pproc import config config.target_values(isExpensive, dict_max_fun_evals) config.config() if (verbose): for i in dsList: if (dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) != inset.instancesOfInterest): warnings.warn('The data of %s do not list ' % (i) + 'the correct instances ' + 'of function F%d.' % (i.funcId)) dictAlg = dsList.dictByAlg() if len(dictAlg) > 1: warnings.warn('Data with multiple algId %s ' % (dictAlg) + 'will be processed together.') # TODO: in this case, all is well as long as for a given problem # (given dimension and function) there is a single instance of # DataSet associated. If there are more than one, the first one only # will be considered... which is probably not what one would expect. # TODO: put some errors where this case would be a problem. # raise Usage? if isfigure or istab or isrldistr or islogloss: if not os.path.exists(outputdir): os.makedirs(outputdir) if verbose: print 'Folder %s was created.' % (outputdir) if isPickled: dsList.pickle(verbose=verbose) if isConv: ppconverrorbars.main(dictAlg, outputdir, verbose) if isfigure: print "Scaling figures...", sys.stdout.flush() # ERT/dim vs dim. plt.rc("axes", **inset.rcaxeslarger) plt.rc("xtick", **inset.rcticklarger) plt.rc("ytick", **inset.rcticklarger) plt.rc("font", **inset.rcfontlarger) plt.rc("legend", **inset.rclegendlarger) ppfigdim.main(dsList, ppfigdim.values_of_interest, outputdir, verbose) plt.rcdefaults() print_done() plt.rc("axes", **inset.rcaxes) plt.rc("xtick", **inset.rctick) plt.rc("ytick", **inset.rctick) plt.rc("font", **inset.rcfont) plt.rc("legend", **inset.rclegend) if istab: print "TeX tables...", sys.stdout.flush() dictNoise = dsList.dictByNoise() for noise, sliceNoise in dictNoise.iteritems(): pptable.main(sliceNoise, inset.tabDimsOfInterest, outputdir, noise, verbose) print_done() if isrldistr: print "ECDF graphs...", sys.stdout.flush() dictNoise = dsList.dictByNoise() if len(dictNoise) > 1: warnings.warn('Data for functions from both the noisy and ' 'non-noisy testbeds have been found. Their ' 'results will be mixed in the "all functions" ' 'ECDF figures.') dictDim = dsList.dictByDim() for dim in inset.rldDimsOfInterest: try: sliceDim = dictDim[dim] except KeyError: continue pprldistr.main(sliceDim, True, outputdir, 'all', verbose) dictNoise = sliceDim.dictByNoise() for noise, sliceNoise in dictNoise.iteritems(): pprldistr.main(sliceNoise, True, outputdir, '%s' % noise, verbose) dictFG = sliceDim.dictByFuncGroup() for fGroup, sliceFuncGroup in dictFG.items(): pprldistr.main(sliceFuncGroup, True, outputdir, '%s' % fGroup, verbose) pprldistr.fmax = None # Resetting the max final value pprldistr.evalfmax = None # Resetting the max #fevalsfactor print_done() if islogloss: print "ERT loss ratio figures and tables...", sys.stdout.flush() for ng, sliceNoise in dsList.dictByNoise().iteritems(): if ng == 'noiselessall': testbed = 'noiseless' elif ng == 'nzall': testbed = 'noisy' txt = ("Please input crafting effort value " + "for %s testbed:\n CrE = " % testbed) CrE = inputCrE while CrE is None: try: CrE = float(raw_input(txt)) except (SyntaxError, NameError, ValueError): print "Float value required." dictDim = sliceNoise.dictByDim() for d in inset.rldDimsOfInterest: try: sliceDim = dictDim[d] except KeyError: continue info = '%s' % ng pplogloss.main(sliceDim, CrE, True, outputdir, info, verbose=verbose) pplogloss.generateTable(sliceDim, CrE, outputdir, info, verbose=verbose) for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup( ).iteritems(): info = '%s' % fGroup pplogloss.main(sliceFuncGroup, CrE, True, outputdir, info, verbose=verbose) pplogloss.evalfmax = None # Resetting the max #fevalsfactor print_done() latex_commands_file = os.path.join( outputdir.split(os.sep)[0], 'bbob_pproc_commands.tex') prepend_to_file(latex_commands_file, [ '\\providecommand{\\bbobloglosstablecaption}[1]{', pplogloss.table_caption, '}' ]) prepend_to_file(latex_commands_file, [ '\\providecommand{\\bbobloglossfigurecaption}[1]{', pplogloss.figure_caption, '}' ]) prepend_to_file( latex_commands_file, [ '\\providecommand{\\bbobpprldistrlegend}[1]{', pprldistr.caption_single( np.max([ val / dim for dim, val in dict_max_fun_evals.iteritems() ]) ), # depends on the config setting, should depend on maxfevals '}' ]) prepend_to_file(latex_commands_file, [ '\\providecommand{\\bbobppfigdimlegend}[1]{', ppfigdim.scaling_figure_caption(), '}' ]) prepend_to_file(latex_commands_file, [ '\\providecommand{\\bbobpptablecaption}[1]{', pptable.table_caption, '}' ]) prepend_to_file(latex_commands_file, ['\\providecommand{\\algfolder}{}' ]) # is overwritten in rungeneric.py prepend_to_file(latex_commands_file, [ '\\providecommand{\\algname}{' + (str_to_latex(strip_pathname(args[0])) if len(args) == 1 else str_to_latex(dsList[0].algId)) + '{}}' ]) if isfigure or istab or isrldistr or islogloss: print "Output data written to folder %s" % outputdir plt.rcdefaults()
def main(argv=None): r"""Routine for post-processing COCO data from two algorithms. Provided with some data, this routine outputs figure and TeX files in a folder needed for the compilation of latex document :file:`template2XXX.tex` or :file:`noisytemplate2XXX.tex`, where :file:`XXX` is either :file:`ecj` or :file:`generic`. The template file needs to be edited so that the command ``\bbobdatapath`` points to the output folder. These output files will contain performance tables, performance scaling figures and empirical cumulative distribution figures. On subsequent executions, new files will be added to the output folder, overwriting existing older files in the process. Keyword arguments: *argv* -- list of strings containing options and arguments. If not given, sys.argv is accessed. *argv* must list folders containing BBOB data files. Each of these folders should correspond to the data of ONE algorithm. Furthermore, argv can begin with, in any order, facultative option flags listed below. -h, --help displays this message. -v, --verbose verbose mode, prints out operations. -o OUTPUTDIR, --output-dir=OUTPUTDIR changes the default output directory (:file:`ppdata`) to :file:`OUTPUTDIR` --settings=SETTING changes the style of the output figures and tables. At the moment only the only differences are in the colors of the output figures. SETTING can be either "grayscale", "color" or "black-white". The default setting is "color". --fig-only, --rld-only, --tab-only, --sca-only these options can be used to output respectively the ERT graphs figures, run length distribution figures or the comparison tables scatter plot figures only. Any combination of these options results in no output. --conv if this option is chosen, additionally convergence plots for each function and algorithm are generated. --rld-single-fcts generate also runlength distribution figures for each single function. --expensive runlength-based f-target values and fixed display limits, useful with comparatively small budgets. By default the setting is based on the budget used in the data. --not-expensive expensive setting off. Exceptions raised: *Usage* -- Gives back a usage message. Examples: * Calling the rungeneric2.py interface from the command line:: $ python bbob_pproc/rungeneric2.py -v Alg0-baseline Alg1-of-interest will post-process the data from folders :file:`Alg0-baseline` and :file:`Alg1-of-interest`, the former containing data for the reference algorithm (zero-th) and the latter data for the algorithm of concern (first). The results will be output in the default output folder. The ``-v`` option adds verbosity. * From the python interpreter (requires that the path to this package is in python search path):: >> import bbob_pproc as bb >> bb.rungeneric2.main('-o outputfolder PSO DEPSO'.split()) This will execute the post-processing on the data found in folder :file:`PSO` and :file:`DEPSO`. The ``-o`` option changes the output folder from the default to :file:`outputfolder`. """ if argv is None: argv = sys.argv[1:] # The zero-th input argument which is the name of the calling script is # disregarded. global ftarget try: try: opts, args = getopt.getopt(argv, shortoptlist, longoptlist) except getopt.error, msg: raise Usage(msg) if not (args): usage() sys.exit() isfigure = True isrldistr = True istable = True isscatter = True isscaleup = True verbose = False outputdir = 'ppdata' inputsettings = 'color' isConv= False isRLbased = None # allows automatic choice isExpensive = None #Process options for o, a in opts: if o in ("-v","--verbose"): verbose = True elif o in ("-h", "--help"): usage() sys.exit() elif o in ("-o", "--output-dir"): outputdir = a #elif o in ("-s", "--style"): # inputsettings = a elif o == "--fig-only": isrldistr = False istable = False isscatter = False elif o == "--rld-only": isfigure = False istable = False isscatter = False elif o == "--tab-only": isfigure = False isrldistr = False isscatter = False elif o == "--sca-only": isfigure = False isrldistr = False istable = False elif o == "--settings": inputsettings = a elif o == "--conv": isConv = True elif o == "--runlength-based": isRLbased = True elif o == "--expensive": isExpensive = True # comprises runlength-based elif o == "--not-expensive": isExpensive = False else: assert False, "unhandled option" # from bbob_pproc import bbob2010 as inset # input settings if inputsettings == "color": from bbob_pproc import genericsettings as inset # input settings config.config() elif inputsettings == "grayscale": # probably very much obsolete from bbob_pproc import grayscalesettings as inset # input settings elif inputsettings == "black-white": # probably very much obsolete from bbob_pproc import bwsettings as inset # input settings else: txt = ('Settings: %s is not an appropriate ' % inputsettings + 'argument for input flag "--settings".') raise Usage(txt) if (not verbose): warnings.simplefilter('module') warnings.simplefilter('ignore') print ("Post-processing will generate comparison " + "data in folder %s" % outputdir) print " this might take several minutes." dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose) if 1 < 3 and len(sortedAlgs) != 2: raise ValueError('rungeneric2.py needs exactly two algorithms to compare, found: ' + str(sortedAlgs) + '\n use rungeneric.py (or rungenericmany.py) to compare more algorithms. ') if not dsList: sys.exit() for i in dictAlg: dictAlg[i] = dictAlg[i].dictAll().get('All', DataSetList()) for i in dsList: if i.dim not in genericsettings.dimensions_to_display: continue if (dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) < inset.instancesOfInterest): warnings.warn('The data of %s do not list ' %(i) + 'the correct instances ' + 'of function F%s.' %(i.funcId)) if len(sortedAlgs) < 2: raise Usage('Expect data from two different algorithms, could ' + 'only find one.') elif len(sortedAlgs) > 2: warnings.warn('Data from folders: %s ' % (sortedAlgs) + 'were found, the first two will be processed.') # Group by algorithm dsList0 = dictAlg[sortedAlgs[0]] if not dsList0: raise Usage('Could not find data for algorithm %s.' % (sortedAlgs[0])) dsList1 = dictAlg[sortedAlgs[1]] if not dsList1: raise Usage('Could not find data for algorithm %s.' % (sortedAlgs[0])) # get the name of each algorithm from the input arguments tmppath0, alg0name = os.path.split(sortedAlgs[0].rstrip(os.sep)) tmppath1, alg1name = os.path.split(sortedAlgs[1].rstrip(os.sep)) for i in dsList0: i.algId = alg0name for i in dsList1: i.algId = alg1name # compute maxfuneval values dict_max_fun_evals1 = {} dict_max_fun_evals2 = {} for ds in dsList0: dict_max_fun_evals1[ds.dim] = np.max((dict_max_fun_evals1.setdefault(ds.dim, 0), float(np.max(ds.maxevals)))) for ds in dsList1: dict_max_fun_evals2[ds.dim] = np.max((dict_max_fun_evals2.setdefault(ds.dim, 0), float(np.max(ds.maxevals)))) if isRLbased is not None: genericsettings.runlength_based_targets = isRLbased config.target_values(isExpensive, {1: min([max([val/dim for dim, val in dict_max_fun_evals1.iteritems()]), max([val/dim for dim, val in dict_max_fun_evals2.iteritems()])] )}) config.config() ######################### Post-processing ############################# if isfigure or isrldistr or istable or isscatter: if not os.path.exists(outputdir): os.mkdir(outputdir) if verbose: print 'Folder %s was created.' % (outputdir) # prepend the algorithm name command to the tex-command file abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' lines = [] for i, alg in enumerate(args): lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' + str_to_latex(strip_pathname(alg)) + '}') prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'), lines, 1000, 'bbob_proc_commands.tex truncated, consider removing the file before the text run' ) # # Check whether both input arguments list noisy and noise-free data # dictFN0 = dsList0.dictAll() # dictFN1 = dsList1.dictAll() # k0 = set(dictFN0.keys()) # k1 = set(dictFN1.keys()) # symdiff = k1 ^ k0 # symmetric difference # if symdiff: # tmpdict = {} # for i, noisegrp in enumerate(symdiff): # if noisegrp == 'nzall': # tmp = 'noisy' # elif noisegrp == 'noiselessall': # tmp = 'noiseless' # # if dictFN0.has_key(noisegrp): # tmp2 = sortedAlgs[0] # elif dictFN1.has_key(noisegrp): # tmp2 = sortedAlgs[1] # # tmpdict.setdefault(tmp2, []).append(tmp) # # txt = [] # for i, j in tmpdict.iteritems(): # txt.append('Only input folder %s lists %s data.' # % (i, ' and '.join(j))) # raise Usage('Data Mismatch: \n ' + ' '.join(txt) # + '\nTry using --noise-free or --noisy flags.') if isfigure: plt.rc("axes", **inset.rcaxeslarger) plt.rc("xtick", **inset.rcticklarger) plt.rc("ytick", **inset.rcticklarger) plt.rc("font", **inset.rcfontlarger) plt.rc("legend", **inset.rclegendlarger) ppfig2.main(dsList0, dsList1, ppfig2_ftarget, outputdir, verbose) print "log ERT1/ERT0 vs target function values done." plt.rc("axes", **inset.rcaxes) plt.rc("xtick", **inset.rctick) plt.rc("ytick", **inset.rctick) plt.rc("font", **inset.rcfont) plt.rc("legend", **inset.rclegend) if isrldistr: # if len(dictFN0) > 1 or len(dictFN1) > 1: # warnings.warn('Data for functions from both the noisy and ' + # 'non-noisy testbeds have been found. Their ' + # 'results will be mixed in the "all functions" ' + # 'ECDF figures.') dictDim0 = dsList0.dictByDim() dictDim1 = dsList1.dictByDim() # ECDFs of ERT ratios for dim in set(dictDim0.keys()) & set(dictDim1.keys()): if dim in inset.rldDimsOfInterest: # ECDF for all functions altogether try: pprldistr2.main(dictDim0[dim], dictDim1[dim], dim, inset.rldValsOfInterest, outputdir, '%02dD_all' % dim, verbose) except KeyError: warnings.warn('Could not find some data in %d-D.' % (dim)) continue # ECDFs per function groups dictFG0 = dictDim0[dim].dictByFuncGroup() dictFG1 = dictDim1[dim].dictByFuncGroup() for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()): pprldistr2.main(dictFG1[fGroup], dictFG0[fGroup], dim, inset.rldValsOfInterest, outputdir, '%02dD_%s' % (dim, fGroup), verbose) # ECDFs for all dictFN0 = dictDim0[dim].dictAll() dictFN1 = dictDim1[dim].dictAll() for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()): pprldistr2.main(dictFN1[fGroup], dictFN0[fGroup], dim, inset.rldValsOfInterest, outputdir, '%02dD_%s' % (dim, fGroup), verbose) prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'), ['\\providecommand{\\bbobpprldistrlegendtwo}[1]{', pprldistr.caption_two(), # depends on the config setting, should depend on maxfevals '}' ]) print "ECDF runlength ratio graphs done." for dim in set(dictDim0.keys()) & set(dictDim1.keys()): pprldistr.fmax = None #Resetting the max final value pprldistr.evalfmax = None #Resetting the max #fevalsfactor # ECDFs of all functions altogether if dim in inset.rldDimsOfInterest: try: pprldistr.comp(dictDim1[dim], dictDim0[dim], inset.rldValsOfInterest, # TODO: let rldVals... possibly be RL-based targets True, outputdir, 'all', verbose) except KeyError: warnings.warn('Could not find some data in %d-D.' % (dim)) continue # ECDFs per function groups dictFG0 = dictDim0[dim].dictByFuncGroup() dictFG1 = dictDim1[dim].dictByFuncGroup() for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()): pprldistr.comp(dictFG1[fGroup], dictFG0[fGroup], inset.rldValsOfInterest, True, outputdir, '%s' % fGroup, verbose) # ECDFs for all dictFN0 = dictDim0[dim].dictAll() dictFN1 = dictDim1[dim].dictAll() for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()): pprldistr.comp(dictFN1[fGroup], dictFN0[fGroup], inset.rldValsOfInterest, True, outputdir, '%s' % fGroup, verbose) print "ECDF runlength graphs done." if isConv: ppconverrorbars.main(dictAlg,outputdir,verbose) ##TODO pptable # if istable: # dictNG0 = dsList0.dictAll() # dictNG1 = dsList1.dictAll() # # for nGroup in set(dictNG0.keys()) & set(dictNG1.keys()): # # split table in as many as necessary # dictFunc0 = dictNG0[nGroup].dictByFunc() # dictFunc1 = dictNG1[nGroup].dictByFunc() # funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys())) # if len(funcs) > 24: # funcs.sort() # nbgroups = int(numpy.ceil(len(funcs)/24.)) # def split_seq(seq, nbgroups): # newseq = [] # splitsize = 1.0/nbgroups*len(seq) # for i in range(nbgroups): # newseq.append(seq[int(round(i*splitsize)):int(round((i+1)*splitsize))]) # return newseq # # groups = split_seq(funcs, nbgroups) # # merge # group0 = [] # group1 = [] # for i, g in enumerate(groups): # tmp0 = DataSetList() # tmp1 = DataSetList() # for f in g: # tmp0.extend(dictFunc0[f]) # tmp1.extend(dictFunc1[f]) # group0.append(tmp0) # group1.append(tmp1) # for i, g in enumerate(zip(group0, group1)): # pptable2.main(g[0], g[1], inset.tabDimsOfInterest, # outputdir, '%s%d' % (nGroup, i), verbose) # else: # if 11 < 3: # future handling: # dictFunc0 = dsList0.dictByFunc() # dictFunc1 = dsList1.dictByFunc() # funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys())) # funcs.sort() # # nbgroups = int(numpy.ceil(len(funcs)/testbedsettings.numberOfFunctions)) # # pptable2.main(dsList0, dsList1, # # testbedsettings.tabDimsOfInterest, outputdir, # # '%s' % (testbedsettings.testbedshortname), verbose) # else: # pptable2.main(dictNG0[nGroup], dictNG1[nGroup], # inset.tabDimsOfInterest, outputdir, # '%s' % (nGroup), verbose) # # if isinstance(pptable2.targetsOfInterest, pproc.RunlengthBasedTargetValues): # prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'), # ['\\providecommand{\\bbobpptablestwolegend}[1]{', # pptable2.table_caption_expensive, # '}' # ]) # else: # prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'), # ['\\providecommand{\\bbobpptablestwolegend}[1]{', # pptable2.table_caption, # '}' # ]) # print "Tables done." if isscatter: if genericsettings.runlength_based_targets: ppscatter.targets = ppscatter.runlength_based_targets ppscatter.main(dsList1, dsList0, outputdir, verbose=verbose) prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'), ['\\providecommand{\\bbobppscatterlegend}[1]{', ppscatter.figure_caption(), '}' ]) print "Scatter plots done." if isscaleup: plt.rc("axes", labelsize=20, titlesize=24) plt.rc("xtick", labelsize=20) plt.rc("ytick", labelsize=20) plt.rc("font", size=20) plt.rc("legend", fontsize=20) if genericsettings.runlength_based_targets: ftarget = RunlengthBasedTargetValues([target_runlength]) # TODO: make this more variable but also consistent ppfigs.main(dictAlg, sortedAlgs, ftarget, outputdir, verbose) plt.rcdefaults() print "Scaling figures done." if isfigure or isrldistr or istable or isscatter or isscaleup: print "Output data written to folder %s" % outputdir plt.rcdefaults()
def main(argv=None): r"""Routine for post-processing COCO data from two algorithms. Provided with some data, this routine outputs figure and TeX files in a folder needed for the compilation of latex document :file:`template2XXX.tex` or :file:`noisytemplate2XXX.tex`, where :file:`XXX` is either :file:`ecj` or :file:`generic`. The template file needs to be edited so that the command ``\bbobdatapath`` points to the output folder. These output files will contain performance tables, performance scaling figures and empirical cumulative distribution figures. On subsequent executions, new files will be added to the output folder, overwriting existing older files in the process. Keyword arguments: *argv* -- list of strings containing options and arguments. If not given, sys.argv is accessed. *argv* must list folders containing BBOB data files. Each of these folders should correspond to the data of ONE algorithm. Furthermore, argv can begin with, in any order, facultative option flags listed below. -h, --help displays this message. -v, --verbose verbose mode, prints out operations. -o OUTPUTDIR, --output-dir=OUTPUTDIR changes the default output directory (:file:`ppdata`) to :file:`OUTPUTDIR` --noise-free, --noisy processes only part of the data. --settings=SETTING changes the style of the output figures and tables. At the moment only the only differences are in the colors of the output figures. SETTING can be either "grayscale", "color" or "black-white". The default setting is "color". --fig-only, --rld-only, --tab-only, --sca-only these options can be used to output respectively the ERT graphs figures, run length distribution figures or the comparison tables scatter plot figures only. Any combination of these options results in no output. --conv if this option is chosen, additionally convergence plots for each function and algorithm are generated. Exceptions raised: *Usage* -- Gives back a usage message. Examples: * Calling the rungeneric2.py interface from the command line:: $ python bbob_pproc/rungeneric2.py -v Alg0-baseline Alg1-of-interest will post-process the data from folders :file:`Alg0-baseline` and :file:`Alg1-of-interest`, the former containing data for the reference algorithm (zero-th) and the latter data for the algorithm of concern (first). The results will be output in the default output folder. The ``-v`` option adds verbosity. * From the python interpreter (requires that the path to this package is in python search path):: >> import bbob_pproc as bb >> bb.rungeneric2.main('-o outputfolder PSO DEPSO'.split()) This will execute the post-processing on the data found in folder :file:`PSO` and :file:`DEPSO`. The ``-o`` option changes the output folder from the default to :file:`outputfolder`. """ if argv is None: argv = sys.argv[1:] # The zero-th input argument which is the name of the calling script is # disregarded. try: try: opts, args = getopt.getopt(argv, shortoptlist, longoptlist) except getopt.error, msg: raise Usage(msg) if not (args): usage() sys.exit() isfigure = True isrldistr = True istable = True isscatter = True isscaleup = True isNoisy = False isNoiseFree = False verbose = False outputdir = 'ppdata' inputsettings = 'color' isConv = False #Process options for o, a in opts: if o in ("-v", "--verbose"): verbose = True elif o in ("-h", "--help"): usage() sys.exit() elif o in ("-o", "--output-dir"): outputdir = a #elif o in ("-s", "--style"): # inputsettings = a elif o == "--fig-only": isrldistr = False istable = False isscatter = False elif o == "--rld-only": isfigure = False istable = False isscatter = False elif o == "--tab-only": isfigure = False isrldistr = False isscatter = False elif o == "--sca-only": isfigure = False isrldistr = False istable = False elif o == "--noisy": isNoisy = True elif o == "--noise-free": isNoiseFree = True elif o == "--settings": inputsettings = a elif o == "--conv": isConv = True else: assert False, "unhandled option" # from bbob_pproc import bbob2010 as inset # input settings if inputsettings == "color": from bbob_pproc import config, genericsettings as inset # input settings config.config() elif inputsettings == "grayscale": from bbob_pproc import grayscalesettings as inset # input settings elif inputsettings == "black-white": from bbob_pproc import bwsettings as inset # input settings else: txt = ('Settings: %s is not an appropriate ' % inputsettings + 'argument for input flag "--settings".') raise Usage(txt) if (not verbose): warnings.simplefilter('module') warnings.simplefilter('ignore') print("Post-processing will generate comparison " + "data in folder %s" % outputdir) print " this might take several minutes." dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose) if 1 < 3 and len(sortedAlgs) != 2: raise ValueError( 'rungeneric2.py needs exactly two algorithms to compare, found: ' + str(sortedAlgs) + '\n use rungeneric.py (or rungenericmany.py) to compare more algorithms. ' ) if not dsList: sys.exit() for i in dictAlg: if isNoisy and not isNoiseFree: dictAlg[i] = dictAlg[i].dictByNoise().get( 'nzall', DataSetList()) if isNoiseFree and not isNoisy: dictAlg[i] = dictAlg[i].dictByNoise().get( 'noiselessall', DataSetList()) for i in dsList: if i.dim not in genericsettings.dimensions_to_display: continue if (dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) < inset.instancesOfInterest): warnings.warn('The data of %s do not list ' % (i) + 'the correct instances ' + 'of function F%d.' % (i.funcId)) if len(sortedAlgs) < 2: raise Usage('Expect data from two different algorithms, could ' + 'only find one.') elif len(sortedAlgs) > 2: warnings.warn('Data from folders: %s ' % (sortedAlgs) + 'were found, the first two will be processed.') # Group by algorithm dsList0 = dictAlg[sortedAlgs[0]] if not dsList0: raise Usage('Could not find data for algorithm %s.' % (sortedAlgs[0])) dsList1 = dictAlg[sortedAlgs[1]] if not dsList1: raise Usage('Could not find data for algorithm %s.' % (sortedAlgs[0])) # get the name of each algorithm from the input arguments tmppath0, alg0name = os.path.split(sortedAlgs[0].rstrip(os.sep)) tmppath1, alg1name = os.path.split(sortedAlgs[1].rstrip(os.sep)) for i in dsList0: i.algId = alg0name for i in dsList1: i.algId = alg1name ######################### Post-processing ############################# if isfigure or isrldistr or istable or isscatter: if not os.path.exists(outputdir): os.mkdir(outputdir) if verbose: print 'Folder %s was created.' % (outputdir) # prepend the algorithm name command to the tex-command file abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' lines = [] for i, alg in enumerate(args): lines.append('\\providecommand{\\algorithm' + abc[i] + '}{' + str_to_latex(strip_pathname(alg)) + '}') prepend_to_file( os.path.join(outputdir, 'bbob_pproc_commands.tex'), lines, 1000, 'bbob_proc_commands.tex truncated, consider removing the file before the text run' ) # Check whether both input arguments list noisy and noise-free data dictFN0 = dsList0.dictByNoise() dictFN1 = dsList1.dictByNoise() k0 = set(dictFN0.keys()) k1 = set(dictFN1.keys()) symdiff = k1 ^ k0 # symmetric difference if symdiff: tmpdict = {} for i, noisegrp in enumerate(symdiff): if noisegrp == 'nzall': tmp = 'noisy' elif noisegrp == 'noiselessall': tmp = 'noiseless' if dictFN0.has_key(noisegrp): tmp2 = sortedAlgs[0] elif dictFN1.has_key(noisegrp): tmp2 = sortedAlgs[1] tmpdict.setdefault(tmp2, []).append(tmp) txt = [] for i, j in tmpdict.iteritems(): txt.append('Only input folder %s lists %s data.' % (i, ' and '.join(j))) raise Usage('Data Mismatch: \n ' + ' '.join(txt) + '\nTry using --noise-free or --noisy flags.') if isfigure: plt.rc("axes", **inset.rcaxeslarger) plt.rc("xtick", **inset.rcticklarger) plt.rc("ytick", **inset.rcticklarger) plt.rc("font", **inset.rcfontlarger) plt.rc("legend", **inset.rclegendlarger) ppfig2.main(dsList0, dsList1, ftarget, outputdir, verbose) print "log ERT1/ERT0 vs target function values done." plt.rc("axes", **inset.rcaxes) plt.rc("xtick", **inset.rctick) plt.rc("ytick", **inset.rctick) plt.rc("font", **inset.rcfont) plt.rc("legend", **inset.rclegend) if isrldistr: if len(dictFN0) > 1 or len(dictFN1) > 1: warnings.warn('Data for functions from both the noisy and ' + 'non-noisy testbeds have been found. Their ' + 'results will be mixed in the "all functions" ' + 'ECDF figures.') dictDim0 = dsList0.dictByDim() dictDim1 = dsList1.dictByDim() # ECDFs of ERT ratios for dim in set(dictDim0.keys()) & set(dictDim1.keys()): if dim in inset.rldDimsOfInterest: # ECDF for all functions altogether try: pprldistr2.main(dictDim0[dim], dictDim1[dim], inset.rldValsOfInterest, outputdir, '%02dD_all' % dim, verbose) except KeyError: warnings.warn('Could not find some data in %d-D.' % (dim)) continue # ECDFs per function groups dictFG0 = dictDim0[dim].dictByFuncGroup() dictFG1 = dictDim1[dim].dictByFuncGroup() for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()): pprldistr2.main(dictFG1[fGroup], dictFG0[fGroup], inset.rldValsOfInterest, outputdir, '%02dD_%s' % (dim, fGroup), verbose) # ECDFs per noise groups dictFN0 = dictDim0[dim].dictByNoise() dictFN1 = dictDim1[dim].dictByNoise() for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()): pprldistr2.main(dictFN1[fGroup], dictFN0[fGroup], inset.rldValsOfInterest, outputdir, '%02dD_%s' % (dim, fGroup), verbose) print "ECDF runlength ratio graphs done." for dim in set(dictDim0.keys()) & set(dictDim1.keys()): pprldistr.fmax = None #Resetting the max final value pprldistr.evalfmax = None #Resetting the max #fevalsfactor # ECDFs of all functions altogether if dim in inset.rldDimsOfInterest: try: pprldistr.comp( dictDim1[dim], dictDim0[dim], inset.rldValsOfInterest if isinstance( inset.rldValsOfInterest, TargetValues) else TargetValues(inset.rldValsOfInterest), True, outputdir, 'all', verbose) except KeyError: warnings.warn('Could not find some data in %d-D.' % (dim)) continue # ECDFs per function groups dictFG0 = dictDim0[dim].dictByFuncGroup() dictFG1 = dictDim1[dim].dictByFuncGroup() for fGroup in set(dictFG0.keys()) & set(dictFG1.keys()): pprldistr.comp( dictFG1[fGroup], dictFG0[fGroup], inset.rldValsOfInterest if isinstance( inset.rldValsOfInterest, TargetValues) else TargetValues(inset.rldValsOfInterest), True, outputdir, '%s' % fGroup, verbose) # ECDFs per noise groups dictFN0 = dictDim0[dim].dictByNoise() dictFN1 = dictDim1[dim].dictByNoise() for fGroup in set(dictFN0.keys()) & set(dictFN1.keys()): pprldistr.comp( dictFN1[fGroup], dictFN0[fGroup], inset.rldValsOfInterest if isinstance( inset.rldValsOfInterest, TargetValues) else TargetValues(inset.rldValsOfInterest), True, outputdir, '%s' % fGroup, verbose) print "ECDF runlength graphs done." if isConv: ppconverrorbars.main(dictAlg, outputdir, verbose) if istable: dictNG0 = dsList0.dictByNoise() dictNG1 = dsList1.dictByNoise() for nGroup in set(dictNG0.keys()) & set(dictNG1.keys()): # split table in as many as necessary dictFunc0 = dictNG0[nGroup].dictByFunc() dictFunc1 = dictNG1[nGroup].dictByFunc() funcs = list(set(dictFunc0.keys()) & set(dictFunc1.keys())) if len(funcs) > 24: funcs.sort() nbgroups = int(numpy.ceil(len(funcs) / 24.)) def split_seq(seq, nbgroups): newseq = [] splitsize = 1.0 / nbgroups * len(seq) for i in range(nbgroups): newseq.append( seq[int(round(i * splitsize) ):int(round((i + 1) * splitsize))]) return newseq groups = split_seq(funcs, nbgroups) # merge group0 = [] group1 = [] for i, g in enumerate(groups): tmp0 = DataSetList() tmp1 = DataSetList() for f in g: tmp0.extend(dictFunc0[f]) tmp1.extend(dictFunc1[f]) group0.append(tmp0) group1.append(tmp1) for i, g in enumerate(zip(group0, group1)): pptable2.main(g[0], g[1], inset.tabDimsOfInterest, outputdir, '%s%d' % (nGroup, i), verbose) else: if 11 < 3: # future handling: dictFunc0 = dsList0.dictByFunc() dictFunc1 = dsList1.dictByFunc() funcs = list( set(dictFunc0.keys()) & set(dictFunc1.keys())) funcs.sort() # nbgroups = int(numpy.ceil(len(funcs)/testbedsettings.numberOfFunctions)) # pptable2.main(dsList0, dsList1, # testbedsettings.tabDimsOfInterest, outputdir, # '%s' % (testbedsettings.testbedshortname), verbose) else: pptable2.main(dictNG0[nGroup], dictNG1[nGroup], inset.tabDimsOfInterest, outputdir, '%s' % (nGroup), verbose) prepend_to_file(os.path.join( outputdir, 'bbob_pproc_commands.tex'), [ '\\providecommand{\\bbobpptablestwolegend}[1]{', pptable2.figure_legend, '}' ]) print "Tables done." if isscatter: ppscatter.main(dsList0, dsList1, outputdir, verbose=verbose) prepend_to_file(os.path.join(outputdir, 'bbob_pproc_commands.tex'), [ '\\providecommand{\\bbobppscatterlegend}[1]{', ppscatter.figure_legend, '}' ]) print "Scatter plots done." if isscaleup: plt.rc("axes", labelsize=20, titlesize=24) plt.rc("xtick", labelsize=20) plt.rc("ytick", labelsize=20) plt.rc("font", size=20) plt.rc("legend", fontsize=20) ppfigs.main(dictAlg, sortedAlgs, ftarget, outputdir, verbose) plt.rcdefaults() print "Scaling figures done." if isfigure or isrldistr or istable or isscatter or isscaleup: print "Output data written to folder %s" % outputdir plt.rcdefaults()
def main(argv=None): r"""Post-processing COCO data of a single algorithm. Provided with some data, this routine outputs figure and TeX files in a folder needed for the compilation of latex document :file:`template1XXX.tex` or :file:`noisytemplate1XXX.tex`, where :file:`XXX` is either :file:`ecj` or :file:`generic`. The template file needs to be edited so that the commands ``\bbobdatapath`` and ``\algfolder`` point to the output folder. These output files will contain performance tables, performance scaling figures and empirical cumulative distribution figures. On subsequent executions, new files will be added to the output folder, overwriting existing older files in the process. Keyword arguments: *argv* -- list of strings containing options and arguments. If not given, sys.argv is accessed. *argv* should list either names of :file:`info` files or folders containing :file:`info` files. argv can also contain post-processed :file:`pickle` files generated by this routine. Furthermore, *argv* can begin with, in any order, facultative option flags listed below. -h, --help displays this message. -v, --verbose verbose mode, prints out all operations. -p, --pickle generates pickle post processed data files. -o OUTPUTDIR, --output-dir=OUTPUTDIR changes the default output directory (:file:`ppdata`) to :file:`OUTPUTDIR`. --crafting-effort=VALUE sets the crafting effort to VALUE (float). Otherwise the default value of 0. will be used. --noise-free, --noisy processes only part of the data. --settings=SETTINGS changes the style of the output figures and tables. At the moment the only differences are in the colors of the output figures. SETTINGS can be either "grayscale", "color" or "black-white". The default setting is "color". --tab-only, --fig-only, --rld-only, --los-only these options can be used to output respectively the TeX tables, convergence and ERTs graphs figures, run length distribution figures, ERT loss ratio figures only. A combination of any two of these options results in no output. --conv if this option is chosen, additionally convergence plots for each function and algorithm are generated. --expensive runlength-based f-target values and fixed display limits, useful with comparatively small budgets. By default the setting is based on the budget used in the data. --not-expensive expensive setting off. --runlength-based runlength-based f-target values, such that the "level of difficulty" is similar for all functions. Exceptions raised: *Usage* -- Gives back a usage message. Examples: * Calling the rungeneric1.py interface from the command line:: $ python bbob_pproc/rungeneric1.py -v experiment1 will post-process the folder experiment1 and all its containing data, base on the .info files found in the folder. The result will appear in the default output folder. The -v option adds verbosity. :: $ python bbob_pproc/rungeneric1.py -o exp2 experiment2/*.info This will execute the post-processing on the info files found in :file:`experiment2`. The result will be located in the alternative location :file:`exp2`. * Loading this package and calling the main from the command line (requires that the path to this package is in python search path):: $ python -m bbob_pproc.rungeneric1 -h This will print out this help message. * From the python interpreter (requires that the path to this package is in python search path):: >> import bbob_pproc as bb >> bb.rungeneric1.main('-o outputfolder folder1'.split()) This will execute the post-processing on the index files found in :file:`folder1`. The ``-o`` option changes the output folder from the default to :file:`outputfolder`. """ if argv is None: argv = sys.argv[1:] # The zero-th input argument which is the name of the calling script is # disregarded. if 1 < 3: opts, args = getopt.getopt(argv, shortoptlist, longoptlist) if 11 < 3: try: opts, args = getopt.getopt(argv, shortoptlist, longoptlist) except getopt.error, msg: raise Usage(msg) if not (args) and not '--help' in argv and not 'h' in argv: print 'not enough input arguments given' print 'cave: the following options also need an argument:' print [o for o in longoptlist if o[-1] == '='] print 'options given:' print opts print 'try --help for help' sys.exit() inputCrE = 0. isfigure = True istab = True isrldistr = True islogloss = True isPostProcessed = False isPickled = False verbose = False outputdir = 'ppdata' isNoisy = False isNoiseFree = False inputsettings = 'color' isConv = False isRLbased = None # allows automatic choice isExpensive = None # Process options for o, a in opts: if o in ("-v", "--verbose"): verbose = True elif o in ("-h", "--help"): usage() sys.exit() elif o in ("-p", "--pickle"): isPickled = True elif o in ("-o", "--output-dir"): outputdir = a elif o == "--noisy": isNoisy = True elif o == "--noise-free": isNoiseFree = True # The next 4 are for testing purpose elif o == "--tab-only": isfigure = False isrldistr = False islogloss = False elif o == "--fig-only": istab = False isrldistr = False islogloss = False elif o == "--rld-only": istab = False isfigure = False islogloss = False elif o == "--los-only": istab = False isfigure = False isrldistr = False elif o == "--crafting-effort": try: inputCrE = float(a) except ValueError: raise Usage('Expect a valid float for flag crafting-effort.') elif o == "--settings": inputsettings = a elif o == "--conv": isConv = True elif o == "--runlength-based": isRLbased = True elif o == "--expensive": isExpensive = True # comprises runlength-based elif o == "--not-expensive": isExpensive = False else: assert False, "unhandled option" # from bbob_pproc import bbob2010 as inset # input settings if inputsettings == "color": from bbob_pproc import genericsettings as inset # input settings elif inputsettings == "grayscale": from bbob_pproc import grayscalesettings as inset # input settings elif inputsettings == "black-white": from bbob_pproc import bwsettings as inset # input settings else: txt = ('Settings: %s is not an appropriate ' % inputsettings + 'argument for input flag "--settings".') raise Usage(txt) if 11 < 3: from bbob_pproc import config # input settings config.config() import imp # import testbedsettings as testbedsettings # input settings try: fp, pathname, description = imp.find_module("testbedsettings") testbedsettings = imp.load_module("testbedsettings", fp, pathname, description) finally: fp.close() if (not verbose): warnings.simplefilter('module') # warnings.simplefilter('ignore') print ("Post-processing (1): will generate output " + "data in folder %s" % outputdir) print " this might take several minutes." filelist = list() for i in args: i = i.strip() if os.path.isdir(i): filelist.extend(findfiles.main(i, verbose)) elif os.path.isfile(i): filelist.append(i) else: txt = 'Input file or folder %s could not be found.' % i print txt raise Usage(txt) dsList = DataSetList(filelist, verbose) if not dsList: raise Usage("Nothing to do: post-processing stopped.") if isNoisy and not isNoiseFree: dsList = dsList.dictByNoise().get('nzall', DataSetList()) if isNoiseFree and not isNoisy: dsList = dsList.dictByNoise().get('noiselessall', DataSetList()) # compute maxfuneval values dict_max_fun_evals = {} for ds in dsList: dict_max_fun_evals[ds.dim] = np.max((dict_max_fun_evals.setdefault(ds.dim, 0), float(np.max(ds.maxevals)))) if isRLbased is not None: genericsettings.runlength_based_targets = isRLbased from bbob_pproc import config config.target_values(isExpensive, dict_max_fun_evals) config.config() if (verbose): for i in dsList: if (dict((j, i.instancenumbers.count(j)) for j in set(i.instancenumbers)) != inset.instancesOfInterest): warnings.warn('The data of %s do not list ' % (i) + 'the correct instances ' + 'of function F%d.' % (i.funcId)) dictAlg = dsList.dictByAlg() if len(dictAlg) > 1: warnings.warn('Data with multiple algId %s ' % (dictAlg) + 'will be processed together.') # TODO: in this case, all is well as long as for a given problem # (given dimension and function) there is a single instance of # DataSet associated. If there are more than one, the first one only # will be considered... which is probably not what one would expect. # TODO: put some errors where this case would be a problem. # raise Usage? if isfigure or istab or isrldistr or islogloss: if not os.path.exists(outputdir): os.makedirs(outputdir) if verbose: print 'Folder %s was created.' % (outputdir) if isPickled: dsList.pickle(verbose=verbose) if isConv: ppconverrorbars.main(dictAlg, outputdir, verbose) if isfigure: print "Scaling figures...", sys.stdout.flush() # ERT/dim vs dim. plt.rc("axes", **inset.rcaxeslarger) plt.rc("xtick", **inset.rcticklarger) plt.rc("ytick", **inset.rcticklarger) plt.rc("font", **inset.rcfontlarger) plt.rc("legend", **inset.rclegendlarger) ppfigdim.main(dsList, ppfigdim.values_of_interest, outputdir, verbose) plt.rcdefaults() print_done() plt.rc("axes", **inset.rcaxes) plt.rc("xtick", **inset.rctick) plt.rc("ytick", **inset.rctick) plt.rc("font", **inset.rcfont) plt.rc("legend", **inset.rclegend) if istab: print "TeX tables...", sys.stdout.flush() dictNoise = dsList.dictByNoise() for noise, sliceNoise in dictNoise.iteritems(): pptable.main(sliceNoise, inset.tabDimsOfInterest, outputdir, noise, verbose) print_done() if isrldistr: print "ECDF graphs...", sys.stdout.flush() dictNoise = dsList.dictByNoise() if len(dictNoise) > 1: warnings.warn('Data for functions from both the noisy and ' 'non-noisy testbeds have been found. Their ' 'results will be mixed in the "all functions" ' 'ECDF figures.') dictDim = dsList.dictByDim() for dim in inset.rldDimsOfInterest: try: sliceDim = dictDim[dim] except KeyError: continue pprldistr.main(sliceDim, True, outputdir, 'all', verbose) dictNoise = sliceDim.dictByNoise() for noise, sliceNoise in dictNoise.iteritems(): pprldistr.main(sliceNoise, True, outputdir, '%s' % noise, verbose) dictFG = sliceDim.dictByFuncGroup() for fGroup, sliceFuncGroup in dictFG.items(): pprldistr.main(sliceFuncGroup, True, outputdir, '%s' % fGroup, verbose) pprldistr.fmax = None # Resetting the max final value pprldistr.evalfmax = None # Resetting the max #fevalsfactor print_done() if islogloss: print "ERT loss ratio figures and tables...", sys.stdout.flush() for ng, sliceNoise in dsList.dictByNoise().iteritems(): if ng == 'noiselessall': testbed = 'noiseless' elif ng == 'nzall': testbed = 'noisy' txt = ("Please input crafting effort value " + "for %s testbed:\n CrE = " % testbed) CrE = inputCrE while CrE is None: try: CrE = float(raw_input(txt)) except (SyntaxError, NameError, ValueError): print "Float value required." dictDim = sliceNoise.dictByDim() for d in inset.rldDimsOfInterest: try: sliceDim = dictDim[d] except KeyError: continue info = '%s' % ng pplogloss.main(sliceDim, CrE, True, outputdir, info, verbose=verbose) pplogloss.generateTable(sliceDim, CrE, outputdir, info, verbose=verbose) for fGroup, sliceFuncGroup in sliceDim.dictByFuncGroup().iteritems(): info = '%s' % fGroup pplogloss.main(sliceFuncGroup, CrE, True, outputdir, info, verbose=verbose) pplogloss.evalfmax = None # Resetting the max #fevalsfactor print_done() latex_commands_file = os.path.join(outputdir.split(os.sep)[0], 'bbob_pproc_commands.tex') prepend_to_file(latex_commands_file, ['\\providecommand{\\bbobloglosstablecaption}[1]{', pplogloss.table_caption, '}']) prepend_to_file(latex_commands_file, ['\\providecommand{\\bbobloglossfigurecaption}[1]{', pplogloss.figure_caption, '}']) prepend_to_file(latex_commands_file, ['\\providecommand{\\bbobpprldistrlegend}[1]{', pprldistr.caption_single(np.max([ val / dim for dim, val in dict_max_fun_evals.iteritems()])), # depends on the config setting, should depend on maxfevals '}']) prepend_to_file(latex_commands_file, ['\\providecommand{\\bbobppfigdimlegend}[1]{', ppfigdim.scaling_figure_caption(), '}']) prepend_to_file(latex_commands_file, ['\\providecommand{\\bbobpptablecaption}[1]{', pptable.table_caption, '}']) prepend_to_file(latex_commands_file, ['\\providecommand{\\algfolder}{}']) # is overwritten in rungeneric.py prepend_to_file(latex_commands_file, ['\\providecommand{\\algname}{' + (str_to_latex(strip_pathname(args[0])) if len(args) == 1 else str_to_latex(dsList[0].algId)) + '{}}']) if isfigure or istab or isrldistr or islogloss: print "Output data written to folder %s" % outputdir plt.rcdefaults()
def main(dictAlg, sortedAlgs, targets, outputdir='.', verbose=True, function_targets_line=True): # [1, 13, 101] """Generate one table per func with results of multiple algorithms.""" """Difference with the first version: * numbers aligned using the decimal separator * premices for dispersion measure * significance test against best algorithm * table width... """ # TODO: method is long, terrible to read, split if possible if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() # Sort data per dimension and function dictData = {} dsListperAlg = list(dictAlg[i] for i in sortedAlgs) for n, entries in enumerate(dsListperAlg): tmpdictdim = entries.dictByDim() for d in tmpdictdim: tmpdictfun = tmpdictdim[d].dictByFunc() for f in tmpdictfun: dictData.setdefault((d, f), {})[n] = tmpdictfun[f] nbtests = len(dictData) for df in dictData: # Generate one table per df # best 2009 refalgentry = bestalg.bestalgentries2009[df] refalgert = refalgentry.detERT(targets) refalgevals = (refalgentry.detEvals((targetf, ))[0][0]) refalgnbruns = len(refalgevals) refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False) # Process the data # The following variables will be lists of elements each corresponding # to an algorithm algnames = [] #algdata = [] algerts = [] algevals = [] algdisp = [] algnbsucc = [] algnbruns = [] algmedmaxevals = [] algmedfinalfunvals = [] algtestres = [] algentries = [] for n in sorted(dictData[df].keys()): entries = dictData[df][n] # the number of datasets for a given dimension and function (df) # should be strictly 1. TODO: find a way to warn # TODO: do this checking before... why wasn't it triggered by ppperprof? if len(entries) > 1: txt = ("There is more than a single entry associated with " "folder %s on %d-D f%d." % (sortedAlgs[n], df[0], df[1])) raise Exception(txt) entry = entries[0] algentries.append(entry) algnames.append(sortedAlgs[n]) evals = entry.detEvals(targets) #tmpdata = [] tmpdisp = [] tmpert = [] for i, e in enumerate(evals): succ = (numpy.isnan(e) == False) e[succ == False] = entry.maxevals[succ == False] ert = toolsstats.sp(e, issuccessful=succ)[0] #tmpdata.append(ert/refalgert[i]) if succ.any(): tmp = toolsstats.drawSP(e[succ], entry.maxevals[succ == False], [10, 50, 90], samplesize=samplesize)[0] tmpdisp.append((tmp[-1] - tmp[0])/2.) else: tmpdisp.append(numpy.nan) tmpert.append(ert) algerts.append(tmpert) algevals.append(evals) #algdata.append(tmpdata) algdisp.append(tmpdisp) algmedmaxevals.append(numpy.median(entry.maxevals)) algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) #algmedmaxevals.append(numpy.median(entry.maxevals)/df[0]) #algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) algtestres.append(significancetest(refalgentry, entry, targets)) # determine success probability for Df = 1e-8 e = entry.detEvals((targetf ,))[0] algnbsucc.append(numpy.sum(numpy.isnan(e) == False)) algnbruns.append(len(e)) # Process over all data # find best values... nalgs = len(dictData[df]) maxRank = 1 + numpy.floor(0.14 * nalgs) # number of algs to be displayed in bold isBoldArray = [] # Point out the best values algfinaldata = [] # Store median function values/median number of function evaluations tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank) for i, erts in enumerate(algerts): tmp = [] for j, ert in enumerate(erts): # algi targetj tmp.append(i in tmptop[j] or (nalgs > 7 and algerts[i][j] <= 3. * refalgert[j])) isBoldArray.append(tmp) algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i])) # significance test of best given algorithm against all others best_alg_idx = numpy.array(algerts).argsort(0)[0, :] # indexed by target index significance_versus_others = significance_all_best_vs_other(algentries, targets, best_alg_idx)[0] # Create the table table = [] spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % (len(targets)) # in case StrLeft not working: replaced c@{} with l@{ } spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % (len(targets)) # in case StrLeft not working: replaced c@{} with l@{ } extraeol = [] # Generate header lines if with_table_heading: header = funInfos[df[1]] if funInfos else 'f%d' % df[1] table.append([r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}' % (2 * len(targets) + 2, header)]) extraeol.append('') if function_targets_line is True or (function_targets_line and df[1] in function_targets_line): curline = [r'$\Delta f_\mathrm{opt}$'] for t in targets[0:-1]: curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}' % writeFEvals2(t, precision=1, isscientific=True)) curline.append(r'\multicolumn{2}{@{\,}X@{}|}{%s}' % writeFEvals2(targets[-1], precision=1, isscientific=True)) curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') table.append(curline) extraeol.append(r'\hline') # extraeol.append(r'\hline\arrayrulecolor{tableShade}') curline = [r'ERT$_{\text{best}}$'] if with_table_heading else [r'\textbf{f%d}' % df[1]] for i in refalgert[0:-1]: curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}' % writeFEvalsMaxPrec(float(i), 2)) curline.append(r'\multicolumn{2}{@{\,}X@{\,}|}{%s}' % writeFEvalsMaxPrec(float(refalgert[-1]), 2)) curline.append('%d' % refalgnbsucc) if refalgnbsucc: curline.append('/%d' % refalgnbruns) #curline.append(curline[0]) table.append(curline) extraeol.append('') #for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))): #print i, gna, gno #set_trace() # Format data #if df == (5, 17): #set_trace() header = r'\providecommand{\ntables}{7}' for i, alg in enumerate(algnames): #algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames, #data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres): commandname = r'\alg%stables' % numtotext(i) # header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg))) header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % (commandname, str_to_latex(strip_pathname(alg))) curline = [commandname + r'\hspace*{\fill}'] # each list element becomes a &-separated table entry? for j, tmp in enumerate(zip(algerts[i], algdisp[i], # j is target index isBoldArray[i], algtestres[i])): ert, dispersion, isBold, testres = tmp alignment = '@{\,}X@{\,}' if j == len(algerts[i]) - 1: alignment = '@{\,}X@{\,}|' data = ert/refalgert[j] # write star for significance against all other algorithms str_significance_subsup = '' if (len(best_alg_idx) > 0 and len(significance_versus_others) > 0 and i == best_alg_idx[j] and nbtests * significance_versus_others[j][1] < 0.05): logp = -numpy.ceil(numpy.log10(nbtests * significance_versus_others[j][1])) str_significance_subsup = r"^{%s%s}" % (significance_vs_others_symbol, str(int(logp)) if logp > 1 else '') # moved out of the above else: this was a bug!? z, p = testres if (nbtests * p) < 0.05 and data < 1. and z < 0.: if not numpy.isinf(refalgert[j]): tmpevals = algevals[i][j].copy() tmpevals[numpy.isnan(tmpevals)] = algentries[i].maxevals[numpy.isnan(tmpevals)] bestevals = refalgentry.detEvals([targets[j]]) bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0]) bestevals[numpy.isnan(bestevals)] = refalgentry.maxevals[bestalgalg][numpy.isnan(bestevals)] tmpevals = numpy.array(sorted(tmpevals))[0:min(len(tmpevals), len(bestevals))] bestevals = numpy.array(sorted(bestevals))[0:min(len(tmpevals), len(bestevals))] #The conditions are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) # tmp2[-1] += r'$^{%s}$' % superscript str_significance_subsup += r'_{%s%s}' % (significance_vs_ref_symbol, str(int(nbstars)) if nbstars > 1 else '') if str_significance_subsup: str_significance_subsup = '$%s$' % str_significance_subsup # format number in variable data if numpy.isnan(data): curline.append(r'\multicolumn{2}{%s}{.}' % alignment) else: if numpy.isinf(refalgert[j]): curline.append(r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}' % (alignment, writeFEvalsMaxPrec(algerts[i][j], 2), writeFEvalsMaxPrec(dispersion, precdispersion), str_significance_subsup)) continue tmp = writeFEvalsMaxPrec(data, precfloat, maxfloatrepr=maxfloatrepr) if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation if numpy.isinf(data) and j == len(algerts[i]) - 1: tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec(algfinaldata[i][1], 0, maxfloatrepr=maxfloatrepr) else: tmp = writeFEvalsMaxPrec(data, precscien, maxfloatrepr=data) if isBold: tmp = r'\textbf{%s}' % tmp if not numpy.isnan(dispersion): tmpdisp = dispersion/refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp += r'\mbox{\tiny (%s)}' % tmpdisp curline.append(r'\multicolumn{2}{%s}{%s%s}' % (alignment, tmp, str_significance_subsup)) else: tmp2 = tmp.split('.', 1) if len(tmp2) < 2: tmp2.append('') else: tmp2[-1] = '.' + tmp2[-1] if isBold: tmp3 = [] for k in tmp2: tmp3.append(r'\textbf{%s}' % k) tmp2 = tmp3 if not numpy.isnan(dispersion): tmpdisp = dispersion/refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.01: tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp)) tmp2[-1] += str_significance_subsup curline.extend(tmp2) curline.append('%d' % algnbsucc[i]) curline.append('/%d' % algnbruns[i]) table.append(curline) extraeol.append('') # Write table res = tableXLaTeX(table, spec=spec, extraeol=extraeol) try: filename = os.path.join(outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0])) f = open(filename, 'w') f.write(header + '\n') f.write(res) if verbose: print 'Wrote table in %s' % filename except: raise else: f.close()
def main(dictAlg, sortedAlgs, targets, outputdir='.', verbose=True, function_targets_line=True): # [1, 13, 101] """Generate one table per func with results of multiple algorithms.""" """Difference with the first version: * numbers aligned using the decimal separator * premices for dispersion measure * significance test against best algorithm * table width... """ # TODO: method is long, terrible to read, split if possible if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() # Sort data per dimension and function dictData = {} dsListperAlg = list(dictAlg[i] for i in sortedAlgs) for n, entries in enumerate(dsListperAlg): tmpdictdim = entries.dictByDim() for d in tmpdictdim: tmpdictfun = tmpdictdim[d].dictByFunc() for f in tmpdictfun: dictData.setdefault((d, f), {})[n] = tmpdictfun[f] nbtests = len(dictData) for df in dictData: # Generate one table per df # best 2009 refalgentry = bestalg.bestalgentries2009[df] refalgert = refalgentry.detERT(targets) refalgevals = (refalgentry.detEvals((targetf, ))[0][0]) refalgnbruns = len(refalgevals) refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False) # Process the data # The following variables will be lists of elements each corresponding # to an algorithm algnames = [] #algdata = [] algerts = [] algevals = [] algdisp = [] algnbsucc = [] algnbruns = [] algmedmaxevals = [] algmedfinalfunvals = [] algtestres = [] algentries = [] for n in sorted(dictData[df].keys()): entries = dictData[df][n] # the number of datasets for a given dimension and function (df) # should be strictly 1. TODO: find a way to warn # TODO: do this checking before... why wasn't it triggered by ppperprof? if len(entries) > 1: txt = ("There is more than a single entry associated with " "folder %s on %d-D f%d." % (sortedAlgs[n], df[0], df[1])) raise Exception(txt) entry = entries[0] algentries.append(entry) algnames.append(sortedAlgs[n]) evals = entry.detEvals(targets) #tmpdata = [] tmpdisp = [] tmpert = [] for i, e in enumerate(evals): succ = (numpy.isnan(e) == False) e[succ == False] = entry.maxevals[succ == False] ert = toolsstats.sp(e, issuccessful=succ)[0] #tmpdata.append(ert/refalgert[i]) if succ.any(): tmp = toolsstats.drawSP(e[succ], entry.maxevals[succ == False], [10, 50, 90], samplesize=samplesize)[0] tmpdisp.append((tmp[-1] - tmp[0]) / 2.) else: tmpdisp.append(numpy.nan) tmpert.append(ert) algerts.append(tmpert) algevals.append(evals) #algdata.append(tmpdata) algdisp.append(tmpdisp) algmedmaxevals.append(numpy.median(entry.maxevals)) algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) #algmedmaxevals.append(numpy.median(entry.maxevals)/df[0]) #algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) algtestres.append(significancetest(refalgentry, entry, targets)) # determine success probability for Df = 1e-8 e = entry.detEvals((targetf, ))[0] algnbsucc.append(numpy.sum(numpy.isnan(e) == False)) algnbruns.append(len(e)) # Process over all data # find best values... nalgs = len(dictData[df]) maxRank = 1 + numpy.floor( 0.14 * nalgs) # number of algs to be displayed in bold isBoldArray = [] # Point out the best values algfinaldata = [ ] # Store median function values/median number of function evaluations tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank) for i, erts in enumerate(algerts): tmp = [] for j, ert in enumerate(erts): # algi targetj tmp.append(i in tmptop[j] or (nalgs > 7 and algerts[i][j] <= 3. * refalgert[j])) isBoldArray.append(tmp) algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i])) # significance test of best given algorithm against all others best_alg_idx = numpy.array(algerts).argsort(0)[ 0, :] # indexed by target index significance_versus_others = significance_all_best_vs_other( algentries, targets, best_alg_idx)[0] # Create the table table = [] spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % ( len(targets) ) # in case StrLeft not working: replaced c@{} with l@{ } spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % ( len(targets) ) # in case StrLeft not working: replaced c@{} with l@{ } extraeol = [] # Generate header lines if with_table_heading: header = funInfos[df[1]] if funInfos else 'f%d' % df[1] table.append([ r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}' % (2 * len(targets) + 2, header) ]) extraeol.append('') if function_targets_line is True or (function_targets_line and df[1] in function_targets_line): curline = [r'$\Delta f_\mathrm{opt}$'] for t in targets[0:-1]: curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}' % writeFEvals2(t, precision=1, isscientific=True)) curline.append( r'\multicolumn{2}{@{\,}X@{}|}{%s}' % writeFEvals2(targets[-1], precision=1, isscientific=True)) curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') table.append(curline) extraeol.append(r'\hline') # extraeol.append(r'\hline\arrayrulecolor{tableShade}') curline = [r'ERT$_{\text{best}}$' ] if with_table_heading else [r'\textbf{f%d}' % df[1]] for i in refalgert[0:-1]: curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}' % writeFEvalsMaxPrec(float(i), 2)) curline.append(r'\multicolumn{2}{@{\,}X@{\,}|}{%s}' % writeFEvalsMaxPrec(float(refalgert[-1]), 2)) curline.append('%d' % refalgnbsucc) if refalgnbsucc: curline.append('/%d' % refalgnbruns) #curline.append(curline[0]) table.append(curline) extraeol.append('') #for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))): #print i, gna, gno #set_trace() # Format data #if df == (5, 17): #set_trace() header = r'\providecommand{\ntables}{7}' for i, alg in enumerate(algnames): #algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames, #data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres): commandname = r'\alg%stables' % numtotext(i) # header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg))) header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % ( commandname, str_to_latex(strip_pathname(alg))) curline = [ commandname + r'\hspace*{\fill}' ] # each list element becomes a &-separated table entry? for j, tmp in enumerate( zip( algerts[i], algdisp[i], # j is target index isBoldArray[i], algtestres[i])): ert, dispersion, isBold, testres = tmp alignment = '@{\,}X@{\,}' if j == len(algerts[i]) - 1: alignment = '@{\,}X@{\,}|' data = ert / refalgert[j] # write star for significance against all other algorithms str_significance_subsup = '' if (len(best_alg_idx) > 0 and len(significance_versus_others) > 0 and i == best_alg_idx[j] and nbtests * significance_versus_others[j][1] < 0.05): logp = -numpy.ceil( numpy.log10( nbtests * significance_versus_others[j][1])) str_significance_subsup = r"^{%s%s}" % ( significance_vs_others_symbol, str(int(logp)) if logp > 1 else '') # moved out of the above else: this was a bug!? z, p = testres if (nbtests * p) < 0.05 and data < 1. and z < 0.: if not numpy.isinf(refalgert[j]): tmpevals = algevals[i][j].copy() tmpevals[numpy.isnan(tmpevals)] = algentries[ i].maxevals[numpy.isnan(tmpevals)] bestevals = refalgentry.detEvals([targets[j]]) bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0]) bestevals[numpy.isnan( bestevals)] = refalgentry.maxevals[bestalgalg][ numpy.isnan(bestevals)] tmpevals = numpy.array(sorted( tmpevals))[0:min(len(tmpevals), len(bestevals))] bestevals = numpy.array(sorted( bestevals))[0:min(len(tmpevals), len(bestevals))] #The conditions are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) # tmp2[-1] += r'$^{%s}$' % superscript str_significance_subsup += r'_{%s%s}' % ( significance_vs_ref_symbol, str(int(nbstars)) if nbstars > 1 else '') if str_significance_subsup: str_significance_subsup = '$%s$' % str_significance_subsup # format number in variable data if numpy.isnan(data): curline.append(r'\multicolumn{2}{%s}{.}' % alignment) else: if numpy.isinf(refalgert[j]): curline.append( r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}' % (alignment, writeFEvalsMaxPrec(algerts[i][j], 2), writeFEvalsMaxPrec(dispersion, precdispersion), str_significance_subsup)) continue tmp = writeFEvalsMaxPrec(data, precfloat, maxfloatrepr=maxfloatrepr) if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation if numpy.isinf(data) and j == len(algerts[i]) - 1: tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec( algfinaldata[i][1], 0, maxfloatrepr=maxfloatrepr) else: tmp = writeFEvalsMaxPrec(data, precscien, maxfloatrepr=data) if isBold: tmp = r'\textbf{%s}' % tmp if not numpy.isnan(dispersion): tmpdisp = dispersion / refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp += r'\mbox{\tiny (%s)}' % tmpdisp curline.append( r'\multicolumn{2}{%s}{%s%s}' % (alignment, tmp, str_significance_subsup)) else: tmp2 = tmp.split('.', 1) if len(tmp2) < 2: tmp2.append('') else: tmp2[-1] = '.' + tmp2[-1] if isBold: tmp3 = [] for k in tmp2: tmp3.append(r'\textbf{%s}' % k) tmp2 = tmp3 if not numpy.isnan(dispersion): tmpdisp = dispersion / refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.01: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp)) tmp2[-1] += str_significance_subsup curline.extend(tmp2) curline.append('%d' % algnbsucc[i]) curline.append('/%d' % algnbruns[i]) table.append(curline) extraeol.append('') # Write table res = tableXLaTeX(table, spec=spec, extraeol=extraeol) try: filename = os.path.join( outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0])) f = open(filename, 'w') f.write(header + '\n') f.write(res) if verbose: print 'Wrote table in %s' % filename except: raise else: f.close()