def build(dictAlg, sortedAlg=None): """Merge datasets in an algorithm portfolio. :param dict dictAlg: dictionary of data sets with algorithm name for keys :param seq sortedAlgs: sequence for sorting the entries of :py:data:`dictAlg`, if not provided, dictAlg.keys() will be instead :returns: an instance of :py:class:`DataSetList` with the porfolio data sets """ if not sortedAlg: sortedAlg = dictAlg.keys() tmpres = [] for f, i in pp.dictAlgByFun(dictAlg).iteritems(): for d, j in pp.dictAlgByDim(i).iteritems(): tmp = [] if sortedAlg: tmplist = list(j[k] for k in sortedAlg) else: tmplist = j.values() for k in tmplist: assert len(k) == 1 # one element list tmp.append(k[0]) try: tmpres.append(DataSet(tmp)) except Usage, err: print >>sys.stderr, err.msg
def generate(dictalg): """Generates dictionary of best algorithm data set. """ # dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose) res = {} for f, i in pproc.dictAlgByFun(dictalg).iteritems(): for d, j in pproc.dictAlgByDim(i).iteritems(): tmp = BestAlgSet(j) res[(d, f)] = tmp return res
def generate(dictalg): """Generates dictionary of best algorithm data set. """ #dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose) res = {} for f, i in pproc.dictAlgByFun(dictalg).iteritems(): for d, j in pproc.dictAlgByDim(i).iteritems(): tmp = BestAlgSet(j) res[(d, f)] = tmp return res
def main(dictAlg, outputdir='.', verbose=True): """Main routine for generating convergence plots """ global warned # bind variable warned into this scope dictFun = pproc.dictAlgByFun(dictAlg) for l in dictFun: # l appears to be the function id!? for i in dictFun[ l]: # please, what is i??? appears to be the algorithm-key plt.figure() if type(i) in (list, tuple): figurename = "ppconv_plot_" + i[0] + "_f" + str(l) else: try: figurename = "ppconv_plot_" + dictFun[l][ i].algId + "_f" + str(l) except AttributeError: # this is a (rather desperate) bug-fix attempt that works for the unit test figurename = "ppconv_plot_" + dictFun[l][i][ 0].algId + "_f" + str(l) plt.xlabel('number of function evaluations / dimension') plt.ylabel('Median of fitness') plt.grid() ax = plt.gca() ax.set_yscale("log") ax.set_xscale("log") for j in dictFun[l][i]: # please, what is j??? a dataset dimList_b = [] dimList_f = [] dimList_b.append(j.funvals[:, 0]) dimList_f.append(j.funvals[:, 1:]) bs, fs = rearrange(dimList_b, dimList_f) labeltext = str(j.dim) + "D" try: if 11 < 3: plt.errorbar(bs[0] / j.dim, fs[0][0], yerr=[fs[0][1], fs[0][2]], label=labeltext) else: plt.errorbar(bs[0] / j.dim, fs[0][0], label=labeltext) except FloatingPointError: # that's a bit of a hack if not warned: print( 'Warning: floating point error when plotting errorbars, ignored' ) warned = True plt.legend(loc=3) saveFigure(os.path.join(outputdir, figurename.replace(' ', '')), genericsettings.fig_formats, verbose=verbose) plt.close() print("Convergence plots done.")
def main(dictAlg, outputdir='.', verbose=True): """Main routine for generating convergence plots """ global warned # bind variable warned into this scope dictFun = pproc.dictAlgByFun(dictAlg) for l in dictFun: # l appears to be the function id!? for i in dictFun[l]: # please, what is i??? appears to be the algorithm-key plt.figure() if 1 < 3: # no algorithm name in filename, as everywhere else figurename = "ppconv_" + "f%03d" % l else: # previous version with algorithm name, but this is not very practical later if type(i) in (list, tuple): figurename = "ppconv_plot_" + i[0] + "_f" + str(l) else: try: figurename = "ppconv_plot_" + dictFun[l][i].algId + "_f" + str(l) except AttributeError: # this is a (rather desperate) bug-fix attempt that works for the unit test figurename = "ppconv_plot_" + dictFun[l][i][0].algId + "_f" + str(l) plt.xlabel('number of function evaluations / dimension') plt.ylabel('Median of fitness') plt.grid() ax = plt.gca() ax.set_yscale("log") ax.set_xscale("log") for j in dictFun[l][i]: # please, what is j??? a dataset dimList_b = [] dimList_f = [] dimList_b.append(j.funvals[:,0]) dimList_f.append(j.funvals[:,1:]) bs, fs= rearrange(dimList_b, dimList_f) labeltext=str(j.dim)+"D" try: if 11 < 3: plt.errorbar(bs[0] / j.dim, fs[0][0], yerr = [fs[0][1], fs[0][2]], label = labeltext) else: plt.errorbar(bs[0] / j.dim, fs[0][0], label = labeltext) except FloatingPointError: # that's a bit of a hack if 1 < 3 or not warned: print('Warning: floating point error when plotting errorbars, ignored') warned = True beautify() saveFigure(os.path.join(outputdir, figurename.replace(' ','')), genericsettings.getFigFormats(), verbose=verbose) plt.close() try: algname = str(dictFun[l].keys()[0][0]) except KeyError: algname = str(dictFun[l].keys()[0]) save_single_functions_html(os.path.join(outputdir, 'ppconv'), algname) # first try print("Convergence plots done.")
def main(argv=None): """ """ if argv is None: argv = sys.argv[1:] try: try: opts, args = getopt.getopt(argv, "h", ["help"]) except getopt.error, msg: raise Usage(msg) args = algs # if not (args): # usage() # sys.exit() outputdir = 'bestAlg' verbose = True #Process options for o, a in opts: if o in ("-h", "--help"): usage() sys.exit() else: assert False, "unhandled option" dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose) #set_trace() if not os.path.exists(outputdir): os.mkdir(outputdir) if verbose: print 'Folder %s was created.' % (outputdir) res = {} for f, i in dictAlgByFun(dictAlg).iteritems(): for d, j in dictAlgByDim(i).iteritems(): tmp = BestAlgSet(j) picklefilename = os.path.join( outputdir, 'bestalg_f%03d_%02d.pickle' % (f, d)) fid = open(picklefilename, 'w') pickle.dump(tmp, fid, 2) fid.close() res[(d, f)] = tmp picklefilename = os.path.join(outputdir, 'bestalg.pickle') fid = open(picklefilename, 'w') pickle.dump(res, fid, 2) fid.close()
def all_single_functions(dictAlg, sortedAlgs=None, outputdir='.', verbose=0): dictFG = pp.dictAlgByFun(dictAlg) for fg, tmpdictAlg in dictFG.iteritems(): dictDim = pp.dictAlgByDim(tmpdictAlg) for d, entries in dictDim.iteritems(): single_fct_output_dir = (outputdir.rstrip(os.sep) + os.sep + 'pprldmany-single-functions' # + os.sep + ('f%03d' % fg) ) if not os.path.exists(single_fct_output_dir): os.makedirs(single_fct_output_dir) main(entries, order=sortedAlgs, outputdir=single_fct_output_dir, info=('f%03d_%02dD' % (fg, d)), verbose=verbose)
def all_single_functions(dictAlg, sortedAlgs=None, outputdir='.', verbose=0): dictFG = pp.dictAlgByFun(dictAlg) for fg, tmpdictAlg in dictFG.iteritems(): dictDim = pp.dictAlgByDim(tmpdictAlg) for d, entries in dictDim.iteritems(): single_fct_output_dir = ( outputdir.rstrip(os.sep) + os.sep + 'pprldmany-single-functions' # + os.sep + ('f%03d' % fg) ) if not os.path.exists(single_fct_output_dir): os.makedirs(single_fct_output_dir) main(entries, order=sortedAlgs, outputdir=single_fct_output_dir, info=('f%03d_%02dD' % (fg, d)), verbose=verbose)
def main(dictAlg, outputdir=".", verbose=True): """Main routine for generating convergence plots """ global warned # bind variable warned into this scope dictFun = pproc.dictAlgByFun(dictAlg) for l in dictFun: # l appears to be the function id!? for i in dictFun[l]: # please, what is i??? appears to be the algorithm-key plt.figure() if type(i) in (list, tuple): figurename = "ppconv_plot_" + i[0] + "_f" + str(l) else: try: figurename = "ppconv_plot_" + dictFun[l][i].algId + "_f" + str(l) except AttributeError: # this is a (rather desperate) bug-fix attempt that works for the unit test figurename = "ppconv_plot_" + dictFun[l][i][0].algId + "_f" + str(l) plt.xlabel("number of function evaluations / dimension") plt.ylabel("Median of fitness") plt.grid() ax = plt.gca() ax.set_yscale("log") ax.set_xscale("log") for j in dictFun[l][i]: # please, what is j??? a dataset dimList_b = [] dimList_f = [] dimList_b.append(j.funvals[:, 0]) dimList_f.append(j.funvals[:, 1:]) bs, fs = rearrange(dimList_b, dimList_f) labeltext = str(j.dim) + "D" try: if 11 < 3: plt.errorbar(bs[0] / j.dim, fs[0][0], yerr=[fs[0][1], fs[0][2]], label=labeltext) else: plt.errorbar(bs[0] / j.dim, fs[0][0], label=labeltext) except FloatingPointError: # that's a bit of a hack if not warned: print("Warning: floating point error when plotting errorbars, ignored") warned = True plt.legend(loc=3) saveFigure( os.path.join(outputdir, figurename.replace(" ", "")), genericsettings.fig_formats, verbose=verbose ) plt.close() print("Convergence plots done.")
outputdir=outputdir, info=('%02dD_%s' % (d, ng)), verbose=verbose) # ECDFs per function groups dictFG = pproc.dictAlgByFuncGroup(dictAlg) for fg, tmpdictAlg in dictFG.iteritems(): dictDim = pproc.dictAlgByDim(tmpdictAlg) for d, entries in dictDim.iteritems(): pprldmany.main(entries, order=sortedAlgs, outputdir=outputdir, info=('%02dD_%s' % (d, fg)), verbose=verbose) if isRldOnSingleFcts: # copy-paste from above, here for each function instead of function groups # ECDFs for each function dictFG = pproc.dictAlgByFun(dictAlg) for fg, tmpdictAlg in dictFG.iteritems(): dictDim = pproc.dictAlgByDim(tmpdictAlg) for d, entries in dictDim.iteritems(): single_fct_output_dir = (outputdir.rstrip(os.sep) + os.sep + 'pprldmany-single-functions' # + os.sep + ('f%03d' % fg) ) if not os.path.exists(single_fct_output_dir): os.makedirs(single_fct_output_dir) pprldmany.main(entries, order=sortedAlgs, outputdir=single_fct_output_dir, info=('f%03d_%02dD' % (fg, d)), verbose=verbose) print "ECDFs of run lengths figures done."
def main(dictAlg, isBiobjective, order=None, outputdir='.', info='default', dimension=None, verbose=True): """Generates a figure showing the performance of algorithms. From a dictionary of :py:class:`DataSetList` sorted by algorithms, generates the cumulative distribution function of the bootstrap distribution of ERT for algorithms on multiple functions for multiple targets altogether. :param dict dictAlg: dictionary of :py:class:`DataSetList` instances one instance is equivalent to one algorithm, :param list targets: target function values :param list order: sorted list of keys to dictAlg for plotting order :param str outputdir: output directory :param str info: output file name suffix :param bool verbose: controls verbosity """ global x_limit # late assignment of default, because it can be set to None in config global divide_by_dimension # not fully implemented/tested yet if 'x_limit' not in globals() or x_limit is None: x_limit = x_limit_default tmp = pp.dictAlgByDim(dictAlg) # tmp = pp.DictAlg(dictAlg).by_dim() if len(tmp) != 1 and dimension is None: raise ValueError('We never integrate over dimension.') if dimension is not None: if dimension not in tmp.keys(): raise ValueError('dimension %d not in dictAlg dimensions %s' % (dimension, str(tmp.keys()))) tmp = {dimension: tmp[dimension]} dim = tmp.keys()[0] divisor = dim if divide_by_dimension else 1 algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []] dictFunc = pp.dictAlgByFun(dictAlg) # Collect data # Crafting effort correction: should we consider any? CrEperAlg = {} for alg in algorithms_with_data: CrE = 0. if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL': tmp = dictAlg[alg].dictByNoise() assert len(tmp.keys()) == 1 if tmp.keys()[0] == 'noiselessall': CrE = 0.5117 elif tmp.keys()[0] == 'nzall': CrE = 0.6572 CrEperAlg[alg] = CrE if CrE != 0.0: print 'Crafting effort for', alg, 'is', CrE dictData = {} # list of (ert per function) per algorithm dictMaxEvals = {} # list of (maxevals per function) per algorithm bestERT = [] # best ert per function # funcsolved = [set()] * len(targets) # number of functions solved per target xbest2009 = [] maxevalsbest2009 = [] for f, dictAlgperFunc in dictFunc.iteritems(): if function_IDs and f not in function_IDs: continue # print target_values((f, dim)) for j, t in enumerate(target_values((f, dim))): # for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)): # funcsolved[j].add(f) for alg in algorithms_with_data: x = [np.inf] * perfprofsamplesize runlengthunsucc = [] try: entry = dictAlgperFunc[alg][0] # one element per fun and per dim. evals = entry.detEvals([t])[0] assert entry.dim == dim runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor if len(runlengthsucc) > 0: x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] except (KeyError, IndexError): #set_trace() warntxt = ('Data for algorithm %s on function %d in %d-D ' % (alg, f, dim) + 'are missing.\n') warnings.warn(warntxt) dictData.setdefault(alg, []).extend(x) dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc) displaybest2009 = not isBiobjective #disabled until we find the bug if displaybest2009: #set_trace() bestalgentries = bestalg.loadBestAlgorithm(isBiobjective) bestalgentry = bestalgentries[(dim, f)] bestalgevals = bestalgentry.detEvals(target_values((f, dim))) # print bestalgevals for j in range(len(bestalgevals[0])): if bestalgevals[1][j]: evals = bestalgevals[0][j] #set_trace() assert dim == bestalgentry.dim runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = bestalgentry.maxevals[bestalgevals[1][j]][np.isnan(evals)] / divisor x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] else: x = perfprofsamplesize * [np.inf] runlengthunsucc = [] xbest2009.extend(x) maxevalsbest2009.extend(runlengthunsucc) if order is None: order = dictData.keys() # Display data lines = [] if displaybest2009: args = {'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11., 'markeredgewidth': 1.5, 'markerfacecolor': refcolor, 'markeredgecolor': refcolor, 'color': refcolor, 'label': 'best 2009', 'zorder': -1} lines.append(plotdata(np.array(xbest2009), x_limit, maxevalsbest2009, CrE = 0., **args)) def algname_to_label(algname, dirname=None): """to be extended to become generally useful""" if isinstance(algname, (tuple, list)): # not sure this is needed return ' '.join([str(name) for name in algname]) return str(algname) for i, alg in enumerate(order): try: data = dictData[alg] maxevals = dictMaxEvals[alg] except KeyError: continue args = styles[(i) % len(styles)] args['linewidth'] = 1.5 args['markersize'] = 12. args['markeredgewidth'] = 1.5 args['markerfacecolor'] = 'None' args['markeredgecolor'] = args['color'] args['label'] = algname_to_label(alg) #args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib #elif len(show_algorithms) > 0: #args['color'] = 'wheat' #args['ls'] = '-' #args['zorder'] = -1 # plotdata calls pprldistr.plotECDF which calls ppfig.plotUnifLog... which does the work lines.append(plotdata(np.array(data), x_limit, maxevals, CrE=CrEperAlg[alg], **args)) labels, handles = plotLegend(lines, x_limit) if True: # isLateXLeg: fileName = os.path.join(outputdir,'pprldmany_%s.tex' % (info)) with open(fileName, 'w') as f: f.write(r'\providecommand{\nperfprof}{7}') algtocommand = {} # latex commands for i, alg in enumerate(order): tmp = r'\alg%sperfprof' % pptex.numtotext(i) f.write(r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' % (tmp, toolsdivers.str_to_latex( toolsdivers.strip_pathname2(algname_to_label(alg))))) algtocommand[algname_to_label(alg)] = tmp if displaybest2009: tmp = r'\algzeroperfprof' f.write(r'\providecommand{%s}{best 2009}' % (tmp)) algtocommand['best 2009'] = tmp commandnames = [] for label in labels: commandnames.append(algtocommand[label]) # f.write(headleg) if len(order) > 28: # latex sidepanel won't work well for more than 25 algorithms, but original labels are also clipped f.write(r'\providecommand{\perfprofsidepanel}{\mbox{%s}\vfill\mbox{%s}}' % (commandnames[0], commandnames[-1])) else: fontsize_command = r'\tiny{}' if len(order) > 19 else '' f.write(r'\providecommand{\perfprofsidepanel}{{%s\mbox{%s}' % (fontsize_command, commandnames[0])) # TODO: check len(labels) > 0 for i in range(1, len(labels)): f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i]) f.write('}}\n') # f.write(footleg) if verbose: print 'Wrote right-hand legend in %s' % fileName figureName = os.path.join(outputdir,'pprldmany_%s' % (info)) #beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat) beautify() text = ppfig.consecutiveNumbers(sorted(dictFunc.keys()), 'f') text += ',%d-D' % dim # TODO: this is strange when different dimensions are plotted plt.text(0.01, 0.98, text, horizontalalignment="left", verticalalignment="top", transform=plt.gca().transAxes) if len(dictFunc) == 1: plt.title(' '.join((str(dictFunc.keys()[0]), genericsettings.current_testbed.short_names[dictFunc.keys()[0]]))) a = plt.gca() plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative) xticks, labels = plt.xticks() tmp = [] for i in xticks: tmp.append('%d' % round(np.log10(i))) a.set_xticklabels(tmp) if save_figure: ppfig.saveFigure(figureName, verbose=verbose) if len(dictFunc) == 1: ppfig.save_single_functions_html( os.path.join(outputdir, 'pprldmany'), '', # algorithms names are clearly visible in the figure add_to_names='_%02dD' %(dim), algorithmCount=ppfig.AlgorithmCount.NON_SPECIFIED ) if close_figure: plt.close()
def main(dictAlg, order=None, outputdir='.', info='default', verbose=True): """Generates a figure showing the performance of algorithms. From a dictionary of :py:class:`DataSetList` sorted by algorithms, generates the cumulative distribution function of the bootstrap distribution of ERT for algorithms on multiple functions for multiple targets altogether. :param dict dictAlg: dictionary of :py:class:`DataSetList` instances one instance is equivalent to one algorithm, :param list targets: target function values :param list order: sorted list of keys to dictAlg for plotting order :param str outputdir: output directory :param str info: output file name suffix :param bool verbose: controls verbosity """ global x_limit # late assignment of default, because it can be set to None in config if 'x_limit' not in globals() or x_limit is None: x_limit = x_limit_default tmp = pp.dictAlgByDim(dictAlg) # tmp = pp.DictAlg(dictAlg).by_dim() if len(tmp) != 1: raise Exception('We never integrate over dimension.') dim = tmp.keys()[0] algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []] dictFunc = pp.dictAlgByFun(dictAlg) # Collect data # Crafting effort correction: should we consider any? CrEperAlg = {} for alg in algorithms_with_data: CrE = 0. if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL': tmp = dictAlg[alg].dictByNoise() assert len(tmp.keys()) == 1 if tmp.keys()[0] == 'noiselessall': CrE = 0.5117 elif tmp.keys()[0] == 'nzall': CrE = 0.6572 CrEperAlg[alg] = CrE if CrE != 0.0: print 'Crafting effort for', alg, 'is', CrE dictData = {} # list of (ert per function) per algorithm dictMaxEvals = {} # list of (maxevals per function) per algorithm bestERT = [] # best ert per function # funcsolved = [set()] * len(targets) # number of functions solved per target xbest2009 = [] maxevalsbest2009 = [] for f, dictAlgperFunc in dictFunc.iteritems(): if function_IDs and f not in function_IDs: continue # print target_values((f, dim)) for j, t in enumerate(target_values((f, dim))): # for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)): # funcsolved[j].add(f) for alg in algorithms_with_data: x = [np.inf] * perfprofsamplesize runlengthunsucc = [] try: entry = dictAlgperFunc[alg][0] # one element per fun and per dim. evals = entry.detEvals([t])[0] runlengthsucc = evals[np.isnan(evals) == False] / entry.dim runlengthunsucc = entry.maxevals[np.isnan(evals)] / entry.dim if len(runlengthsucc) > 0: x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] except (KeyError, IndexError): #set_trace() warntxt = ('Data for algorithm %s on function %d in %d-D ' % (alg, f, dim) + 'are missing.\n') warnings.warn(warntxt) dictData.setdefault(alg, []).extend(x) dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc) if displaybest2009: #set_trace() if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() bestalgentry = bestalg.bestalgentries2009[(dim, f)] bestalgevals = bestalgentry.detEvals(target_values((f, dim))) # print bestalgevals for j in range(len(bestalgevals[0])): if bestalgevals[1][j]: evals = bestalgevals[0][j] #set_trace() runlengthsucc = evals[np.isnan(evals) == False] / bestalgentry.dim runlengthunsucc = bestalgentry.maxevals[bestalgevals[1][j]][np.isnan(evals)] / bestalgentry.dim x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] else: x = perfprofsamplesize * [np.inf] runlengthunsucc = [] xbest2009.extend(x) maxevalsbest2009.extend(runlengthunsucc) if order is None: order = dictData.keys() # Display data lines = [] if displaybest2009: args = {'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11., 'markeredgewidth': 1.5, 'markerfacecolor': refcolor, 'markeredgecolor': refcolor, 'color': refcolor, 'label': 'best 2009', 'zorder': -1} lines.append(plotdata(np.array(xbest2009), x_limit, maxevalsbest2009, CrE = 0., **args)) for i, alg in enumerate(order): try: data = dictData[alg] maxevals = dictMaxEvals[alg] except KeyError: continue args = styles[(i) % len(styles)] args['linewidth'] = 1.5 args['markersize'] = 12. args['markeredgewidth'] = 1.5 args['markerfacecolor'] = 'None' args['markeredgecolor'] = args['color'] args['label'] = alg #args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib #elif len(show_algorithms) > 0: #args['color'] = 'wheat' #args['ls'] = '-' #args['zorder'] = -1 lines.append(plotdata(np.array(data), x_limit, maxevals, CrE=CrEperAlg[alg], **args)) labels, handles = plotLegend(lines, x_limit) if True: #isLateXLeg: fileName = os.path.join(outputdir,'pprldmany_%s.tex' % (info)) try: f = open(fileName, 'w') f.write(r'\providecommand{\nperfprof}{7}') algtocommand = {} for i, alg in enumerate(order): tmp = r'\alg%sperfprof' % pptex.numtotext(i) f.write(r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' % (tmp, toolsdivers.str_to_latex(toolsdivers.strip_pathname2(alg)))) algtocommand[alg] = tmp commandnames = [] if displaybest2009: tmp = r'\algzeroperfprof' f.write(r'\providecommand{%s}{best 2009}' % (tmp)) algtocommand['best 2009'] = tmp for l in labels: commandnames.append(algtocommand[l]) # f.write(headleg) f.write(r'\providecommand{\perfprofsidepanel}{\mbox{%s}' % commandnames[0]) # TODO: check len(labels) > 0 for i in range(1, len(labels)): f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i]) f.write('}\n') # f.write(footleg) if verbose: print 'Wrote right-hand legend in %s' % fileName except: raise # TODO: Does this make sense? else: f.close() figureName = os.path.join(outputdir,'pprldmany_%s' % (info)) #beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat) beautify() text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys()))) text += ',%d-D' % dim plt.text(0.01, 0.98, text, horizontalalignment="left", verticalalignment="top", transform=plt.gca().transAxes) a = plt.gca() plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative) xticks, labels = plt.xticks() tmp = [] for i in xticks: tmp.append('%d' % round(np.log10(i))) a.set_xticklabels(tmp) ppfig.saveFigure(figureName, verbose=verbose) plt.close()
def main(dictAlg, sortedAlgs, target=1e-8, outputdir='ppdata', verbose=True): """From a DataSetList, returns figures showing the scaling: ERT/dim vs dim. One function and one target per figure. sortedAlgs is a list of string-identifies (folder names) """ dictFunc = pproc.dictAlgByFun(dictAlg) for f in dictFunc: filename = os.path.join(outputdir,'ppfigs_f%03d' % (f)) handles = [] fix_styles(len(sortedAlgs)) # for i, alg in enumerate(sortedAlgs): dictDim = dictFunc[f][alg].dictByDim() #Collect data dimert = [] ert = [] dimnbsucc = [] ynbsucc = [] nbsucc = [] dimmaxevals = [] maxevals = [] dimmedian = [] medianfes = [] for dim in sorted(dictDim): assert len(dictDim[dim]) == 1 entry = dictDim[dim][0] data = generateData(entry, target) # TODO: here we might want a different target for each function if 1 < 3 or data[2] == 0: # No success dimmaxevals.append(dim) maxevals.append(float(data[3])/dim) if data[2] > 0: dimmedian.append(dim) medianfes.append(data[4]/dim) dimert.append(dim) ert.append(float(data[0])/dim) if data[1] < 1.: dimnbsucc.append(dim) ynbsucc.append(float(data[0])/dim) nbsucc.append('%d' % data[2]) # Draw lines tmp = plt.plot(dimert, ert, **styles[i]) #label=alg, ) plt.setp(tmp[0], markeredgecolor=plt.getp(tmp[0], 'color')) # For legend # tmp = plt.plot([], [], label=alg.replace('..' + os.sep, '').strip(os.sep), **styles[i]) tmp = plt.plot([], [], label=alg.split(os.sep)[-1], **styles[i]) plt.setp(tmp[0], markersize=12., markeredgecolor=plt.getp(tmp[0], 'color')) if dimmaxevals: tmp = plt.plot(dimmaxevals, maxevals, **styles[i]) plt.setp(tmp[0], markersize=20, #label=alg, markeredgecolor=plt.getp(tmp[0], 'color'), markeredgewidth=1, markerfacecolor='None', linestyle='None') handles.append(tmp) #tmp2 = plt.plot(dimmedian, medianfes, ls='', marker='+', # markersize=30, markeredgewidth=5, # markeredgecolor=plt.getp(tmp, 'color'))[0] #for i, n in enumerate(nbsucc): # plt.text(dimnbsucc[i], numpy.array(ynbsucc[i])*1.85, n, # verticalalignment='bottom', # horizontalalignment='center') if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() bestalgdata = [] dimbestalg = list(df[0] for df in bestalg.bestalgentries2009 if df[1] == f) dimbestalg.sort() dimbestalg2 = [] for d in dimbestalg: entry = bestalg.bestalgentries2009[(d, f)] tmp = entry.detERT([target])[0] if numpy.isfinite(tmp): bestalgdata.append(float(tmp)/d) dimbestalg2.append(d) tmp = plt.plot(dimbestalg2, bestalgdata, color=refcolor, linewidth=10, marker='d', markersize=25, markeredgecolor=refcolor, zorder=-1 #label='best 2009', ) handles.append(tmp) if show_significance: # plot significance-stars xstar, ystar = [], [] dims = sorted(pproc.dictAlgByDim(dictFunc[f])) for i, dim in enumerate(dims): datasets = pproc.dictAlgByDim(dictFunc[f])[dim] assert all([len(datasets[ialg]) == 1 for ialg in sortedAlgs if datasets[ialg]]) dsetlist = [datasets[ialg][0] for ialg in sortedAlgs if datasets[ialg]] if len(dsetlist) > 1: arzp, arialg = toolsstats.significance_all_best_vs_other(dsetlist, [target]) if arzp[0][1] * len(dims) < 0.05: ert = dsetlist[arialg[0]].detERT([target])[0] if ert < numpy.inf: xstar.append(dim) ystar.append(ert/dim) plt.plot(xstar, ystar, 'k*', markerfacecolor=None, markeredgewidth=2, markersize=0.5*styles[0]['markersize']) if funInfos: plt.gca().set_title(funInfos[f]) isLegend = False if legend: plotLegend(handles) elif 1 < 3: if f in (1, 24, 101, 130) and len(sortedAlgs) < 6: # 6 elements at most in the boxed legend isLegend = True beautify(legend=isLegend, rightlegend=legend) plt.text(plt.xlim()[0], plt.ylim()[0], 'ftarget=%.0e' % target) saveFigure(filename, verbose=verbose) plt.close() # generate commands in tex file: try: abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' alg_definitions = [] for i in range(len(sortedAlgs)): symb = r'{%s%s}' % (color_to_latex(styles[i]['color']), marker_to_latex(styles[i]['marker'])) alg_definitions.append((', ' if i > 0 else '') + '%s:%s' % (symb, '\\algorithm' + abc[i % len(abc)])) filename = os.path.join(outputdir, 'bbob_pproc_commands.tex') toolsdivers.prepend_to_file(filename, ['\\providecommand{\\bbobppfigsftarget}{\\ensuremath{10^{%d}}}' % int(numpy.round(numpy.log10(target))), '\\providecommand{\\bbobppfigslegend}[1]{', scaling_figure_legend, 'Legend: '] + alg_definitions + ['}'] ) if verbose: print 'Wrote commands and legend to %s' % filename # this is obsolete (however check templates) filename = os.path.join(outputdir,'ppfigs.tex') f = open(filename, 'w') f.write('% Do not modify this file: calls to post-processing software' + ' will overwrite any modification.\n') f.write('Legend: ') for i in range(0, len(sortedAlgs)): symb = r'{%s%s}' % (color_to_latex(styles[i]['color']), marker_to_latex(styles[i]['marker'])) f.write((', ' if i > 0 else '') + '%s:%s' % (symb, writeLabels(sortedAlgs[i]))) f.close() if verbose: print '(obsolete) Wrote legend in %s' % filename except IOError: raise handles.append(tmp) if funInfos: plt.gca().set_title(funInfos[f]) beautify(rightlegend=legend) if legend: plotLegend(handles) else: if f in (1, 24, 101, 130): plt.legend() saveFigure(filename, figFormat=genericsettings.fig_formats, verbose=verbose) plt.close()
def extractBestAlgorithms(args = algs2009, f_factor=2, target_lb=1e-8, target_ub=1e22): """Returns (and prints) per dimension a list of algorithms within algorithm list args that contains an algorithm if for any dimension/target/function pair this algorithm: - is the best algorithm wrt ERT - its own ERT lies within a factor f_factor of the best ERT - there is no algorithm within a factor of f_factor of the best ERT and the current algorithm is the second best. """ # TODO: use pproc.TargetValues class as input target values # default target values: targets = pproc.TargetValues( 10**np.arange(np.log10(max((1e-8, target_lb))), np.log10(target_ub) + 1e-9, 0.2)) # there should be a simpler way to express this to become the # interface of this function print 'Loading algorithm data from given algorithm list...\n' verbose = True dsList, sortedAlgs, dictAlg = pproc.processInputArgs(args, verbose=verbose) print 'This may take a while (depending on the number of algorithms)' selectedAlgsPerProblem = {} for f, i in pproc.dictAlgByFun(dictAlg).iteritems(): for d, j in pproc.dictAlgByDim(i).iteritems(): selectedAlgsPerProblemDF = [] best = BestAlgSet(j) for i in range(0, len(best.target)): t = best.target[i] # if ((t <= target_ub) and (t >= target_lb)): if toolsstats.in_approximately(t, targets((f, d), discretize=True)): # add best for this target: selectedAlgsPerProblemDF.append(best.algs[i]) # add second best or all algorithms that have an ERT # within a factor of f_factor of the best: secondbest_ERT = np.infty secondbest_str = '' secondbest_included = False for astring in j: currdictalg = dictAlg[astring].dictByDim() if currdictalg.has_key(d): curralgdata = currdictalg[d][f-1] currERT = curralgdata.detERT([t])[0] if (astring != best.algs[i]): if (currERT < secondbest_ERT): secondbest_ERT = currERT secondbest_str = astring if (currERT <= best.detERT([t])[0] * f_factor): selectedAlgsPerProblemDF.append(astring) secondbest_included = True if not (secondbest_included) and (secondbest_str != ''): selectedAlgsPerProblemDF.append(secondbest_str) if len(selectedAlgsPerProblemDF) > 0: selectedAlgsPerProblem[(d, f)] = selectedAlgsPerProblemDF print 'pre-processing of function', f, 'done.' print 'loading of best algorithm(s) data done.' countsperalgorithm = {} for (d, f) in selectedAlgsPerProblem: print 'dimension:', d, ', function:', f setofalgs = set(selectedAlgsPerProblem[d,f]) # now count how often algorithm a is best for the extracted targets for a in setofalgs: # use setdefault to initialize with zero if a entry not existant: countsperalgorithm.setdefault((d, a), 0) countsperalgorithm[(d,a)] += selectedAlgsPerProblem[d,f].count(a) selectedalgsperdimension = {} for (d,a) in sorted(countsperalgorithm): if not selectedalgsperdimension.has_key(d): selectedalgsperdimension[d] = [] selectedalgsperdimension[d].append((countsperalgorithm[(d,a)], a)) for d in sorted(selectedalgsperdimension): print d, 'D:' for (count, alg) in sorted(selectedalgsperdimension[d], reverse=True): print count, alg print '\n' print " done." return selectedalgsperdimension
def extractBestAlgorithms(args=algs2009, f_factor=2, target_lb=1e-8, target_ub=1e22): """Returns (and prints) per dimension a list of algorithms within algorithm list args that contains an algorithm if for any dimension/target/function pair this algorithm: - is the best algorithm wrt ERT - its own ERT lies within a factor f_factor of the best ERT - there is no algorithm within a factor of f_factor of the best ERT and the current algorithm is the second best. """ # TODO: use pproc.TargetValues class as input target values # default target values: targets = pproc.TargetValues(10**np.arange( np.log10(max((1e-8, target_lb))), np.log10(target_ub) + 1e-9, 0.2)) # there should be a simpler way to express this to become the # interface of this function print 'Loading algorithm data from given algorithm list...\n' verbose = True dsList, sortedAlgs, dictAlg = pproc.processInputArgs(args, verbose=verbose) print 'This may take a while (depending on the number of algorithms)' selectedAlgsPerProblem = {} for f, i in pproc.dictAlgByFun(dictAlg).iteritems(): for d, j in pproc.dictAlgByDim(i).iteritems(): selectedAlgsPerProblemDF = [] best = BestAlgSet(j) for i in range(0, len(best.target)): t = best.target[i] # if ((t <= target_ub) and (t >= target_lb)): if toolsstats.in_approximately( t, targets((f, d), discretize=True)): # add best for this target: selectedAlgsPerProblemDF.append(best.algs[i]) # add second best or all algorithms that have an ERT # within a factor of f_factor of the best: secondbest_ERT = np.infty secondbest_str = '' secondbest_included = False for astring in j: currdictalg = dictAlg[astring].dictByDim() if currdictalg.has_key(d): curralgdata = currdictalg[d][f - 1] currERT = curralgdata.detERT([t])[0] if (astring != best.algs[i]): if (currERT < secondbest_ERT): secondbest_ERT = currERT secondbest_str = astring if (currERT <= best.detERT([t])[0] * f_factor): selectedAlgsPerProblemDF.append(astring) secondbest_included = True if not (secondbest_included) and (secondbest_str != ''): selectedAlgsPerProblemDF.append(secondbest_str) if len(selectedAlgsPerProblemDF) > 0: selectedAlgsPerProblem[(d, f)] = selectedAlgsPerProblemDF print 'pre-processing of function', f, 'done.' print 'loading of best algorithm(s) data done.' countsperalgorithm = {} for (d, f) in selectedAlgsPerProblem: print 'dimension:', d, ', function:', f setofalgs = set(selectedAlgsPerProblem[d, f]) # now count how often algorithm a is best for the extracted targets for a in setofalgs: # use setdefault to initialize with zero if a entry not existant: countsperalgorithm.setdefault((d, a), 0) countsperalgorithm[(d, a)] += selectedAlgsPerProblem[d, f].count(a) selectedalgsperdimension = {} for (d, a) in sorted(countsperalgorithm): if not selectedalgsperdimension.has_key(d): selectedalgsperdimension[d] = [] selectedalgsperdimension[d].append((countsperalgorithm[(d, a)], a)) for d in sorted(selectedalgsperdimension): print d, 'D:' for (count, alg) in sorted(selectedalgsperdimension[d], reverse=True): print count, alg print '\n' print " done." return selectedalgsperdimension
def main(dictAlg, sortedAlgs=None, target=ftarget_default, outputdir='ppdata', verbose=True): """From a DataSetList, returns figures showing the scaling: ERT/dim vs dim. One function and one target per figure. ``target`` can be a scalar, a list with one element or a ``pproc.TargetValues`` instance with one target. ``sortedAlgs`` is a list of string-identifies (folder names) """ # target becomes a TargetValues "list" with one element target = pproc.TargetValues.cast([target] if numpy.isscalar(target) else target) latex_commands_filename = os.path.join(outputdir, 'bbob_pproc_commands.tex') assert isinstance(target, pproc.TargetValues) if len(target) != 1: raise ValueError('only a single target can be managed in ppfigs, ' + str(len(target)) + ' targets were given') dictFunc = pproc.dictAlgByFun(dictAlg) if sortedAlgs is None: sortedAlgs = sorted(dictAlg.keys()) if not os.path.isdir(outputdir): os.mkdir(outputdir) for f in dictFunc: filename = os.path.join(outputdir,'ppfigs_f%03d' % (f)) handles = [] fix_styles(len(sortedAlgs)) # for i, alg in enumerate(sortedAlgs): dictDim = dictFunc[f][alg].dictByDim() # this does not look like the most obvious solution #Collect data dimert = [] ert = [] dimnbsucc = [] ynbsucc = [] nbsucc = [] dimmaxevals = [] maxevals = [] dimmedian = [] medianfes = [] for dim in sorted(dictDim): assert len(dictDim[dim]) == 1 entry = dictDim[dim][0] data = generateData(entry, target((f, dim))[0]) # TODO: here we might want a different target for each function if 1 < 3 or data[2] == 0: # No success dimmaxevals.append(dim) maxevals.append(float(data[3])/dim) if data[2] > 0: dimmedian.append(dim) medianfes.append(data[4]/dim) dimert.append(dim) ert.append(float(data[0])/dim) if data[1] < 1.: dimnbsucc.append(dim) ynbsucc.append(float(data[0])/dim) nbsucc.append('%d' % data[2]) # Draw lines tmp = plt.plot(dimert, ert, **styles[i]) #label=alg, ) plt.setp(tmp[0], markeredgecolor=plt.getp(tmp[0], 'color')) # For legend # tmp = plt.plot([], [], label=alg.replace('..' + os.sep, '').strip(os.sep), **styles[i]) tmp = plt.plot([], [], label=alg.split(os.sep)[-1], **styles[i]) plt.setp(tmp[0], markersize=12., markeredgecolor=plt.getp(tmp[0], 'color')) if dimmaxevals: tmp = plt.plot(dimmaxevals, maxevals, **styles[i]) plt.setp(tmp[0], markersize=20, #label=alg, markeredgecolor=plt.getp(tmp[0], 'color'), markeredgewidth=1, markerfacecolor='None', linestyle='None') handles.append(tmp) #tmp2 = plt.plot(dimmedian, medianfes, ls='', marker='+', # markersize=30, markeredgewidth=5, # markeredgecolor=plt.getp(tmp, 'color'))[0] #for i, n in enumerate(nbsucc): # plt.text(dimnbsucc[i], numpy.array(ynbsucc[i])*1.85, n, # verticalalignment='bottom', # horizontalalignment='center') if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() bestalgdata = [] dimbestalg = list(df[0] for df in bestalg.bestalgentries2009 if df[1] == f) dimbestalg.sort() dimbestalg2 = [] for d in dimbestalg: entry = bestalg.bestalgentries2009[(d, f)] tmp = entry.detERT(target((f, d)))[0] if numpy.isfinite(tmp): bestalgdata.append(float(tmp)/d) dimbestalg2.append(d) tmp = plt.plot(dimbestalg2, bestalgdata, color=refcolor, linewidth=10, marker='d', markersize=25, markeredgecolor=refcolor, zorder=-1 #label='best 2009', ) handles.append(tmp) if show_significance: # plot significance-stars xstar, ystar = [], [] dims = sorted(pproc.dictAlgByDim(dictFunc[f])) for i, dim in enumerate(dims): datasets = pproc.dictAlgByDim(dictFunc[f])[dim] assert all([len(datasets[ialg]) == 1 for ialg in sortedAlgs if datasets[ialg]]) dsetlist = [datasets[ialg][0] for ialg in sortedAlgs if datasets[ialg]] if len(dsetlist) > 1: arzp, arialg = toolsstats.significance_all_best_vs_other(dsetlist, target((f, dim))) if arzp[0][1] * len(dims) < show_significance: ert = dsetlist[arialg[0]].detERT(target((f, dim)))[0] if ert < numpy.inf: xstar.append(dim) ystar.append(ert/dim) plt.plot(xstar, ystar, 'k*', markerfacecolor=None, markeredgewidth=2, markersize=0.5*styles[0]['markersize']) if funInfos: plt.gca().set_title(funInfos[f]) isLegend = False if legend: plotLegend(handles) elif 1 < 3: if f in (1, 24, 101, 130) and len(sortedAlgs) < 6: # 6 elements at most in the boxed legend isLegend = True beautify(legend=isLegend, rightlegend=legend) plt.text(plt.xlim()[0], plt.ylim()[0], 'target ' + target.label_name() + ': ' + target.label(0)) # TODO: check saveFigure(filename, verbose=verbose) plt.close() # generate commands in tex file: try: abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' alg_definitions = [] for i in range(len(sortedAlgs)): symb = r'{%s%s}' % (color_to_latex(styles[i]['color']), marker_to_latex(styles[i]['marker'])) alg_definitions.append((', ' if i > 0 else '') + '%s:%s' % (symb, '\\algorithm' + abc[i % len(abc)])) toolsdivers.prepend_to_file(latex_commands_filename, [#'\\providecommand{\\bbobppfigsftarget}{\\ensuremath{10^{%s}}}' # % target.loglabel(0), # int(numpy.round(numpy.log10(target))), '\\providecommand{\\bbobppfigslegend}[1]{', scaling_figure_caption(target), 'Legend: '] + alg_definitions + ['}'] ) toolsdivers.prepend_to_file(latex_commands_filename, ['\\providecommand{\\bbobECDFslegend}[1]{', ecdfs_figure_caption(target), '}'] ) if verbose: print 'Wrote commands and legend to %s' % filename # this is obsolete (however check templates) filename = os.path.join(outputdir,'ppfigs.tex') f = open(filename, 'w') f.write('% Do not modify this file: calls to post-processing software' + ' will overwrite any modification.\n') f.write('Legend: ') for i in range(0, len(sortedAlgs)): symb = r'{%s%s}' % (color_to_latex(styles[i]['color']), marker_to_latex(styles[i]['marker'])) f.write((', ' if i > 0 else '') + '%s:%s' % (symb, writeLabels(sortedAlgs[i]))) f.close() if verbose: print '(obsolete) Wrote legend in %s' % filename except IOError: raise handles.append(tmp) if funInfos: plt.gca().set_title(funInfos[f]) beautify(rightlegend=legend) if legend: plotLegend(handles) else: if f in (1, 24, 101, 130): plt.legend() saveFigure(filename, figFormat=genericsettings.fig_formats, verbose=verbose) plt.close()
def main(dictAlg, order=None, outputdir='.', info='default', dimension=None, verbose=True): """Generates a figure showing the performance of algorithms. From a dictionary of :py:class:`DataSetList` sorted by algorithms, generates the cumulative distribution function of the bootstrap distribution of ERT for algorithms on multiple functions for multiple targets altogether. :param dict dictAlg: dictionary of :py:class:`DataSetList` instances one instance is equivalent to one algorithm, :param list targets: target function values :param list order: sorted list of keys to dictAlg for plotting order :param str outputdir: output directory :param str info: output file name suffix :param bool verbose: controls verbosity """ global x_limit # late assignment of default, because it can be set to None in config global divide_by_dimension # not fully implemented/tested yet if 'x_limit' not in globals() or x_limit is None: x_limit = x_limit_default tmp = pp.dictAlgByDim(dictAlg) # tmp = pp.DictAlg(dictAlg).by_dim() if len(tmp) != 1 and dimension is None: raise ValueError('We never integrate over dimension.') if dimension is not None: if dimension not in tmp.keys(): raise ValueError('dimension %d not in dictAlg dimensions %s' % (dimension, str(tmp.keys()))) tmp = {dimension: tmp[dimension]} dim = tmp.keys()[0] divisor = dim if divide_by_dimension else 1 algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []] dictFunc = pp.dictAlgByFun(dictAlg) # Collect data # Crafting effort correction: should we consider any? CrEperAlg = {} for alg in algorithms_with_data: CrE = 0. if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL': tmp = dictAlg[alg].dictByNoise() assert len(tmp.keys()) == 1 if tmp.keys()[0] == 'noiselessall': CrE = 0.5117 elif tmp.keys()[0] == 'nzall': CrE = 0.6572 CrEperAlg[alg] = CrE if CrE != 0.0: print 'Crafting effort for', alg, 'is', CrE dictData = {} # list of (ert per function) per algorithm dictMaxEvals = {} # list of (maxevals per function) per algorithm bestERT = [] # best ert per function # funcsolved = [set()] * len(targets) # number of functions solved per target xbest2009 = [] maxevalsbest2009 = [] for f, dictAlgperFunc in dictFunc.iteritems(): if function_IDs and f not in function_IDs: continue # print target_values((f, dim)) for j, t in enumerate(target_values((f, dim))): # for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)): # funcsolved[j].add(f) for alg in algorithms_with_data: x = [np.inf] * perfprofsamplesize runlengthunsucc = [] try: entry = dictAlgperFunc[alg][ 0] # one element per fun and per dim. evals = entry.detEvals([t])[0] assert entry.dim == dim runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor if len(runlengthsucc) > 0: x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] except (KeyError, IndexError): #set_trace() warntxt = ( 'Data for algorithm %s on function %d in %d-D ' % (alg, f, dim) + 'are missing.\n') warnings.warn(warntxt) dictData.setdefault(alg, []).extend(x) dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc) if displaybest2009: #set_trace() if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() bestalgentry = bestalg.bestalgentries2009[(dim, f)] bestalgevals = bestalgentry.detEvals(target_values((f, dim))) # print bestalgevals for j in range(len(bestalgevals[0])): if bestalgevals[1][j]: evals = bestalgevals[0][j] #set_trace() assert dim == bestalgentry.dim runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = bestalgentry.maxevals[ bestalgevals[1][j]][np.isnan(evals)] / divisor x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] else: x = perfprofsamplesize * [np.inf] runlengthunsucc = [] xbest2009.extend(x) maxevalsbest2009.extend(runlengthunsucc) if order is None: order = dictData.keys() # Display data lines = [] if displaybest2009: args = { 'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11., 'markeredgewidth': 1.5, 'markerfacecolor': refcolor, 'markeredgecolor': refcolor, 'color': refcolor, 'label': 'best 2009', 'zorder': -1 } lines.append( plotdata(np.array(xbest2009), x_limit, maxevalsbest2009, CrE=0., **args)) def algname_to_label(algname, dirname=None): """to be extended to become generally useful""" if isinstance(algname, (tuple, list)): # not sure this is needed return ' '.join([str(name) for name in algname]) return str(algname) for i, alg in enumerate(order): try: data = dictData[alg] maxevals = dictMaxEvals[alg] except KeyError: continue args = styles[(i) % len(styles)] args['linewidth'] = 1.5 args['markersize'] = 12. args['markeredgewidth'] = 1.5 args['markerfacecolor'] = 'None' args['markeredgecolor'] = args['color'] args['label'] = algname_to_label(alg) #args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib #elif len(show_algorithms) > 0: #args['color'] = 'wheat' #args['ls'] = '-' #args['zorder'] = -1 # plotdata calls pprldistr.plotECDF which calls ppfig.plotUnifLog... which does the work lines.append( plotdata(np.array(data), x_limit, maxevals, CrE=CrEperAlg[alg], **args)) labels, handles = plotLegend(lines, x_limit) if True: # isLateXLeg: fileName = os.path.join(outputdir, 'pprldmany_%s.tex' % (info)) with open(fileName, 'w') as f: f.write(r'\providecommand{\nperfprof}{7}') algtocommand = {} # latex commands for i, alg in enumerate(order): tmp = r'\alg%sperfprof' % pptex.numtotext(i) f.write( r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' % (tmp, toolsdivers.str_to_latex( toolsdivers.strip_pathname2(algname_to_label(alg))))) algtocommand[algname_to_label(alg)] = tmp if displaybest2009: tmp = r'\algzeroperfprof' f.write(r'\providecommand{%s}{best 2009}' % (tmp)) algtocommand['best 2009'] = tmp commandnames = [] for label in labels: commandnames.append(algtocommand[label]) # f.write(headleg) if len( order ) > 28: # latex sidepanel won't work well for more than 25 algorithms, but original labels are also clipped f.write( r'\providecommand{\perfprofsidepanel}{\mbox{%s}\vfill\mbox{%s}}' % (commandnames[0], commandnames[-1])) else: fontsize_command = r'\tiny{}' if len(order) > 19 else '' f.write(r'\providecommand{\perfprofsidepanel}{{%s\mbox{%s}' % (fontsize_command, commandnames[0])) # TODO: check len(labels) > 0 for i in range(1, len(labels)): f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i]) f.write('}}\n') # f.write(footleg) if verbose: print 'Wrote right-hand legend in %s' % fileName figureName = os.path.join(outputdir, 'pprldmany_%s' % (info)) #beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat) beautify() text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys()))) text += ',%d-D' % dim # TODO: this is strange when different dimensions are plotted plt.text(0.01, 0.98, text, horizontalalignment="left", verticalalignment="top", transform=plt.gca().transAxes) if len(dictFunc) == 1: plt.title(' '.join( (str(dictFunc.keys()[0]), genericsettings.current_testbed.short_names[dictFunc.keys()[0]]))) a = plt.gca() plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative) xticks, labels = plt.xticks() tmp = [] for i in xticks: tmp.append('%d' % round(np.log10(i))) a.set_xticklabels(tmp) if save_figure: ppfig.saveFigure(figureName, verbose=verbose) if len(dictFunc) == 1: ppfig.save_single_functions_html( os.path.join(outputdir, 'pprldmany'), '', # algorithms names are clearly visible in the figure add_to_names='_%02dD' % (dim), algorithmCount=ppfig.AlgorithmCount.NON_SPECIFIED) if close_figure: plt.close()