def main(dsList0, dsList1, dim, targetsOfInterest=None, outputdir='', info='default', verbose=True): """Generate figures of empirical cumulative distribution functions. :param DataSetList dsList0: data set of reference algorithm :param DataSetList dsList1: data set of algorithm of concern :param int dim: dimension :param TargetValues targetsOfInterest: target function values to be displayed :param bool isStoringXMax: if set to True, the first call BeautifyVD sets the globals :py:data:`fmax` and :py:data:`maxEvals` and all subsequent calls will use these values as rightmost xlim in the generated figures. :param string outputdir: output directory (must exist) :param string info: string suffix for output file names. :param bool verbose: control verbosity Outputs: Image files of the empirical cumulative distribution functions. """ #plt.rc("axes", labelsize=20, titlesize=24) #plt.rc("xtick", labelsize=20) #plt.rc("ytick", labelsize=20) #plt.rc("font", size=20) #plt.rc("legend", fontsize=20) figureName = os.path.join(outputdir, 'pplogabs_%s' % (info)) handles = plotLogAbs(dsList0, dsList1, dim, targetsOfInterest, verbose=verbose) beautify(handles) funcs = set(dsList0.dictByFunc().keys()) & set(dsList1.dictByFunc().keys()) text = 'f%s' % consecutiveNumbers(sorted(funcs)) if len(dsList0.dictByDim().keys()) == len(dsList1.dictByDim().keys()) == 1: text += ',%d-D' % dsList0.dictByDim().keys()[0] plt.text(0.98, 0.02, text, horizontalalignment="right", transform=plt.gca().transAxes) saveFigure(figureName, verbose=verbose) plt.close()
def plot(dsList, targets=single_target_values, **plotArgs): """Plot ECDF of evaluations and final function values in a single figure for demonstration purposes.""" # targets = targets() # TODO: this needs to be rectified # targets = targets.target_values dsList = pproc.DataSetList(dsList) assert len(dsList.dictByDim()) == 1, ('Cannot display different ' 'dimensionalities together') res = [] plt.subplot(121) maxEvalsFactor = max(i.mMaxEvals() / i.dim for i in dsList) evalfmax = maxEvalsFactor for j in range(len(targets)): tmpplotArgs = dict(plotArgs, **rldStyles[j % len(rldStyles)]) tmp = plotRLDistr(dsList, lambda fun_dim: targets(fun_dim)[j], **tmpplotArgs) res.extend(tmp) res.append(plt.axvline(x=maxEvalsFactor, color='k', **plotArgs)) funcs = list(i.funcId for i in dsList) text = 'f%s' % (consecutiveNumbers(sorted(funcs))) res.append( plt.text(0.5, 0.98, text, horizontalalignment="center", verticalalignment="top", transform=plt.gca().transAxes)) plt.subplot(122) for j in [range(len(targets))[-1]]: tmpplotArgs = dict(plotArgs, **rldStyles[j % len(rldStyles)]) tmp = plotFVDistr(dsList, evalfmax, lambda fun_dim: targets(fun_dim)[j], **tmpplotArgs) res.extend(tmp) tmp = np.floor(np.log10(evalfmax)) # coloring right to left: maxEvalsF = np.power(10, np.arange(0, tmp)) for j in range(len(maxEvalsF)): tmpplotArgs = dict(plotArgs, **rldUnsuccStyles[j % len(rldUnsuccStyles)]) tmp = plotFVDistr(dsList, maxEvalsF[j], lambda fun_dim: targets(fun_dim)[-1], **tmpplotArgs) res.extend(tmp) res.append( plt.text(0.98, 0.02, text, horizontalalignment="right", transform=plt.gca().transAxes)) return res
def main(dsList0, dsList1, valuesOfInterest=None, outputdir='', info='default', verbose=True): """Generate figures of empirical cumulative distribution functions. :param DataSetList dsList0: data set of reference algorithm :param DataSetList dsList1: data set of algorithm of concern :param sequence valuesOfInterest: target function values to be displayed :param bool isStoringXMax: if set to True, the first call BeautifyVD sets the globals :py:data:`fmax` and :py:data:`maxEvals` and all subsequent calls will use these values as rightmost xlim in the generated figures. :param string outputdir: output directory (must exist) :param string info: string suffix for output file names. :param bool verbose: control verbosity Outputs: Image files of the empirical cumulative distribution functions. """ #plt.rc("axes", labelsize=20, titlesize=24) #plt.rc("xtick", labelsize=20) #plt.rc("ytick", labelsize=20) #plt.rc("font", size=20) #plt.rc("legend", fontsize=20) figureName = os.path.join(outputdir,'pplogabs_%s' %(info)) handles = plotLogAbs(dsList0, dsList1, valuesOfInterest, verbose=verbose) beautify(handles) funcs = set(dsList0.dictByFunc().keys()) & set(dsList1.dictByFunc().keys()) text = 'f%s' % consecutiveNumbers(sorted(funcs)) if len(dsList0.dictByDim().keys()) == len(dsList1.dictByDim().keys()) == 1: text += ',%d-D' % dsList0.dictByDim().keys()[0] plt.text(0.98, 0.02, text, horizontalalignment="right", transform=plt.gca().transAxes) saveFigure(figureName, verbose=verbose) plt.close()
def plot(dsList, targets = single_target_values, **plotArgs): """Plot ECDF of evaluations and final function values in a single figure for demonstration purposes.""" # targets = targets() # TODO: this needs to be rectified # targets = targets.target_values dsList = pproc.DataSetList(dsList) assert len(dsList.dictByDim()) == 1, ('Cannot display different ' 'dimensionalities together') res = [] plt.subplot(121) maxEvalsFactor = max(i.mMaxEvals() / i.dim for i in dsList) evalfmax = maxEvalsFactor for j in range(len(targets)): tmpplotArgs = dict(plotArgs, **rldStyles[j % len(rldStyles)]) tmp = plotRLDistr(dsList, lambda fun_dim: targets(fun_dim)[j], **tmpplotArgs) res.extend(tmp) res.append(plt.axvline(x = maxEvalsFactor, color = 'k', **plotArgs)) funcs = list(i.funcId for i in dsList) text = consecutiveNumbers(sorted(funcs), 'f') res.append(plt.text(0.5, 0.98, text, horizontalalignment = "center", verticalalignment = "top", transform = plt.gca().transAxes)) plt.subplot(122) for j in [range(len(targets))[-1]]: tmpplotArgs = dict(plotArgs, **rldStyles[j % len(rldStyles)]) tmp = plotFVDistr(dsList, evalfmax, lambda fun_dim: targets(fun_dim)[j], **tmpplotArgs) if tmp: res.extend(tmp) tmp = np.floor(np.log10(evalfmax)) # coloring right to left: maxEvalsF = np.power(10, np.arange(0, tmp)) for j in range(len(maxEvalsF)): tmpplotArgs = dict(plotArgs, **rldUnsuccStyles[j % len(rldUnsuccStyles)]) tmp = plotFVDistr(dsList, maxEvalsF[j], lambda fun_dim: targets(fun_dim)[-1], **tmpplotArgs) if tmp: res.extend(tmp) res.append(plt.text(0.98, 0.02, text, horizontalalignment = "right", transform = plt.gca().transAxes)) return res
def main(dsList, isStoringXMax=False, outputdir='', info='default', verbose=True): """Generate figures of empirical cumulative distribution functions. This method has a feature which allows to keep the same boundaries for the x-axis, if ``isStoringXMax==True``. This makes sense when dealing with different functions or subsets of functions for one given dimension. CAVE: this is bug-prone, as some data depend on the maximum evaluations and the appearence therefore depends on the calling order. :param DataSetList dsList: list of DataSet instances to process. :param bool isStoringXMax: if set to True, the first call :py:func:`beautifyFVD` sets the globals :py:data:`fmax` and :py:data:`maxEvals` and all subsequent calls will use these values as rightmost xlim in the generated figures. :param string outputdir: output directory (must exist) :param string info: string suffix for output file names. :param bool verbose: control verbosity """ # plt.rc("axes", labelsize=20, titlesize=24) # plt.rc("xtick", labelsize=20) # plt.rc("ytick", labelsize=20) # plt.rc("font", size=20) # plt.rc("legend", fontsize=20) targets = single_target_values # convenience abbreviation for d, dictdim in dsList.dictByDim().iteritems(): maxEvalsFactor = max(i.mMaxEvals() / d for i in dictdim) if isStoringXMax: global evalfmax else: evalfmax = None if not evalfmax: evalfmax = maxEvalsFactor if runlen_xlimits_max is not None: evalfmax = runlen_xlimits_max # first figure: Run Length Distribution filename = os.path.join(outputdir, 'pprldistr_%02dD_%s' % (d, info)) fig = plt.figure() for j in range(len(targets)): plotRLDistr( dictdim, lambda fun_dim: targets(fun_dim)[j], targets.label(j) if isinstance( targets, pproc.RunlengthBasedTargetValues) else targets.loglabel(j), evalfmax, # can be larger maxEvalsFactor with no effect **rldStyles[j % len(rldStyles)]) funcs = list(i.funcId for i in dictdim) text = 'f%s' % (consecutiveNumbers(sorted(funcs))) text += ',%d-D' % d if (1): # try: if not isinstance(targets, pproc.RunlengthBasedTargetValues): # if targets.target_values[-1] == 1e-8: # this is a hack plot_previous_algorithms(d, funcs) else: plotRLB_previous_algorithms(d, funcs) # except: # pass plt.axvline(x=maxEvalsFactor, color='k') # vertical line at maxevals plt.legend(loc='best') plt.text(0.5, 0.98, text, horizontalalignment="center", verticalalignment="top", transform=plt.gca().transAxes # bbox=dict(ec='k', fill=False) ) try: # was never tested, so let's make it safe if len(funcs) == 1: plt.title(genericsettings.current_testbed.info(funcs[0])[:27]) except: warnings.warn('could not print title') beautifyRLD(evalfmax) saveFigure(filename, verbose=verbose) plt.close(fig) # second figure: Function Value Distribution filename = os.path.join(outputdir, 'ppfvdistr_%02dD_%s' % (d, info)) fig = plt.figure() plotFVDistr(dictdim, np.inf, 1e-8, **rldStyles[-1]) # coloring right to left for j, max_eval_factor in enumerate(single_runlength_factors): if max_eval_factor > maxEvalsFactor: break plotFVDistr(dictdim, max_eval_factor, 1e-8, **rldUnsuccStyles[j % len(rldUnsuccStyles)]) plt.text( 0.98, 0.02, text, horizontalalignment="right", transform=plt.gca().transAxes) # bbox=dict(ec='k', fill=False), beautifyFVD(isStoringXMax=isStoringXMax, ylabel=False) saveFigure(filename, verbose=verbose) plt.close(fig)
def comp(dsList0, dsList1, targets, isStoringXMax=False, outputdir='', info='default', verbose=True): """Generate figures of ECDF that compare 2 algorithms. :param DataSetList dsList0: list of DataSet instances for ALG0 :param DataSetList dsList1: list of DataSet instances for ALG1 :param seq targets: target function values to be displayed :param bool isStoringXMax: if set to True, the first call :py:func:`beautifyFVD` sets the globals :py:data:`fmax` and :py:data:`maxEvals` and all subsequent calls will use these values as rightmost xlim in the generated figures. :param string outputdir: output directory (must exist) :param string info: string suffix for output file names. :param bool verbose: control verbosity """ # plt.rc("axes", labelsize=20, titlesize=24) # plt.rc("xtick", labelsize=20) # plt.rc("ytick", labelsize=20) # plt.rc("font", size=20) # plt.rc("legend", fontsize=20) if not isinstance(targets, pproc.RunlengthBasedTargetValues): targets = pproc.TargetValues.cast(targets) dictdim0 = dsList0.dictByDim() dictdim1 = dsList1.dictByDim() for d in set(dictdim0.keys()) & set(dictdim1.keys()): maxEvalsFactor = max(max(i.mMaxEvals() / d for i in dictdim0[d]), max(i.mMaxEvals() / d for i in dictdim1[d])) if isStoringXMax: global evalfmax else: evalfmax = None if not evalfmax: evalfmax = maxEvalsFactor**1.05 if runlen_xlimits_max is not None: evalfmax = runlen_xlimits_max filename = os.path.join(outputdir, 'pprldistr_%02dD_%s' % (d, info)) fig = plt.figure() for j in range(len(targets)): tmp = plotRLDistr(dictdim0[d], lambda fun_dim: targets(fun_dim)[j], targets.label(j) if isinstance( targets, pproc.RunlengthBasedTargetValues) else targets.loglabel(j), marker=genericsettings.line_styles[1]['marker'], **rldStyles[j % len(rldStyles)]) plt.setp(tmp[-1], label=None) # Remove automatic legend # Mods are added after to prevent them from appearing in the legend plt.setp(tmp, markersize=20., markeredgewidth=plt.getp(tmp[-1], 'linewidth'), markeredgecolor=plt.getp(tmp[-1], 'color'), markerfacecolor='none') tmp = plotRLDistr(dictdim1[d], lambda fun_dim: targets(fun_dim)[j], targets.label(j) if isinstance( targets, pproc.RunlengthBasedTargetValues) else targets.loglabel(j), marker=genericsettings.line_styles[0]['marker'], **rldStyles[j % len(rldStyles)]) # modify the automatic legend: remover marker and change text plt.setp(tmp[-1], marker='', label=targets.label(j) if isinstance( targets, pproc.RunlengthBasedTargetValues) else targets.loglabel(j)) # Mods are added after to prevent them from appearing in the legend plt.setp(tmp, markersize=15., markeredgewidth=plt.getp(tmp[-1], 'linewidth'), markeredgecolor=plt.getp(tmp[-1], 'color'), markerfacecolor='none') funcs = set(i.funcId for i in dictdim0[d]) | set(i.funcId for i in dictdim1[d]) text = 'f%s' % (consecutiveNumbers(sorted(funcs))) if not isinstance(targets, pproc.RunlengthBasedTargetValues): plot_previous_algorithms(d, funcs) else: plotRLB_previous_algorithms(d, funcs) # plt.axvline(max(i.mMaxEvals()/i.dim for i in dictdim0[d]), ls='--', color='k') # plt.axvline(max(i.mMaxEvals()/i.dim for i in dictdim1[d]), color='k') plt.axvline(max(i.mMaxEvals() / i.dim for i in dictdim0[d]), marker='+', markersize=20., color='k', markeredgewidth=plt.getp( tmp[-1], 'linewidth', )) plt.axvline(max(i.mMaxEvals() / i.dim for i in dictdim1[d]), marker='o', markersize=15., color='k', markerfacecolor='None', markeredgewidth=plt.getp(tmp[-1], 'linewidth')) plt.legend(loc='best') plt.text( 0.5, 0.98, text, horizontalalignment="center", verticalalignment="top", transform=plt.gca().transAxes) # bbox=dict(ec='k', fill=False), beautifyRLD(evalfmax) saveFigure(filename, verbose=verbose) plt.close(fig)
def main(dictAlg, order=None, outputdir='.', info='default', dimension=None, verbose=True): """Generates a figure showing the performance of algorithms. From a dictionary of :py:class:`DataSetList` sorted by algorithms, generates the cumulative distribution function of the bootstrap distribution of ERT for algorithms on multiple functions for multiple targets altogether. :param dict dictAlg: dictionary of :py:class:`DataSetList` instances one instance is equivalent to one algorithm, :param list targets: target function values :param list order: sorted list of keys to dictAlg for plotting order :param str outputdir: output directory :param str info: output file name suffix :param bool verbose: controls verbosity """ global x_limit # late assignment of default, because it can be set to None in config global divide_by_dimension # not fully implemented/tested yet if 'x_limit' not in globals() or x_limit is None: x_limit = x_limit_default tmp = pp.dictAlgByDim(dictAlg) # tmp = pp.DictAlg(dictAlg).by_dim() if len(tmp) != 1 and dimension is None: raise ValueError('We never integrate over dimension.') if dimension is not None: if dimension not in tmp.keys(): raise ValueError('dimension %d not in dictAlg dimensions %s' % (dimension, str(tmp.keys()))) tmp = {dimension: tmp[dimension]} dim = tmp.keys()[0] divisor = dim if divide_by_dimension else 1 algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []] dictFunc = pp.dictAlgByFun(dictAlg) # Collect data # Crafting effort correction: should we consider any? CrEperAlg = {} for alg in algorithms_with_data: CrE = 0. if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL': tmp = dictAlg[alg].dictByNoise() assert len(tmp.keys()) == 1 if tmp.keys()[0] == 'noiselessall': CrE = 0.5117 elif tmp.keys()[0] == 'nzall': CrE = 0.6572 CrEperAlg[alg] = CrE if CrE != 0.0: print 'Crafting effort for', alg, 'is', CrE dictData = {} # list of (ert per function) per algorithm dictMaxEvals = {} # list of (maxevals per function) per algorithm bestERT = [] # best ert per function # funcsolved = [set()] * len(targets) # number of functions solved per target xbest2009 = [] maxevalsbest2009 = [] for f, dictAlgperFunc in dictFunc.iteritems(): if function_IDs and f not in function_IDs: continue # print target_values((f, dim)) for j, t in enumerate(target_values((f, dim))): # for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)): # funcsolved[j].add(f) for alg in algorithms_with_data: x = [np.inf] * perfprofsamplesize runlengthunsucc = [] try: entry = dictAlgperFunc[alg][ 0] # one element per fun and per dim. evals = entry.detEvals([t])[0] assert entry.dim == dim runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor if len(runlengthsucc) > 0: x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] except (KeyError, IndexError): #set_trace() warntxt = ( 'Data for algorithm %s on function %d in %d-D ' % (alg, f, dim) + 'are missing.\n') warnings.warn(warntxt) dictData.setdefault(alg, []).extend(x) dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc) if displaybest2009: #set_trace() if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() bestalgentry = bestalg.bestalgentries2009[(dim, f)] bestalgevals = bestalgentry.detEvals(target_values((f, dim))) # print bestalgevals for j in range(len(bestalgevals[0])): if bestalgevals[1][j]: evals = bestalgevals[0][j] #set_trace() assert dim == bestalgentry.dim runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = bestalgentry.maxevals[ bestalgevals[1][j]][np.isnan(evals)] / divisor x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] else: x = perfprofsamplesize * [np.inf] runlengthunsucc = [] xbest2009.extend(x) maxevalsbest2009.extend(runlengthunsucc) if order is None: order = dictData.keys() # Display data lines = [] if displaybest2009: args = { 'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11., 'markeredgewidth': 1.5, 'markerfacecolor': refcolor, 'markeredgecolor': refcolor, 'color': refcolor, 'label': 'best 2009', 'zorder': -1 } lines.append( plotdata(np.array(xbest2009), x_limit, maxevalsbest2009, CrE=0., **args)) def algname_to_label(algname, dirname=None): """to be extended to become generally useful""" if isinstance(algname, (tuple, list)): # not sure this is needed return ' '.join([str(name) for name in algname]) return str(algname) for i, alg in enumerate(order): try: data = dictData[alg] maxevals = dictMaxEvals[alg] except KeyError: continue args = styles[(i) % len(styles)] args['linewidth'] = 1.5 args['markersize'] = 12. args['markeredgewidth'] = 1.5 args['markerfacecolor'] = 'None' args['markeredgecolor'] = args['color'] args['label'] = algname_to_label(alg) #args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib #elif len(show_algorithms) > 0: #args['color'] = 'wheat' #args['ls'] = '-' #args['zorder'] = -1 # plotdata calls pprldistr.plotECDF which calls ppfig.plotUnifLog... which does the work lines.append( plotdata(np.array(data), x_limit, maxevals, CrE=CrEperAlg[alg], **args)) labels, handles = plotLegend(lines, x_limit) if True: # isLateXLeg: fileName = os.path.join(outputdir, 'pprldmany_%s.tex' % (info)) with open(fileName, 'w') as f: f.write(r'\providecommand{\nperfprof}{7}') algtocommand = {} # latex commands for i, alg in enumerate(order): tmp = r'\alg%sperfprof' % pptex.numtotext(i) f.write( r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' % (tmp, toolsdivers.str_to_latex( toolsdivers.strip_pathname2(algname_to_label(alg))))) algtocommand[algname_to_label(alg)] = tmp if displaybest2009: tmp = r'\algzeroperfprof' f.write(r'\providecommand{%s}{best 2009}' % (tmp)) algtocommand['best 2009'] = tmp commandnames = [] for label in labels: commandnames.append(algtocommand[label]) # f.write(headleg) if len( order ) > 28: # latex sidepanel won't work well for more than 25 algorithms, but original labels are also clipped f.write( r'\providecommand{\perfprofsidepanel}{\mbox{%s}\vfill\mbox{%s}}' % (commandnames[0], commandnames[-1])) else: fontsize_command = r'\tiny{}' if len(order) > 19 else '' f.write(r'\providecommand{\perfprofsidepanel}{{%s\mbox{%s}' % (fontsize_command, commandnames[0])) # TODO: check len(labels) > 0 for i in range(1, len(labels)): f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i]) f.write('}}\n') # f.write(footleg) if verbose: print 'Wrote right-hand legend in %s' % fileName figureName = os.path.join(outputdir, 'pprldmany_%s' % (info)) #beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat) beautify() text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys()))) text += ',%d-D' % dim # TODO: this is strange when different dimensions are plotted plt.text(0.01, 0.98, text, horizontalalignment="left", verticalalignment="top", transform=plt.gca().transAxes) if len(dictFunc) == 1: plt.title(' '.join( (str(dictFunc.keys()[0]), genericsettings.current_testbed.short_names[dictFunc.keys()[0]]))) a = plt.gca() plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative) xticks, labels = plt.xticks() tmp = [] for i in xticks: tmp.append('%d' % round(np.log10(i))) a.set_xticklabels(tmp) if save_figure: ppfig.saveFigure(figureName, verbose=verbose) if len(dictFunc) == 1: ppfig.save_single_functions_html( os.path.join(outputdir, 'pprldmany'), '', # algorithms names are clearly visible in the figure add_to_names='_%02dD' % (dim), algorithmCount=ppfig.AlgorithmCount.NON_SPECIFIED) if close_figure: plt.close()
def main(dsList, valuesOfInterest, isStoringXMax=False, outputdir='', info='default', verbose=True): """Generate figures of empirical cumulative distribution functions. Keyword arguments: dsList -- list of DataSet instances to process. valuesOfInterest -- target function values to be displayed. isStoringXMax -- if set to True, the first call BeautifyVD sets the globals fmax and maxEvals and all subsequent calls will use these values as rightmost xlim in the generated figures. outputdir -- output directory (must exist) info --- string suffix for output file names. Outputs: Image files of the empirical cumulative distribution functions. """ plt.rc("axes", labelsize=20, titlesize=24) plt.rc("xtick", labelsize=20) plt.rc("ytick", labelsize=20) plt.rc("font", size=20) plt.rc("legend", fontsize=20) maxEvalsFactor = max(i.mMaxEvals() / i.dim for i in dsList) #maxEvalsFactorCeil = numpy.power(10, #numpy.ceil(numpy.log10(maxEvalsFactor))) if isStoringXMax: global evalfmax else: evalfmax = None if not evalfmax: evalfmax = maxEvalsFactor figureName = os.path.join(outputdir, 'pprldistr_%s' % (info)) fig = plt.figure() legend = [] for j in range(len(valuesOfInterest)): tmp = plotRLDistr(dsList, valuesOfInterest[j], evalfmax, verbose=verbose) if not tmp is None: plt.setp(tmp, 'color', rldColors[j]) if rldColors[j] == 'r': # 1e-8 in bold plt.setp(tmp, 'linewidth', 3) funcs = list(i.funcId for i in dsList) text = 'f%s' % (consecutiveNumbers(sorted(funcs))) if isAlgorithm2009Found: d = set(i.dim for i in dsList).pop() # Get only one element... for alg in dict2009: x = [] nn = 0 try: tmp = dict2009[alg] for f in funcs: tmp[f][d] # simply test that they exists except KeyError: continue for f in funcs: tmp2 = tmp[f][d][0][1:] # [0], because the maximum #evals is also recorded # [1:] because the target function value is recorded x.append(tmp2[numpy.isnan(tmp2) == False]) nn += len(tmp2) if x: x.append([(evalfmax * d)**1.05]) x = numpy.hstack(x) plotECDF(x[numpy.isfinite(x)] / float(d), nn, { 'color': 'wheat', 'ls': '-', 'zorder': -1 }) plt.axvline(x=maxEvalsFactor, color='k') beautifyRLD(fig, figureName, evalfmax, fileFormat=figformat, text=text, verbose=verbose) plt.close(fig) figureName = os.path.join(outputdir, 'ppfvdistr_%s' % (info)) fig = plt.figure() for j in range(len(valuesOfInterest)): tmp = plotFVDistr(dsList, valuesOfInterest[j], evalfmax, verbose=verbose) plt.setp(tmp, 'color', rldColors[j]) if rldColors[j] == 'r': # 1e-8 in bold plt.setp(tmp, 'linewidth', 3) tmp = numpy.floor(numpy.log10(evalfmax)) # coloring left to right: #maxEvalsF = numpy.power(10, numpy.arange(tmp, 0, -1) - 1) # coloring right to left: maxEvalsF = numpy.power(10, numpy.arange(0, tmp)) for k in range(len(maxEvalsF)): tmp = plotFVDistr(dsList, valuesOfInterest[-1], maxEvalsF=maxEvalsF[k], verbose=verbose) plt.setp(tmp, 'color', rldUnsuccColors[k]) beautifyFVD(fig, figureName, fileFormat=figformat, text=text, isStoringXMax=isStoringXMax, verbose=verbose) plt.close(fig) plt.rcdefaults()
def comp(dsList0, dsList1, valuesOfInterest, isStoringXMax=False, outputdir='', info='default', verbose=True): """Generate figures of empirical cumulative distribution functions. Dashed lines will correspond to ALG0 and solid lines to ALG1. Keyword arguments: dsList0 -- list of DataSet instances for ALG0. dsList1 -- list of DataSet instances for ALG1 valuesOfInterest -- target function values to be displayed. isStoringXMax -- if set to True, the first call BeautifyVD sets the globals fmax and maxEvals and all subsequent calls will use these values as rightmost xlim in the generated figures. -- if set to True, the first call BeautifyVD sets the global fmax and all subsequent call will have the same maximum xlim. outputdir -- output directory (must exist) info --- string suffix for output file names. Outputs: Image files of the empirical cumulative distribution functions. """ plt.rc("axes", labelsize=20, titlesize=24) plt.rc("xtick", labelsize=20) plt.rc("ytick", labelsize=20) plt.rc("font", size=20) plt.rc("legend", fontsize=20) maxEvalsFactor = max(max(i.mMaxEvals() / i.dim for i in dsList0), max(i.mMaxEvals() / i.dim for i in dsList1)) if isStoringXMax: global evalfmax else: evalfmax = None if not evalfmax: evalfmax = maxEvalsFactor figureName = os.path.join(outputdir, 'pprldistr2_%s' % (info)) fig = plt.figure() legend = [] for j in range(len(valuesOfInterest)): tmp = plotRLDistr(dsList0, valuesOfInterest[j], evalfmax, verbose=verbose) if not tmp is None: plt.setp(tmp, 'color', rldColors[j]) plt.setp(tmp, 'ls', '--') plt.setp(tmp, 'label', None) # Hack for the legend if rldColors[j] == 'r': # 1e-8 in bold plt.setp(tmp, 'linewidth', 3) tmp = plotRLDistr(dsList1, valuesOfInterest[j], evalfmax, verbose=verbose) if not tmp is None: plt.setp(tmp, 'color', rldColors[j]) # Hack for the legend. plt.setp(tmp, 'label', ('%+d' % (numpy.log10(valuesOfInterest[j][1])))) if rldColors[j] == 'r': # 1e-8 in bold plt.setp(tmp, 'linewidth', 3) funcs = set(i.funcId for i in dsList0) | set(i.funcId for i in dsList1) text = 'f%s' % (consecutiveNumbers(sorted(funcs))) if isAlgorithm2009Found: d = set.union(set(i.dim for i in dsList0), set(i.dim for i in dsList1)).pop() # Get only one element... for alg in dict2009: x = [] nn = 0 try: tmp = dict2009[alg] for f in funcs: tmp[f][d] # simply test that they exists except KeyError: continue for f in funcs: tmp2 = tmp[f][d][0][1:] # [0], because the maximum #evals is also recorded # [1:] because the target function value is recorded x.append(tmp2[numpy.isnan(tmp2) == False]) nn += len(tmp2) if x: x.append([(evalfmax * d)**1.05]) x = numpy.hstack(x) plotECDF(x[numpy.isfinite(x)] / d, nn, { 'color': 'wheat', 'ls': '-', 'zorder': -1 }) plt.axvline(max(i.mMaxEvals() / i.dim for i in dsList0), ls='--', color='k') plt.axvline(max(i.mMaxEvals() / i.dim for i in dsList1), color='k') beautifyRLD(fig, figureName, evalfmax, fileFormat=figformat, text=text, verbose=verbose) plt.close(fig) # The figures generated by the lines below are not displayed in the # BBOB2010 template: should they be done? #figureName = os.path.join(outputdir,'ppfvdistr2_%s' %(info)) #fig = plt.figure() #for j in range(len(valuesOfInterest)): ##set_trace() #tmp = plotFVDistr(dsList0, valuesOfInterest[j], #evalfmax, verbose=verbose) ##if not tmp is None: #plt.setp(tmp, 'color', rldColors[j]) #plt.setp(tmp, 'ls', '--') #if rldColors [j] == 'r': # 1e-8 in bold #plt.setp(tmp, 'linewidth', 3) #tmp = plotFVDistr(dsList1, valuesOfInterest[j], #evalfmax, verbose=verbose) ##if not tmp is None: #plt.setp(tmp, 'color', rldColors[j]) #plt.setp(tmp, 'ls', '--') #if rldColors [j] == 'r': # 1e-8 in bold #plt.setp(tmp, 'linewidth', 3) #tmp = numpy.floor(numpy.log10(evalfmax)) ## coloring left to right: ##maxEvalsF = numpy.power(10, numpy.arange(tmp, 0, -1) - 1) ## coloring right to left: #maxEvalsF = numpy.power(10, numpy.arange(0, tmp)) ##The last index of valuesOfInterest is still used in this loop. ##set_trace() ## Plot lines for different number of function evaluations #for k in range(len(maxEvalsF)): #tmp = plotFVDistr(dsList0, valuesOfInterest[j], #maxEvalsF=maxEvalsF[k], verbose=verbose) #plt.setp(tmp, 'color', rldUnsuccColors[k]) #plt.setp(tmp, 'ls', '--') #tmp = plotFVDistr(dsList1, valuesOfInterest[j], #maxEvalsF=maxEvalsF[k], verbose=verbose) #plt.setp(tmp, 'color', rldUnsuccColors[k]) #beautifyFVD(fig, figureName, fileFormat=figformat, text=text, #isStoringXMax=isStoringXMax, verbose=verbose) #plt.close(fig) plt.rcdefaults()
def main(dictAlg, isBiobjective, order=None, outputdir='.', info='default', dimension=None, verbose=True): """Generates a figure showing the performance of algorithms. From a dictionary of :py:class:`DataSetList` sorted by algorithms, generates the cumulative distribution function of the bootstrap distribution of ERT for algorithms on multiple functions for multiple targets altogether. :param dict dictAlg: dictionary of :py:class:`DataSetList` instances one instance is equivalent to one algorithm, :param list targets: target function values :param list order: sorted list of keys to dictAlg for plotting order :param str outputdir: output directory :param str info: output file name suffix :param bool verbose: controls verbosity """ global x_limit # late assignment of default, because it can be set to None in config global divide_by_dimension # not fully implemented/tested yet if 'x_limit' not in globals() or x_limit is None: x_limit = x_limit_default tmp = pp.dictAlgByDim(dictAlg) # tmp = pp.DictAlg(dictAlg).by_dim() if len(tmp) != 1 and dimension is None: raise ValueError('We never integrate over dimension.') if dimension is not None: if dimension not in tmp.keys(): raise ValueError('dimension %d not in dictAlg dimensions %s' % (dimension, str(tmp.keys()))) tmp = {dimension: tmp[dimension]} dim = tmp.keys()[0] divisor = dim if divide_by_dimension else 1 algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []] dictFunc = pp.dictAlgByFun(dictAlg) # Collect data # Crafting effort correction: should we consider any? CrEperAlg = {} for alg in algorithms_with_data: CrE = 0. if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL': tmp = dictAlg[alg].dictByNoise() assert len(tmp.keys()) == 1 if tmp.keys()[0] == 'noiselessall': CrE = 0.5117 elif tmp.keys()[0] == 'nzall': CrE = 0.6572 CrEperAlg[alg] = CrE if CrE != 0.0: print 'Crafting effort for', alg, 'is', CrE dictData = {} # list of (ert per function) per algorithm dictMaxEvals = {} # list of (maxevals per function) per algorithm bestERT = [] # best ert per function # funcsolved = [set()] * len(targets) # number of functions solved per target xbest2009 = [] maxevalsbest2009 = [] for f, dictAlgperFunc in dictFunc.iteritems(): if function_IDs and f not in function_IDs: continue # print target_values((f, dim)) for j, t in enumerate(target_values((f, dim))): # for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)): # funcsolved[j].add(f) for alg in algorithms_with_data: x = [np.inf] * perfprofsamplesize runlengthunsucc = [] try: entry = dictAlgperFunc[alg][0] # one element per fun and per dim. evals = entry.detEvals([t])[0] assert entry.dim == dim runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor if len(runlengthsucc) > 0: x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] except (KeyError, IndexError): #set_trace() warntxt = ('Data for algorithm %s on function %d in %d-D ' % (alg, f, dim) + 'are missing.\n') warnings.warn(warntxt) dictData.setdefault(alg, []).extend(x) dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc) displaybest2009 = not isBiobjective #disabled until we find the bug if displaybest2009: #set_trace() bestalgentries = bestalg.loadBestAlgorithm(isBiobjective) bestalgentry = bestalgentries[(dim, f)] bestalgevals = bestalgentry.detEvals(target_values((f, dim))) # print bestalgevals for j in range(len(bestalgevals[0])): if bestalgevals[1][j]: evals = bestalgevals[0][j] #set_trace() assert dim == bestalgentry.dim runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = bestalgentry.maxevals[bestalgevals[1][j]][np.isnan(evals)] / divisor x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] else: x = perfprofsamplesize * [np.inf] runlengthunsucc = [] xbest2009.extend(x) maxevalsbest2009.extend(runlengthunsucc) if order is None: order = dictData.keys() # Display data lines = [] if displaybest2009: args = {'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11., 'markeredgewidth': 1.5, 'markerfacecolor': refcolor, 'markeredgecolor': refcolor, 'color': refcolor, 'label': 'best 2009', 'zorder': -1} lines.append(plotdata(np.array(xbest2009), x_limit, maxevalsbest2009, CrE = 0., **args)) def algname_to_label(algname, dirname=None): """to be extended to become generally useful""" if isinstance(algname, (tuple, list)): # not sure this is needed return ' '.join([str(name) for name in algname]) return str(algname) for i, alg in enumerate(order): try: data = dictData[alg] maxevals = dictMaxEvals[alg] except KeyError: continue args = styles[(i) % len(styles)] args['linewidth'] = 1.5 args['markersize'] = 12. args['markeredgewidth'] = 1.5 args['markerfacecolor'] = 'None' args['markeredgecolor'] = args['color'] args['label'] = algname_to_label(alg) #args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib #elif len(show_algorithms) > 0: #args['color'] = 'wheat' #args['ls'] = '-' #args['zorder'] = -1 # plotdata calls pprldistr.plotECDF which calls ppfig.plotUnifLog... which does the work lines.append(plotdata(np.array(data), x_limit, maxevals, CrE=CrEperAlg[alg], **args)) labels, handles = plotLegend(lines, x_limit) if True: # isLateXLeg: fileName = os.path.join(outputdir,'pprldmany_%s.tex' % (info)) with open(fileName, 'w') as f: f.write(r'\providecommand{\nperfprof}{7}') algtocommand = {} # latex commands for i, alg in enumerate(order): tmp = r'\alg%sperfprof' % pptex.numtotext(i) f.write(r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' % (tmp, toolsdivers.str_to_latex( toolsdivers.strip_pathname2(algname_to_label(alg))))) algtocommand[algname_to_label(alg)] = tmp if displaybest2009: tmp = r'\algzeroperfprof' f.write(r'\providecommand{%s}{best 2009}' % (tmp)) algtocommand['best 2009'] = tmp commandnames = [] for label in labels: commandnames.append(algtocommand[label]) # f.write(headleg) if len(order) > 28: # latex sidepanel won't work well for more than 25 algorithms, but original labels are also clipped f.write(r'\providecommand{\perfprofsidepanel}{\mbox{%s}\vfill\mbox{%s}}' % (commandnames[0], commandnames[-1])) else: fontsize_command = r'\tiny{}' if len(order) > 19 else '' f.write(r'\providecommand{\perfprofsidepanel}{{%s\mbox{%s}' % (fontsize_command, commandnames[0])) # TODO: check len(labels) > 0 for i in range(1, len(labels)): f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i]) f.write('}}\n') # f.write(footleg) if verbose: print 'Wrote right-hand legend in %s' % fileName figureName = os.path.join(outputdir,'pprldmany_%s' % (info)) #beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat) beautify() text = ppfig.consecutiveNumbers(sorted(dictFunc.keys()), 'f') text += ',%d-D' % dim # TODO: this is strange when different dimensions are plotted plt.text(0.01, 0.98, text, horizontalalignment="left", verticalalignment="top", transform=plt.gca().transAxes) if len(dictFunc) == 1: plt.title(' '.join((str(dictFunc.keys()[0]), genericsettings.current_testbed.short_names[dictFunc.keys()[0]]))) a = plt.gca() plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative) xticks, labels = plt.xticks() tmp = [] for i in xticks: tmp.append('%d' % round(np.log10(i))) a.set_xticklabels(tmp) if save_figure: ppfig.saveFigure(figureName, verbose=verbose) if len(dictFunc) == 1: ppfig.save_single_functions_html( os.path.join(outputdir, 'pprldmany'), '', # algorithms names are clearly visible in the figure add_to_names='_%02dD' %(dim), algorithmCount=ppfig.AlgorithmCount.NON_SPECIFIED ) if close_figure: plt.close()
def main(dsList, isStoringXMax = False, outputdir = '', info = 'default', verbose = True): """Generate figures of empirical cumulative distribution functions. This method has a feature which allows to keep the same boundaries for the x-axis, if ``isStoringXMax==True``. This makes sense when dealing with different functions or subsets of functions for one given dimension. CAVE: this is bug-prone, as some data depend on the maximum evaluations and the appearence therefore depends on the calling order. :param DataSetList dsList: list of DataSet instances to process. :param bool isStoringXMax: if set to True, the first call :py:func:`beautifyFVD` sets the globals :py:data:`fmax` and :py:data:`maxEvals` and all subsequent calls will use these values as rightmost xlim in the generated figures. :param string outputdir: output directory (must exist) :param string info: string suffix for output file names. :param bool verbose: control verbosity """ # plt.rc("axes", labelsize=20, titlesize=24) # plt.rc("xtick", labelsize=20) # plt.rc("ytick", labelsize=20) # plt.rc("font", size=20) # plt.rc("legend", fontsize=20) targets = single_target_values # convenience abbreviation for d, dictdim in dsList.dictByDim().iteritems(): maxEvalsFactor = max(i.mMaxEvals() / d for i in dictdim) if isStoringXMax: global evalfmax else: evalfmax = None if not evalfmax: evalfmax = maxEvalsFactor if runlen_xlimits_max is not None: evalfmax = runlen_xlimits_max # first figure: Run Length Distribution filename = os.path.join(outputdir, 'pprldistr_%02dD_%s' % (d, info)) fig = plt.figure() for j in range(len(targets)): plotRLDistr(dictdim, lambda fun_dim: targets(fun_dim)[j], targets.label(j) if isinstance(targets, pproc.RunlengthBasedTargetValues) else targets.loglabel(j), evalfmax, # can be larger maxEvalsFactor with no effect ** rldStyles[j % len(rldStyles)]) funcs = list(i.funcId for i in dictdim) text = 'f%s' % (consecutiveNumbers(sorted(funcs))) text += ',%d-D' % d if(1): # try: if not isinstance(targets, pproc.RunlengthBasedTargetValues): # if targets.target_values[-1] == 1e-8: # this is a hack plot_previous_algorithms(d, funcs) else: plotRLB_previous_algorithms(d, funcs) # except: # pass plt.axvline(x = maxEvalsFactor, color = 'k') # vertical line at maxevals plt.legend(loc = 'best') plt.text(0.5, 0.98, text, horizontalalignment = "center", verticalalignment = "top", transform = plt.gca().transAxes # bbox=dict(ec='k', fill=False) ) try: # was never tested, so let's make it safe if len(funcs) == 1: plt.title(genericsettings.current_testbed.info(funcs[0])[:27]) except: warnings.warn('could not print title') beautifyRLD(evalfmax) saveFigure(filename, verbose = verbose) plt.close(fig) # second figure: Function Value Distribution filename = os.path.join(outputdir, 'ppfvdistr_%02dD_%s' % (d, info)) fig = plt.figure() plotFVDistr(dictdim, np.inf, 1e-8, **rldStyles[-1]) # coloring right to left for j, max_eval_factor in enumerate(single_runlength_factors): if max_eval_factor > maxEvalsFactor: break plotFVDistr(dictdim, max_eval_factor, 1e-8, **rldUnsuccStyles[j % len(rldUnsuccStyles)]) plt.text(0.98, 0.02, text, horizontalalignment = "right", transform = plt.gca().transAxes) # bbox=dict(ec='k', fill=False), beautifyFVD(isStoringXMax = isStoringXMax, ylabel = False) saveFigure(filename, verbose = verbose) plt.close(fig)
def comp(dsList0, dsList1, targets, isStoringXMax = False, outputdir = '', info = 'default', verbose = True): """Generate figures of ECDF that compare 2 algorithms. :param DataSetList dsList0: list of DataSet instances for ALG0 :param DataSetList dsList1: list of DataSet instances for ALG1 :param seq targets: target function values to be displayed :param bool isStoringXMax: if set to True, the first call :py:func:`beautifyFVD` sets the globals :py:data:`fmax` and :py:data:`maxEvals` and all subsequent calls will use these values as rightmost xlim in the generated figures. :param string outputdir: output directory (must exist) :param string info: string suffix for output file names. :param bool verbose: control verbosity """ # plt.rc("axes", labelsize=20, titlesize=24) # plt.rc("xtick", labelsize=20) # plt.rc("ytick", labelsize=20) # plt.rc("font", size=20) # plt.rc("legend", fontsize=20) if not isinstance(targets, pproc.RunlengthBasedTargetValues): targets = pproc.TargetValues.cast(targets) dictdim0 = dsList0.dictByDim() dictdim1 = dsList1.dictByDim() for d in set(dictdim0.keys()) & set(dictdim1.keys()): maxEvalsFactor = max(max(i.mMaxEvals() / d for i in dictdim0[d]), max(i.mMaxEvals() / d for i in dictdim1[d])) if isStoringXMax: global evalfmax else: evalfmax = None if not evalfmax: evalfmax = maxEvalsFactor ** 1.05 if runlen_xlimits_max is not None: evalfmax = runlen_xlimits_max filename = os.path.join(outputdir, 'pprldistr_%02dD_%s' % (d, info)) fig = plt.figure() for j in range(len(targets)): tmp = plotRLDistr(dictdim0[d], lambda fun_dim: targets(fun_dim)[j], targets.label(j) if isinstance(targets, pproc.RunlengthBasedTargetValues) else targets.loglabel(j), marker = genericsettings.line_styles[1]['marker'], **rldStyles[j % len(rldStyles)]) plt.setp(tmp[-1], label = None) # Remove automatic legend # Mods are added after to prevent them from appearing in the legend plt.setp(tmp, markersize = 20., markeredgewidth = plt.getp(tmp[-1], 'linewidth'), markeredgecolor = plt.getp(tmp[-1], 'color'), markerfacecolor = 'none') tmp = plotRLDistr(dictdim1[d], lambda fun_dim: targets(fun_dim)[j], targets.label(j) if isinstance(targets, pproc.RunlengthBasedTargetValues) else targets.loglabel(j), marker = genericsettings.line_styles[0]['marker'], **rldStyles[j % len(rldStyles)]) # modify the automatic legend: remover marker and change text plt.setp(tmp[-1], marker = '', label = targets.label(j) if isinstance(targets, pproc.RunlengthBasedTargetValues) else targets.loglabel(j)) # Mods are added after to prevent them from appearing in the legend plt.setp(tmp, markersize = 15., markeredgewidth = plt.getp(tmp[-1], 'linewidth'), markeredgecolor = plt.getp(tmp[-1], 'color'), markerfacecolor = 'none') funcs = set(i.funcId for i in dictdim0[d]) | set(i.funcId for i in dictdim1[d]) text = 'f%s' % (consecutiveNumbers(sorted(funcs))) if not isinstance(targets, pproc.RunlengthBasedTargetValues): plot_previous_algorithms(d, funcs) else: plotRLB_previous_algorithms(d, funcs) # plt.axvline(max(i.mMaxEvals()/i.dim for i in dictdim0[d]), ls='--', color='k') # plt.axvline(max(i.mMaxEvals()/i.dim for i in dictdim1[d]), color='k') plt.axvline(max(i.mMaxEvals() / i.dim for i in dictdim0[d]), marker = '+', markersize = 20., color = 'k', markeredgewidth = plt.getp(tmp[-1], 'linewidth',)) plt.axvline(max(i.mMaxEvals() / i.dim for i in dictdim1[d]), marker = 'o', markersize = 15., color = 'k', markerfacecolor = 'None', markeredgewidth = plt.getp(tmp[-1], 'linewidth')) plt.legend(loc = 'best') plt.text(0.5, 0.98, text, horizontalalignment = "center", verticalalignment = "top", transform = plt.gca().transAxes) # bbox=dict(ec='k', fill=False), beautifyRLD(evalfmax) saveFigure(filename, verbose = verbose) plt.close(fig)
def main(dictAlg, order=None, outputdir='.', info='default', verbose=True): """Generates a figure showing the performance of algorithms. From a dictionary of :py:class:`DataSetList` sorted by algorithms, generates the cumulative distribution function of the bootstrap distribution of ERT for algorithms on multiple functions for multiple targets altogether. :param dict dictAlg: dictionary of :py:class:`DataSetList` instances one instance is equivalent to one algorithm, :param list targets: target function values :param list order: sorted list of keys to dictAlg for plotting order :param str outputdir: output directory :param str info: output file name suffix :param bool verbose: controls verbosity """ global x_limit # late assignment of default, because it can be set to None in config if 'x_limit' not in globals() or x_limit is None: x_limit = x_limit_default tmp = pp.dictAlgByDim(dictAlg) # tmp = pp.DictAlg(dictAlg).by_dim() if len(tmp) != 1: raise Exception('We never integrate over dimension.') dim = tmp.keys()[0] algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []] dictFunc = pp.dictAlgByFun(dictAlg) # Collect data # Crafting effort correction: should we consider any? CrEperAlg = {} for alg in algorithms_with_data: CrE = 0. if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL': tmp = dictAlg[alg].dictByNoise() assert len(tmp.keys()) == 1 if tmp.keys()[0] == 'noiselessall': CrE = 0.5117 elif tmp.keys()[0] == 'nzall': CrE = 0.6572 CrEperAlg[alg] = CrE if CrE != 0.0: print 'Crafting effort for', alg, 'is', CrE dictData = {} # list of (ert per function) per algorithm dictMaxEvals = {} # list of (maxevals per function) per algorithm bestERT = [] # best ert per function # funcsolved = [set()] * len(targets) # number of functions solved per target xbest2009 = [] maxevalsbest2009 = [] for f, dictAlgperFunc in dictFunc.iteritems(): if function_IDs and f not in function_IDs: continue # print target_values((f, dim)) for j, t in enumerate(target_values((f, dim))): # for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)): # funcsolved[j].add(f) for alg in algorithms_with_data: x = [np.inf] * perfprofsamplesize runlengthunsucc = [] try: entry = dictAlgperFunc[alg][0] # one element per fun and per dim. evals = entry.detEvals([t])[0] runlengthsucc = evals[np.isnan(evals) == False] / entry.dim runlengthunsucc = entry.maxevals[np.isnan(evals)] / entry.dim if len(runlengthsucc) > 0: x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] except (KeyError, IndexError): #set_trace() warntxt = ('Data for algorithm %s on function %d in %d-D ' % (alg, f, dim) + 'are missing.\n') warnings.warn(warntxt) dictData.setdefault(alg, []).extend(x) dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc) if displaybest2009: #set_trace() if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() bestalgentry = bestalg.bestalgentries2009[(dim, f)] bestalgevals = bestalgentry.detEvals(target_values((f, dim))) # print bestalgevals for j in range(len(bestalgevals[0])): if bestalgevals[1][j]: evals = bestalgevals[0][j] #set_trace() runlengthsucc = evals[np.isnan(evals) == False] / bestalgentry.dim runlengthunsucc = bestalgentry.maxevals[bestalgevals[1][j]][np.isnan(evals)] / bestalgentry.dim x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] else: x = perfprofsamplesize * [np.inf] runlengthunsucc = [] xbest2009.extend(x) maxevalsbest2009.extend(runlengthunsucc) if order is None: order = dictData.keys() # Display data lines = [] if displaybest2009: args = {'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11., 'markeredgewidth': 1.5, 'markerfacecolor': refcolor, 'markeredgecolor': refcolor, 'color': refcolor, 'label': 'best 2009', 'zorder': -1} lines.append(plotdata(np.array(xbest2009), x_limit, maxevalsbest2009, CrE = 0., **args)) for i, alg in enumerate(order): try: data = dictData[alg] maxevals = dictMaxEvals[alg] except KeyError: continue args = styles[(i) % len(styles)] args['linewidth'] = 1.5 args['markersize'] = 12. args['markeredgewidth'] = 1.5 args['markerfacecolor'] = 'None' args['markeredgecolor'] = args['color'] args['label'] = alg #args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib #elif len(show_algorithms) > 0: #args['color'] = 'wheat' #args['ls'] = '-' #args['zorder'] = -1 lines.append(plotdata(np.array(data), x_limit, maxevals, CrE=CrEperAlg[alg], **args)) labels, handles = plotLegend(lines, x_limit) if True: #isLateXLeg: fileName = os.path.join(outputdir,'pprldmany_%s.tex' % (info)) try: f = open(fileName, 'w') f.write(r'\providecommand{\nperfprof}{7}') algtocommand = {} for i, alg in enumerate(order): tmp = r'\alg%sperfprof' % pptex.numtotext(i) f.write(r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' % (tmp, toolsdivers.str_to_latex(toolsdivers.strip_pathname2(alg)))) algtocommand[alg] = tmp commandnames = [] if displaybest2009: tmp = r'\algzeroperfprof' f.write(r'\providecommand{%s}{best 2009}' % (tmp)) algtocommand['best 2009'] = tmp for l in labels: commandnames.append(algtocommand[l]) # f.write(headleg) f.write(r'\providecommand{\perfprofsidepanel}{\mbox{%s}' % commandnames[0]) # TODO: check len(labels) > 0 for i in range(1, len(labels)): f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i]) f.write('}\n') # f.write(footleg) if verbose: print 'Wrote right-hand legend in %s' % fileName except: raise # TODO: Does this make sense? else: f.close() figureName = os.path.join(outputdir,'pprldmany_%s' % (info)) #beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat) beautify() text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys()))) text += ',%d-D' % dim plt.text(0.01, 0.98, text, horizontalalignment="left", verticalalignment="top", transform=plt.gca().transAxes) a = plt.gca() plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative) xticks, labels = plt.xticks() tmp = [] for i in xticks: tmp.append('%d' % round(np.log10(i))) a.set_xticklabels(tmp) ppfig.saveFigure(figureName, verbose=verbose) plt.close()
def main2(dsList0, dsList1, valuesOfInterest=None, outputdir='', info='default', verbose=True): """Generate figures of empirical cumulative distribution functions. Keyword arguments: indexEntries -- list of IndexEntry instances to process. valuesOfInterest -- target function values to be displayed. isStoringXMax -- if set to True, the first call BeautifyVD sets the globals fmax and maxEvals and all subsequent calls will use these values as rightmost xlim in the generated figures. -- if set to True, the first call BeautifyVD sets the global fmax and all subsequent call will have the same maximum xlim. outputdir -- output directory (must exist) info --- string suffix for output file names. Outputs: Image files of the empirical cumulative distribution functions. """ plt.rc("axes", labelsize=20, titlesize=24) plt.rc("xtick", labelsize=20) plt.rc("ytick", labelsize=20) plt.rc("font", size=20) plt.rc("legend", fontsize=20) figureName = os.path.join(outputdir, 'pplogabs_%s' % (info)) tmp = plotLogAbs2(dsList0, dsList1, valuesOfInterest, verbose=verbose) beautify2() # Prolong to the boundary xmin, xmax = plt.xlim() for i in tmp: try: xdata, ydata = i.get_data() except AttributeError: xdata = i.get_xdata() ydata = i.get_ydata() if len(xdata) == 0 or len(ydata) == 0: continue xdata = numpy.insert(xdata, 0, xmin) try: xdata = numpy.insert(xdata, len(xdata), xmax) except OverflowError: xdata = xdata + 0.0 # TODO: Hack for float conversion, compatibility with 0.8 xdata = numpy.insert(xdata, len(xdata), xmax) ydata = numpy.insert(ydata, 0, ydata[0]) ydata = numpy.insert(ydata, len(ydata), ydata[-1]) i.set_data(xdata, ydata) plt.legend(loc='best') #plt.text(0.5, 0.93, text, horizontalalignment="center", # transform=axisHandle.transAxes) funcs = set(dsList0.dictByFunc().keys()) & set(dsList1.dictByFunc().keys()) text = 'f%s' % consecutiveNumbers(sorted(funcs)) plt.text(0.98, 0.02, text, horizontalalignment="right", transform=plt.gca().transAxes) #set_trace() saveFigure(figureName, figFormat=figformat, verbose=verbose) plt.close() #set_trace() plt.rcdefaults()
def generateFigure(dsList, CrE=0., isStoringXRange=True, outputdir='.', info='default', verbose=True): """Generates ERT loss ratio figures. :param DataSetList dsList: input data set :param float CrE: crafting effort (see COCO documentation) :param bool isStoringXRange: if set to True, the first call to this function sets the global :py:data:`evalf` and all subsequent calls will use this value as boundaries in the generated figures. :param string outputdir: output folder (must exist) :param string info: string suffix for output file names :param bool verbose: controls verbosity """ #plt.rc("axes", labelsize=20, titlesize=24) #plt.rc("xtick", labelsize=20) #plt.rc("ytick", labelsize=20) #plt.rc("font", size=20) #plt.rc("legend", fontsize=20) if isStoringXRange: global evalf else: evalf = None # do not aggregate over dimensions for d, dsdim in dsList.dictByDim().iteritems(): maxevals = max(max(i.ert[np.isinf(i.ert)==False]) for i in dsdim) EVALS = [2.*d] EVALS.extend(10.**(np.arange(1, np.ceil(1e-9 + np.log10(maxevals * 1./d))))*d) if not evalf: evalf = (np.log10(EVALS[0]/d), np.log10(EVALS[-1]/d)) data = generateData(dsdim, EVALS, CrE) ydata = [] for i in range(len(EVALS)): #Aggregate over functions. ydata.append(np.log10(list(data[f][i] for f in data))) xdata = np.log10(np.array(EVALS)/d) xticklabels = [''] xticklabels.extend('%d' % i for i in xdata[1:]) plot(xdata, ydata) filename = os.path.join(outputdir, 'pplogloss_%02dD_%s' % (d, info)) plt.xticks(xdata, xticklabels) #Is there an upper bound? if CrE > 0 and len(set(dsdim.dictByFunc().keys())) >= 20: #TODO: hopefully this means we are not considering function groups. plt.text(0.01, 0.98, 'CrE = %5g' % CrE, fontsize=20, horizontalalignment='left', verticalalignment='top', transform = plt.gca().transAxes, bbox=dict(facecolor='w')) plt.axhline(1., color='k', ls='-', zorder=-1) plt.axvline(x=np.log10(max(i.mMaxEvals()/d for i in dsdim)), color='k') funcs = set(i.funcId for i in dsdim) if len(funcs) > 1: text = consecutiveNumbers(sorted(funcs), 'f') else: text = 'f%d' % (funcs.pop()) plt.text(0.5, 0.93, text, horizontalalignment="center", transform=plt.gca().transAxes) beautify() if evalf: plt.xlim(xmin=evalf[0]-0.5, xmax=evalf[1]+0.5) saveFigure(filename, verbose=verbose) #plt.show() plt.close()