def plot(dsList, targets=None, craftingeffort=0., **kwargs): """This function is obsolete? Generates a graph of the run length distribution of an algorithm. We display the empirical cumulative distribution function ECDF of the bootstrapped distribution of the runlength for an algorithm (in number of function evaluations) to reach the target functions value :py:data:`targets`. :param DataSetList dsList: data set for one algorithm :param seq targets: target function values :param float crafting effort: the data will be multiplied by the exponential of this value :param dict kwargs: additional parameters provided to plot function. :returns: handles """ if targets is None: targets = target_values # set above or in config.py try: if np.min(targets) >= 1: ValueError( 'smallest target f-value is not smaller than one, use ``pproc.TargetValues(targets)`` to prevent this error' ) targets = pp.TargetValues(targets) except TypeError: pass res = [] assert len(pp.DataSetList( dsList).dictByDim()) == 1 # We never integrate over dimensions... data = [] maxevals = [] for entry in dsList: for t in targets((entry.funcId, entry.dim)): divisor = entry.dim if divide_by_dimension else 1 x = [np.inf] * perfprofsamplesize runlengthunsucc = [] evals = entry.detEvals([t])[0] runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor if len(runlengthsucc) > 0: x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] data.extend(x) maxevals.extend(runlengthunsucc) # Display data data = np.array(data) data = data[np.isnan(data) == False] # Take away the nans n = len(data) data = data[np.isinf(data) == False] # Take away the infs # data = data[data <= maxval] # Take away rightmost data data = np.exp(craftingeffort) * data # correction by crafting effort CrE if len(data) == 0: # data is empty. res = pprldistr.plotECDF(np.array((1., )), n=np.inf, **kwargs) else: res = pprldistr.plotECDF(np.array(data), n=n, **kwargs) #plotdata(np.array(data), x_limit, maxevals, # CrE=0., **kwargs) if maxevals: # Should cover the case where maxevals is None or empty x3 = np.median(maxevals) if np.any(data > x3): y3 = float(np.sum(data <= x3)) / n h = plt_plot((x3, ), (y3, ), marker='x', markersize=24, markeredgewidth=3, markeredgecolor=plt.getp(res[0], 'color'), ls='', color=plt.getp(res[0], 'color')) h.extend(res) res = h # so the last element in res still has the label. return res
CAVEAT: the naming conventions in this module mix up ERT (an estimate of the expected running length) and run lengths. """ from __future__ import absolute_import import os import warnings # I don't know what I am doing here import numpy as np import pickle, gzip import matplotlib.pyplot as plt from pdb import set_trace from bbob_pproc import toolsstats, genericsettings, pproc from bbob_pproc.ppfig import consecutiveNumbers, plotUnifLogXMarkers, saveFigure, logxticks single_target_values = pproc.TargetValues( (10., 1e-1, 1e-4, 1e-8)) # possibly changed in config single_runlength_factors = [0.5, 1.2, 3, 10] + [10**i for i in range(2, 12)] # TODO: the method names in this module seem to be overly unclear or misleading and should be revised. refcolor = 'wheat' nbperdecade = 1 # markers in x-axis decades in ecdfs runlen_xlimits_max = None # is possibly manipulated in config runlen_xlimits_min = 1 # set to 10**-0.5 in runlength case in config # Used as a global to store the largest xmax and align the FV ECD figures. fmax = None evalfmax = runlen_xlimits_max # is manipulated/stored in this module # TODO: the target function values and the styles of the line only make sense # together. Therefore we should either: # 1. keep the targets as input argument and make rldStyles depend on them or
""" from __future__ import absolute_import import os import numpy as np import matplotlib.pyplot as plt from bbob_pproc import genericsettings, bestalg, toolsstats, pproc from bbob_pproc.pptex import tableLaTeX, tableLaTeXStar, writeFEvals2, writeFEvalsMaxPrec from bbob_pproc.toolsstats import significancetest from pdb import set_trace targets = (10., 1., 1e-1, 1e-3, 1e-5, 1e-7) # targets of the table finaltarget = 1e-8 # value for determining the success ratio targetsOfInterest = (10., 1., 1e-1, 1e-3, 1e-5, 1e-7) # targets of the table targetsOfInterest = pproc.TargetValues((10, 1, 1e-1, 1e-2, 1e-3, 1e-5, 1e-7)) targetf = 1e-8 # value for determining the success ratio samplesize = genericsettings.simulated_runlength_bootstrap_sample_size # TODO: change samplesize # def tablespec(targets): # # i = 0 # tspec = {'col%d' % i: {'what': 'fname', 'header': r'$\Delta f$', 'format': None}} # for t in targets: # i = i + 1 # tspec.update({'col%d' % i: {'what': 'ERT ratio for df=%e' % t, # 'header': r'\multicolumn{2}{@{}c@{}}{1e%+d}' % (int(np.log10(t)), # 'format': writeFEval}}) # i = i + 1 # tspec.update({'col%d' % i: {'what': 'nb of success', 'header': r'\#succ', # 'format': '%d'}})
from __future__ import absolute_import import os import warnings from pdb import set_trace import numpy as np import matplotlib.pyplot as plt from bbob_pproc import toolsstats, bestalg, genericsettings from bbob_pproc import pproc as pp # import dictAlgByDim, dictAlgByFun from bbob_pproc import toolsdivers # strip_pathname, str_to_latex from bbob_pproc import pprldistr # plotECDF, beautifyECDF from bbob_pproc import ppfig # consecutiveNumbers, saveFigure, plotUnifLogXMarkers, logxticks from bbob_pproc import pptex # numtotex displaybest2009 = True target_values = pp.TargetValues(10**np.arange( 2, -8, -0.2)) # possibly changed in config x_limit = None # not sure whether this is necessary/useful x_limit_default = 1e7 # better: 10 * genericsettings.evaluation_setting[1], noisy: 1e8, otherwise: 1e7. maximal run length shown divide_by_dimension = True annotation_line_end_relative = 1.11 # lines between graph and annotation annotation_space_end_relative = 1.24 # figure space end relative to x_limit save_zoom = False # save zoom into left and right part of the figures perfprofsamplesize = genericsettings.simulated_runlength_bootstrap_sample_size_rld # number of bootstrap samples drawn for each fct+target in the performance profile dpi_global_var = 100 # 100 ==> 800x600 (~160KB), 120 ==> 960x720 (~200KB), 150 ==> 1200x900 (~300KB) looks ugly in latex nbperdecade = 1 median_max_evals_marker_format = ['x', 24, 3] label_fontsize = 18 styles = [d.copy() for d in genericsettings.line_styles] # deep copy refcolor = 'wheat' """color of reference (best) algorithm"""
def plotLogAbs(dsList0, dsList1, dim, targetValuesToReach, verbose=True): """Creates ECDF of run length ratios. :param DataSetList dsList0: reference :param DataSetList dsList1: data set list of algorithm of interest :param int dim: dimension :param TargetValues targetValuesToReach: target function values :param bool verbose: controls verbosity :returns: handles """ res = [] dictFunc0 = dsList0.dictByFunc() dictFunc1 = dsList1.dictByFunc() evals0 = {} evals1 = {} if not isinstance(targetValuesToReach, pproc.TargetValues): targetValuesToReach = pproc.TargetValues(targetValuesToReach) succ0 = [0] * len(targetValuesToReach) succ1 = [0] * len(targetValuesToReach) # TODO: check all functions are there... for func in set(dictFunc0.keys()) & set(dictFunc1.keys()): i0 = dictFunc0[func][0] i1 = dictFunc1[func][0] #tmp = list(i[func] for i in fvalueToReach) #if tmp != targets: # set_trace() # should not occur # warnings.warn('') evals0[func] = i0.detEvals(targetValuesToReach((func, dim))) for i, evals in enumerate(evals0[func]): tmp = numpy.isnan(evals) evals[tmp] = numpy.inf if not tmp.all(): succ0[i] += 1 evals1[func] = i1.detEvals(targetValuesToReach((func, dim))) for i, evals in enumerate(evals1[func]): tmp = numpy.isnan(evals) evals[tmp] = numpy.inf if not tmp.all(): succ1[i] += 1 for j in range(len(targetValuesToReach)): x = [] for func in evals0: # Compute the pair-wise ratio tmp1 = numpy.reshape(evals1[func][j], (1, len(evals1[func][j]))) tmp0 = numpy.reshape(evals0[func][j], (len(evals0[func][j]), 1)) try: x.append((tmp1 / tmp0).flatten()) # inf/inf results in nan except FloatingPointError: if numpy.isfinite(tmp1).all() or numpy.isfinite(tmp1).all(): raise #TODO: check division, check numpy.inf... if isinstance(targetValuesToReach, pproc.RunlengthBasedTargetValues): label = '%s: %d/%d' % (targetValuesToReach.label(j), succ1[j], succ0[j]) else: label = '%s: %d/%d' % (targetValuesToReach.loglabel(j), succ1[j], succ0[j]) if len(x) > 0: # prevent warning/error x = numpy.hstack(x) x = x[numpy.isnan(x) == False] # Is it correct? n = len(x) if n == 0: res.extend( plt.plot([], [], label=label, linewidth=3., **rldStyles[j])) continue # no plot? x.sort() #Catch negative values: zeros are not a problem... tmp = len(list(i for i in x if i <= 0)) x = x[tmp:] #Catch inf, those could be a problem with the log scale... #tmp2 = 0 tmp2 = len(list(i for i in x if i > 0 and numpy.isinf(i))) if tmp2 > 0: x = x[:-tmp2] #xbound = max(abs(numpy.floor(numpy.log10(x[0]))), # abs(numpy.ceil(numpy.log10(x[-1])))) if len(x) == 0: res.append( plt.axhline(tmp / float(n), label=label, linewidth=3., **rldStyles[j])) # tmp/float(n) == (n-tmp2)/float(n) # TODO: check else: x2 = numpy.hstack([numpy.repeat(x, 2)]) #maxEvalsF: used for the limit of the plot. y2 = numpy.hstack([ tmp / float(n), numpy.repeat(numpy.arange(tmp + 1, n - tmp2) / float(n), 2), (n - tmp2) / float(n) ]) #res.extend(plt.plot(x2, y2, label=label, linewidth=3., **rldStyles[i])) plotArgs = rldStyles[j].copy() plotArgs['label'] = label plotArgs['linewidth'] = 3. #res.extend(plotUnifLogXMarkers(x2, y2, 3, plotArgs)) res.append( plotUnifLogXMarkers(x2, y2, nbperdecade=3, logscale=False, **plotArgs)[0]) # TODO: check if all of evalsX[func] is numpy.inf and so on... return res
def extractBestAlgorithms(args=algs2009, f_factor=2, target_lb=1e-8, target_ub=1e22): """Returns (and prints) per dimension a list of algorithms within algorithm list args that contains an algorithm if for any dimension/target/function pair this algorithm: - is the best algorithm wrt ERT - its own ERT lies within a factor f_factor of the best ERT - there is no algorithm within a factor of f_factor of the best ERT and the current algorithm is the second best. """ # TODO: use pproc.TargetValues class as input target values # default target values: targets = pproc.TargetValues(10**np.arange( np.log10(max((1e-8, target_lb))), np.log10(target_ub) + 1e-9, 0.2)) # there should be a simpler way to express this to become the # interface of this function print 'Loading algorithm data from given algorithm list...\n' verbose = True dsList, sortedAlgs, dictAlg = pproc.processInputArgs(args, verbose=verbose) print 'This may take a while (depending on the number of algorithms)' selectedAlgsPerProblem = {} for f, i in pproc.dictAlgByFun(dictAlg).iteritems(): for d, j in pproc.dictAlgByDim(i).iteritems(): selectedAlgsPerProblemDF = [] best = BestAlgSet(j) for i in range(0, len(best.target)): t = best.target[i] # if ((t <= target_ub) and (t >= target_lb)): if toolsstats.in_approximately( t, targets((f, d), discretize=True)): # add best for this target: selectedAlgsPerProblemDF.append(best.algs[i]) # add second best or all algorithms that have an ERT # within a factor of f_factor of the best: secondbest_ERT = np.infty secondbest_str = '' secondbest_included = False for astring in j: currdictalg = dictAlg[astring].dictByDim() if currdictalg.has_key(d): curralgdata = currdictalg[d][f - 1] currERT = curralgdata.detERT([t])[0] if (astring != best.algs[i]): if (currERT < secondbest_ERT): secondbest_ERT = currERT secondbest_str = astring if (currERT <= best.detERT([t])[0] * f_factor): selectedAlgsPerProblemDF.append(astring) secondbest_included = True if not (secondbest_included) and (secondbest_str != ''): selectedAlgsPerProblemDF.append(secondbest_str) if len(selectedAlgsPerProblemDF) > 0: selectedAlgsPerProblem[(d, f)] = selectedAlgsPerProblemDF print 'pre-processing of function', f, 'done.' print 'loading of best algorithm(s) data done.' countsperalgorithm = {} for (d, f) in selectedAlgsPerProblem: print 'dimension:', d, ', function:', f setofalgs = set(selectedAlgsPerProblem[d, f]) # now count how often algorithm a is best for the extracted targets for a in setofalgs: # use setdefault to initialize with zero if a entry not existant: countsperalgorithm.setdefault((d, a), 0) countsperalgorithm[(d, a)] += selectedAlgsPerProblem[d, f].count(a) selectedalgsperdimension = {} for (d, a) in sorted(countsperalgorithm): if not selectedalgsperdimension.has_key(d): selectedalgsperdimension[d] = [] selectedalgsperdimension[d].append((countsperalgorithm[(d, a)], a)) for d in sorted(selectedalgsperdimension): print d, 'D:' for (count, alg) in sorted(selectedalgsperdimension[d], reverse=True): print count, alg print '\n' print " done." return selectedalgsperdimension
figure() bb.ppfigdim.plot(ds) bb.ppfigdim.beautify() bb.ppfigdim.plot_previous_algorithms(2) # plot BBOB 2009 best algorithm on fun 2 """ import os import sys import matplotlib.pyplot as plt import numpy as np from pdb import set_trace from bbob_pproc import genericsettings, toolsstats, bestalg, pproc from bbob_pproc.ppfig import saveFigure, groupByRange values_of_interest = pproc.TargetValues((10, 1, 1e-1, 1e-2, 1e-3, 1e-5, 1e-8)) # to rename!? xlim_max = None ynormalize_by_dimension = True # not at all tested yet styles = [ # sort of rainbow style, most difficult (red) first {'color': 'r', 'marker': 'o', 'markeredgecolor': 'k', 'markeredgewidth': 2, 'linewidth': 4}, {'color': 'm', 'marker': '.', 'linewidth': 4}, {'color': 'y', 'marker': '^', 'markeredgecolor': 'k', 'markeredgewidth': 2, 'linewidth': 4}, {'color': 'g', 'marker': '.', 'linewidth': 4}, {'color': 'c', 'marker': 'v', 'markeredgecolor': 'k', 'markeredgewidth': 2, 'linewidth': 4}, {'color': 'b', 'marker': '.', 'linewidth': 4}, {'color': 'k', 'marker': 'o', 'markeredgecolor': 'k', 'markeredgewidth': 2, 'linewidth': 4}, ] refcolor = 'wheat'
import numpy import numpy as np from pdb import set_trace from matplotlib import pyplot as plt try: from matplotlib.transforms import blended_transform_factory as blend except ImportError: # compatibility matplotlib 0.8 from matplotlib.transforms import blend_xy_sep_transform as blend from bbob_pproc import readalign from bbob_pproc.ppfig import saveFigure from bbob_pproc import toolsdivers from bbob_pproc import pproc dimensions = (2, 3, 5, 10, 20, 40) fixed_targets = pproc.TargetValues(np.logspace(-8, 2, 46)) runlength_based_targets = pproc.RunlengthBasedTargetValues(np.logspace(numpy.log10(0.5), numpy.log10(50), 8)) # runlength_based_targets = pproc.RunlengthBasedTargetValues([0.5, 1, 3, 10, 50]) targets = fixed_targets # default # formattings colors = ('c', 'g', 'b', 'k', 'r', 'm', 'k', 'y', 'k', 'c', 'r', 'm') markers = ('+', 'v', '*', 'o', 's', 'D', 'x') markersize = 14 # modified in config.py linewidth = 3 max_evals_line_length = 9 # length away from the diagonal as a factor, line indicates maximal evaluations for each data offset = 0. #0.02 offset provides a way to move away the box boundaries to display the outer markers fully, clip_on=False is more effective caption_start_fixed = r"""Expected running time (\ERT\ in $\log_{10}$ of number of function evaluations) of \algorithmB\ ($x$-axis) versus \algorithmA\ ($y$-axis) for $NBTARGETS$ target values $\Df \in [NBLOW, NBUP]$ in each dimension on functions #1. """
from __future__ import absolute_import import os import warnings from pdb import set_trace import numpy as np import matplotlib.pyplot as plt from bbob_pproc import toolsstats, bestalg, genericsettings from bbob_pproc import pproc as pp # import dictAlgByDim, dictAlgByFun from bbob_pproc import toolsdivers # strip_pathname, str_to_latex from bbob_pproc import pprldistr # plotECDF, beautifyECDF from bbob_pproc import ppfig # consecutiveNumbers, saveFigure, plotUnifLogXMarkers, logxticks from bbob_pproc import pptex # numtotex displaybest2009 = True target_values = pp.TargetValues(10**np.arange(2, -8, -0.2)) # changed in config.py x_limit = None # not sure whether this is necessary/useful x_limit_default = 1e7 # better: 10 * genericsettings.evaluation_setting[1], noisy: 1e8, otherwise: 1e7. maximal run length shown annotation_line_end_relative = 1.11 # lines between graph and annotation annotation_space_end_relative = 1.24 # figure space end relative to x_limit save_zoom = False # save zoom into left and right part of the figures perfprofsamplesize = genericsettings.simulated_runlength_bootstrap_sample_size_rld # number of bootstrap samples drawn for each fct+target in the performance profile dpi_global_var = 100 # 100 ==> 800x600 (~160KB), 120 ==> 960x720 (~200KB), 150 ==> 1200x900 (~300KB) looks ugly in latex nbperdecade = 1 median_max_evals_marker_format = ['x', 24, 3] styles = [d.copy() for d in genericsettings.line_styles] # deep copy refcolor = 'wheat' # TODO: update the list below which are not relevant anymore