def plot(dsList, targets=None, craftingeffort=0., **kwargs):
"""This function is obsolete?
Generates a graph of the run length distribution of an algorithm.
We display the empirical cumulative distribution function ECDF of
the bootstrapped distribution of the runlength for an algorithm
(in number of function evaluations) to reach the target functions
value :py:data:`targets`.
:param DataSetList dsList: data set for one algorithm
:param seq targets: target function values
:param float crafting effort: the data will be multiplied by the
exponential of this value
:param dict kwargs: additional parameters provided to plot function.
:returns: handles
"""
if targets is None:
targets = target_values # set above or in config.py
try:
if np.min(targets) >= 1:
ValueError(
'smallest target f-value is not smaller than one, use ``pproc.TargetValues(targets)`` to prevent this error'
)
targets = pp.TargetValues(targets)
except TypeError:
pass
res = []
assert len(pp.DataSetList(
dsList).dictByDim()) == 1 # We never integrate over dimensions...
data = []
maxevals = []
for entry in dsList:
for t in targets((entry.funcId, entry.dim)):
divisor = entry.dim if divide_by_dimension else 1
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
evals = entry.detEvals([t])[0]
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(runlengthsucc,
runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
data.extend(x)
maxevals.extend(runlengthunsucc)
# Display data
data = np.array(data)
data = data[np.isnan(data) == False] # Take away the nans
n = len(data)
data = data[np.isinf(data) == False] # Take away the infs
# data = data[data <= maxval] # Take away rightmost data
data = np.exp(craftingeffort) * data # correction by crafting effort CrE
if len(data) == 0: # data is empty.
res = pprldistr.plotECDF(np.array((1., )), n=np.inf, **kwargs)
else:
res = pprldistr.plotECDF(np.array(data), n=n, **kwargs)
#plotdata(np.array(data), x_limit, maxevals,
# CrE=0., **kwargs)
if maxevals: # Should cover the case where maxevals is None or empty
x3 = np.median(maxevals)
if np.any(data > x3):
y3 = float(np.sum(data <= x3)) / n
h = plt_plot((x3, ), (y3, ),
marker='x',
markersize=24,
markeredgewidth=3,
markeredgecolor=plt.getp(res[0], 'color'),
ls='',
color=plt.getp(res[0], 'color'))
h.extend(res)
res = h # so the last element in res still has the label.
return res
CAVEAT: the naming conventions in this module mix up ERT (an estimate
of the expected running length) and run lengths.
"""
from __future__ import absolute_import
import os
import warnings # I don't know what I am doing here
import numpy as np
import pickle, gzip
import matplotlib.pyplot as plt
from pdb import set_trace
from bbob_pproc import toolsstats, genericsettings, pproc
from bbob_pproc.ppfig import consecutiveNumbers, plotUnifLogXMarkers, saveFigure, logxticks
single_target_values = pproc.TargetValues(
(10., 1e-1, 1e-4, 1e-8)) # possibly changed in config
single_runlength_factors = [0.5, 1.2, 3, 10] + [10**i for i in range(2, 12)]
# TODO: the method names in this module seem to be overly unclear or misleading and should be revised.
refcolor = 'wheat'
nbperdecade = 1 # markers in x-axis decades in ecdfs
runlen_xlimits_max = None # is possibly manipulated in config
runlen_xlimits_min = 1 # set to 10**-0.5 in runlength case in config
# Used as a global to store the largest xmax and align the FV ECD figures.
fmax = None
evalfmax = runlen_xlimits_max # is manipulated/stored in this module
# TODO: the target function values and the styles of the line only make sense
# together. Therefore we should either:
# 1. keep the targets as input argument and make rldStyles depend on them or
"""
from __future__ import absolute_import
import os
import numpy as np
import matplotlib.pyplot as plt
from bbob_pproc import genericsettings, bestalg, toolsstats, pproc
from bbob_pproc.pptex import tableLaTeX, tableLaTeXStar, writeFEvals2, writeFEvalsMaxPrec
from bbob_pproc.toolsstats import significancetest
from pdb import set_trace
targets = (10., 1., 1e-1, 1e-3, 1e-5, 1e-7) # targets of the table
finaltarget = 1e-8 # value for determining the success ratio
targetsOfInterest = (10., 1., 1e-1, 1e-3, 1e-5, 1e-7) # targets of the table
targetsOfInterest = pproc.TargetValues((10, 1, 1e-1, 1e-2, 1e-3, 1e-5, 1e-7))
targetf = 1e-8 # value for determining the success ratio
samplesize = genericsettings.simulated_runlength_bootstrap_sample_size # TODO: change samplesize
# def tablespec(targets):
#
# i = 0
# tspec = {'col%d' % i: {'what': 'fname', 'header': r'$\Delta f$', 'format': None}}
# for t in targets:
# i = i + 1
# tspec.update({'col%d' % i: {'what': 'ERT ratio for df=%e' % t,
# 'header': r'\multicolumn{2}{@{}c@{}}{1e%+d}' % (int(np.log10(t)),
# 'format': writeFEval}})
# i = i + 1
# tspec.update({'col%d' % i: {'what': 'nb of success', 'header': r'\#succ',
# 'format': '%d'}})
from __future__ import absolute_import
import os
import warnings
from pdb import set_trace
import numpy as np
import matplotlib.pyplot as plt
from bbob_pproc import toolsstats, bestalg, genericsettings
from bbob_pproc import pproc as pp # import dictAlgByDim, dictAlgByFun
from bbob_pproc import toolsdivers # strip_pathname, str_to_latex
from bbob_pproc import pprldistr # plotECDF, beautifyECDF
from bbob_pproc import ppfig # consecutiveNumbers, saveFigure, plotUnifLogXMarkers, logxticks
from bbob_pproc import pptex # numtotex
displaybest2009 = True
target_values = pp.TargetValues(10**np.arange(
2, -8, -0.2)) # possibly changed in config
x_limit = None # not sure whether this is necessary/useful
x_limit_default = 1e7 # better: 10 * genericsettings.evaluation_setting[1], noisy: 1e8, otherwise: 1e7. maximal run length shown
divide_by_dimension = True
annotation_line_end_relative = 1.11 # lines between graph and annotation
annotation_space_end_relative = 1.24 # figure space end relative to x_limit
save_zoom = False # save zoom into left and right part of the figures
perfprofsamplesize = genericsettings.simulated_runlength_bootstrap_sample_size_rld # number of bootstrap samples drawn for each fct+target in the performance profile
dpi_global_var = 100 # 100 ==> 800x600 (~160KB), 120 ==> 960x720 (~200KB), 150 ==> 1200x900 (~300KB) looks ugly in latex
nbperdecade = 1
median_max_evals_marker_format = ['x', 24, 3]
label_fontsize = 18
styles = [d.copy() for d in genericsettings.line_styles] # deep copy
refcolor = 'wheat'
"""color of reference (best) algorithm"""
def plotLogAbs(dsList0, dsList1, dim, targetValuesToReach, verbose=True):
"""Creates ECDF of run length ratios.
:param DataSetList dsList0: reference
:param DataSetList dsList1: data set list of algorithm of interest
:param int dim: dimension
:param TargetValues targetValuesToReach: target function values
:param bool verbose: controls verbosity
:returns: handles
"""
res = []
dictFunc0 = dsList0.dictByFunc()
dictFunc1 = dsList1.dictByFunc()
evals0 = {}
evals1 = {}
if not isinstance(targetValuesToReach, pproc.TargetValues):
targetValuesToReach = pproc.TargetValues(targetValuesToReach)
succ0 = [0] * len(targetValuesToReach)
succ1 = [0] * len(targetValuesToReach)
# TODO: check all functions are there...
for func in set(dictFunc0.keys()) & set(dictFunc1.keys()):
i0 = dictFunc0[func][0]
i1 = dictFunc1[func][0]
#tmp = list(i[func] for i in fvalueToReach)
#if tmp != targets:
# set_trace() # should not occur
# warnings.warn('')
evals0[func] = i0.detEvals(targetValuesToReach((func, dim)))
for i, evals in enumerate(evals0[func]):
tmp = numpy.isnan(evals)
evals[tmp] = numpy.inf
if not tmp.all():
succ0[i] += 1
evals1[func] = i1.detEvals(targetValuesToReach((func, dim)))
for i, evals in enumerate(evals1[func]):
tmp = numpy.isnan(evals)
evals[tmp] = numpy.inf
if not tmp.all():
succ1[i] += 1
for j in range(len(targetValuesToReach)):
x = []
for func in evals0:
# Compute the pair-wise ratio
tmp1 = numpy.reshape(evals1[func][j], (1, len(evals1[func][j])))
tmp0 = numpy.reshape(evals0[func][j], (len(evals0[func][j]), 1))
try:
x.append((tmp1 / tmp0).flatten()) # inf/inf results in nan
except FloatingPointError:
if numpy.isfinite(tmp1).all() or numpy.isfinite(tmp1).all():
raise
#TODO: check division, check numpy.inf...
if isinstance(targetValuesToReach, pproc.RunlengthBasedTargetValues):
label = '%s: %d/%d' % (targetValuesToReach.label(j), succ1[j],
succ0[j])
else:
label = '%s: %d/%d' % (targetValuesToReach.loglabel(j), succ1[j],
succ0[j])
if len(x) > 0: # prevent warning/error
x = numpy.hstack(x)
x = x[numpy.isnan(x) == False] # Is it correct?
n = len(x)
if n == 0:
res.extend(
plt.plot([], [], label=label, linewidth=3., **rldStyles[j]))
continue # no plot?
x.sort()
#Catch negative values: zeros are not a problem...
tmp = len(list(i for i in x if i <= 0))
x = x[tmp:]
#Catch inf, those could be a problem with the log scale...
#tmp2 = 0
tmp2 = len(list(i for i in x if i > 0 and numpy.isinf(i)))
if tmp2 > 0:
x = x[:-tmp2]
#xbound = max(abs(numpy.floor(numpy.log10(x[0]))),
# abs(numpy.ceil(numpy.log10(x[-1]))))
if len(x) == 0:
res.append(
plt.axhline(tmp / float(n),
label=label,
linewidth=3.,
**rldStyles[j]))
# tmp/float(n) == (n-tmp2)/float(n) # TODO: check
else:
x2 = numpy.hstack([numpy.repeat(x, 2)])
#maxEvalsF: used for the limit of the plot.
y2 = numpy.hstack([
tmp / float(n),
numpy.repeat(numpy.arange(tmp + 1, n - tmp2) / float(n), 2),
(n - tmp2) / float(n)
])
#res.extend(plt.plot(x2, y2, label=label, linewidth=3., **rldStyles[i]))
plotArgs = rldStyles[j].copy()
plotArgs['label'] = label
plotArgs['linewidth'] = 3.
#res.extend(plotUnifLogXMarkers(x2, y2, 3, plotArgs))
res.append(
plotUnifLogXMarkers(x2,
y2,
nbperdecade=3,
logscale=False,
**plotArgs)[0])
# TODO: check if all of evalsX[func] is numpy.inf and so on...
return res
def extractBestAlgorithms(args=algs2009,
f_factor=2,
target_lb=1e-8,
target_ub=1e22):
"""Returns (and prints) per dimension a list of algorithms within
algorithm list args that contains an algorithm if for any
dimension/target/function pair this algorithm:
- is the best algorithm wrt ERT
- its own ERT lies within a factor f_factor of the best ERT
- there is no algorithm within a factor of f_factor of the best ERT
and the current algorithm is the second best.
"""
# TODO: use pproc.TargetValues class as input target values
# default target values:
targets = pproc.TargetValues(10**np.arange(
np.log10(max((1e-8, target_lb))),
np.log10(target_ub) + 1e-9, 0.2))
# there should be a simpler way to express this to become the
# interface of this function
print 'Loading algorithm data from given algorithm list...\n'
verbose = True
dsList, sortedAlgs, dictAlg = pproc.processInputArgs(args, verbose=verbose)
print 'This may take a while (depending on the number of algorithms)'
selectedAlgsPerProblem = {}
for f, i in pproc.dictAlgByFun(dictAlg).iteritems():
for d, j in pproc.dictAlgByDim(i).iteritems():
selectedAlgsPerProblemDF = []
best = BestAlgSet(j)
for i in range(0, len(best.target)):
t = best.target[i]
# if ((t <= target_ub) and (t >= target_lb)):
if toolsstats.in_approximately(
t, targets((f, d), discretize=True)):
# add best for this target:
selectedAlgsPerProblemDF.append(best.algs[i])
# add second best or all algorithms that have an ERT
# within a factor of f_factor of the best:
secondbest_ERT = np.infty
secondbest_str = ''
secondbest_included = False
for astring in j:
currdictalg = dictAlg[astring].dictByDim()
if currdictalg.has_key(d):
curralgdata = currdictalg[d][f - 1]
currERT = curralgdata.detERT([t])[0]
if (astring != best.algs[i]):
if (currERT < secondbest_ERT):
secondbest_ERT = currERT
secondbest_str = astring
if (currERT <= best.detERT([t])[0] * f_factor):
selectedAlgsPerProblemDF.append(astring)
secondbest_included = True
if not (secondbest_included) and (secondbest_str != ''):
selectedAlgsPerProblemDF.append(secondbest_str)
if len(selectedAlgsPerProblemDF) > 0:
selectedAlgsPerProblem[(d, f)] = selectedAlgsPerProblemDF
print 'pre-processing of function', f, 'done.'
print 'loading of best algorithm(s) data done.'
countsperalgorithm = {}
for (d, f) in selectedAlgsPerProblem:
print 'dimension:', d, ', function:', f
setofalgs = set(selectedAlgsPerProblem[d, f])
# now count how often algorithm a is best for the extracted targets
for a in setofalgs:
# use setdefault to initialize with zero if a entry not existant:
countsperalgorithm.setdefault((d, a), 0)
countsperalgorithm[(d, a)] += selectedAlgsPerProblem[d, f].count(a)
selectedalgsperdimension = {}
for (d, a) in sorted(countsperalgorithm):
if not selectedalgsperdimension.has_key(d):
selectedalgsperdimension[d] = []
selectedalgsperdimension[d].append((countsperalgorithm[(d, a)], a))
for d in sorted(selectedalgsperdimension):
print d, 'D:'
for (count, alg) in sorted(selectedalgsperdimension[d], reverse=True):
print count, alg
print '\n'
print " done."
return selectedalgsperdimension
figure()
bb.ppfigdim.plot(ds)
bb.ppfigdim.beautify()
bb.ppfigdim.plot_previous_algorithms(2) # plot BBOB 2009 best algorithm on fun 2
"""
import os
import sys
import matplotlib.pyplot as plt
import numpy as np
from pdb import set_trace
from bbob_pproc import genericsettings, toolsstats, bestalg, pproc
from bbob_pproc.ppfig import saveFigure, groupByRange
values_of_interest = pproc.TargetValues((10, 1, 1e-1, 1e-2, 1e-3, 1e-5, 1e-8)) # to rename!?
xlim_max = None
ynormalize_by_dimension = True # not at all tested yet
styles = [ # sort of rainbow style, most difficult (red) first
{'color': 'r', 'marker': 'o', 'markeredgecolor': 'k', 'markeredgewidth': 2, 'linewidth': 4},
{'color': 'm', 'marker': '.', 'linewidth': 4},
{'color': 'y', 'marker': '^', 'markeredgecolor': 'k', 'markeredgewidth': 2, 'linewidth': 4},
{'color': 'g', 'marker': '.', 'linewidth': 4},
{'color': 'c', 'marker': 'v', 'markeredgecolor': 'k', 'markeredgewidth': 2, 'linewidth': 4},
{'color': 'b', 'marker': '.', 'linewidth': 4},
{'color': 'k', 'marker': 'o', 'markeredgecolor': 'k', 'markeredgewidth': 2, 'linewidth': 4},
]
refcolor = 'wheat'
import numpy
import numpy as np
from pdb import set_trace
from matplotlib import pyplot as plt
try:
from matplotlib.transforms import blended_transform_factory as blend
except ImportError:
# compatibility matplotlib 0.8
from matplotlib.transforms import blend_xy_sep_transform as blend
from bbob_pproc import readalign
from bbob_pproc.ppfig import saveFigure
from bbob_pproc import toolsdivers
from bbob_pproc import pproc
dimensions = (2, 3, 5, 10, 20, 40)
fixed_targets = pproc.TargetValues(np.logspace(-8, 2, 46))
runlength_based_targets = pproc.RunlengthBasedTargetValues(np.logspace(numpy.log10(0.5), numpy.log10(50), 8))
# runlength_based_targets = pproc.RunlengthBasedTargetValues([0.5, 1, 3, 10, 50])
targets = fixed_targets # default
# formattings
colors = ('c', 'g', 'b', 'k', 'r', 'm', 'k', 'y', 'k', 'c', 'r', 'm')
markers = ('+', 'v', '*', 'o', 's', 'D', 'x')
markersize = 14 # modified in config.py
linewidth = 3
max_evals_line_length = 9 # length away from the diagonal as a factor, line indicates maximal evaluations for each data
offset = 0. #0.02 offset provides a way to move away the box boundaries to display the outer markers fully, clip_on=False is more effective
caption_start_fixed = r"""Expected running time (\ERT\ in $\log_{10}$ of number of function evaluations)
of \algorithmB\ ($x$-axis) versus \algorithmA\ ($y$-axis) for $NBTARGETS$ target values
$\Df \in [NBLOW, NBUP]$ in each dimension on functions #1. """
from __future__ import absolute_import import os import warnings from pdb import set_trace import numpy as np import matplotlib.pyplot as plt from bbob_pproc import toolsstats, bestalg, genericsettings from bbob_pproc import pproc as pp # import dictAlgByDim, dictAlgByFun from bbob_pproc import toolsdivers # strip_pathname, str_to_latex from bbob_pproc import pprldistr # plotECDF, beautifyECDF from bbob_pproc import ppfig # consecutiveNumbers, saveFigure, plotUnifLogXMarkers, logxticks from bbob_pproc import pptex # numtotex displaybest2009 = True target_values = pp.TargetValues(10**np.arange(2, -8, -0.2)) # changed in config.py x_limit = None # not sure whether this is necessary/useful x_limit_default = 1e7 # better: 10 * genericsettings.evaluation_setting[1], noisy: 1e8, otherwise: 1e7. maximal run length shown annotation_line_end_relative = 1.11 # lines between graph and annotation annotation_space_end_relative = 1.24 # figure space end relative to x_limit save_zoom = False # save zoom into left and right part of the figures perfprofsamplesize = genericsettings.simulated_runlength_bootstrap_sample_size_rld # number of bootstrap samples drawn for each fct+target in the performance profile dpi_global_var = 100 # 100 ==> 800x600 (~160KB), 120 ==> 960x720 (~200KB), 150 ==> 1200x900 (~300KB) looks ugly in latex nbperdecade = 1 median_max_evals_marker_format = ['x', 24, 3] styles = [d.copy() for d in genericsettings.line_styles] # deep copy refcolor = 'wheat' # TODO: update the list below which are not relevant anymore
