def main(dsList0,
dsList1,
dim,
targetsOfInterest=None,
outputdir='',
info='default',
verbose=True):
"""Generate figures of empirical cumulative distribution functions.
:param DataSetList dsList0: data set of reference algorithm
:param DataSetList dsList1: data set of algorithm of concern
:param int dim: dimension
:param TargetValues targetsOfInterest: target function values to be
displayed
:param bool isStoringXMax: if set to True, the first call BeautifyVD
sets the globals :py:data:`fmax` and
:py:data:`maxEvals` and all subsequent
calls will use these values as rightmost
xlim in the generated figures.
:param string outputdir: output directory (must exist)
:param string info: string suffix for output file names.
:param bool verbose: control verbosity
Outputs:
Image files of the empirical cumulative distribution functions.
"""
#plt.rc("axes", labelsize=20, titlesize=24)
#plt.rc("xtick", labelsize=20)
#plt.rc("ytick", labelsize=20)
#plt.rc("font", size=20)
#plt.rc("legend", fontsize=20)
figureName = os.path.join(outputdir, 'pplogabs_%s' % (info))
handles = plotLogAbs(dsList0,
dsList1,
dim,
targetsOfInterest,
verbose=verbose)
beautify(handles)
funcs = set(dsList0.dictByFunc().keys()) & set(dsList1.dictByFunc().keys())
text = 'f%s' % consecutiveNumbers(sorted(funcs))
if len(dsList0.dictByDim().keys()) == len(dsList1.dictByDim().keys()) == 1:
text += ',%d-D' % dsList0.dictByDim().keys()[0]
plt.text(0.98,
0.02,
text,
horizontalalignment="right",
transform=plt.gca().transAxes)
saveFigure(figureName, verbose=verbose)
plt.close()
def plot(dsList, targets=single_target_values, **plotArgs):
"""Plot ECDF of evaluations and final function values
in a single figure for demonstration purposes."""
# targets = targets() # TODO: this needs to be rectified
# targets = targets.target_values
dsList = pproc.DataSetList(dsList)
assert len(dsList.dictByDim()) == 1, ('Cannot display different '
'dimensionalities together')
res = []
plt.subplot(121)
maxEvalsFactor = max(i.mMaxEvals() / i.dim for i in dsList)
evalfmax = maxEvalsFactor
for j in range(len(targets)):
tmpplotArgs = dict(plotArgs, **rldStyles[j % len(rldStyles)])
tmp = plotRLDistr(dsList, lambda fun_dim: targets(fun_dim)[j],
**tmpplotArgs)
res.extend(tmp)
res.append(plt.axvline(x=maxEvalsFactor, color='k', **plotArgs))
funcs = list(i.funcId for i in dsList)
text = 'f%s' % (consecutiveNumbers(sorted(funcs)))
res.append(
plt.text(0.5,
0.98,
text,
horizontalalignment="center",
verticalalignment="top",
transform=plt.gca().transAxes))
plt.subplot(122)
for j in [range(len(targets))[-1]]:
tmpplotArgs = dict(plotArgs, **rldStyles[j % len(rldStyles)])
tmp = plotFVDistr(dsList, evalfmax,
lambda fun_dim: targets(fun_dim)[j], **tmpplotArgs)
res.extend(tmp)
tmp = np.floor(np.log10(evalfmax))
# coloring right to left:
maxEvalsF = np.power(10, np.arange(0, tmp))
for j in range(len(maxEvalsF)):
tmpplotArgs = dict(plotArgs,
**rldUnsuccStyles[j % len(rldUnsuccStyles)])
tmp = plotFVDistr(dsList, maxEvalsF[j],
lambda fun_dim: targets(fun_dim)[-1], **tmpplotArgs)
res.extend(tmp)
res.append(
plt.text(0.98,
0.02,
text,
horizontalalignment="right",
transform=plt.gca().transAxes))
return res
def main(dsList0, dsList1, valuesOfInterest=None,
outputdir='', info='default', verbose=True):
"""Generate figures of empirical cumulative distribution functions.
:param DataSetList dsList0: data set of reference algorithm
:param DataSetList dsList1: data set of algorithm of concern
:param sequence valuesOfInterest: target function values to be
displayed
:param bool isStoringXMax: if set to True, the first call BeautifyVD
sets the globals :py:data:`fmax` and
:py:data:`maxEvals` and all subsequent
calls will use these values as rightmost
xlim in the generated figures.
:param string outputdir: output directory (must exist)
:param string info: string suffix for output file names.
:param bool verbose: control verbosity
Outputs:
Image files of the empirical cumulative distribution functions.
"""
#plt.rc("axes", labelsize=20, titlesize=24)
#plt.rc("xtick", labelsize=20)
#plt.rc("ytick", labelsize=20)
#plt.rc("font", size=20)
#plt.rc("legend", fontsize=20)
figureName = os.path.join(outputdir,'pplogabs_%s' %(info))
handles = plotLogAbs(dsList0, dsList1,
valuesOfInterest, verbose=verbose)
beautify(handles)
funcs = set(dsList0.dictByFunc().keys()) & set(dsList1.dictByFunc().keys())
text = 'f%s' % consecutiveNumbers(sorted(funcs))
if len(dsList0.dictByDim().keys()) == len(dsList1.dictByDim().keys()) == 1:
text += ',%d-D' % dsList0.dictByDim().keys()[0]
plt.text(0.98, 0.02, text, horizontalalignment="right",
transform=plt.gca().transAxes)
saveFigure(figureName, verbose=verbose)
plt.close()
def plot(dsList, targets = single_target_values, **plotArgs):
"""Plot ECDF of evaluations and final function values
in a single figure for demonstration purposes."""
# targets = targets() # TODO: this needs to be rectified
# targets = targets.target_values
dsList = pproc.DataSetList(dsList)
assert len(dsList.dictByDim()) == 1, ('Cannot display different '
'dimensionalities together')
res = []
plt.subplot(121)
maxEvalsFactor = max(i.mMaxEvals() / i.dim for i in dsList)
evalfmax = maxEvalsFactor
for j in range(len(targets)):
tmpplotArgs = dict(plotArgs, **rldStyles[j % len(rldStyles)])
tmp = plotRLDistr(dsList, lambda fun_dim: targets(fun_dim)[j], **tmpplotArgs)
res.extend(tmp)
res.append(plt.axvline(x = maxEvalsFactor, color = 'k', **plotArgs))
funcs = list(i.funcId for i in dsList)
text = consecutiveNumbers(sorted(funcs), 'f')
res.append(plt.text(0.5, 0.98, text, horizontalalignment = "center",
verticalalignment = "top", transform = plt.gca().transAxes))
plt.subplot(122)
for j in [range(len(targets))[-1]]:
tmpplotArgs = dict(plotArgs, **rldStyles[j % len(rldStyles)])
tmp = plotFVDistr(dsList, evalfmax, lambda fun_dim: targets(fun_dim)[j], **tmpplotArgs)
if tmp:
res.extend(tmp)
tmp = np.floor(np.log10(evalfmax))
# coloring right to left:
maxEvalsF = np.power(10, np.arange(0, tmp))
for j in range(len(maxEvalsF)):
tmpplotArgs = dict(plotArgs, **rldUnsuccStyles[j % len(rldUnsuccStyles)])
tmp = plotFVDistr(dsList, maxEvalsF[j], lambda fun_dim: targets(fun_dim)[-1], **tmpplotArgs)
if tmp:
res.extend(tmp)
res.append(plt.text(0.98, 0.02, text, horizontalalignment = "right",
transform = plt.gca().transAxes))
return res
def main(dsList,
isStoringXMax=False,
outputdir='',
info='default',
verbose=True):
"""Generate figures of empirical cumulative distribution functions.
This method has a feature which allows to keep the same boundaries
for the x-axis, if ``isStoringXMax==True``. This makes sense when
dealing with different functions or subsets of functions for one
given dimension.
CAVE: this is bug-prone, as some data depend on the maximum
evaluations and the appearence therefore depends on the
calling order.
:param DataSetList dsList: list of DataSet instances to process.
:param bool isStoringXMax: if set to True, the first call
:py:func:`beautifyFVD` sets the
globals :py:data:`fmax` and
:py:data:`maxEvals` and all subsequent
calls will use these values as rightmost
xlim in the generated figures.
:param string outputdir: output directory (must exist)
:param string info: string suffix for output file names.
:param bool verbose: control verbosity
"""
# plt.rc("axes", labelsize=20, titlesize=24)
# plt.rc("xtick", labelsize=20)
# plt.rc("ytick", labelsize=20)
# plt.rc("font", size=20)
# plt.rc("legend", fontsize=20)
targets = single_target_values # convenience abbreviation
for d, dictdim in dsList.dictByDim().iteritems():
maxEvalsFactor = max(i.mMaxEvals() / d for i in dictdim)
if isStoringXMax:
global evalfmax
else:
evalfmax = None
if not evalfmax:
evalfmax = maxEvalsFactor
if runlen_xlimits_max is not None:
evalfmax = runlen_xlimits_max
# first figure: Run Length Distribution
filename = os.path.join(outputdir, 'pprldistr_%02dD_%s' % (d, info))
fig = plt.figure()
for j in range(len(targets)):
plotRLDistr(
dictdim,
lambda fun_dim: targets(fun_dim)[j],
targets.label(j) if isinstance(
targets, pproc.RunlengthBasedTargetValues) else
targets.loglabel(j),
evalfmax, # can be larger maxEvalsFactor with no effect
**rldStyles[j % len(rldStyles)])
funcs = list(i.funcId for i in dictdim)
text = 'f%s' % (consecutiveNumbers(sorted(funcs)))
text += ',%d-D' % d
if (1):
# try:
if not isinstance(targets, pproc.RunlengthBasedTargetValues):
# if targets.target_values[-1] == 1e-8: # this is a hack
plot_previous_algorithms(d, funcs)
else:
plotRLB_previous_algorithms(d, funcs)
# except:
# pass
plt.axvline(x=maxEvalsFactor, color='k') # vertical line at maxevals
plt.legend(loc='best')
plt.text(0.5,
0.98,
text,
horizontalalignment="center",
verticalalignment="top",
transform=plt.gca().transAxes
# bbox=dict(ec='k', fill=False)
)
try: # was never tested, so let's make it safe
if len(funcs) == 1:
plt.title(genericsettings.current_testbed.info(funcs[0])[:27])
except:
warnings.warn('could not print title')
beautifyRLD(evalfmax)
saveFigure(filename, verbose=verbose)
plt.close(fig)
# second figure: Function Value Distribution
filename = os.path.join(outputdir, 'ppfvdistr_%02dD_%s' % (d, info))
fig = plt.figure()
plotFVDistr(dictdim, np.inf, 1e-8, **rldStyles[-1])
# coloring right to left
for j, max_eval_factor in enumerate(single_runlength_factors):
if max_eval_factor > maxEvalsFactor:
break
plotFVDistr(dictdim, max_eval_factor, 1e-8,
**rldUnsuccStyles[j % len(rldUnsuccStyles)])
plt.text(
0.98,
0.02,
text,
horizontalalignment="right",
transform=plt.gca().transAxes) # bbox=dict(ec='k', fill=False),
beautifyFVD(isStoringXMax=isStoringXMax, ylabel=False)
saveFigure(filename, verbose=verbose)
plt.close(fig)
def comp(dsList0,
dsList1,
targets,
isStoringXMax=False,
outputdir='',
info='default',
verbose=True):
"""Generate figures of ECDF that compare 2 algorithms.
:param DataSetList dsList0: list of DataSet instances for ALG0
:param DataSetList dsList1: list of DataSet instances for ALG1
:param seq targets: target function values to be displayed
:param bool isStoringXMax: if set to True, the first call
:py:func:`beautifyFVD` sets the globals
:py:data:`fmax` and :py:data:`maxEvals`
and all subsequent calls will use these
values as rightmost xlim in the generated
figures.
:param string outputdir: output directory (must exist)
:param string info: string suffix for output file names.
:param bool verbose: control verbosity
"""
# plt.rc("axes", labelsize=20, titlesize=24)
# plt.rc("xtick", labelsize=20)
# plt.rc("ytick", labelsize=20)
# plt.rc("font", size=20)
# plt.rc("legend", fontsize=20)
if not isinstance(targets, pproc.RunlengthBasedTargetValues):
targets = pproc.TargetValues.cast(targets)
dictdim0 = dsList0.dictByDim()
dictdim1 = dsList1.dictByDim()
for d in set(dictdim0.keys()) & set(dictdim1.keys()):
maxEvalsFactor = max(max(i.mMaxEvals() / d for i in dictdim0[d]),
max(i.mMaxEvals() / d for i in dictdim1[d]))
if isStoringXMax:
global evalfmax
else:
evalfmax = None
if not evalfmax:
evalfmax = maxEvalsFactor**1.05
if runlen_xlimits_max is not None:
evalfmax = runlen_xlimits_max
filename = os.path.join(outputdir, 'pprldistr_%02dD_%s' % (d, info))
fig = plt.figure()
for j in range(len(targets)):
tmp = plotRLDistr(dictdim0[d],
lambda fun_dim: targets(fun_dim)[j],
targets.label(j) if isinstance(
targets, pproc.RunlengthBasedTargetValues)
else targets.loglabel(j),
marker=genericsettings.line_styles[1]['marker'],
**rldStyles[j % len(rldStyles)])
plt.setp(tmp[-1], label=None) # Remove automatic legend
# Mods are added after to prevent them from appearing in the legend
plt.setp(tmp,
markersize=20.,
markeredgewidth=plt.getp(tmp[-1], 'linewidth'),
markeredgecolor=plt.getp(tmp[-1], 'color'),
markerfacecolor='none')
tmp = plotRLDistr(dictdim1[d],
lambda fun_dim: targets(fun_dim)[j],
targets.label(j) if isinstance(
targets, pproc.RunlengthBasedTargetValues)
else targets.loglabel(j),
marker=genericsettings.line_styles[0]['marker'],
**rldStyles[j % len(rldStyles)])
# modify the automatic legend: remover marker and change text
plt.setp(tmp[-1],
marker='',
label=targets.label(j) if isinstance(
targets, pproc.RunlengthBasedTargetValues) else
targets.loglabel(j))
# Mods are added after to prevent them from appearing in the legend
plt.setp(tmp,
markersize=15.,
markeredgewidth=plt.getp(tmp[-1], 'linewidth'),
markeredgecolor=plt.getp(tmp[-1], 'color'),
markerfacecolor='none')
funcs = set(i.funcId for i in dictdim0[d]) | set(i.funcId
for i in dictdim1[d])
text = 'f%s' % (consecutiveNumbers(sorted(funcs)))
if not isinstance(targets, pproc.RunlengthBasedTargetValues):
plot_previous_algorithms(d, funcs)
else:
plotRLB_previous_algorithms(d, funcs)
# plt.axvline(max(i.mMaxEvals()/i.dim for i in dictdim0[d]), ls='--', color='k')
# plt.axvline(max(i.mMaxEvals()/i.dim for i in dictdim1[d]), color='k')
plt.axvline(max(i.mMaxEvals() / i.dim for i in dictdim0[d]),
marker='+',
markersize=20.,
color='k',
markeredgewidth=plt.getp(
tmp[-1],
'linewidth',
))
plt.axvline(max(i.mMaxEvals() / i.dim for i in dictdim1[d]),
marker='o',
markersize=15.,
color='k',
markerfacecolor='None',
markeredgewidth=plt.getp(tmp[-1], 'linewidth'))
plt.legend(loc='best')
plt.text(
0.5,
0.98,
text,
horizontalalignment="center",
verticalalignment="top",
transform=plt.gca().transAxes) # bbox=dict(ec='k', fill=False),
beautifyRLD(evalfmax)
saveFigure(filename, verbose=verbose)
plt.close(fig)
def main(dictAlg,
order=None,
outputdir='.',
info='default',
dimension=None,
verbose=True):
"""Generates a figure showing the performance of algorithms.
From a dictionary of :py:class:`DataSetList` sorted by algorithms,
generates the cumulative distribution function of the bootstrap
distribution of ERT for algorithms on multiple functions for
multiple targets altogether.
:param dict dictAlg: dictionary of :py:class:`DataSetList` instances
one instance is equivalent to one algorithm,
:param list targets: target function values
:param list order: sorted list of keys to dictAlg for plotting order
:param str outputdir: output directory
:param str info: output file name suffix
:param bool verbose: controls verbosity
"""
global x_limit # late assignment of default, because it can be set to None in config
global divide_by_dimension # not fully implemented/tested yet
if 'x_limit' not in globals() or x_limit is None:
x_limit = x_limit_default
tmp = pp.dictAlgByDim(dictAlg)
# tmp = pp.DictAlg(dictAlg).by_dim()
if len(tmp) != 1 and dimension is None:
raise ValueError('We never integrate over dimension.')
if dimension is not None:
if dimension not in tmp.keys():
raise ValueError('dimension %d not in dictAlg dimensions %s' %
(dimension, str(tmp.keys())))
tmp = {dimension: tmp[dimension]}
dim = tmp.keys()[0]
divisor = dim if divide_by_dimension else 1
algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []]
dictFunc = pp.dictAlgByFun(dictAlg)
# Collect data
# Crafting effort correction: should we consider any?
CrEperAlg = {}
for alg in algorithms_with_data:
CrE = 0.
if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL':
tmp = dictAlg[alg].dictByNoise()
assert len(tmp.keys()) == 1
if tmp.keys()[0] == 'noiselessall':
CrE = 0.5117
elif tmp.keys()[0] == 'nzall':
CrE = 0.6572
CrEperAlg[alg] = CrE
if CrE != 0.0:
print 'Crafting effort for', alg, 'is', CrE
dictData = {} # list of (ert per function) per algorithm
dictMaxEvals = {} # list of (maxevals per function) per algorithm
bestERT = [] # best ert per function
# funcsolved = [set()] * len(targets) # number of functions solved per target
xbest2009 = []
maxevalsbest2009 = []
for f, dictAlgperFunc in dictFunc.iteritems():
if function_IDs and f not in function_IDs:
continue
# print target_values((f, dim))
for j, t in enumerate(target_values((f, dim))):
# for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)):
# funcsolved[j].add(f)
for alg in algorithms_with_data:
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
try:
entry = dictAlgperFunc[alg][
0] # one element per fun and per dim.
evals = entry.detEvals([t])[0]
assert entry.dim == dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(runlengthsucc,
runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
except (KeyError, IndexError):
#set_trace()
warntxt = (
'Data for algorithm %s on function %d in %d-D ' %
(alg, f, dim) + 'are missing.\n')
warnings.warn(warntxt)
dictData.setdefault(alg, []).extend(x)
dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc)
if displaybest2009:
#set_trace()
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgentry = bestalg.bestalgentries2009[(dim, f)]
bestalgevals = bestalgentry.detEvals(target_values((f, dim)))
# print bestalgevals
for j in range(len(bestalgevals[0])):
if bestalgevals[1][j]:
evals = bestalgevals[0][j]
#set_trace()
assert dim == bestalgentry.dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = bestalgentry.maxevals[
bestalgevals[1][j]][np.isnan(evals)] / divisor
x = toolsstats.drawSP(runlengthsucc,
runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
else:
x = perfprofsamplesize * [np.inf]
runlengthunsucc = []
xbest2009.extend(x)
maxevalsbest2009.extend(runlengthunsucc)
if order is None:
order = dictData.keys()
# Display data
lines = []
if displaybest2009:
args = {
'ls': '-',
'linewidth': 6,
'marker': 'D',
'markersize': 11.,
'markeredgewidth': 1.5,
'markerfacecolor': refcolor,
'markeredgecolor': refcolor,
'color': refcolor,
'label': 'best 2009',
'zorder': -1
}
lines.append(
plotdata(np.array(xbest2009),
x_limit,
maxevalsbest2009,
CrE=0.,
**args))
def algname_to_label(algname, dirname=None):
"""to be extended to become generally useful"""
if isinstance(algname, (tuple, list)): # not sure this is needed
return ' '.join([str(name) for name in algname])
return str(algname)
for i, alg in enumerate(order):
try:
data = dictData[alg]
maxevals = dictMaxEvals[alg]
except KeyError:
continue
args = styles[(i) % len(styles)]
args['linewidth'] = 1.5
args['markersize'] = 12.
args['markeredgewidth'] = 1.5
args['markerfacecolor'] = 'None'
args['markeredgecolor'] = args['color']
args['label'] = algname_to_label(alg)
#args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib
#elif len(show_algorithms) > 0:
#args['color'] = 'wheat'
#args['ls'] = '-'
#args['zorder'] = -1
# plotdata calls pprldistr.plotECDF which calls ppfig.plotUnifLog... which does the work
lines.append(
plotdata(np.array(data),
x_limit,
maxevals,
CrE=CrEperAlg[alg],
**args))
labels, handles = plotLegend(lines, x_limit)
if True: # isLateXLeg:
fileName = os.path.join(outputdir, 'pprldmany_%s.tex' % (info))
with open(fileName, 'w') as f:
f.write(r'\providecommand{\nperfprof}{7}')
algtocommand = {} # latex commands
for i, alg in enumerate(order):
tmp = r'\alg%sperfprof' % pptex.numtotext(i)
f.write(
r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' %
(tmp,
toolsdivers.str_to_latex(
toolsdivers.strip_pathname2(algname_to_label(alg)))))
algtocommand[algname_to_label(alg)] = tmp
if displaybest2009:
tmp = r'\algzeroperfprof'
f.write(r'\providecommand{%s}{best 2009}' % (tmp))
algtocommand['best 2009'] = tmp
commandnames = []
for label in labels:
commandnames.append(algtocommand[label])
# f.write(headleg)
if len(
order
) > 28: # latex sidepanel won't work well for more than 25 algorithms, but original labels are also clipped
f.write(
r'\providecommand{\perfprofsidepanel}{\mbox{%s}\vfill\mbox{%s}}'
% (commandnames[0], commandnames[-1]))
else:
fontsize_command = r'\tiny{}' if len(order) > 19 else ''
f.write(r'\providecommand{\perfprofsidepanel}{{%s\mbox{%s}' %
(fontsize_command,
commandnames[0])) # TODO: check len(labels) > 0
for i in range(1, len(labels)):
f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i])
f.write('}}\n')
# f.write(footleg)
if verbose:
print 'Wrote right-hand legend in %s' % fileName
figureName = os.path.join(outputdir, 'pprldmany_%s' % (info))
#beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat)
beautify()
text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys())))
text += ',%d-D' % dim # TODO: this is strange when different dimensions are plotted
plt.text(0.01,
0.98,
text,
horizontalalignment="left",
verticalalignment="top",
transform=plt.gca().transAxes)
if len(dictFunc) == 1:
plt.title(' '.join(
(str(dictFunc.keys()[0]),
genericsettings.current_testbed.short_names[dictFunc.keys()[0]])))
a = plt.gca()
plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative)
xticks, labels = plt.xticks()
tmp = []
for i in xticks:
tmp.append('%d' % round(np.log10(i)))
a.set_xticklabels(tmp)
if save_figure:
ppfig.saveFigure(figureName, verbose=verbose)
if len(dictFunc) == 1:
ppfig.save_single_functions_html(
os.path.join(outputdir, 'pprldmany'),
'', # algorithms names are clearly visible in the figure
add_to_names='_%02dD' % (dim),
algorithmCount=ppfig.AlgorithmCount.NON_SPECIFIED)
if close_figure:
plt.close()
def main(dsList,
valuesOfInterest,
isStoringXMax=False,
outputdir='',
info='default',
verbose=True):
"""Generate figures of empirical cumulative distribution functions.
Keyword arguments:
dsList -- list of DataSet instances to process.
valuesOfInterest -- target function values to be displayed.
isStoringXMax -- if set to True, the first call BeautifyVD sets the globals
fmax and maxEvals and all subsequent calls will use these
values as rightmost xlim in the generated figures.
outputdir -- output directory (must exist)
info --- string suffix for output file names.
Outputs:
Image files of the empirical cumulative distribution functions.
"""
plt.rc("axes", labelsize=20, titlesize=24)
plt.rc("xtick", labelsize=20)
plt.rc("ytick", labelsize=20)
plt.rc("font", size=20)
plt.rc("legend", fontsize=20)
maxEvalsFactor = max(i.mMaxEvals() / i.dim for i in dsList)
#maxEvalsFactorCeil = numpy.power(10,
#numpy.ceil(numpy.log10(maxEvalsFactor)))
if isStoringXMax:
global evalfmax
else:
evalfmax = None
if not evalfmax:
evalfmax = maxEvalsFactor
figureName = os.path.join(outputdir, 'pprldistr_%s' % (info))
fig = plt.figure()
legend = []
for j in range(len(valuesOfInterest)):
tmp = plotRLDistr(dsList,
valuesOfInterest[j],
evalfmax,
verbose=verbose)
if not tmp is None:
plt.setp(tmp, 'color', rldColors[j])
if rldColors[j] == 'r': # 1e-8 in bold
plt.setp(tmp, 'linewidth', 3)
funcs = list(i.funcId for i in dsList)
text = 'f%s' % (consecutiveNumbers(sorted(funcs)))
if isAlgorithm2009Found:
d = set(i.dim for i in dsList).pop() # Get only one element...
for alg in dict2009:
x = []
nn = 0
try:
tmp = dict2009[alg]
for f in funcs:
tmp[f][d] # simply test that they exists
except KeyError:
continue
for f in funcs:
tmp2 = tmp[f][d][0][1:]
# [0], because the maximum #evals is also recorded
# [1:] because the target function value is recorded
x.append(tmp2[numpy.isnan(tmp2) == False])
nn += len(tmp2)
if x:
x.append([(evalfmax * d)**1.05])
x = numpy.hstack(x)
plotECDF(x[numpy.isfinite(x)] / float(d), nn, {
'color': 'wheat',
'ls': '-',
'zorder': -1
})
plt.axvline(x=maxEvalsFactor, color='k')
beautifyRLD(fig,
figureName,
evalfmax,
fileFormat=figformat,
text=text,
verbose=verbose)
plt.close(fig)
figureName = os.path.join(outputdir, 'ppfvdistr_%s' % (info))
fig = plt.figure()
for j in range(len(valuesOfInterest)):
tmp = plotFVDistr(dsList,
valuesOfInterest[j],
evalfmax,
verbose=verbose)
plt.setp(tmp, 'color', rldColors[j])
if rldColors[j] == 'r': # 1e-8 in bold
plt.setp(tmp, 'linewidth', 3)
tmp = numpy.floor(numpy.log10(evalfmax))
# coloring left to right:
#maxEvalsF = numpy.power(10, numpy.arange(tmp, 0, -1) - 1)
# coloring right to left:
maxEvalsF = numpy.power(10, numpy.arange(0, tmp))
for k in range(len(maxEvalsF)):
tmp = plotFVDistr(dsList,
valuesOfInterest[-1],
maxEvalsF=maxEvalsF[k],
verbose=verbose)
plt.setp(tmp, 'color', rldUnsuccColors[k])
beautifyFVD(fig,
figureName,
fileFormat=figformat,
text=text,
isStoringXMax=isStoringXMax,
verbose=verbose)
plt.close(fig)
plt.rcdefaults()
def comp(dsList0,
dsList1,
valuesOfInterest,
isStoringXMax=False,
outputdir='',
info='default',
verbose=True):
"""Generate figures of empirical cumulative distribution functions.
Dashed lines will correspond to ALG0 and solid lines to ALG1.
Keyword arguments:
dsList0 -- list of DataSet instances for ALG0.
dsList1 -- list of DataSet instances for ALG1
valuesOfInterest -- target function values to be displayed.
isStoringXMax -- if set to True, the first call BeautifyVD sets the globals
fmax and maxEvals and all subsequent calls will use these
values as rightmost xlim in the generated figures.
-- if set to True, the first call BeautifyVD sets the global
fmax and all subsequent call will have the same maximum
xlim.
outputdir -- output directory (must exist)
info --- string suffix for output file names.
Outputs:
Image files of the empirical cumulative distribution functions.
"""
plt.rc("axes", labelsize=20, titlesize=24)
plt.rc("xtick", labelsize=20)
plt.rc("ytick", labelsize=20)
plt.rc("font", size=20)
plt.rc("legend", fontsize=20)
maxEvalsFactor = max(max(i.mMaxEvals() / i.dim for i in dsList0),
max(i.mMaxEvals() / i.dim for i in dsList1))
if isStoringXMax:
global evalfmax
else:
evalfmax = None
if not evalfmax:
evalfmax = maxEvalsFactor
figureName = os.path.join(outputdir, 'pprldistr2_%s' % (info))
fig = plt.figure()
legend = []
for j in range(len(valuesOfInterest)):
tmp = plotRLDistr(dsList0,
valuesOfInterest[j],
evalfmax,
verbose=verbose)
if not tmp is None:
plt.setp(tmp, 'color', rldColors[j])
plt.setp(tmp, 'ls', '--')
plt.setp(tmp, 'label', None) # Hack for the legend
if rldColors[j] == 'r': # 1e-8 in bold
plt.setp(tmp, 'linewidth', 3)
tmp = plotRLDistr(dsList1,
valuesOfInterest[j],
evalfmax,
verbose=verbose)
if not tmp is None:
plt.setp(tmp, 'color', rldColors[j])
# Hack for the legend.
plt.setp(tmp, 'label',
('%+d' % (numpy.log10(valuesOfInterest[j][1]))))
if rldColors[j] == 'r': # 1e-8 in bold
plt.setp(tmp, 'linewidth', 3)
funcs = set(i.funcId for i in dsList0) | set(i.funcId for i in dsList1)
text = 'f%s' % (consecutiveNumbers(sorted(funcs)))
if isAlgorithm2009Found:
d = set.union(set(i.dim for i in dsList0),
set(i.dim
for i in dsList1)).pop() # Get only one element...
for alg in dict2009:
x = []
nn = 0
try:
tmp = dict2009[alg]
for f in funcs:
tmp[f][d] # simply test that they exists
except KeyError:
continue
for f in funcs:
tmp2 = tmp[f][d][0][1:]
# [0], because the maximum #evals is also recorded
# [1:] because the target function value is recorded
x.append(tmp2[numpy.isnan(tmp2) == False])
nn += len(tmp2)
if x:
x.append([(evalfmax * d)**1.05])
x = numpy.hstack(x)
plotECDF(x[numpy.isfinite(x)] / d, nn, {
'color': 'wheat',
'ls': '-',
'zorder': -1
})
plt.axvline(max(i.mMaxEvals() / i.dim for i in dsList0),
ls='--',
color='k')
plt.axvline(max(i.mMaxEvals() / i.dim for i in dsList1), color='k')
beautifyRLD(fig,
figureName,
evalfmax,
fileFormat=figformat,
text=text,
verbose=verbose)
plt.close(fig)
# The figures generated by the lines below are not displayed in the
# BBOB2010 template: should they be done?
#figureName = os.path.join(outputdir,'ppfvdistr2_%s' %(info))
#fig = plt.figure()
#for j in range(len(valuesOfInterest)):
##set_trace()
#tmp = plotFVDistr(dsList0, valuesOfInterest[j],
#evalfmax, verbose=verbose)
##if not tmp is None:
#plt.setp(tmp, 'color', rldColors[j])
#plt.setp(tmp, 'ls', '--')
#if rldColors [j] == 'r': # 1e-8 in bold
#plt.setp(tmp, 'linewidth', 3)
#tmp = plotFVDistr(dsList1, valuesOfInterest[j],
#evalfmax, verbose=verbose)
##if not tmp is None:
#plt.setp(tmp, 'color', rldColors[j])
#plt.setp(tmp, 'ls', '--')
#if rldColors [j] == 'r': # 1e-8 in bold
#plt.setp(tmp, 'linewidth', 3)
#tmp = numpy.floor(numpy.log10(evalfmax))
## coloring left to right:
##maxEvalsF = numpy.power(10, numpy.arange(tmp, 0, -1) - 1)
## coloring right to left:
#maxEvalsF = numpy.power(10, numpy.arange(0, tmp))
##The last index of valuesOfInterest is still used in this loop.
##set_trace()
## Plot lines for different number of function evaluations
#for k in range(len(maxEvalsF)):
#tmp = plotFVDistr(dsList0, valuesOfInterest[j],
#maxEvalsF=maxEvalsF[k], verbose=verbose)
#plt.setp(tmp, 'color', rldUnsuccColors[k])
#plt.setp(tmp, 'ls', '--')
#tmp = plotFVDistr(dsList1, valuesOfInterest[j],
#maxEvalsF=maxEvalsF[k], verbose=verbose)
#plt.setp(tmp, 'color', rldUnsuccColors[k])
#beautifyFVD(fig, figureName, fileFormat=figformat, text=text,
#isStoringXMax=isStoringXMax, verbose=verbose)
#plt.close(fig)
plt.rcdefaults()
def main(dictAlg, isBiobjective, order=None, outputdir='.', info='default',
dimension=None, verbose=True):
"""Generates a figure showing the performance of algorithms.
From a dictionary of :py:class:`DataSetList` sorted by algorithms,
generates the cumulative distribution function of the bootstrap
distribution of ERT for algorithms on multiple functions for
multiple targets altogether.
:param dict dictAlg: dictionary of :py:class:`DataSetList` instances
one instance is equivalent to one algorithm,
:param list targets: target function values
:param list order: sorted list of keys to dictAlg for plotting order
:param str outputdir: output directory
:param str info: output file name suffix
:param bool verbose: controls verbosity
"""
global x_limit # late assignment of default, because it can be set to None in config
global divide_by_dimension # not fully implemented/tested yet
if 'x_limit' not in globals() or x_limit is None:
x_limit = x_limit_default
tmp = pp.dictAlgByDim(dictAlg)
# tmp = pp.DictAlg(dictAlg).by_dim()
if len(tmp) != 1 and dimension is None:
raise ValueError('We never integrate over dimension.')
if dimension is not None:
if dimension not in tmp.keys():
raise ValueError('dimension %d not in dictAlg dimensions %s'
% (dimension, str(tmp.keys())))
tmp = {dimension: tmp[dimension]}
dim = tmp.keys()[0]
divisor = dim if divide_by_dimension else 1
algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []]
dictFunc = pp.dictAlgByFun(dictAlg)
# Collect data
# Crafting effort correction: should we consider any?
CrEperAlg = {}
for alg in algorithms_with_data:
CrE = 0.
if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL':
tmp = dictAlg[alg].dictByNoise()
assert len(tmp.keys()) == 1
if tmp.keys()[0] == 'noiselessall':
CrE = 0.5117
elif tmp.keys()[0] == 'nzall':
CrE = 0.6572
CrEperAlg[alg] = CrE
if CrE != 0.0:
print 'Crafting effort for', alg, 'is', CrE
dictData = {} # list of (ert per function) per algorithm
dictMaxEvals = {} # list of (maxevals per function) per algorithm
bestERT = [] # best ert per function
# funcsolved = [set()] * len(targets) # number of functions solved per target
xbest2009 = []
maxevalsbest2009 = []
for f, dictAlgperFunc in dictFunc.iteritems():
if function_IDs and f not in function_IDs:
continue
# print target_values((f, dim))
for j, t in enumerate(target_values((f, dim))):
# for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)):
# funcsolved[j].add(f)
for alg in algorithms_with_data:
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
try:
entry = dictAlgperFunc[alg][0] # one element per fun and per dim.
evals = entry.detEvals([t])[0]
assert entry.dim == dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
except (KeyError, IndexError):
#set_trace()
warntxt = ('Data for algorithm %s on function %d in %d-D '
% (alg, f, dim)
+ 'are missing.\n')
warnings.warn(warntxt)
dictData.setdefault(alg, []).extend(x)
dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc)
displaybest2009 = not isBiobjective #disabled until we find the bug
if displaybest2009:
#set_trace()
bestalgentries = bestalg.loadBestAlgorithm(isBiobjective)
bestalgentry = bestalgentries[(dim, f)]
bestalgevals = bestalgentry.detEvals(target_values((f, dim)))
# print bestalgevals
for j in range(len(bestalgevals[0])):
if bestalgevals[1][j]:
evals = bestalgevals[0][j]
#set_trace()
assert dim == bestalgentry.dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = bestalgentry.maxevals[bestalgevals[1][j]][np.isnan(evals)] / divisor
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
else:
x = perfprofsamplesize * [np.inf]
runlengthunsucc = []
xbest2009.extend(x)
maxevalsbest2009.extend(runlengthunsucc)
if order is None:
order = dictData.keys()
# Display data
lines = []
if displaybest2009:
args = {'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11.,
'markeredgewidth': 1.5, 'markerfacecolor': refcolor,
'markeredgecolor': refcolor, 'color': refcolor,
'label': 'best 2009', 'zorder': -1}
lines.append(plotdata(np.array(xbest2009), x_limit, maxevalsbest2009,
CrE = 0., **args))
def algname_to_label(algname, dirname=None):
"""to be extended to become generally useful"""
if isinstance(algname, (tuple, list)): # not sure this is needed
return ' '.join([str(name) for name in algname])
return str(algname)
for i, alg in enumerate(order):
try:
data = dictData[alg]
maxevals = dictMaxEvals[alg]
except KeyError:
continue
args = styles[(i) % len(styles)]
args['linewidth'] = 1.5
args['markersize'] = 12.
args['markeredgewidth'] = 1.5
args['markerfacecolor'] = 'None'
args['markeredgecolor'] = args['color']
args['label'] = algname_to_label(alg)
#args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib
#elif len(show_algorithms) > 0:
#args['color'] = 'wheat'
#args['ls'] = '-'
#args['zorder'] = -1
# plotdata calls pprldistr.plotECDF which calls ppfig.plotUnifLog... which does the work
lines.append(plotdata(np.array(data), x_limit, maxevals,
CrE=CrEperAlg[alg], **args))
labels, handles = plotLegend(lines, x_limit)
if True: # isLateXLeg:
fileName = os.path.join(outputdir,'pprldmany_%s.tex' % (info))
with open(fileName, 'w') as f:
f.write(r'\providecommand{\nperfprof}{7}')
algtocommand = {} # latex commands
for i, alg in enumerate(order):
tmp = r'\alg%sperfprof' % pptex.numtotext(i)
f.write(r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' %
(tmp, toolsdivers.str_to_latex(
toolsdivers.strip_pathname2(algname_to_label(alg)))))
algtocommand[algname_to_label(alg)] = tmp
if displaybest2009:
tmp = r'\algzeroperfprof'
f.write(r'\providecommand{%s}{best 2009}' % (tmp))
algtocommand['best 2009'] = tmp
commandnames = []
for label in labels:
commandnames.append(algtocommand[label])
# f.write(headleg)
if len(order) > 28: # latex sidepanel won't work well for more than 25 algorithms, but original labels are also clipped
f.write(r'\providecommand{\perfprofsidepanel}{\mbox{%s}\vfill\mbox{%s}}'
% (commandnames[0], commandnames[-1]))
else:
fontsize_command = r'\tiny{}' if len(order) > 19 else ''
f.write(r'\providecommand{\perfprofsidepanel}{{%s\mbox{%s}' %
(fontsize_command, commandnames[0])) # TODO: check len(labels) > 0
for i in range(1, len(labels)):
f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i])
f.write('}}\n')
# f.write(footleg)
if verbose:
print 'Wrote right-hand legend in %s' % fileName
figureName = os.path.join(outputdir,'pprldmany_%s' % (info))
#beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat)
beautify()
text = ppfig.consecutiveNumbers(sorted(dictFunc.keys()), 'f')
text += ',%d-D' % dim # TODO: this is strange when different dimensions are plotted
plt.text(0.01, 0.98, text, horizontalalignment="left",
verticalalignment="top", transform=plt.gca().transAxes)
if len(dictFunc) == 1:
plt.title(' '.join((str(dictFunc.keys()[0]),
genericsettings.current_testbed.short_names[dictFunc.keys()[0]])))
a = plt.gca()
plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative)
xticks, labels = plt.xticks()
tmp = []
for i in xticks:
tmp.append('%d' % round(np.log10(i)))
a.set_xticklabels(tmp)
if save_figure:
ppfig.saveFigure(figureName, verbose=verbose)
if len(dictFunc) == 1:
ppfig.save_single_functions_html(
os.path.join(outputdir, 'pprldmany'),
'', # algorithms names are clearly visible in the figure
add_to_names='_%02dD' %(dim),
algorithmCount=ppfig.AlgorithmCount.NON_SPECIFIED
)
if close_figure:
plt.close()
def main(dsList, isStoringXMax = False, outputdir = '',
info = 'default', verbose = True):
"""Generate figures of empirical cumulative distribution functions.
This method has a feature which allows to keep the same boundaries
for the x-axis, if ``isStoringXMax==True``. This makes sense when
dealing with different functions or subsets of functions for one
given dimension.
CAVE: this is bug-prone, as some data depend on the maximum
evaluations and the appearence therefore depends on the
calling order.
:param DataSetList dsList: list of DataSet instances to process.
:param bool isStoringXMax: if set to True, the first call
:py:func:`beautifyFVD` sets the
globals :py:data:`fmax` and
:py:data:`maxEvals` and all subsequent
calls will use these values as rightmost
xlim in the generated figures.
:param string outputdir: output directory (must exist)
:param string info: string suffix for output file names.
:param bool verbose: control verbosity
"""
# plt.rc("axes", labelsize=20, titlesize=24)
# plt.rc("xtick", labelsize=20)
# plt.rc("ytick", labelsize=20)
# plt.rc("font", size=20)
# plt.rc("legend", fontsize=20)
targets = single_target_values # convenience abbreviation
for d, dictdim in dsList.dictByDim().iteritems():
maxEvalsFactor = max(i.mMaxEvals() / d for i in dictdim)
if isStoringXMax:
global evalfmax
else:
evalfmax = None
if not evalfmax:
evalfmax = maxEvalsFactor
if runlen_xlimits_max is not None:
evalfmax = runlen_xlimits_max
# first figure: Run Length Distribution
filename = os.path.join(outputdir, 'pprldistr_%02dD_%s' % (d, info))
fig = plt.figure()
for j in range(len(targets)):
plotRLDistr(dictdim,
lambda fun_dim: targets(fun_dim)[j],
targets.label(j) if isinstance(targets, pproc.RunlengthBasedTargetValues) else targets.loglabel(j),
evalfmax, # can be larger maxEvalsFactor with no effect
** rldStyles[j % len(rldStyles)])
funcs = list(i.funcId for i in dictdim)
text = 'f%s' % (consecutiveNumbers(sorted(funcs)))
text += ',%d-D' % d
if(1):
# try:
if not isinstance(targets, pproc.RunlengthBasedTargetValues):
# if targets.target_values[-1] == 1e-8: # this is a hack
plot_previous_algorithms(d, funcs)
else:
plotRLB_previous_algorithms(d, funcs)
# except:
# pass
plt.axvline(x = maxEvalsFactor, color = 'k') # vertical line at maxevals
plt.legend(loc = 'best')
plt.text(0.5, 0.98, text, horizontalalignment = "center",
verticalalignment = "top",
transform = plt.gca().transAxes
# bbox=dict(ec='k', fill=False)
)
try: # was never tested, so let's make it safe
if len(funcs) == 1:
plt.title(genericsettings.current_testbed.info(funcs[0])[:27])
except:
warnings.warn('could not print title')
beautifyRLD(evalfmax)
saveFigure(filename, verbose = verbose)
plt.close(fig)
# second figure: Function Value Distribution
filename = os.path.join(outputdir, 'ppfvdistr_%02dD_%s' % (d, info))
fig = plt.figure()
plotFVDistr(dictdim, np.inf, 1e-8, **rldStyles[-1])
# coloring right to left
for j, max_eval_factor in enumerate(single_runlength_factors):
if max_eval_factor > maxEvalsFactor:
break
plotFVDistr(dictdim, max_eval_factor, 1e-8,
**rldUnsuccStyles[j % len(rldUnsuccStyles)])
plt.text(0.98, 0.02, text, horizontalalignment = "right",
transform = plt.gca().transAxes) # bbox=dict(ec='k', fill=False),
beautifyFVD(isStoringXMax = isStoringXMax, ylabel = False)
saveFigure(filename, verbose = verbose)
plt.close(fig)
def comp(dsList0, dsList1, targets, isStoringXMax = False,
outputdir = '', info = 'default', verbose = True):
"""Generate figures of ECDF that compare 2 algorithms.
:param DataSetList dsList0: list of DataSet instances for ALG0
:param DataSetList dsList1: list of DataSet instances for ALG1
:param seq targets: target function values to be displayed
:param bool isStoringXMax: if set to True, the first call
:py:func:`beautifyFVD` sets the globals
:py:data:`fmax` and :py:data:`maxEvals`
and all subsequent calls will use these
values as rightmost xlim in the generated
figures.
:param string outputdir: output directory (must exist)
:param string info: string suffix for output file names.
:param bool verbose: control verbosity
"""
# plt.rc("axes", labelsize=20, titlesize=24)
# plt.rc("xtick", labelsize=20)
# plt.rc("ytick", labelsize=20)
# plt.rc("font", size=20)
# plt.rc("legend", fontsize=20)
if not isinstance(targets, pproc.RunlengthBasedTargetValues):
targets = pproc.TargetValues.cast(targets)
dictdim0 = dsList0.dictByDim()
dictdim1 = dsList1.dictByDim()
for d in set(dictdim0.keys()) & set(dictdim1.keys()):
maxEvalsFactor = max(max(i.mMaxEvals() / d for i in dictdim0[d]),
max(i.mMaxEvals() / d for i in dictdim1[d]))
if isStoringXMax:
global evalfmax
else:
evalfmax = None
if not evalfmax:
evalfmax = maxEvalsFactor ** 1.05
if runlen_xlimits_max is not None:
evalfmax = runlen_xlimits_max
filename = os.path.join(outputdir, 'pprldistr_%02dD_%s' % (d, info))
fig = plt.figure()
for j in range(len(targets)):
tmp = plotRLDistr(dictdim0[d], lambda fun_dim: targets(fun_dim)[j],
targets.label(j) if isinstance(targets, pproc.RunlengthBasedTargetValues) else targets.loglabel(j),
marker = genericsettings.line_styles[1]['marker'],
**rldStyles[j % len(rldStyles)])
plt.setp(tmp[-1], label = None) # Remove automatic legend
# Mods are added after to prevent them from appearing in the legend
plt.setp(tmp, markersize = 20.,
markeredgewidth = plt.getp(tmp[-1], 'linewidth'),
markeredgecolor = plt.getp(tmp[-1], 'color'),
markerfacecolor = 'none')
tmp = plotRLDistr(dictdim1[d], lambda fun_dim: targets(fun_dim)[j],
targets.label(j) if isinstance(targets, pproc.RunlengthBasedTargetValues) else targets.loglabel(j),
marker = genericsettings.line_styles[0]['marker'],
**rldStyles[j % len(rldStyles)])
# modify the automatic legend: remover marker and change text
plt.setp(tmp[-1], marker = '',
label = targets.label(j) if isinstance(targets, pproc.RunlengthBasedTargetValues) else targets.loglabel(j))
# Mods are added after to prevent them from appearing in the legend
plt.setp(tmp, markersize = 15.,
markeredgewidth = plt.getp(tmp[-1], 'linewidth'),
markeredgecolor = plt.getp(tmp[-1], 'color'),
markerfacecolor = 'none')
funcs = set(i.funcId for i in dictdim0[d]) | set(i.funcId for i in dictdim1[d])
text = 'f%s' % (consecutiveNumbers(sorted(funcs)))
if not isinstance(targets, pproc.RunlengthBasedTargetValues):
plot_previous_algorithms(d, funcs)
else:
plotRLB_previous_algorithms(d, funcs)
# plt.axvline(max(i.mMaxEvals()/i.dim for i in dictdim0[d]), ls='--', color='k')
# plt.axvline(max(i.mMaxEvals()/i.dim for i in dictdim1[d]), color='k')
plt.axvline(max(i.mMaxEvals() / i.dim for i in dictdim0[d]),
marker = '+', markersize = 20., color = 'k',
markeredgewidth = plt.getp(tmp[-1], 'linewidth',))
plt.axvline(max(i.mMaxEvals() / i.dim for i in dictdim1[d]),
marker = 'o', markersize = 15., color = 'k', markerfacecolor = 'None',
markeredgewidth = plt.getp(tmp[-1], 'linewidth'))
plt.legend(loc = 'best')
plt.text(0.5, 0.98, text, horizontalalignment = "center",
verticalalignment = "top", transform = plt.gca().transAxes) # bbox=dict(ec='k', fill=False),
beautifyRLD(evalfmax)
saveFigure(filename, verbose = verbose)
plt.close(fig)
def main(dictAlg, order=None, outputdir='.', info='default',
verbose=True):
"""Generates a figure showing the performance of algorithms.
From a dictionary of :py:class:`DataSetList` sorted by algorithms,
generates the cumulative distribution function of the bootstrap
distribution of ERT for algorithms on multiple functions for
multiple targets altogether.
:param dict dictAlg: dictionary of :py:class:`DataSetList` instances
one instance is equivalent to one algorithm,
:param list targets: target function values
:param list order: sorted list of keys to dictAlg for plotting order
:param str outputdir: output directory
:param str info: output file name suffix
:param bool verbose: controls verbosity
"""
global x_limit # late assignment of default, because it can be set to None in config
if 'x_limit' not in globals() or x_limit is None:
x_limit = x_limit_default
tmp = pp.dictAlgByDim(dictAlg)
# tmp = pp.DictAlg(dictAlg).by_dim()
if len(tmp) != 1:
raise Exception('We never integrate over dimension.')
dim = tmp.keys()[0]
algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []]
dictFunc = pp.dictAlgByFun(dictAlg)
# Collect data
# Crafting effort correction: should we consider any?
CrEperAlg = {}
for alg in algorithms_with_data:
CrE = 0.
if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL':
tmp = dictAlg[alg].dictByNoise()
assert len(tmp.keys()) == 1
if tmp.keys()[0] == 'noiselessall':
CrE = 0.5117
elif tmp.keys()[0] == 'nzall':
CrE = 0.6572
CrEperAlg[alg] = CrE
if CrE != 0.0:
print 'Crafting effort for', alg, 'is', CrE
dictData = {} # list of (ert per function) per algorithm
dictMaxEvals = {} # list of (maxevals per function) per algorithm
bestERT = [] # best ert per function
# funcsolved = [set()] * len(targets) # number of functions solved per target
xbest2009 = []
maxevalsbest2009 = []
for f, dictAlgperFunc in dictFunc.iteritems():
if function_IDs and f not in function_IDs:
continue
# print target_values((f, dim))
for j, t in enumerate(target_values((f, dim))):
# for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)):
# funcsolved[j].add(f)
for alg in algorithms_with_data:
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
try:
entry = dictAlgperFunc[alg][0] # one element per fun and per dim.
evals = entry.detEvals([t])[0]
runlengthsucc = evals[np.isnan(evals) == False] / entry.dim
runlengthunsucc = entry.maxevals[np.isnan(evals)] / entry.dim
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
except (KeyError, IndexError):
#set_trace()
warntxt = ('Data for algorithm %s on function %d in %d-D '
% (alg, f, dim)
+ 'are missing.\n')
warnings.warn(warntxt)
dictData.setdefault(alg, []).extend(x)
dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc)
if displaybest2009:
#set_trace()
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgentry = bestalg.bestalgentries2009[(dim, f)]
bestalgevals = bestalgentry.detEvals(target_values((f, dim)))
# print bestalgevals
for j in range(len(bestalgevals[0])):
if bestalgevals[1][j]:
evals = bestalgevals[0][j]
#set_trace()
runlengthsucc = evals[np.isnan(evals) == False] / bestalgentry.dim
runlengthunsucc = bestalgentry.maxevals[bestalgevals[1][j]][np.isnan(evals)] / bestalgentry.dim
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
else:
x = perfprofsamplesize * [np.inf]
runlengthunsucc = []
xbest2009.extend(x)
maxevalsbest2009.extend(runlengthunsucc)
if order is None:
order = dictData.keys()
# Display data
lines = []
if displaybest2009:
args = {'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11.,
'markeredgewidth': 1.5, 'markerfacecolor': refcolor,
'markeredgecolor': refcolor, 'color': refcolor,
'label': 'best 2009', 'zorder': -1}
lines.append(plotdata(np.array(xbest2009), x_limit, maxevalsbest2009,
CrE = 0., **args))
for i, alg in enumerate(order):
try:
data = dictData[alg]
maxevals = dictMaxEvals[alg]
except KeyError:
continue
args = styles[(i) % len(styles)]
args['linewidth'] = 1.5
args['markersize'] = 12.
args['markeredgewidth'] = 1.5
args['markerfacecolor'] = 'None'
args['markeredgecolor'] = args['color']
args['label'] = alg
#args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib
#elif len(show_algorithms) > 0:
#args['color'] = 'wheat'
#args['ls'] = '-'
#args['zorder'] = -1
lines.append(plotdata(np.array(data), x_limit, maxevals,
CrE=CrEperAlg[alg], **args))
labels, handles = plotLegend(lines, x_limit)
if True: #isLateXLeg:
fileName = os.path.join(outputdir,'pprldmany_%s.tex' % (info))
try:
f = open(fileName, 'w')
f.write(r'\providecommand{\nperfprof}{7}')
algtocommand = {}
for i, alg in enumerate(order):
tmp = r'\alg%sperfprof' % pptex.numtotext(i)
f.write(r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' % (tmp, toolsdivers.str_to_latex(toolsdivers.strip_pathname2(alg))))
algtocommand[alg] = tmp
commandnames = []
if displaybest2009:
tmp = r'\algzeroperfprof'
f.write(r'\providecommand{%s}{best 2009}' % (tmp))
algtocommand['best 2009'] = tmp
for l in labels:
commandnames.append(algtocommand[l])
# f.write(headleg)
f.write(r'\providecommand{\perfprofsidepanel}{\mbox{%s}' % commandnames[0]) # TODO: check len(labels) > 0
for i in range(1, len(labels)):
f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i])
f.write('}\n')
# f.write(footleg)
if verbose:
print 'Wrote right-hand legend in %s' % fileName
except:
raise # TODO: Does this make sense?
else:
f.close()
figureName = os.path.join(outputdir,'pprldmany_%s' % (info))
#beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat)
beautify()
text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys())))
text += ',%d-D' % dim
plt.text(0.01, 0.98, text, horizontalalignment="left",
verticalalignment="top", transform=plt.gca().transAxes)
a = plt.gca()
plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative)
xticks, labels = plt.xticks()
tmp = []
for i in xticks:
tmp.append('%d' % round(np.log10(i)))
a.set_xticklabels(tmp)
ppfig.saveFigure(figureName, verbose=verbose)
plt.close()
def main2(dsList0,
dsList1,
valuesOfInterest=None,
outputdir='',
info='default',
verbose=True):
"""Generate figures of empirical cumulative distribution functions.
Keyword arguments:
indexEntries -- list of IndexEntry instances to process.
valuesOfInterest -- target function values to be displayed.
isStoringXMax -- if set to True, the first call BeautifyVD sets the globals
fmax and maxEvals and all subsequent calls will use these
values as rightmost xlim in the generated figures.
-- if set to True, the first call BeautifyVD sets the global
fmax and all subsequent call will have the same maximum
xlim.
outputdir -- output directory (must exist)
info --- string suffix for output file names.
Outputs:
Image files of the empirical cumulative distribution functions.
"""
plt.rc("axes", labelsize=20, titlesize=24)
plt.rc("xtick", labelsize=20)
plt.rc("ytick", labelsize=20)
plt.rc("font", size=20)
plt.rc("legend", fontsize=20)
figureName = os.path.join(outputdir, 'pplogabs_%s' % (info))
tmp = plotLogAbs2(dsList0, dsList1, valuesOfInterest, verbose=verbose)
beautify2()
# Prolong to the boundary
xmin, xmax = plt.xlim()
for i in tmp:
try:
xdata, ydata = i.get_data()
except AttributeError:
xdata = i.get_xdata()
ydata = i.get_ydata()
if len(xdata) == 0 or len(ydata) == 0:
continue
xdata = numpy.insert(xdata, 0, xmin)
try:
xdata = numpy.insert(xdata, len(xdata), xmax)
except OverflowError:
xdata = xdata + 0.0
# TODO: Hack for float conversion, compatibility with 0.8
xdata = numpy.insert(xdata, len(xdata), xmax)
ydata = numpy.insert(ydata, 0, ydata[0])
ydata = numpy.insert(ydata, len(ydata), ydata[-1])
i.set_data(xdata, ydata)
plt.legend(loc='best')
#plt.text(0.5, 0.93, text, horizontalalignment="center",
# transform=axisHandle.transAxes)
funcs = set(dsList0.dictByFunc().keys()) & set(dsList1.dictByFunc().keys())
text = 'f%s' % consecutiveNumbers(sorted(funcs))
plt.text(0.98,
0.02,
text,
horizontalalignment="right",
transform=plt.gca().transAxes)
#set_trace()
saveFigure(figureName, figFormat=figformat, verbose=verbose)
plt.close()
#set_trace()
plt.rcdefaults()
def generateFigure(dsList, CrE=0., isStoringXRange=True, outputdir='.',
info='default', verbose=True):
"""Generates ERT loss ratio figures.
:param DataSetList dsList: input data set
:param float CrE: crafting effort (see COCO documentation)
:param bool isStoringXRange: if set to True, the first call to this
function sets the global
:py:data:`evalf` and all subsequent
calls will use this value as boundaries
in the generated figures.
:param string outputdir: output folder (must exist)
:param string info: string suffix for output file names
:param bool verbose: controls verbosity
"""
#plt.rc("axes", labelsize=20, titlesize=24)
#plt.rc("xtick", labelsize=20)
#plt.rc("ytick", labelsize=20)
#plt.rc("font", size=20)
#plt.rc("legend", fontsize=20)
if isStoringXRange:
global evalf
else:
evalf = None
# do not aggregate over dimensions
for d, dsdim in dsList.dictByDim().iteritems():
maxevals = max(max(i.ert[np.isinf(i.ert)==False]) for i in dsdim)
EVALS = [2.*d]
EVALS.extend(10.**(np.arange(1, np.ceil(1e-9 + np.log10(maxevals * 1./d))))*d)
if not evalf:
evalf = (np.log10(EVALS[0]/d), np.log10(EVALS[-1]/d))
data = generateData(dsdim, EVALS, CrE)
ydata = []
for i in range(len(EVALS)):
#Aggregate over functions.
ydata.append(np.log10(list(data[f][i] for f in data)))
xdata = np.log10(np.array(EVALS)/d)
xticklabels = ['']
xticklabels.extend('%d' % i for i in xdata[1:])
plot(xdata, ydata)
filename = os.path.join(outputdir, 'pplogloss_%02dD_%s' % (d, info))
plt.xticks(xdata, xticklabels)
#Is there an upper bound?
if CrE > 0 and len(set(dsdim.dictByFunc().keys())) >= 20:
#TODO: hopefully this means we are not considering function groups.
plt.text(0.01, 0.98, 'CrE = %5g' % CrE, fontsize=20,
horizontalalignment='left', verticalalignment='top',
transform = plt.gca().transAxes,
bbox=dict(facecolor='w'))
plt.axhline(1., color='k', ls='-', zorder=-1)
plt.axvline(x=np.log10(max(i.mMaxEvals()/d for i in dsdim)), color='k')
funcs = set(i.funcId for i in dsdim)
if len(funcs) > 1:
text = consecutiveNumbers(sorted(funcs), 'f')
else:
text = 'f%d' % (funcs.pop())
plt.text(0.5, 0.93, text, horizontalalignment="center",
transform=plt.gca().transAxes)
beautify()
if evalf:
plt.xlim(xmin=evalf[0]-0.5, xmax=evalf[1]+0.5)
saveFigure(filename, verbose=verbose)
#plt.show()
plt.close()
