def main2(dsList, dimsOfInterest, outputdir='.', info='', verbose=True):
"""Generate a table of ratio ERT/ERTbest vs target precision.
1 table per dimension will be generated.
Rank-sum tests table on "Final Data Points" for only one algorithm.
that is, for example, using 1/#fevals(ftarget) if ftarget was
reached and -f_final otherwise as input for the rank-sum test, where
obviously the larger the better.
"""
# TODO: remove dimsOfInterest, was added just for compatibility's sake
if info:
info = '_' + info
# insert a separator between the default file name and the additional
# information string.
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
for d, dsdim in dsList.dictByDim().iteritems():
dictfun = dsdim.dictByFunc()
res = []
for f, dsfun in sorted(dsdim.dictByFunc().iteritems()):
assert len(dsfun) == 1, ('Expect one-element DataSetList for a '
'given dimension and function')
ds = dsfun[0]
data = _treat(ds)
res = _table(data)
res = []
outputfile = os.path.join(outputdir, 'pptable_%02dD%s.tex' % (d, info))
f = open(outputfile, 'w')
f.write(res)
f.close()
if verbose:
print "Table written in %s" % outputfile
def generateData(dsList, evals, CrE_A):
res = {}
D = set(i.dim for i in dsList).pop() # should have only one element
#if D == 3:
#set_trace()
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
for fun, tmpdsList in dsList.dictByFunc().iteritems():
assert len(tmpdsList) == 1
entry = tmpdsList[0]
bestalgentry = bestalg.bestalgentries2009[(D, fun)]
#ERT_A
f_A = detf(entry, evals)
ERT_best = detERT(bestalgentry, f_A)
ERT_A = detERT(entry, f_A)
nextbestf = []
for i in f_A:
if i == 0.:
nextbestf.append(0.)
else:
tmp = bestalgentry.target[bestalgentry.target < i]
try:
nextbestf.append(tmp[0])
except IndexError:
nextbestf.append(i * 10.**(-0.2)) # TODO: this is a hack
ERT_best_nextbestf = detERT(bestalgentry, nextbestf)
for i in range(len(ERT_A)):
# nextbestf[i] >= f_thresh: this is tested because if it is not true
# ERT_best_nextbestf[i] is supposed to be infinite.
if nextbestf[i] >= f_thresh and ERT_best_nextbestf[i] < evals[
i]: # is different from the specification...
ERT_A[i] = evals[i]
# For test purpose:
#if fun % 10 == 0:
# ERT_A[-2] = 1.
# ERT_best[-2] = numpy.inf
ERT_A = numpy.array(ERT_A)
ERT_best = numpy.array(ERT_best)
loss_A = numpy.exp(CrE_A) * ERT_A / ERT_best
assert (numpy.isnan(loss_A) == False).all()
#set_trace()
#if numpy.isnan(loss_A).any() or numpy.isinf(loss_A).any() or (loss_A == 0.).any():
# txt = 'Problem with entry %s' % str(entry)
# warnings.warn(txt)
# #set_trace()
res[fun] = loss_A
return res
def generateData(dsList, evals, CrE_A):
res = {}
D = set(i.dim for i in dsList).pop() # should have only one element
#if D == 3:
#set_trace()
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
for fun, tmpdsList in dsList.dictByFunc().iteritems():
assert len(tmpdsList) == 1
entry = tmpdsList[0]
bestalgentry = bestalg.bestalgentries2009[(D, fun)]
#ERT_A
f_A = detf(entry, evals)
ERT_best = detERT(bestalgentry, f_A)
ERT_A = detERT(entry, f_A)
nextbestf = []
for i in f_A:
if i == 0.:
nextbestf.append(0.)
else:
tmp = bestalgentry.target[bestalgentry.target < i]
try:
nextbestf.append(tmp[0])
except IndexError:
nextbestf.append(i * 10.**(-0.2)) # TODO: this is a hack
ERT_best_nextbestf = detERT(bestalgentry, nextbestf)
for i in range(len(ERT_A)):
# nextbestf[i] >= f_thresh: this is tested because if it is not true
# ERT_best_nextbestf[i] is supposed to be infinite.
if nextbestf[i] >= f_thresh and ERT_best_nextbestf[i] < evals[i]: # is different from the specification...
ERT_A[i] = evals[i]
# For test purpose:
#if fun % 10 == 0:
# ERT_A[-2] = 1.
# ERT_best[-2] = numpy.inf
ERT_A = numpy.array(ERT_A)
ERT_best = numpy.array(ERT_best)
loss_A = numpy.exp(CrE_A) * ERT_A / ERT_best
assert (numpy.isnan(loss_A) == False).all()
#set_trace()
#if numpy.isnan(loss_A).any() or numpy.isinf(loss_A).any() or (loss_A == 0.).any():
# txt = 'Problem with entry %s' % str(entry)
# warnings.warn(txt)
# #set_trace()
res[fun] = loss_A
return res
def plot_previous_algorithms(func, target=lambda x: [1e-8]):
"""Add graph of the BBOB-2009 virtual best algorithm."""
if isinstance(values_of_interest, pproc.RunlengthBasedTargetValues):
return None
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgdata = []
for d in dimensions:
entry = bestalg.bestalgentries2009[(d, func)]
tmp = entry.detERT([target((func, d))[-1]])[0]
if not np.isinf(tmp):
bestalgdata.append(tmp / d)
else:
bestalgdata.append(None)
res = plt.plot(dimensions, bestalgdata, color=refcolor, linewidth=10,
marker='d', markersize=25, markeredgecolor='k',
zorder= -2)
return res
def main(dsList, _valuesOfInterest, outputdir, verbose=True):
"""From a DataSetList, returns a convergence and ERT/dim figure vs dim.
Uses data of BBOB 2009 (:py:mod:`bbob_pproc.bestalg`).
:param DataSetList dsList: data sets
:param seq _valuesOfInterest: target precisions, there might be as
many graphs as there are elements in
this input
:param string outputdir: output directory
:param bool verbose: controls verbosity
"""
# plt.rc("axes", labelsize=20, titlesize=24)
# plt.rc("xtick", labelsize=20)
# plt.rc("ytick", labelsize=20)
# plt.rc("font", size=20)
# plt.rc("legend", fontsize=20)
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
dictFunc = dsList.dictByFunc()
for func in dictFunc:
plot(dictFunc[func], _valuesOfInterest,
styles=styles) # styles might have changed via config
beautify(axesLabel=False)
plt.text(plt.xlim()[0],
plt.ylim()[0],
_valuesOfInterest.short_info,
fontsize=14)
if func in functions_with_legend:
plt.legend(loc="best")
if isBenchmarkinfosFound:
plt.gca().set_title(funInfos[func])
plot_previous_algorithms(func, _valuesOfInterest)
filename = os.path.join(outputdir, 'ppfigdim_f%03d' % (func))
saveFigure(filename, verbose=verbose)
plt.close()
def main(dsList, _valuesOfInterest, outputdir, verbose=True):
"""From a DataSetList, returns a convergence and ERT/dim figure vs dim.
Uses data of BBOB 2009 (:py:mod:`bbob_pproc.bestalg`).
:param DataSetList dsList: data sets
:param seq _valuesOfInterest: target precisions, either as list or as
``pproc.TargetValues`` class instance.
There will be as many graphs as there are
elements in this input.
:param string outputdir: output directory
:param bool verbose: controls verbosity
"""
# plt.rc("axes", labelsize=20, titlesize=24)
# plt.rc("xtick", labelsize=20)
# plt.rc("ytick", labelsize=20)
# plt.rc("font", size=20)
# plt.rc("legend", fontsize=20)
_valuesOfInterest = pproc.TargetValues.cast(_valuesOfInterest)
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
dictFunc = dsList.dictByFunc()
for func in dictFunc:
plot(dictFunc[func], _valuesOfInterest, styles=styles) # styles might have changed via config
beautify(axesLabel=False)
plt.text(plt.xlim()[0], plt.ylim()[0], _valuesOfInterest.short_info, fontsize=14)
if func in functions_with_legend:
plt.legend(loc="best")
if isBenchmarkinfosFound:
plt.gca().set_title(funInfos[func])
plot_previous_algorithms(func, _valuesOfInterest)
filename = os.path.join(outputdir, 'ppfigdim_f%03d' % (func))
saveFigure(filename, verbose=verbose)
plt.close()
def plot_previous_algorithms(func, target=values_of_interest): # lambda x: [1e-8]):
"""Add graph of the BBOB-2009 virtual best algorithm using the
last, most difficult target in ``target``."""
target = pproc.TargetValues.cast(target)
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgdata = []
for d in dimensions:
try:
entry = bestalg.bestalgentries2009[(d, func)]
tmp = entry.detERT([target((func, d))[-1]])[0]
if not np.isinf(tmp):
bestalgdata.append(tmp / d)
else:
bestalgdata.append(None)
except KeyError: #dimension not in bestalg
bestalgdata.append(None)
res = plt.plot(dimensions, bestalgdata, color=refcolor, linewidth=10,
marker='d', markersize=25, markeredgecolor='k',
zorder= -2)
return res
def plot_previous_algorithms(func, target=lambda x: [1e-8]):
"""Add graph of the BBOB-2009 virtual best algorithm."""
if isinstance(values_of_interest, pproc.RunlengthBasedTargetValues):
return None
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgdata = []
for d in dimensions:
entry = bestalg.bestalgentries2009[(d, func)]
tmp = entry.detERT([target((func, d))[-1]])[0]
if not np.isinf(tmp):
bestalgdata.append(tmp / d)
else:
bestalgdata.append(None)
res = plt.plot(dimensions,
bestalgdata,
color=refcolor,
linewidth=10,
marker='d',
markersize=25,
markeredgecolor='k',
zorder=-2)
return res
def plot_previous_algorithms(func, target=values_of_interest): # lambda x: [1e-8]):
"""Add graph of the BBOB-2009 virtual best algorithm using the
last, most difficult target in ``target``."""
target = pproc.TargetValues.cast(target)
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgdata = []
for d in dimensions:
try:
entry = bestalg.bestalgentries2009[(d, func)]
tmp = entry.detERT([target((func, d))[-1]])[0]
if not np.isinf(tmp):
bestalgdata.append(tmp / d)
else:
bestalgdata.append(None)
except KeyError: #dimension not in bestalg
bestalgdata.append(None)
res = plt.plot(dimensions, bestalgdata, color=refcolor, linewidth=10,
marker='d', markersize=25, markeredgecolor='k',
zorder= -2)
return res
def mainnew(dsList0,
dsList1,
dimsOfInterest,
outputdir,
info='',
verbose=True):
"""Still in shambles: new cleaned up version.
One table per dimension...
"""
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3]
alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3]
if info:
info = '_' + info
dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys()))
if not bestalg.bestalgentries:
bestalg.loadBBOB2009()
header = [r'$\Delta f$']
for i in targetsOfInterest:
#header.append(r'\multicolumn{2}{@{}c@{}}{$10^{%d}$}' % (int(numpy.log10(i))))
header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' %
(int(numpy.log10(i))))
header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}')
for d in dimsOfInterest: # TODO set as input arguments
table = [header]
extraeol = [r'\hline']
dictFunc0 = dictDim0[d].dictByFunc()
dictFunc1 = dictDim1[d].dictByFunc()
funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys()))
nbtests = len(funcs) * 2. #len(dimsOfInterest)
for f in sorted(funcs):
bestalgentry = bestalg.bestalgentries[(d, f)]
curline = [r'${\bf f_{%d}}$' % f]
bestalgdata = bestalgentry.detERT(targetsOfInterest)
bestalgevals, bestalgalgs = bestalgentry.detEvals(
targetsOfInterest)
for i in bestalgdata[:-1]:
#if numpy.isnan(i):
#set_trace()
curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' %
writeFEvals2(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' %
writeFEvals2(bestalgdata[-1], 2))
tmp = bestalgentry.detEvals([targetf])[0][0]
tmp2 = numpy.sum(numpy.isnan(tmp) == False)
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(tmp))
table.append(curline[:])
extraeol.append('')
rankdata0 = []
# generate all data from ranksum test
entries = []
ertdata = {}
for nb, dsList in enumerate((dictFunc0, dictFunc1)):
try:
entry = dsList[f][0] # take the first element
except KeyError:
continue # TODO: problem here!
ertdata[nb] = entry.detERT(targetsOfInterest)
entries.append(entry)
testres0vs1 = significancetest(entries[0], entries[1],
targetsOfInterest)
testresbestvs1 = significancetest(bestalgentry, entries[1],
targetsOfInterest)
for nb, entry in enumerate(entries):
if nb == 0:
curline = [r'0:\:\algzeroshort\hspace*{\fill}']
else:
curline = [r'1:\:\algoneshort\hspace*{\fill}']
#curline = [r'\alg%sshort' % tmp]
#curline = [r'Alg%d' % nb]
#curline = [r'%.3s%d' % (entry.algId, nb)]
data = entry.detERT(targetsOfInterest)
if nb == 0:
assert not isinstance(data, numpy.ndarray)
data0 = data[:] # check if it is not an array
for i, j in enumerate(
data): # is j an appropriate identifier here?
#if numpy.isnan(float(j)/bestalgdata[i]):
# set_trace()
z, p = testres0vs1[
i] #TODO: there is something with the sign that I don't get
# assign significance flag
significance0vs1 = 0
if nb == 0:
istat0 = 0
istat1 = 1
else:
z = -z
istat0 = 1
istat1 = 0
# TODO: I don't understand the thing with the sign of significance0vs1
if (nbtests * p < 0.05 and z > 0
and not numpy.isinf(ertdata[istat0][i])
and z * (ertdata[istat1][i] - ertdata[istat0][i]) >
0): # z-value and ERT-ratio must agree
significance0vs1 = -int(
numpy.ceil(numpy.log10(nbtests * p)))
elif nbtests * p < 0.05 and z < 0 and z * (
ertdata[istat1][i] - ertdata[istat0][i]) > 0:
significance0vs1 = int(
numpy.ceil(numpy.log10(nbtests * p)))
alignment = 'c'
if i == len(data) - 1: # last element
alignment = 'c|'
if numpy.isinf(bestalgdata[i]):
tableentry = (
r'\multicolumn{2}{@{}%s@{}}{\textbf{\textit{%s}}}'
% (alignment, writeFEvals2(float(j), 2)))
# TODO: is this the desired behaviour?
else:
# Formatting
tmp = float(j) / bestalgdata[i]
assert not numpy.isnan(tmp)
isscientific = False
if tmp >= 1000:
isscientific = True
tableentry = writeFEvals2(tmp,
2,
isscientific=isscientific)
isBold = False
if significance0vs1 > 0:
isBold = True
if numpy.isinf(tmp) and i == len(data) - 1:
tableentry = (
tableentry + r'\textit{%s}' %
writeFEvals2(numpy.median(entry.maxevals), 2))
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3 and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' %
(alignment, tableentry))
elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and
i != len(data) - 1):
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3 and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' %
(alignment, tableentry))
else:
tmp = tableentry.split('.', 1)
if isBold:
tmp = list(r'\textbf{%s}' % i for i in tmp)
elif 11 < 3 and significance0vs1 < 0:
tmp = list(r'\textit{%s}' % i for i in tmp)
tableentry = ' & .'.join(tmp)
if len(tmp) == 1:
tableentry += '&'
superscript = ''
z, p = testresbestvs1[i]
#z, p = ranksums(rankdatabest[i], currankdata)
#if ((nbtests * p) < 0.05
# and ((numpy.isinf(bestalgdata[i]) and numpy.isinf(j))
# or z * (j - bestalgdata[i]) > 0)): # z-value and ERT-ratio must agree
#The conditions are now that ERT < ERT_best and
# all(sorted(FEvals_best) > sorted(FEvals_current)).
if j - bestalgdata[i] < 0. and not numpy.isinf(
bestalgdata[i]):
evals = entry.detEvals([targetsOfInterest[i]])[0]
evals[numpy.isnan(evals)] = entry.maxevals[numpy.isnan(
evals)]
bestevals = bestalgentry.detEvals(
[targetsOfInterest[i]])
bestevals, bestalgalg = (bestevals[0][0],
bestevals[1][0])
bestevals[numpy.isnan(
bestevals)] = bestalgentry.maxevals[bestalgalg][
numpy.isnan(bestevals)]
evals = numpy.array(
sorted(evals))[0:min(len(evals), len(bestevals))]
bestevals = numpy.array(sorted(
bestevals))[0:min(len(evals), len(bestevals))]
#The conditions are now that ERT < ERT_best and
# all(sorted(FEvals_best) > sorted(FEvals_current)).
if ((nbtests * p) < 0.05 and j - bestalgdata[i] < 0.
and z < 0. and (numpy.isinf(bestalgdata[i])
or all(evals < bestevals))):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
#tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star'])
if z > 0:
superscript = r'\uparrow' #* nbstars
else:
superscript = r'\downarrow' #* nbstars
# print z, linebest[i], line1
if nbstars > 1:
superscript += str(int(nbstars))
addition = ''
if superscript or significance0vs1:
s = ''
if significance0vs1 > 0:
s = '\star'
if significance0vs1 > 1:
s += str(significance0vs1)
s = r'$^{' + s + superscript + r'}$'
if tableentry.endswith('}'):
tableentry = tableentry[:-1] + s + r'}'
else:
tableentry += s
curline.append(tableentry)
# Two cases: both tabular give an overfull hbox
# AND generate a LaTeX Warning: Float too large for page by 16.9236pt on input line 421. (noisy)
# OR generate a LaTeX Warning: Float too large for page by 33.57658pt on input line 421. (noisy)
tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:]
try:
tmp = tmp[0]
curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False))
except IndexError:
curline.append('%d' % 0)
curline.append('/%d' % entry.nbRuns())
table.append(curline[:])
extraeol.append('')
extraeol[-1] = r'\hline'
extraeol[-1] = ''
outputfile = os.path.join(outputdir,
'cmptable_%02dD%s.tex' % (d, info))
spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(
targetsOfInterest) + '|@{}r@{}@{}l@{}'
res = r'\providecommand{\algzeroshort}{%s}' % alg0 + '\n'
res += r'\providecommand{\algoneshort}{%s}' % alg1 + '\n'
#res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec,
#extraeol=extraeol)
res += tableLaTeX(table, spec=spec, extraeol=extraeol)
f = open(outputfile, 'w')
f.write(res)
f.close()
if verbose:
print "Table written in %s" % outputfile
def main2(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True):
"""Generate comparison tables.
One table per dimension...
"""
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3]
alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3]
if info:
info = '_' + info
dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys()))
if not bestalg.bestalgentries:
bestalg.loadBBOB2009()
header = [r'$\Delta f$']
for i in targetsOfInterest:
#header.append(r'\multicolumn{2}{@{}c@{}}{$10^{%d}$}' % (int(numpy.log10(i))))
header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' %
(int(numpy.log10(i))))
header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}')
for d in dimsOfInterest: # TODO set as input arguments
table = [header]
extraeol = [r'\hline']
dictFunc0 = dictDim0[d].dictByFunc()
dictFunc1 = dictDim1[d].dictByFunc()
funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys()))
nbtests = len(funcs) * 2. #len(dimsOfInterest)
for f in sorted(funcs):
bestalgentry = bestalg.bestalgentries[(d, f)]
curline = [r'${\bf f_{%d}}$' % f]
bestalgdata = bestalgentry.detERT(targetsOfInterest)
bestalgevals, bestalgalgs = bestalgentry.detEvals(
targetsOfInterest)
for i in bestalgdata[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' %
writeFEvals2(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' %
writeFEvals2(bestalgdata[-1], 2))
rankdatabest = []
for i, j in enumerate(bestalgevals):
if bestalgalgs[i] is None:
tmp = -bestalgentry.finalfunvals[bestalgentry.algs[-1]]
else:
tmp = numpy.power(j, -1.)
tmp[numpy.isnan(tmp)] = -bestalgentry.finalfunvals[
bestalgalgs[i]][numpy.isnan(tmp)]
rankdatabest.append(tmp)
tmp = bestalgentry.detEvals([targetf])[0][0]
tmp2 = numpy.sum(numpy.isnan(tmp) == False)
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(tmp))
table.append(curline[:])
extraeol.append('')
rankdata0 = []
# generate all data for ranksum test
rankdata = {} # rankdata[nb +/- 1,i] is of interest later
ertdata = {}
for nb, entries in enumerate(
(dictFunc0, dictFunc1)): # copy paste of loop below
try:
entry = entries[f][0] # take the first element
except KeyError:
continue
ertdata[nb] = entry.detERT(targetsOfInterest)
evals = entry.detEvals(targetsOfInterest)
for i, tmp in enumerate(ertdata[nb]):
# print i
rankdata[nb, i] = numpy.power(evals[i], -1.)
rankdata[nb, i][numpy.isnan(
rankdata[nb, i])] = -entry.finalfunvals[numpy.isnan(
rankdata[nb, i])]
for nb, entries in enumerate((dictFunc0, dictFunc1)):
try:
entry = entries[f][0] # take the first element
except KeyError:
continue
if nb == 0:
curline = [r'0:\:\algzeroshort\hspace*{\fill}']
else:
curline = [r'1:\:\algoneshort\hspace*{\fill}']
#curline = [r'\alg%sshort' % tmp]
#curline = [r'Alg%d' % nb]
#curline = [r'%.3s%d' % (entry.algId, nb)]
data = entry.detERT(targetsOfInterest)
evals = entry.detEvals(targetsOfInterest)
if nb == 0:
assert not isinstance(data, numpy.ndarray)
data0 = data[:] # check if it is not an array
for i, j in enumerate(
data): # is j an appropriate identifier here?
#if numpy.isnan(float(j)/bestalgdata[i]):
# set_trace()
# assign significance flag
significance0vs1 = 0
if nb == 0:
istat0 = 0
istat1 = 1
else:
istat0 = 1
istat1 = 0
#set_trace()
z, p = ranksums(rankdata[istat0, i], rankdata[istat1, i])
if (nbtests * p < 0.05 and z > 0
and not numpy.isinf(ertdata[istat0][i])
and z * (ertdata[istat1][i] - ertdata[istat0][i]) >
0): # z-value and ERT-ratio must agree
significance0vs1 = -int(
numpy.ceil(numpy.log10(nbtests * p)))
if nbtests * p < 0.05 and z < 0 and z * (
ertdata[istat1][i] - ertdata[istat0][i]) > 0:
significance0vs1 = int(
numpy.ceil(numpy.log10(nbtests * p)))
alignment = 'c'
if i == len(data) - 1: # last element
alignment = 'c|'
if numpy.isinf(bestalgdata[i]):
tableentry = (
r'\multicolumn{2}{@{}%s@{}}{\textbf{\textit{%s}}}'
% (alignment, writeFEvals2(float(j), 2)))
# TODO: is this the desired behaviour?
else:
# Formatting
assert not numpy.isnan(float(j) / bestalgdata[i])
tmp = float(j) / bestalgdata[i]
tableentry = writeFEvals2(tmp, 2)
isBold = False
if 11 < 3 and tmp <= 3:
isBold = True
if significance0vs1 > 0:
isBold = True
if tableentry.find('e') > -1:
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' %
(alignment, tableentry))
else:
tmp = tableentry.split('.', 1)
if isBold:
tmp = list(r'\textbf{%s}' % i for i in tmp)
elif significance0vs1 < 0:
tmp = list(r'\textit{%s}' % i for i in tmp)
tableentry = ' & .'.join(tmp)
if len(tmp) == 1:
tableentry += '&'
currankdata = numpy.power(evals[i], -1.)
currankdata[numpy.isnan(
currankdata
)] = -entry.finalfunvals[numpy.isnan(currankdata)]
if 11 < 3 and nb == 0:
rankdata0.append(currankdata.copy())
addition = ''
elif 11 < 3: #nb==1
z, p = ranksums(rankdata0[i], currankdata)
tmp = '.'
#addition = '' # Uncomment to gain space. Meanwhile it displays the worst case scenario
if ((nbtests * p) < 0.05 and
((numpy.isinf(data0[i]) and numpy.isinf(j)) or z *
(j - data0[i]) > 0)
): # z-value and ERT-ratio must agree
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
if z > 0:
tmp = r'\uparrow' #* nbstars
else:
tmp = r'\downarrow' #* nbstars
# print z, linebest[i], line1
if nbstars > 1:
tmp += str(int(nbstars))
#addition = tmp # uncomment to gain space.
addition = r'/' + tmp # Comment and uncomment line above to gain space
#addition = tmp # Comment and uncomment line above to gain space
superscript = '\uparrow'
superscript = ''
#if addition:
#superscript = '.'
z, p = ranksums(rankdatabest[i], currankdata)
if ((nbtests * p) < 0.05 and
((numpy.isinf(bestalgdata[i]) and numpy.isinf(j))
or z * (j - bestalgdata[i]) > 0)
): # z-value and ERT-ratio must agree
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
#tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star'])
#set_trace()
if z > 0:
superscript = r'\uparrow' #* nbstars
else:
superscript = r'\downarrow' #* nbstars
# print z, linebest[i], line1
if nbstars > 1:
superscript += str(int(nbstars))
addition = ''
if superscript or significance0vs1:
s = ''
if significance0vs1 > 0:
s = '\star'
if significance0vs1 > 1:
s += str(significance0vs1)
s = r'$^{' + s + superscript + r'}$'
if tableentry.endswith('}'):
tableentry = tableentry[:-1] + s + r'}'
else:
tableentry += s
if 11 < 3 and (superscript or addition):
isClosingBrace = False
if tableentry.endswith('}'):
isClosingBrace = True
tableentry = tableentry[:-1]
#tableentry += r'$^{' + superscript + '}_{' + addition + '}$'
tableentry += r'$^{' + superscript + addition + '}$'
if isClosingBrace:
tableentry += '}'
curline.append(tableentry)
# Two cases: both tabular give an overfull hbox
# AND generate a LaTeX Warning: Float too large for page by 16.9236pt on input line 421. (noisy)
# OR generate a LaTeX Warning: Float too large for page by 33.57658pt on input line 421. (noisy)
tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:]
try:
tmp = tmp[0]
curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False))
except IndexError:
curline.append('%d' % 0)
curline.append('/%d' % entry.nbRuns())
table.append(curline[:])
extraeol.append('')
extraeol[-1] = r'\hline'
extraeol[-1] = ''
outputfile = os.path.join(outputdir,
'cmptable_%02dD%s.tex' % (d, info))
spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(
targetsOfInterest) + '|@{}r@{}@{}l@{}'
res = r'\providecommand{\algzeroshort}{%s}' % alg0 + '\n'
res += r'\providecommand{\algoneshort}{%s}' % alg1 + '\n'
#res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec,
#extraeol=extraeol)
res += tableLaTeX(table, spec=spec, extraeol=extraeol)
f = open(outputfile, 'w')
f.write(res)
f.close()
if verbose:
print "Table written in %s" % outputfile
def main(dsList, _valuesOfInterest, outputdir, verbose=True):
"""From a DataSetList, returns a convergence and ERT figure vs dim."""
plt.rc("axes", labelsize=20, titlesize=24)
plt.rc("xtick", labelsize=20)
plt.rc("ytick", labelsize=20)
plt.rc("font", size=20)
plt.rc("legend", fontsize=20)
dictFunc = dsList.dictByFunc()
for func in dictFunc:
dictFunc[func] = dictFunc[func].dictByDim()
filename = os.path.join(outputdir, 'ppdata_f%d' % (func))
fig = plt.figure()
#legend = []
line = []
valuesOfInterest = list(j[func] for j in _valuesOfInterest)
valuesOfInterest.sort(reverse=True)
for i in range(len(valuesOfInterest)):
succ = []
unsucc = []
displaynumber = []
data = []
#Collect data that have the same function and different dimension.
for dim in sorted(dictFunc[func]):
tmp = generateData(dictFunc[func][dim][0], valuesOfInterest[i])
#data.append(numpy.append(dim, tmp))
if tmp[2] > 0: #Number of success is larger than 0
succ.append(numpy.append(dim, tmp))
if tmp[2] < dictFunc[func][dim][0].nbRuns():
displaynumber.append((dim, tmp[0], tmp[2]))
else:
unsucc.append(numpy.append(dim, tmp))
if succ:
tmp = numpy.vstack(succ)
#ERT
plt.plot(tmp[:, 0],
tmp[:, 1],
figure=fig,
color=colors[i],
marker='o',
markersize=20)
#median
plt.plot(tmp[:, 0],
tmp[:, -1],
figure=fig,
color=colors[i],
linestyle='',
marker='+',
markersize=30,
markeredgewidth=5)
# To have the legend displayed whatever happens with the data.
plt.plot([], [],
color=colors[i],
label=' %+d' % (numpy.log10(valuesOfInterest[i])))
#Only for the last target function value...
if unsucc:
#set_trace()
tmp = numpy.vstack(unsucc)
plt.plot(tmp[:, 0],
tmp[:, -2],
figure=fig,
color=colors[i],
marker='x',
markersize=20)
if not bestalg.bestalgentries:
bestalg.loadBBOB2009()
bestalgdata = []
for d in dimsBBOB:
entry = bestalg.bestalgentries[(d, func)]
tmp = entry.ert[entry.target <= 1e-8]
try:
bestalgdata.append(tmp[0])
except IndexError:
bestalgdata.append(None)
plt.plot(dimsBBOB,
bestalgdata,
figure=fig,
color='wheat',
linewidth=10,
zorder=-1)
plt.plot(dimsBBOB,
bestalgdata,
figure=fig,
ls='',
marker='d',
markersize=25,
color='wheat',
zorder=-1)
if displaynumber: #displayed only for the smallest valuesOfInterest
a = fig.gca()
for j in displaynumber:
plt.text(j[0],
j[1] * 1.85,
"%.0f" % j[2],
axes=a,
horizontalalignment="center",
verticalalignment="bottom")
if isBenchmarkinfosFound:
title = funInfos[func]
else:
title = ''
legend = func in (1, 24, 101, 130)
beautify(fig,
filename,
title=title,
legend=legend,
fileFormat=figformat,
labels=['', ''],
scale=['log', 'log'],
verbose=verbose)
plt.close(fig)
plt.rcdefaults()
def main(dictAlg, sortedAlgs, outputdir='.', verbose=True, function_targets_line=True): # [1, 13, 101]
"""Generate one table per func with results of multiple algorithms."""
"""Difference with the first version:
* numbers aligned using the decimal separator
* premices for dispersion measure
* significance test against best algorithm
* table width...
Takes ``targetsOfInterest`` from this file as "input argument" to compute
the desired target values. ``targetsOfInterest`` might be configured via
config.
"""
# TODO: method is long, terrible to read, split if possible
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
# Sort data per dimension and function
dictData = {}
dsListperAlg = list(dictAlg[i] for i in sortedAlgs)
for n, entries in enumerate(dsListperAlg):
tmpdictdim = entries.dictByDim()
for d in tmpdictdim:
tmpdictfun = tmpdictdim[d].dictByFunc()
for f in tmpdictfun:
dictData.setdefault((d, f), {})[n] = tmpdictfun[f]
nbtests = len(dictData)
for df in dictData:
# Generate one table per df
# first update targets for each dimension-function pair if needed:
targets = targetsOfInterest((df[1], df[0]))
targetf = targets[-1]
# best 2009
refalgentry = bestalg.bestalgentries2009[df]
refalgert = refalgentry.detERT(targets)
refalgevals = (refalgentry.detEvals((targetf, ))[0][0])
refalgnbruns = len(refalgevals)
refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False)
# Process the data
# The following variables will be lists of elements each corresponding
# to an algorithm
algnames = []
#algdata = []
algerts = []
algevals = []
algdisp = []
algnbsucc = []
algnbruns = []
algmedmaxevals = []
algmedfinalfunvals = []
algtestres = []
algentries = []
for n in sorted(dictData[df].keys()):
entries = dictData[df][n]
# the number of datasets for a given dimension and function (df)
# should be strictly 1. TODO: find a way to warn
# TODO: do this checking before... why wasn't it triggered by ppperprof?
if len(entries) > 1:
txt = ("There is more than a single entry associated with "
"folder %s on %d-D f%d." % (sortedAlgs[n], df[0], df[1]))
raise Exception(txt)
entry = entries[0]
algentries.append(entry)
algnames.append(sortedAlgs[n])
evals = entry.detEvals(targets)
#tmpdata = []
tmpdisp = []
tmpert = []
for i, e in enumerate(evals):
succ = (numpy.isnan(e) == False)
ec = e.copy() # note: here was the previous bug (changes made in e also appeared in evals !)
ec[succ == False] = entry.maxevals[succ == False]
ert = toolsstats.sp(ec, issuccessful=succ)[0]
#tmpdata.append(ert/refalgert[i])
if succ.any():
tmp = toolsstats.drawSP(ec[succ], entry.maxevals[succ == False],
[10, 50, 90], samplesize=samplesize)[0]
tmpdisp.append((tmp[-1] - tmp[0])/2.)
else:
tmpdisp.append(numpy.nan)
tmpert.append(ert)
algerts.append(tmpert)
algevals.append(evals)
#algdata.append(tmpdata)
algdisp.append(tmpdisp)
algmedmaxevals.append(numpy.median(entry.maxevals))
algmedfinalfunvals.append(numpy.median(entry.finalfunvals))
#algmedmaxevals.append(numpy.median(entry.maxevals)/df[0])
#algmedfinalfunvals.append(numpy.median(entry.finalfunvals))
algtestres.append(significancetest(refalgentry, entry, targets))
# determine success probability for Df = 1e-8
e = entry.detEvals((targetf ,))[0]
algnbsucc.append(numpy.sum(numpy.isnan(e) == False))
algnbruns.append(len(e))
# Process over all data
# find best values...
nalgs = len(dictData[df])
maxRank = 1 + numpy.floor(0.14 * nalgs) # number of algs to be displayed in bold
isBoldArray = [] # Point out the best values
algfinaldata = [] # Store median function values/median number of function evaluations
tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank)
for i, erts in enumerate(algerts):
tmp = []
for j, ert in enumerate(erts): # algi targetj
tmp.append(i in tmptop[j] or (nalgs > 7 and algerts[i][j] <= 3. * refalgert[j]))
isBoldArray.append(tmp)
algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i]))
# significance test of best given algorithm against all others
best_alg_idx = numpy.array(algerts).argsort(0)[0, :] # indexed by target index
significance_versus_others = significance_all_best_vs_other(algentries, targets, best_alg_idx)[0]
# Create the table
table = []
spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % (len(targets)) # in case StrLeft not working: replaced c@{} with l@{ }
spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % (len(targets)) # in case StrLeft not working: replaced c@{} with l@{ }
extraeol = []
# Generate header lines
if with_table_heading:
header = funInfos[df[1]] if funInfos else 'f%d' % df[1]
table.append([r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}'
% (2 * len(targets) + 2, header)])
extraeol.append('')
if function_targets_line is True or (function_targets_line and df[1] in function_targets_line):
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
curline = [r'\#FEs/D']
for i in targetsOfInterest.labels():
curline.append(r'\multicolumn{2}{@{}c@{}}{%s}'
% i)
else:
curline = [r'$\Delta f_\mathrm{opt}$']
for t in targets:
curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}'
% writeFEvals2(t, precision=1, isscientific=True))
# curline.append(r'\multicolumn{2}{@{\,}X@{}|}{%s}'
# % writeFEvals2(targets[-1], precision=1, isscientific=True))
curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}')
table.append(curline)
extraeol.append(r'\hline')
# extraeol.append(r'\hline\arrayrulecolor{tableShade}')
curline = [r'ERT$_{\text{best}}$'] if with_table_heading else [r'\textbf{f%d}' % df[1]]
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
# write ftarget:fevals
for i in xrange(len(refalgert[:-1])):
temp="%.1e" %targetsOfInterest((df[1], df[0]))[i]
if temp[-2]=="0":
temp=temp[:-2]+temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}'
% (temp,writeFEvalsMaxPrec(refalgert[i], 2)))
temp="%.1e" %targetsOfInterest((df[1], df[0]))[-1]
if temp[-2]=="0":
temp=temp[:-2]+temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }'
% (temp,writeFEvalsMaxPrec(refalgert[-1], 2)))
else:
# write #fevals of the reference alg
for i in refalgert[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}'
% writeFEvalsMaxPrec(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}'
% writeFEvalsMaxPrec(refalgert[-1], 2))
# write the success ratio for the reference alg
tmp2 = numpy.sum(numpy.isnan(refalgevals) == False) # count the nb of success
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(refalgevals))
table.append(curline[:])
extraeol.append('')
#for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))):
#print i, gna, gno
#set_trace()
# Format data
#if df == (5, 17):
#set_trace()
header = r'\providecommand{\ntables}{7}'
for i, alg in enumerate(algnames):
#algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames,
#data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres):
commandname = r'\alg%stables' % numtotext(i)
# header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg)))
header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % (commandname, str_to_latex(strip_pathname2(alg)))
curline = [commandname + r'\hspace*{\fill}'] # each list element becomes a &-separated table entry?
for j, tmp in enumerate(zip(algerts[i], algdisp[i], # j is target index
isBoldArray[i], algtestres[i])):
ert, dispersion, isBold, testres = tmp
alignment = '@{\,}X@{\,}'
if j == len(algerts[i]) - 1:
alignment = '@{\,}X@{\,}|'
data = ert/refalgert[j]
# write star for significance against all other algorithms
str_significance_subsup = ''
if (len(best_alg_idx) > 0 and len(significance_versus_others) > 0 and
i == best_alg_idx[j] and nbtests * significance_versus_others[j][1] < 0.05):
logp = -numpy.ceil(numpy.log10(nbtests * significance_versus_others[j][1]))
str_significance_subsup = r"^{%s%s}" % (significance_vs_others_symbol, str(int(logp)) if logp > 1 else '')
# moved out of the above else: this was a bug!?
z, p = testres
if (nbtests * p) < 0.05 and data < 1. and z < 0.:
if not numpy.isinf(refalgert[j]):
tmpevals = algevals[i][j].copy()
tmpevals[numpy.isnan(tmpevals)] = algentries[i].maxevals[numpy.isnan(tmpevals)]
bestevals = refalgentry.detEvals(targets)
bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0])
bestevals[numpy.isnan(bestevals)] = refalgentry.maxevals[bestalgalg][numpy.isnan(bestevals)]
tmpevals = numpy.array(sorted(tmpevals))[0:min(len(tmpevals), len(bestevals))]
bestevals = numpy.array(sorted(bestevals))[0:min(len(tmpevals), len(bestevals))]
#The conditions are now that ERT < ERT_best and
# all(sorted(FEvals_best) > sorted(FEvals_current)).
if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
# tmp2[-1] += r'$^{%s}$' % superscript
str_significance_subsup += r'_{%s%s}' % (significance_vs_ref_symbol,
str(int(nbstars)) if nbstars > 1 else '')
if str_significance_subsup:
str_significance_subsup = '$%s$' % str_significance_subsup
# format number in variable data
if numpy.isnan(data):
curline.append(r'\multicolumn{2}{%s}{.}' % alignment)
else:
if numpy.isinf(refalgert[j]):
curline.append(r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}'
% (alignment,
writeFEvalsMaxPrec(algerts[i][j], 2),
writeFEvalsMaxPrec(dispersion, precdispersion),
str_significance_subsup))
continue
tmp = writeFEvalsMaxPrec(data, precfloat, maxfloatrepr=maxfloatrepr)
if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation
if numpy.isinf(data) and j == len(algerts[i]) - 1:
tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec(algfinaldata[i][1], 0, maxfloatrepr=maxfloatrepr)
else:
tmp = writeFEvalsMaxPrec(data, precscien, maxfloatrepr=data)
if isBold:
tmp = r'\textbf{%s}' % tmp
if not numpy.isnan(dispersion):
tmpdisp = dispersion/refalgert[j]
if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack
tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=tmpdisp)
else:
tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr)
tmp += r'\mbox{\tiny (%s)}' % tmpdisp
curline.append(r'\multicolumn{2}{%s}{%s%s}' % (alignment, tmp, str_significance_subsup))
else:
tmp2 = tmp.split('.', 1)
if len(tmp2) < 2:
tmp2.append('')
else:
tmp2[-1] = '.' + tmp2[-1]
if isBold:
tmp3 = []
for k in tmp2:
tmp3.append(r'\textbf{%s}' % k)
tmp2 = tmp3
if not numpy.isnan(dispersion):
tmpdisp = dispersion/refalgert[j]
if tmpdisp >= maxfloatrepr or tmpdisp < 0.01:
tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=tmpdisp)
else:
tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr)
tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp))
tmp2[-1] += str_significance_subsup
curline.extend(tmp2)
curline.append('%d' % algnbsucc[i])
curline.append('/%d' % algnbruns[i])
table.append(curline)
extraeol.append('')
# Write table
res = tableXLaTeX(table, spec=spec, extraeol=extraeol)
try:
filename = os.path.join(outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0]))
f = open(filename, 'w')
f.write(header + '\n')
f.write(res)
if verbose:
print 'Wrote table in %s' % filename
except:
raise
else:
f.close()
def main(dictAlg,
sortedAlgs,
outputdir='.',
verbose=True,
function_targets_line=True): # [1, 13, 101]
"""Generate one table per func with results of multiple algorithms."""
"""Difference with the first version:
* numbers aligned using the decimal separator
* premices for dispersion measure
* significance test against best algorithm
* table width...
Takes ``targetsOfInterest`` from this file as "input argument" to compute
the desired target values. ``targetsOfInterest`` might be configured via
config.
"""
# TODO: method is long, terrible to read, split if possible
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
# Sort data per dimension and function
dictData = {}
dsListperAlg = list(dictAlg[i] for i in sortedAlgs)
for n, entries in enumerate(dsListperAlg):
tmpdictdim = entries.dictByDim()
for d in tmpdictdim:
tmpdictfun = tmpdictdim[d].dictByFunc()
for f in tmpdictfun:
dictData.setdefault((d, f), {})[n] = tmpdictfun[f]
nbtests = len(dictData)
for df in dictData:
# Generate one table per df
# first update targets for each dimension-function pair if needed:
targets = targetsOfInterest((df[1], df[0]))
targetf = targets[-1]
# best 2009
refalgentry = bestalg.bestalgentries2009[df]
refalgert = refalgentry.detERT(targets)
refalgevals = (refalgentry.detEvals((targetf, ))[0][0])
refalgnbruns = len(refalgevals)
refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False)
# Process the data
# The following variables will be lists of elements each corresponding
# to an algorithm
algnames = []
#algdata = []
algerts = []
algevals = []
algdisp = []
algnbsucc = []
algnbruns = []
algmedmaxevals = []
algmedfinalfunvals = []
algtestres = []
algentries = []
for n in sorted(dictData[df].keys()):
entries = dictData[df][n]
# the number of datasets for a given dimension and function (df)
# should be strictly 1. TODO: find a way to warn
# TODO: do this checking before... why wasn't it triggered by ppperprof?
if len(entries) > 1:
print entries
txt = ("There is more than a single entry associated with "
"folder %s on %d-D f%d." %
(sortedAlgs[n], df[0], df[1]))
raise Exception(txt)
entry = entries[0]
algentries.append(entry)
algnames.append(sortedAlgs[n])
evals = entry.detEvals(targets)
#tmpdata = []
tmpdisp = []
tmpert = []
for i, e in enumerate(evals):
succ = (numpy.isnan(e) == False)
ec = e.copy(
) # note: here was the previous bug (changes made in e also appeared in evals !)
ec[succ == False] = entry.maxevals[succ == False]
ert = toolsstats.sp(ec, issuccessful=succ)[0]
#tmpdata.append(ert/refalgert[i])
if succ.any():
tmp = toolsstats.drawSP(ec[succ],
entry.maxevals[succ == False],
[10, 50, 90],
samplesize=samplesize)[0]
tmpdisp.append((tmp[-1] - tmp[0]) / 2.)
else:
tmpdisp.append(numpy.nan)
tmpert.append(ert)
algerts.append(tmpert)
algevals.append(evals)
#algdata.append(tmpdata)
algdisp.append(tmpdisp)
algmedmaxevals.append(numpy.median(entry.maxevals))
algmedfinalfunvals.append(numpy.median(entry.finalfunvals))
#algmedmaxevals.append(numpy.median(entry.maxevals)/df[0])
#algmedfinalfunvals.append(numpy.median(entry.finalfunvals))
algtestres.append(significancetest(refalgentry, entry, targets))
# determine success probability for Df = 1e-8
e = entry.detEvals((targetf, ))[0]
algnbsucc.append(numpy.sum(numpy.isnan(e) == False))
algnbruns.append(len(e))
# Process over all data
# find best values...
nalgs = len(dictData[df])
maxRank = 1 + numpy.floor(
0.14 * nalgs) # number of algs to be displayed in bold
isBoldArray = [] # Point out the best values
algfinaldata = [
] # Store median function values/median number of function evaluations
tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank)
for i, erts in enumerate(algerts):
tmp = []
for j, ert in enumerate(erts): # algi targetj
tmp.append(i in tmptop[j] or
(nalgs > 7 and algerts[i][j] <= 3. * refalgert[j]))
isBoldArray.append(tmp)
algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i]))
# significance test of best given algorithm against all others
best_alg_idx = numpy.array(algerts).argsort(0)[
0, :] # indexed by target index
significance_versus_others = significance_all_best_vs_other(
algentries, targets, best_alg_idx)[0]
# Create the table
table = []
tableHtml = []
spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % (
len(targets)
) # in case StrLeft not working: replaced c@{} with l@{ }
spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % (
len(targets)
) # in case StrLeft not working: replaced c@{} with l@{ }
extraeol = []
# Generate header lines
if with_table_heading:
header = funInfos[df[1]] if funInfos else 'f%d' % df[1]
table.append([
r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}' %
(2 * len(targets) + 2, header)
])
extraeol.append('')
if function_targets_line is True or (function_targets_line and df[1]
in function_targets_line):
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
curline = [r'\#FEs/D']
curlineHtml = ['<thead>\n<tr>\n<th>#FEs/D<br>REPLACEH</th>\n']
counter = 1
for i in targetsOfInterest.labels():
curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' % i)
curlineHtml.append('<td>%s<br>REPLACE%d</td>\n' %
(i, counter))
counter += 1
else:
curline = [r'$\Delta f_\mathrm{opt}$']
curlineHtml = [
'<thead>\n<tr>\n<th>Δ f<sub>opt</sub><br>REPLACEH</th>\n'
]
counter = 1
for t in targets:
curline.append(
r'\multicolumn{2}{@{\,}X@{\,}}{%s}' %
writeFEvals2(t, precision=1, isscientific=True))
curlineHtml.append(
'<td>%s<br>REPLACE%d</td>\n' % (writeFEvals2(
t, precision=1, isscientific=True), counter))
counter += 1
# curline.append(r'\multicolumn{2}{@{\,}X@{}|}{%s}'
# % writeFEvals2(targets[-1], precision=1, isscientific=True))
curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}')
curlineHtml.append('<td>#succ<br>REPLACEF</td>\n</tr>\n</thead>\n')
table.append(curline)
extraeol.append(r'\hline')
# extraeol.append(r'\hline\arrayrulecolor{tableShade}')
curline = [r'ERT$_{\text{best}}$'
] if with_table_heading else [r'\textbf{f%d}' % df[1]]
replaceValue = 'ERT<sub>best</sub>' if with_table_heading else (
'<b>f%d</b>' % df[1])
curlineHtml = [
item.replace('REPLACEH', replaceValue) for item in curlineHtml
]
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
# write ftarget:fevals
counter = 1
for i in xrange(len(refalgert[:-1])):
temp = "%.1e" % targetsOfInterest((df[1], df[0]))[i]
if temp[-2] == "0":
temp = temp[:-2] + temp[-1]
curline.append(
r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' %
(temp, writeFEvalsMaxPrec(refalgert[i], 2)))
replaceValue = '<i>%s</i>:%s' % (
temp, writeFEvalsMaxPrec(refalgert[i], 2))
curlineHtml = [
item.replace('REPLACE%d' % counter, replaceValue)
for item in curlineHtml
]
counter += 1
temp = "%.1e" % targetsOfInterest((df[1], df[0]))[-1]
if temp[-2] == "0":
temp = temp[:-2] + temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' %
(temp, writeFEvalsMaxPrec(refalgert[-1], 2)))
replaceValue = '<i>%s</i>:%s' % (
temp, writeFEvalsMaxPrec(refalgert[-1], 2))
curlineHtml = [
item.replace('REPLACE%d' % counter, replaceValue)
for item in curlineHtml
]
else:
# write #fevals of the reference alg
counter = 1
for i in refalgert[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' %
writeFEvalsMaxPrec(i, 2))
curlineHtml = [
item.replace('REPLACE%d' % counter,
writeFEvalsMaxPrec(i, 2))
for item in curlineHtml
]
counter += 1
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' %
writeFEvalsMaxPrec(refalgert[-1], 2))
curlineHtml = [
item.replace('REPLACE%d' % counter,
writeFEvalsMaxPrec(refalgert[-1], 2))
for item in curlineHtml
]
# write the success ratio for the reference alg
tmp2 = numpy.sum(
numpy.isnan(refalgevals) == False) # count the nb of success
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(refalgevals))
replaceValue = '%d/%d' % (tmp2, len(refalgevals))
else:
replaceValue = '%d' % tmp2
curlineHtml = [
item.replace('REPLACEF', replaceValue) for item in curlineHtml
]
table.append(curline[:])
tableHtml.extend(curlineHtml[:])
tableHtml.append('<tbody>\n')
extraeol.append('')
#for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))):
#print i, gna, gno
#set_trace()
# Format data
#if df == (5, 17):
#set_trace()
header = r'\providecommand{\ntables}{7}'
for i, alg in enumerate(algnames):
tableHtml.append('<tr>\n')
#algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames,
#data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres):
commandname = r'\alg%stables' % numtotext(i)
# header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg)))
header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % (
commandname, str_to_latex(strip_pathname1(alg)))
curline = [
commandname + r'\hspace*{\fill}'
] # each list element becomes a &-separated table entry?
curlineHtml = [
'<th>%s</th>\n' % str_to_latex(strip_pathname1(alg))
]
for j, tmp in enumerate(
zip(
algerts[i],
algdisp[i], # j is target index
isBoldArray[i],
algtestres[i])):
ert, dispersion, isBold, testres = tmp
alignment = '@{\,}X@{\,}'
if j == len(algerts[i]) - 1:
alignment = '@{\,}X@{\,}|'
data = ert / refalgert[j]
# write star for significance against all other algorithms
str_significance_subsup = ''
str_significance_subsup_html = ''
if (len(best_alg_idx) > 0
and len(significance_versus_others) > 0
and i == best_alg_idx[j]
and nbtests * significance_versus_others[j][1] < 0.05):
logp = -numpy.ceil(
numpy.log10(
nbtests * significance_versus_others[j][1]))
logp = numpy.min(
(9,
logp)) # not messing up the format and handling inf
str_significance_subsup = r"^{%s%s}" % (
significance_vs_others_symbol,
str(int(logp)) if logp > 1 else '')
str_significance_subsup_html = '<sup>%s%s</sup>' % (
significance_vs_others_symbol_html,
str(int(logp)) if logp > 1 else '')
# moved out of the above else: this was a bug!?
z, p = testres
if (nbtests * p) < 0.05 and data < 1. and z < 0.:
if not numpy.isinf(refalgert[j]):
tmpevals = algevals[i][j].copy()
tmpevals[numpy.isnan(tmpevals)] = algentries[
i].maxevals[numpy.isnan(tmpevals)]
bestevals = refalgentry.detEvals(targets)
bestevals, bestalgalg = (bestevals[0][0],
bestevals[1][0])
bestevals[numpy.isnan(
bestevals)] = refalgentry.maxevals[bestalgalg][
numpy.isnan(bestevals)]
tmpevals = numpy.array(sorted(
tmpevals))[0:min(len(tmpevals), len(bestevals))]
bestevals = numpy.array(sorted(
bestevals))[0:min(len(tmpevals), len(bestevals))]
#The conditions are now that ERT < ERT_best and
# all(sorted(FEvals_best) > sorted(FEvals_current)).
if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
# tmp2[-1] += r'$^{%s}$' % superscript
str_significance_subsup += r'_{%s%s}' % (
significance_vs_ref_symbol,
str(int(nbstars)) if nbstars > 1 else '')
str_significance_subsup_html = '<sub>%s%s</sub>' % (
significance_vs_ref_symbol_html,
str(int(nbstars)) if nbstars > 1 else '')
if str_significance_subsup:
str_significance_subsup = '$%s$' % str_significance_subsup
# format number in variable data
if numpy.isnan(data):
curline.append(r'\multicolumn{2}{%s}{.}' % alignment)
else:
if numpy.isinf(refalgert[j]):
curline.append(
r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}'
% (alignment, writeFEvalsMaxPrec(algerts[i][j], 2),
writeFEvalsMaxPrec(dispersion, precdispersion),
str_significance_subsup))
curlineHtml.append(
'<td sorttable_customkey=\"%f\"><b>%s</b> (%s)%s</td>\n'
% (algerts[i][j],
writeFEvalsMaxPrec(algerts[i][j], 2),
writeFEvalsMaxPrec(dispersion, precdispersion),
str_significance_subsup_html))
continue
tmp = writeFEvalsMaxPrec(data,
precfloat,
maxfloatrepr=maxfloatrepr)
tmpHtml = writeFEvalsMaxPrec(data,
precfloat,
maxfloatrepr=maxfloatrepr)
sortKey = data
if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation
if numpy.isinf(data) and j == len(algerts[i]) - 1:
tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec(
algfinaldata[i][1],
0,
maxfloatrepr=maxfloatrepr)
tmpHtml += '<i>%s</i>' % writeFEvalsMaxPrec(
algfinaldata[i][1],
0,
maxfloatrepr=maxfloatrepr)
sortKey = algfinaldata[i][1]
else:
tmp = writeFEvalsMaxPrec(data,
precscien,
maxfloatrepr=data)
if isBold:
tmpHtml = '<b>%s</b>' % tmp
tmp = r'\textbf{%s}' % tmp
if not numpy.isnan(dispersion):
tmpdisp = dispersion / refalgert[j]
if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=tmpdisp)
else:
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=maxfloatrepr)
tmp += r'\mbox{\tiny (%s)}' % tmpdisp
tmpHtml += ' (%s)' % tmpdisp
curline.append(
r'\multicolumn{2}{%s}{%s%s}' %
(alignment, tmp, str_significance_subsup))
tmpHtml = tmpHtml.replace('$\infty$', '∞')
if (numpy.isinf(sortKey)):
sortKey = sys.maxint
curlineHtml.append(
'<td sorttable_customkey=\"%f\">%s%s</td>' %
(sortKey, tmpHtml, str_significance_subsup_html))
else:
tmp2 = tmp.split('.', 1)
if len(tmp2) < 2:
tmp2.append('')
else:
tmp2[-1] = '.' + tmp2[-1]
if isBold:
tmp3 = []
tmp3html = []
for k in tmp2:
tmp3.append(r'\textbf{%s}' % k)
tmp3html.append('<b>%s</b>' % k)
tmp2 = tmp3
tmp2html = tmp3html
else:
tmp2html = []
tmp2html.extend(tmp2)
if not numpy.isnan(dispersion):
tmpdisp = dispersion / refalgert[j]
if tmpdisp >= maxfloatrepr or tmpdisp < 0.01:
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=tmpdisp)
else:
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=maxfloatrepr)
tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp))
tmp2html[-1] += ' (%s)' % tmpdisp
tmp2[-1] += str_significance_subsup
tmp2html[-1] += str_significance_subsup_html
curline.extend(tmp2)
tmp2html = ("").join(str(item) for item in tmp2html)
tmp2html = tmp2html.replace('$\infty$', '∞')
curlineHtml.append(
'<td sorttable_customkey=\"%f\">%s</td>' %
(data, tmp2html))
curline.append('%d' % algnbsucc[i])
curline.append('/%d' % algnbruns[i])
table.append(curline)
curlineHtml.append('<td sorttable_customkey=\"%d\">%d/%d</td>\n' %
(algnbsucc[i], algnbsucc[i], algnbruns[i]))
tableHtml.extend(curlineHtml[:])
extraeol.append('')
# Write table
res = tableXLaTeX(table, spec=spec, extraeol=extraeol)
try:
filename = os.path.join(
outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0]))
f = open(filename, 'w')
f.write(header + '\n')
f.write(res)
res = ("").join(str(item) for item in tableHtml)
res = '\n<table class=\"sortable\" style=\"width:800px \">\n%s</table>\n<p/>\n' % res
if df[0] in (5, 20):
filename = os.path.join(
outputdir,
genericsettings.many_algorithm_file_name + '.html')
lines = []
with open(filename) as infile:
for line in infile:
if '<!--' + 'pptablesf%03d%02dDHtml' % (
df[1], df[0]) + '-->' in line:
lines.append(res)
lines.append(line)
with open(filename, 'w') as outfile:
for line in lines:
outfile.write(line)
if verbose:
print 'Wrote table in %s' % filename
except:
raise
else:
f.close()
def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True):
"""One table per dimension, modified to fit in 1 page per table."""
#TODO: method is long, split if possible
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3]
alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3]
open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(
r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' +
r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n')
if info:
info = '_' + info
dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys()))
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
header = []
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
header = [r'\#FEs/D']
for label in targetsOfInterest.labels():
header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % label)
else:
header = [r'$\Delta f_\mathrm{opt}$']
for label in targetsOfInterest.labels():
header.append(r'\multicolumn{2}{@{\,}c@{\,}}{%s}' % label)
header.append(r'\multicolumn{2}{@{}l@{}}{\#succ}')
for d in dimsOfInterest: # TODO set as input arguments
table = [header]
extraeol = [r'\hline']
try:
dictFunc0 = dictDim0[d].dictByFunc()
dictFunc1 = dictDim1[d].dictByFunc()
except KeyError:
continue
funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys()))
nbtests = len(funcs) * 2. #len(dimsOfInterest)
for f in sorted(funcs):
targets = targetsOfInterest((f, d))
targetf = targets[-1]
bestalgentry = bestalg.bestalgentries2009[(d, f)]
curline = [r'${\bf f_{%d}}$' % f]
bestalgdata = bestalgentry.detERT(targets)
bestalgevals, bestalgalgs = bestalgentry.detEvals(targets)
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
# write ftarget:fevals
for i in xrange(len(bestalgdata[:-1])):
temp = "%.1e" % targetsOfInterest((f, d))[i]
if temp[-2] == "0":
temp = temp[:-2] + temp[-1]
curline.append(
r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' %
(temp, writeFEvalsMaxPrec(bestalgdata[i], 2)))
temp = "%.1e" % targetsOfInterest((f, d))[-1]
if temp[-2] == "0":
temp = temp[:-2] + temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' %
(temp, writeFEvalsMaxPrec(bestalgdata[-1], 2)))
else:
# write #fevals of the reference alg
for i in bestalgdata[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' %
writeFEvalsMaxPrec(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' %
writeFEvalsMaxPrec(bestalgdata[-1], 2))
tmp = bestalgentry.detEvals([targetf])[0][0]
tmp2 = numpy.sum(numpy.isnan(tmp) == False)
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(tmp))
table.append(curline[:])
extraeol.append('')
rankdata0 = [] # never used
# generate all data from ranksum test
entries = []
ertdata = {}
for nb, dsList in enumerate((dictFunc0, dictFunc1)):
try:
entry = dsList[f][
0] # take the first DataSet, there should be only one?
except KeyError:
warnings.warn('data missing for data set ' + str(nb) +
' and function ' + str(f))
print('*** Warning: data missing for data set ' + str(nb) +
' and function ' + str(f) + '***')
continue # TODO: problem here!
ertdata[nb] = entry.detERT(targets)
entries.append(entry)
for _t in ertdata.values():
for _tt in _t:
if _tt is None:
raise ValueError
if len(entries) < 2: # funcion not available for *both* algorithms
continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following
testres0vs1 = significancetest(entries[0], entries[1], targets)
testresbestvs1 = significancetest(bestalgentry, entries[1],
targets)
testresbestvs0 = significancetest(bestalgentry, entries[0],
targets)
for nb, entry in enumerate(entries):
if nb == 0:
curline = [r'1:\:\algorithmAshort\hspace*{\fill}']
else:
curline = [r'2:\:\algorithmBshort\hspace*{\fill}']
#data = entry.detERT(targetsOfInterest)
dispersion = []
data = []
evals = entry.detEvals(targets)
for i in evals:
succ = (numpy.isnan(i) == False)
tmp = i.copy()
tmp[succ == False] = entry.maxevals[numpy.isnan(i)]
#set_trace()
data.append(toolsstats.sp(tmp, issuccessful=succ)[0])
#if not any(succ):
#set_trace()
if any(succ):
tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ == False],
(10, 50, 90), samplesize)[0]
dispersion.append((tmp2[-1] - tmp2[0]) / 2.)
else:
dispersion.append(None)
if nb == 0:
assert not isinstance(data, numpy.ndarray)
data0 = data[:] # TODO: check if it is not an array, it's never used anyway?
for i, dati in enumerate(data):
z, p = testres0vs1[
i] # TODO: there is something with the sign that I don't get
# assign significance flag, which is the -log10(p)
significance0vs1 = 0
if nb != 0:
z = -z # the test is symmetric
if nbtests * p < 0.05 and z > 0:
significance0vs1 = -int(
numpy.ceil(numpy.log10(min([
1.0, nbtests * p
])))) # this is the larger the more significant
isBold = significance0vs1 > 0
alignment = 'c'
if i == len(data) - 1: # last element
alignment = 'c|'
if numpy.isinf(
bestalgdata[i]
): # if the 2009 best did not solve the problem
tmp = writeFEvalsMaxPrec(float(dati), 2)
if not numpy.isinf(dati):
tmp = r'\textit{%s}' % (tmp)
if isBold:
tmp = r'\textbf{%s}' % tmp
if dispersion[i] and numpy.isfinite(dispersion[i]):
tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(
dispersion[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' %
(alignment, tmp))
else:
# Formatting
tmp = float(dati) / bestalgdata[i]
assert not numpy.isnan(tmp)
isscientific = False
if tmp >= 1000:
isscientific = True
tableentry = writeFEvals2(tmp,
2,
isscientific=isscientific)
tableentry = writeFEvalsMaxPrec(tmp, 2)
if numpy.isinf(tmp) and i == len(data) - 1:
tableentry = (
tableentry + r'\textit{%s}' %
writeFEvals2(numpy.median(entry.maxevals), 2))
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore
tableentry = r'\textit{%s}' % tableentry
if dispersion[i] and numpy.isfinite(
dispersion[i] / bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(
dispersion[i] / bestalgdata[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' %
(alignment, tableentry))
elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and
i != len(data) - 1):
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3 and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
if dispersion[i] and numpy.isfinite(
dispersion[i] / bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(
dispersion[i] / bestalgdata[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' %
(alignment, tableentry))
else:
tmp = tableentry.split('.', 1)
if isBold:
tmp = list(r'\textbf{%s}' % i for i in tmp)
elif 11 < 3 and significance0vs1 < 0:
tmp = list(r'\textit{%s}' % i for i in tmp)
tableentry = ' & .'.join(tmp)
if len(tmp) == 1:
tableentry += '&'
if dispersion[i] and numpy.isfinite(
dispersion[i] / bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(
dispersion[i] / bestalgdata[i], 1)
superscript = ''
if nb == 0:
z, p = testresbestvs0[i]
else:
z, p = testresbestvs1[i]
#The conditions are now that ERT < ERT_best
if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0.
and z < 0.):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
#tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star'])
if z > 0:
superscript = r'\uparrow' #* nbstars
else:
superscript = r'\downarrow' #* nbstars
# print z, linebest[i], line1
if nbstars > 1:
superscript += str(int(nbstars))
if superscript or significance0vs1:
s = ''
if significance0vs1 > 0:
s = '\star'
if significance0vs1 > 1:
s += str(significance0vs1)
s = r'$^{' + s + superscript + r'}$'
if tableentry.endswith('}'):
tableentry = tableentry[:-1] + s + r'}'
else:
tableentry += s
curline.append(tableentry)
#curline.append(tableentry)
#if dispersion[i] is None or numpy.isinf(bestalgdata[i]):
#curline.append('')
#else:
#tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2)
#curline.append('(%s)' % tmp)
tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:]
try:
tmp = tmp[0]
curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False))
except IndexError:
curline.append('%d' % 0)
curline.append('/%d' % entry.nbRuns())
table.append(curline[:])
extraeol.append('')
extraeol[-1] = r'\hline'
extraeol[-1] = ''
outputfile = os.path.join(outputdir,
'pptable2_%02dD%s.tex' % (d, info))
spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(
targetsOfInterest) + '|@{}r@{}@{}l@{}'
res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels(
alg0) + '\n'
res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels(
alg1) + '\n'
# open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res)
#res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec,
#extraeol=extraeol)
res += tableLaTeX(table, spec=spec, extraeol=extraeol)
f = open(outputfile, 'w')
f.write(res)
f.close()
if verbose:
print "Table written in %s" % outputfile
def main(dictAlg, order=None, outputdir='.', info='default',
verbose=True):
"""Generates a figure showing the performance of algorithms.
From a dictionary of :py:class:`DataSetList` sorted by algorithms,
generates the cumulative distribution function of the bootstrap
distribution of ERT for algorithms on multiple functions for
multiple targets altogether.
:param dict dictAlg: dictionary of :py:class:`DataSetList` instances
one instance is equivalent to one algorithm,
:param list targets: target function values
:param list order: sorted list of keys to dictAlg for plotting order
:param str outputdir: output directory
:param str info: output file name suffix
:param bool verbose: controls verbosity
"""
global x_limit # late assignment of default, because it can be set to None in config
if 'x_limit' not in globals() or x_limit is None:
x_limit = x_limit_default
tmp = pp.dictAlgByDim(dictAlg)
# tmp = pp.DictAlg(dictAlg).by_dim()
if len(tmp) != 1:
raise Exception('We never integrate over dimension.')
dim = tmp.keys()[0]
algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []]
dictFunc = pp.dictAlgByFun(dictAlg)
# Collect data
# Crafting effort correction: should we consider any?
CrEperAlg = {}
for alg in algorithms_with_data:
CrE = 0.
if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL':
tmp = dictAlg[alg].dictByNoise()
assert len(tmp.keys()) == 1
if tmp.keys()[0] == 'noiselessall':
CrE = 0.5117
elif tmp.keys()[0] == 'nzall':
CrE = 0.6572
CrEperAlg[alg] = CrE
if CrE != 0.0:
print 'Crafting effort for', alg, 'is', CrE
dictData = {} # list of (ert per function) per algorithm
dictMaxEvals = {} # list of (maxevals per function) per algorithm
bestERT = [] # best ert per function
# funcsolved = [set()] * len(targets) # number of functions solved per target
xbest2009 = []
maxevalsbest2009 = []
for f, dictAlgperFunc in dictFunc.iteritems():
if function_IDs and f not in function_IDs:
continue
# print target_values((f, dim))
for j, t in enumerate(target_values((f, dim))):
# for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)):
# funcsolved[j].add(f)
for alg in algorithms_with_data:
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
try:
entry = dictAlgperFunc[alg][0] # one element per fun and per dim.
evals = entry.detEvals([t])[0]
runlengthsucc = evals[np.isnan(evals) == False] / entry.dim
runlengthunsucc = entry.maxevals[np.isnan(evals)] / entry.dim
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
except (KeyError, IndexError):
#set_trace()
warntxt = ('Data for algorithm %s on function %d in %d-D '
% (alg, f, dim)
+ 'are missing.\n')
warnings.warn(warntxt)
dictData.setdefault(alg, []).extend(x)
dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc)
if displaybest2009:
#set_trace()
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgentry = bestalg.bestalgentries2009[(dim, f)]
bestalgevals = bestalgentry.detEvals(target_values((f, dim)))
# print bestalgevals
for j in range(len(bestalgevals[0])):
if bestalgevals[1][j]:
evals = bestalgevals[0][j]
#set_trace()
runlengthsucc = evals[np.isnan(evals) == False] / bestalgentry.dim
runlengthunsucc = bestalgentry.maxevals[bestalgevals[1][j]][np.isnan(evals)] / bestalgentry.dim
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
else:
x = perfprofsamplesize * [np.inf]
runlengthunsucc = []
xbest2009.extend(x)
maxevalsbest2009.extend(runlengthunsucc)
if order is None:
order = dictData.keys()
# Display data
lines = []
if displaybest2009:
args = {'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11.,
'markeredgewidth': 1.5, 'markerfacecolor': refcolor,
'markeredgecolor': refcolor, 'color': refcolor,
'label': 'best 2009', 'zorder': -1}
lines.append(plotdata(np.array(xbest2009), x_limit, maxevalsbest2009,
CrE = 0., **args))
for i, alg in enumerate(order):
try:
data = dictData[alg]
maxevals = dictMaxEvals[alg]
except KeyError:
continue
args = styles[(i) % len(styles)]
args['linewidth'] = 1.5
args['markersize'] = 12.
args['markeredgewidth'] = 1.5
args['markerfacecolor'] = 'None'
args['markeredgecolor'] = args['color']
args['label'] = alg
#args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib
#elif len(show_algorithms) > 0:
#args['color'] = 'wheat'
#args['ls'] = '-'
#args['zorder'] = -1
lines.append(plotdata(np.array(data), x_limit, maxevals,
CrE=CrEperAlg[alg], **args))
labels, handles = plotLegend(lines, x_limit)
if True: #isLateXLeg:
fileName = os.path.join(outputdir,'pprldmany_%s.tex' % (info))
try:
f = open(fileName, 'w')
f.write(r'\providecommand{\nperfprof}{7}')
algtocommand = {}
for i, alg in enumerate(order):
tmp = r'\alg%sperfprof' % pptex.numtotext(i)
f.write(r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' % (tmp, toolsdivers.str_to_latex(toolsdivers.strip_pathname2(alg))))
algtocommand[alg] = tmp
commandnames = []
if displaybest2009:
tmp = r'\algzeroperfprof'
f.write(r'\providecommand{%s}{best 2009}' % (tmp))
algtocommand['best 2009'] = tmp
for l in labels:
commandnames.append(algtocommand[l])
# f.write(headleg)
f.write(r'\providecommand{\perfprofsidepanel}{\mbox{%s}' % commandnames[0]) # TODO: check len(labels) > 0
for i in range(1, len(labels)):
f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i])
f.write('}\n')
# f.write(footleg)
if verbose:
print 'Wrote right-hand legend in %s' % fileName
except:
raise # TODO: Does this make sense?
else:
f.close()
figureName = os.path.join(outputdir,'pprldmany_%s' % (info))
#beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat)
beautify()
text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys())))
text += ',%d-D' % dim
plt.text(0.01, 0.98, text, horizontalalignment="left",
verticalalignment="top", transform=plt.gca().transAxes)
a = plt.gca()
plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative)
xticks, labels = plt.xticks()
tmp = []
for i in xticks:
tmp.append('%d' % round(np.log10(i)))
a.set_xticklabels(tmp)
ppfig.saveFigure(figureName, verbose=verbose)
plt.close()
def main(dictAlg,
sortedAlgs,
targets,
outputdir='.',
verbose=True,
function_targets_line=True): # [1, 13, 101]
"""Generate one table per func with results of multiple algorithms."""
"""Difference with the first version:
* numbers aligned using the decimal separator
* premices for dispersion measure
* significance test against best algorithm
* table width...
"""
# TODO: method is long, terrible to read, split if possible
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
# Sort data per dimension and function
dictData = {}
dsListperAlg = list(dictAlg[i] for i in sortedAlgs)
for n, entries in enumerate(dsListperAlg):
tmpdictdim = entries.dictByDim()
for d in tmpdictdim:
tmpdictfun = tmpdictdim[d].dictByFunc()
for f in tmpdictfun:
dictData.setdefault((d, f), {})[n] = tmpdictfun[f]
nbtests = len(dictData)
for df in dictData:
# Generate one table per df
# best 2009
refalgentry = bestalg.bestalgentries2009[df]
refalgert = refalgentry.detERT(targets)
refalgevals = (refalgentry.detEvals((targetf, ))[0][0])
refalgnbruns = len(refalgevals)
refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False)
# Process the data
# The following variables will be lists of elements each corresponding
# to an algorithm
algnames = []
#algdata = []
algerts = []
algevals = []
algdisp = []
algnbsucc = []
algnbruns = []
algmedmaxevals = []
algmedfinalfunvals = []
algtestres = []
algentries = []
for n in sorted(dictData[df].keys()):
entries = dictData[df][n]
# the number of datasets for a given dimension and function (df)
# should be strictly 1. TODO: find a way to warn
# TODO: do this checking before... why wasn't it triggered by ppperprof?
if len(entries) > 1:
txt = ("There is more than a single entry associated with "
"folder %s on %d-D f%d." %
(sortedAlgs[n], df[0], df[1]))
raise Exception(txt)
entry = entries[0]
algentries.append(entry)
algnames.append(sortedAlgs[n])
evals = entry.detEvals(targets)
#tmpdata = []
tmpdisp = []
tmpert = []
for i, e in enumerate(evals):
succ = (numpy.isnan(e) == False)
e[succ == False] = entry.maxevals[succ == False]
ert = toolsstats.sp(e, issuccessful=succ)[0]
#tmpdata.append(ert/refalgert[i])
if succ.any():
tmp = toolsstats.drawSP(e[succ],
entry.maxevals[succ == False],
[10, 50, 90],
samplesize=samplesize)[0]
tmpdisp.append((tmp[-1] - tmp[0]) / 2.)
else:
tmpdisp.append(numpy.nan)
tmpert.append(ert)
algerts.append(tmpert)
algevals.append(evals)
#algdata.append(tmpdata)
algdisp.append(tmpdisp)
algmedmaxevals.append(numpy.median(entry.maxevals))
algmedfinalfunvals.append(numpy.median(entry.finalfunvals))
#algmedmaxevals.append(numpy.median(entry.maxevals)/df[0])
#algmedfinalfunvals.append(numpy.median(entry.finalfunvals))
algtestres.append(significancetest(refalgentry, entry, targets))
# determine success probability for Df = 1e-8
e = entry.detEvals((targetf, ))[0]
algnbsucc.append(numpy.sum(numpy.isnan(e) == False))
algnbruns.append(len(e))
# Process over all data
# find best values...
nalgs = len(dictData[df])
maxRank = 1 + numpy.floor(
0.14 * nalgs) # number of algs to be displayed in bold
isBoldArray = [] # Point out the best values
algfinaldata = [
] # Store median function values/median number of function evaluations
tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank)
for i, erts in enumerate(algerts):
tmp = []
for j, ert in enumerate(erts): # algi targetj
tmp.append(i in tmptop[j] or
(nalgs > 7 and algerts[i][j] <= 3. * refalgert[j]))
isBoldArray.append(tmp)
algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i]))
# significance test of best given algorithm against all others
best_alg_idx = numpy.array(algerts).argsort(0)[
0, :] # indexed by target index
significance_versus_others = significance_all_best_vs_other(
algentries, targets, best_alg_idx)[0]
# Create the table
table = []
spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % (
len(targets)
) # in case StrLeft not working: replaced c@{} with l@{ }
spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % (
len(targets)
) # in case StrLeft not working: replaced c@{} with l@{ }
extraeol = []
# Generate header lines
if with_table_heading:
header = funInfos[df[1]] if funInfos else 'f%d' % df[1]
table.append([
r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}' %
(2 * len(targets) + 2, header)
])
extraeol.append('')
if function_targets_line is True or (function_targets_line and df[1]
in function_targets_line):
curline = [r'$\Delta f_\mathrm{opt}$']
for t in targets[0:-1]:
curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}' %
writeFEvals2(t, precision=1, isscientific=True))
curline.append(
r'\multicolumn{2}{@{\,}X@{}|}{%s}' %
writeFEvals2(targets[-1], precision=1, isscientific=True))
curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}')
table.append(curline)
extraeol.append(r'\hline')
# extraeol.append(r'\hline\arrayrulecolor{tableShade}')
curline = [r'ERT$_{\text{best}}$'
] if with_table_heading else [r'\textbf{f%d}' % df[1]]
for i in refalgert[0:-1]:
curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}' %
writeFEvalsMaxPrec(float(i), 2))
curline.append(r'\multicolumn{2}{@{\,}X@{\,}|}{%s}' %
writeFEvalsMaxPrec(float(refalgert[-1]), 2))
curline.append('%d' % refalgnbsucc)
if refalgnbsucc:
curline.append('/%d' % refalgnbruns)
#curline.append(curline[0])
table.append(curline)
extraeol.append('')
#for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))):
#print i, gna, gno
#set_trace()
# Format data
#if df == (5, 17):
#set_trace()
header = r'\providecommand{\ntables}{7}'
for i, alg in enumerate(algnames):
#algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames,
#data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres):
commandname = r'\alg%stables' % numtotext(i)
# header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg)))
header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % (
commandname, str_to_latex(strip_pathname(alg)))
curline = [
commandname + r'\hspace*{\fill}'
] # each list element becomes a &-separated table entry?
for j, tmp in enumerate(
zip(
algerts[i],
algdisp[i], # j is target index
isBoldArray[i],
algtestres[i])):
ert, dispersion, isBold, testres = tmp
alignment = '@{\,}X@{\,}'
if j == len(algerts[i]) - 1:
alignment = '@{\,}X@{\,}|'
data = ert / refalgert[j]
# write star for significance against all other algorithms
str_significance_subsup = ''
if (len(best_alg_idx) > 0
and len(significance_versus_others) > 0
and i == best_alg_idx[j]
and nbtests * significance_versus_others[j][1] < 0.05):
logp = -numpy.ceil(
numpy.log10(
nbtests * significance_versus_others[j][1]))
str_significance_subsup = r"^{%s%s}" % (
significance_vs_others_symbol,
str(int(logp)) if logp > 1 else '')
# moved out of the above else: this was a bug!?
z, p = testres
if (nbtests * p) < 0.05 and data < 1. and z < 0.:
if not numpy.isinf(refalgert[j]):
tmpevals = algevals[i][j].copy()
tmpevals[numpy.isnan(tmpevals)] = algentries[
i].maxevals[numpy.isnan(tmpevals)]
bestevals = refalgentry.detEvals([targets[j]])
bestevals, bestalgalg = (bestevals[0][0],
bestevals[1][0])
bestevals[numpy.isnan(
bestevals)] = refalgentry.maxevals[bestalgalg][
numpy.isnan(bestevals)]
tmpevals = numpy.array(sorted(
tmpevals))[0:min(len(tmpevals), len(bestevals))]
bestevals = numpy.array(sorted(
bestevals))[0:min(len(tmpevals), len(bestevals))]
#The conditions are now that ERT < ERT_best and
# all(sorted(FEvals_best) > sorted(FEvals_current)).
if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
# tmp2[-1] += r'$^{%s}$' % superscript
str_significance_subsup += r'_{%s%s}' % (
significance_vs_ref_symbol,
str(int(nbstars)) if nbstars > 1 else '')
if str_significance_subsup:
str_significance_subsup = '$%s$' % str_significance_subsup
# format number in variable data
if numpy.isnan(data):
curline.append(r'\multicolumn{2}{%s}{.}' % alignment)
else:
if numpy.isinf(refalgert[j]):
curline.append(
r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}'
% (alignment, writeFEvalsMaxPrec(algerts[i][j], 2),
writeFEvalsMaxPrec(dispersion, precdispersion),
str_significance_subsup))
continue
tmp = writeFEvalsMaxPrec(data,
precfloat,
maxfloatrepr=maxfloatrepr)
if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation
if numpy.isinf(data) and j == len(algerts[i]) - 1:
tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec(
algfinaldata[i][1],
0,
maxfloatrepr=maxfloatrepr)
else:
tmp = writeFEvalsMaxPrec(data,
precscien,
maxfloatrepr=data)
if isBold:
tmp = r'\textbf{%s}' % tmp
if not numpy.isnan(dispersion):
tmpdisp = dispersion / refalgert[j]
if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=tmpdisp)
else:
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=maxfloatrepr)
tmp += r'\mbox{\tiny (%s)}' % tmpdisp
curline.append(
r'\multicolumn{2}{%s}{%s%s}' %
(alignment, tmp, str_significance_subsup))
else:
tmp2 = tmp.split('.', 1)
if len(tmp2) < 2:
tmp2.append('')
else:
tmp2[-1] = '.' + tmp2[-1]
if isBold:
tmp3 = []
for k in tmp2:
tmp3.append(r'\textbf{%s}' % k)
tmp2 = tmp3
if not numpy.isnan(dispersion):
tmpdisp = dispersion / refalgert[j]
if tmpdisp >= maxfloatrepr or tmpdisp < 0.01:
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=tmpdisp)
else:
tmpdisp = writeFEvalsMaxPrec(
tmpdisp,
precdispersion,
maxfloatrepr=maxfloatrepr)
tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp))
tmp2[-1] += str_significance_subsup
curline.extend(tmp2)
curline.append('%d' % algnbsucc[i])
curline.append('/%d' % algnbruns[i])
table.append(curline)
extraeol.append('')
# Write table
res = tableXLaTeX(table, spec=spec, extraeol=extraeol)
try:
filename = os.path.join(
outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0]))
f = open(filename, 'w')
f.write(header + '\n')
f.write(res)
if verbose:
print 'Wrote table in %s' % filename
except:
raise
else:
f.close()
def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True):
"""One table per dimension, modified to fit in 1 page per table."""
#TODO: method is long, split if possible
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3]
alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3]
open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a'
).write(r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' +
r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n')
if info:
info = '_' + info
dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys()))
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
header = [r'$\Delta f$']
for i in targetsOfInterest:
#header.append(r'\multicolumn{2}{@{}c@{}}{$10^{%d}$}' % (int(numpy.log10(i))))
header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' % (int(numpy.log10(i))))
header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}')
for d in dimsOfInterest: # TODO set as input arguments
table = [header]
extraeol = [r'\hline']
try:
dictFunc0 = dictDim0[d].dictByFunc()
dictFunc1 = dictDim1[d].dictByFunc()
except KeyError:
continue
funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys()))
nbtests = len(funcs) * 2. #len(dimsOfInterest)
for f in sorted(funcs):
bestalgentry = bestalg.bestalgentries2009[(d, f)]
curline = [r'${\bf f_{%d}}$' % f]
bestalgdata = bestalgentry.detERT(targetsOfInterest)
bestalgevals, bestalgalgs = bestalgentry.detEvals(targetsOfInterest)
for i in bestalgdata[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' % writeFEvalsMaxPrec(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2))
tmp = bestalgentry.detEvals([targetf])[0][0]
tmp2 = numpy.sum(numpy.isnan(tmp) == False)
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(tmp))
table.append(curline[:])
extraeol.append('')
rankdata0 = [] # never used
# generate all data from ranksum test
entries = []
ertdata = {}
for nb, dsList in enumerate((dictFunc0, dictFunc1)):
try:
entry = dsList[f][0] # take the first DataSet, there should be only one?
except KeyError:
warnings.warn('data missing for data set ' + str(nb) + ' and function ' + str(f))
print('*** Warning: data missing for data set ' + str(nb) + ' and function ' + str(f) + '***')
continue # TODO: problem here!
ertdata[nb] = entry.detERT(targetsOfInterest)
entries.append(entry)
for _t in ertdata.values():
for _tt in _t:
if _tt is None:
raise ValueError
if len(entries) < 2: # funcion not available for *both* algorithms
continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following
testres0vs1 = significancetest(entries[0], entries[1], targetsOfInterest)
testresbestvs1 = significancetest(bestalgentry, entries[1], targetsOfInterest)
testresbestvs0 = significancetest(bestalgentry, entries[0], targetsOfInterest)
for nb, entry in enumerate(entries):
if nb == 0:
curline = [r'1:\:\algorithmAshort\hspace*{\fill}']
else:
curline = [r'2:\:\algorithmBshort\hspace*{\fill}']
#data = entry.detERT(targetsOfInterest)
dispersion = []
data = []
evals = entry.detEvals(targetsOfInterest)
for i in evals:
succ = (numpy.isnan(i) == False)
tmp = i.copy()
tmp[succ==False] = entry.maxevals[numpy.isnan(i)]
#set_trace()
data.append(toolsstats.sp(tmp, issuccessful=succ)[0])
#if not any(succ):
#set_trace()
if any(succ):
tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ==False],
(10, 50, 90), samplesize)[0]
dispersion.append((tmp2[-1]-tmp2[0])/2.)
else:
dispersion.append(None)
if nb == 0:
assert not isinstance(data, numpy.ndarray)
data0 = data[:] # TODO: check if it is not an array, it's never used anyway?
for i, dati in enumerate(data):
z, p = testres0vs1[i] # TODO: there is something with the sign that I don't get
# assign significance flag, which is the -log10(p)
significance0vs1 = 0
if nb != 0:
z = -z # the test is symmetric
if nbtests * p < 0.05 and z > 0:
significance0vs1 = -int(numpy.ceil(numpy.log10(min([1.0, nbtests * p])))) # this is the larger the more significant
isBold = significance0vs1 > 0
alignment = 'c'
if i == len(data) - 1: # last element
alignment = 'c|'
if numpy.isinf(bestalgdata[i]): # if the 2009 best did not solve the problem
tmp = writeFEvalsMaxPrec(float(dati), 2)
if not numpy.isinf(dati):
tmp = r'\textit{%s}' % (tmp)
if isBold:
tmp = r'\textbf{%s}' % tmp
if dispersion[i] and numpy.isfinite(dispersion[i]):
tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tmp))
else:
# Formatting
tmp = float(dati)/bestalgdata[i]
assert not numpy.isnan(tmp)
isscientific = False
if tmp >= 1000:
isscientific = True
tableentry = writeFEvals2(tmp, 2, isscientific=isscientific)
tableentry = writeFEvalsMaxPrec(tmp, 2)
if numpy.isinf(tmp) and i == len(data)-1:
tableentry = (tableentry
+ r'\textit{%s}' % writeFEvals2(numpy.median(entry.maxevals), 2))
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore
tableentry = r'\textit{%s}' % tableentry
if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tableentry))
elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and i != len(data) - 1):
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3 and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tableentry))
else:
tmp = tableentry.split('.', 1)
if isBold:
tmp = list(r'\textbf{%s}' % i for i in tmp)
elif 11 < 3 and significance0vs1 < 0:
tmp = list(r'\textit{%s}' % i for i in tmp)
tableentry = ' & .'.join(tmp)
if len(tmp) == 1:
tableentry += '&'
if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
superscript = ''
if nb == 0:
z, p = testresbestvs0[i]
else:
z, p = testresbestvs1[i]
#The conditions are now that ERT < ERT_best
if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0.
and z < 0.):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
#tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star'])
if z > 0:
superscript = r'\uparrow' #* nbstars
else:
superscript = r'\downarrow' #* nbstars
# print z, linebest[i], line1
if nbstars > 1:
superscript += str(int(nbstars))
if superscript or significance0vs1:
s = ''
if significance0vs1 > 0:
s = '\star'
if significance0vs1 > 1:
s += str(significance0vs1)
s = r'$^{' + s + superscript + r'}$'
if tableentry.endswith('}'):
tableentry = tableentry[:-1] + s + r'}'
else:
tableentry += s
curline.append(tableentry)
#curline.append(tableentry)
#if dispersion[i] is None or numpy.isinf(bestalgdata[i]):
#curline.append('')
#else:
#tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2)
#curline.append('(%s)' % tmp)
tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:]
try:
tmp = tmp[0]
curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False))
except IndexError:
curline.append('%d' % 0)
curline.append('/%d' % entry.nbRuns())
table.append(curline[:])
extraeol.append('')
extraeol[-1] = r'\hline'
extraeol[-1] = ''
outputfile = os.path.join(outputdir, 'pptable2_%02dD%s.tex' % (d, info))
spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}'
res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n'
res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n'
# open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res)
#res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec,
#extraeol=extraeol)
res += tableLaTeX(table, spec=spec, extraeol=extraeol)
f = open(outputfile, 'w')
f.write(res)
f.close()
if verbose:
print "Table written in %s" % outputfile
def main(dictAlg, sortedAlgs=None, target=ftarget_default, outputdir='ppdata', verbose=True):
"""From a DataSetList, returns figures showing the scaling: ERT/dim vs dim.
One function and one target per figure.
``target`` can be a scalar, a list with one element or a
``pproc.TargetValues`` instance with one target.
``sortedAlgs`` is a list of string-identifies (folder names)
"""
# target becomes a TargetValues "list" with one element
target = pproc.TargetValues.cast([target] if numpy.isscalar(target) else target)
latex_commands_filename = os.path.join(outputdir, 'bbob_pproc_commands.tex')
assert isinstance(target, pproc.TargetValues)
if len(target) != 1:
raise ValueError('only a single target can be managed in ppfigs, ' + str(len(target)) + ' targets were given')
dictFunc = pproc.dictAlgByFun(dictAlg)
if sortedAlgs is None:
sortedAlgs = sorted(dictAlg.keys())
if not os.path.isdir(outputdir):
os.mkdir(outputdir)
for f in dictFunc:
filename = os.path.join(outputdir,'ppfigs_f%03d' % (f))
handles = []
fix_styles(len(sortedAlgs)) #
for i, alg in enumerate(sortedAlgs):
dictDim = dictFunc[f][alg].dictByDim() # this does not look like the most obvious solution
#Collect data
dimert = []
ert = []
dimnbsucc = []
ynbsucc = []
nbsucc = []
dimmaxevals = []
maxevals = []
dimmedian = []
medianfes = []
for dim in sorted(dictDim):
assert len(dictDim[dim]) == 1
entry = dictDim[dim][0]
data = generateData(entry, target((f, dim))[0]) # TODO: here we might want a different target for each function
if 1 < 3 or data[2] == 0: # No success
dimmaxevals.append(dim)
maxevals.append(float(data[3])/dim)
if data[2] > 0:
dimmedian.append(dim)
medianfes.append(data[4]/dim)
dimert.append(dim)
ert.append(float(data[0])/dim)
if data[1] < 1.:
dimnbsucc.append(dim)
ynbsucc.append(float(data[0])/dim)
nbsucc.append('%d' % data[2])
# Draw lines
tmp = plt.plot(dimert, ert, **styles[i]) #label=alg, )
plt.setp(tmp[0], markeredgecolor=plt.getp(tmp[0], 'color'))
# For legend
# tmp = plt.plot([], [], label=alg.replace('..' + os.sep, '').strip(os.sep), **styles[i])
tmp = plt.plot([], [], label=alg.split(os.sep)[-1], **styles[i])
plt.setp(tmp[0], markersize=12.,
markeredgecolor=plt.getp(tmp[0], 'color'))
if dimmaxevals:
tmp = plt.plot(dimmaxevals, maxevals, **styles[i])
plt.setp(tmp[0], markersize=20, #label=alg,
markeredgecolor=plt.getp(tmp[0], 'color'),
markeredgewidth=1,
markerfacecolor='None', linestyle='None')
handles.append(tmp)
#tmp2 = plt.plot(dimmedian, medianfes, ls='', marker='+',
# markersize=30, markeredgewidth=5,
# markeredgecolor=plt.getp(tmp, 'color'))[0]
#for i, n in enumerate(nbsucc):
# plt.text(dimnbsucc[i], numpy.array(ynbsucc[i])*1.85, n,
# verticalalignment='bottom',
# horizontalalignment='center')
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgdata = []
dimbestalg = list(df[0] for df in bestalg.bestalgentries2009 if df[1] == f)
dimbestalg.sort()
dimbestalg2 = []
for d in dimbestalg:
entry = bestalg.bestalgentries2009[(d, f)]
tmp = entry.detERT(target((f, d)))[0]
if numpy.isfinite(tmp):
bestalgdata.append(float(tmp)/d)
dimbestalg2.append(d)
tmp = plt.plot(dimbestalg2, bestalgdata, color=refcolor, linewidth=10,
marker='d', markersize=25, markeredgecolor=refcolor, zorder=-1
#label='best 2009',
)
handles.append(tmp)
if show_significance: # plot significance-stars
xstar, ystar = [], []
dims = sorted(pproc.dictAlgByDim(dictFunc[f]))
for i, dim in enumerate(dims):
datasets = pproc.dictAlgByDim(dictFunc[f])[dim]
assert all([len(datasets[ialg]) == 1 for ialg in sortedAlgs if datasets[ialg]])
dsetlist = [datasets[ialg][0] for ialg in sortedAlgs if datasets[ialg]]
if len(dsetlist) > 1:
arzp, arialg = toolsstats.significance_all_best_vs_other(dsetlist, target((f, dim)))
if arzp[0][1] * len(dims) < show_significance:
ert = dsetlist[arialg[0]].detERT(target((f, dim)))[0]
if ert < numpy.inf:
xstar.append(dim)
ystar.append(ert/dim)
plt.plot(xstar, ystar, 'k*', markerfacecolor=None, markeredgewidth=2, markersize=0.5*styles[0]['markersize'])
if funInfos:
plt.gca().set_title(funInfos[f])
isLegend = False
if legend:
plotLegend(handles)
elif 1 < 3:
if f in (1, 24, 101, 130) and len(sortedAlgs) < 6: # 6 elements at most in the boxed legend
isLegend = True
beautify(legend=isLegend, rightlegend=legend)
plt.text(plt.xlim()[0], plt.ylim()[0], 'target ' + target.label_name() + ': ' + target.label(0)) # TODO: check
saveFigure(filename, verbose=verbose)
plt.close()
# generate commands in tex file:
try:
abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
alg_definitions = []
for i in range(len(sortedAlgs)):
symb = r'{%s%s}' % (color_to_latex(styles[i]['color']),
marker_to_latex(styles[i]['marker']))
alg_definitions.append((', ' if i > 0 else '') + '%s:%s' % (symb, '\\algorithm' + abc[i % len(abc)]))
toolsdivers.prepend_to_file(latex_commands_filename,
[#'\\providecommand{\\bbobppfigsftarget}{\\ensuremath{10^{%s}}}'
# % target.loglabel(0), # int(numpy.round(numpy.log10(target))),
'\\providecommand{\\bbobppfigslegend}[1]{',
scaling_figure_caption(target),
'Legend: '] + alg_definitions + ['}']
)
toolsdivers.prepend_to_file(latex_commands_filename,
['\\providecommand{\\bbobECDFslegend}[1]{',
ecdfs_figure_caption(target), '}']
)
if verbose:
print 'Wrote commands and legend to %s' % filename
# this is obsolete (however check templates)
filename = os.path.join(outputdir,'ppfigs.tex')
f = open(filename, 'w')
f.write('% Do not modify this file: calls to post-processing software'
+ ' will overwrite any modification.\n')
f.write('Legend: ')
for i in range(0, len(sortedAlgs)):
symb = r'{%s%s}' % (color_to_latex(styles[i]['color']),
marker_to_latex(styles[i]['marker']))
f.write((', ' if i > 0 else '') + '%s:%s' % (symb, writeLabels(sortedAlgs[i])))
f.close()
if verbose:
print '(obsolete) Wrote legend in %s' % filename
except IOError:
raise
handles.append(tmp)
if funInfos:
plt.gca().set_title(funInfos[f])
beautify(rightlegend=legend)
if legend:
plotLegend(handles)
else:
if f in (1, 24, 101, 130):
plt.legend()
saveFigure(filename, figFormat=genericsettings.fig_formats, verbose=verbose)
plt.close()
def main(dsList, dimsOfInterest, outputdir, info='', verbose=True):
"""Generate a table of ratio ERT/ERTbest vs target precision.
1 table per dimension will be generated.
Rank-sum tests table on "Final Data Points" for only one algorithm.
that is, for example, using 1/#fevals(ftarget) if ftarget was
reached and -f_final otherwise as input for the rank-sum test, where
obviously the larger the better.
"""
#TODO: check that it works for any reference algorithm?
#in the following the reference algorithm is the one given in
#bestalg.bestalgentries which is the virtual best of BBOB
dictDim = dsList.dictByDim()
targetf=1e-8
if info:
info = '_' + info
# insert a separator between the default file name and the additional
# information string.
dims = set(dictDim.keys())
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
header = [r'\#FEs/D']
for i in targetsOfInterest.labels():
header.append(r'\multicolumn{2}{@{}c@{}}{%s}'
% i)
else:
header = [r'$\Delta f$']
for i in targetsOfInterest.target_values:
header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}'
% (int(np.log10(i))))
header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}')
for d in dimsOfInterest:
table = [header]
extraeol = [r'\hline']
try:
dictFunc = dictDim[d].dictByFunc()
except KeyError:
continue
funcs = set(dictFunc.keys())
nbtests = float(len(funcs)) # #funcs tests times one algorithm
for f in sorted(funcs):
bestalgentry = bestalg.bestalgentries2009[(d, f)]
curline = [r'${\bf f_{%d}}$' % f]
bestalgdata = bestalgentry.detERT(targetsOfInterest((f,d)))
bestalgevals, bestalgalgs = bestalgentry.detEvals(targetsOfInterest((f,d)))
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
#write ftarget:fevals
for i in xrange(len(bestalgdata[:-1])):
temp="%.1e" %targetsOfInterest((f,d))[i]
if temp[-2]=="0":
temp=temp[:-2]+temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}'
% (temp,writeFEvalsMaxPrec(bestalgdata[i], 2)))
temp="%.1e" %targetsOfInterest((f,d))[-1]
if temp[-2]=="0":
temp=temp[:-2]+temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }'
% (temp,writeFEvalsMaxPrec(bestalgdata[-1], 2)))
#success
targetf=targetsOfInterest((f,d))[-1]
else:
# write #fevals of the reference alg
for i in bestalgdata[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}'
% writeFEvalsMaxPrec(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}'
% writeFEvalsMaxPrec(bestalgdata[-1], 2))
# write the success ratio for the reference alg
tmp = bestalgentry.detEvals([targetf])[0][0]
tmp2 = np.sum(np.isnan(tmp) == False) # count the nb of success
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(tmp))
table.append(curline[:])
extraeol.append('')
# generate all data for ranksum test
assert len(dictFunc[f]) == 1
entry = dictFunc[f][0] # take the first element
ertdata = entry.detERT(targetsOfInterest((f, d)))
testresbestvs1 = significancetest(bestalgentry, entry,
targetsOfInterest((f, d)))
#for nb, entry in enumerate(entries):
#curline = [r'\algshort\hspace*{\fill}']
curline = ['']
#data = entry.detERT(targetsOfInterest)
evals = entry.detEvals(targetsOfInterest((f,d)))
dispersion = []
data = []
for i in evals:
succ = (np.isnan(i) == False)
tmp = i.copy()
tmp[succ==False] = entry.maxevals[np.isnan(i)]
#set_trace()
# TODO: what is the difference between data and ertdata?
data.append(toolsstats.sp(tmp, issuccessful=succ)[0])
#if not any(succ):
#set_trace()
if any(succ):
tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ==False],
(10, 50, 90), samplesize)[0]
dispersion.append((tmp2[-1] - tmp2[0]) / 2.)
else:
dispersion.append(None)
assert data == ertdata
for i, ert in enumerate(data):
alignment = 'c'
if i == len(data) - 1: # last element
alignment = 'c|'
nbstars = 0
z, p = testresbestvs1[i]
if ert - bestalgdata[i] < 0. and not np.isinf(bestalgdata[i]):
evals = entry.detEvals([targetsOfInterest((f,d))[i]])[0]
evals[np.isnan(evals)] = entry.maxevals[np.isnan(evals)]
bestevals = bestalgentry.detEvals([targetsOfInterest((f,d))[i]])
bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0])
bestevals[np.isnan(bestevals)] = bestalgentry.maxevals[bestalgalg][np.isnan(bestevals)]
evals = np.array(sorted(evals))[0:min(len(evals), len(bestevals))]
bestevals = np.array(sorted(bestevals))[0:min(len(evals), len(bestevals))]
#The conditions for significance are now that ERT < ERT_best and
# all(sorted(FEvals_best) > sorted(FEvals_current)).
if ((nbtests * p) < 0.05 and ert - bestalgdata[i] < 0.
and z < 0.
and (np.isinf(bestalgdata[i])
or all(evals < bestevals))):
nbstars = -np.ceil(np.log10(nbtests * p))
isBold = False
if nbstars > 0:
isBold = True
if np.isinf(bestalgdata[i]): # if the best did not solve the problem
tmp = writeFEvalsMaxPrec(float(ert), 2)
if not np.isinf(ert):
tmp = r'\textit{%s}' % (tmp)
if isBold:
tmp = r'\textbf{%s}' % tmp
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tmp))
else:
# Formatting
tmp = float(ert) / bestalgdata[i]
assert not np.isnan(tmp)
tableentry = writeFEvalsMaxPrec(tmp, 2)
if np.isinf(tmp) and i == len(data)-1:
tableentry = (tableentry
+ r'\textit{%s}' % writeFEvals2(np.median(entry.maxevals), 2))
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3: # and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tableentry))
elif tableentry.find('e') > -1 or (np.isinf(tmp) and i != len(data) - 1):
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3: # and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tableentry))
else:
tmp = tableentry.split('.', 1)
if isBold:
tmp = list(r'\textbf{%s}' % i for i in tmp)
elif 11 < 3: # and significance0vs1 < 0:
tmp = list(r'\textit{%s}' % i for i in tmp)
tableentry = ' & .'.join(tmp)
if len(tmp) == 1:
tableentry += '&'
superscript = ''
if nbstars > 0:
#tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star'])
if z > 0:
superscript = r'\uparrow' #* nbstars
else:
superscript = r'\downarrow' #* nbstars
# print z, linebest[i], line1
if nbstars > 1:
superscript += str(int(min((9, nbstars))))
# superscript += str(int(nbstars))
#if superscript or significance0vs1:
#s = ''
#if significance0vs1 > 0:
#s = '\star'
#if significance0vs1 > 1:
#s += str(significance0vs1)
#s = r'$^{' + s + superscript + r'}$'
#if tableentry.endswith('}'):
#tableentry = tableentry[:-1] + s + r'}'
#else:
#tableentry += s
if dispersion[i]:
if not np.isinf(bestalgdata[i]):
tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
else:
tmp = writeFEvalsMaxPrec(dispersion[i], 1)
tableentry += (r'${\scriptscriptstyle(%s)}$' % tmp)
if superscript:
s = r'$^{' + superscript + r'}$'
if tableentry.endswith('}'):
tableentry = tableentry[:-1] + s + r'}'
else:
tableentry += s
curline.append(tableentry)
#curline.append(tableentry)
#if dispersion[i] is None or np.isinf(bestalgdata[i]):
#curline.append('')
#else:
#tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2)
#curline.append('(%s)' % tmp)
tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:]
try:
tmp = tmp[0]
curline.append('%d' % np.sum(np.isnan(tmp) == False))
except IndexError:
curline.append('%d' % 0)
curline.append('/%d' % entry.nbRuns())
table.append(curline[:])
extraeol.append(r'\hline')
extraeol[-1] = ''
outputfile = os.path.join(outputdir, 'pptable_%02dD%s.tex' % (d, info))
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
spec = r'@{}c@{}|' + '*{%d}{@{ }r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}'
else:
spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}'
#res = r'\providecommand{\algshort}{%s}' % alg1 + '\n'
#res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec,
#extraeol=extraeol)
res = tableLaTeX(table, spec=spec, extraeol=extraeol)
f = open(outputfile, 'w')
f.write(res)
f.close()
if verbose:
print "Table written in %s" % outputfile
def main(dictAlg, order=None, outputdir=".", info="default", dimension=None, verbose=True):
"""Generates a figure showing the performance of algorithms.
From a dictionary of :py:class:`DataSetList` sorted by algorithms,
generates the cumulative distribution function of the bootstrap
distribution of ERT for algorithms on multiple functions for
multiple targets altogether.
:param dict dictAlg: dictionary of :py:class:`DataSetList` instances
one instance is equivalent to one algorithm,
:param list targets: target function values
:param list order: sorted list of keys to dictAlg for plotting order
:param str outputdir: output directory
:param str info: output file name suffix
:param bool verbose: controls verbosity
"""
global x_limit # late assignment of default, because it can be set to None in config
global divide_by_dimension # not fully implemented/tested yet
if "x_limit" not in globals() or x_limit is None:
x_limit = x_limit_default
tmp = pp.dictAlgByDim(dictAlg)
# tmp = pp.DictAlg(dictAlg).by_dim()
if len(tmp) != 1 and dimension is None:
raise ValueError("We never integrate over dimension.")
if dimension is not None:
if dimension not in tmp.keys():
raise ValueError("dimension %d not in dictAlg dimensions %s" % (dimension, str(tmp.keys())))
tmp = {dimension: tmp[dimension]}
dim = tmp.keys()[0]
divisor = dim if divide_by_dimension else 1
algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []]
dictFunc = pp.dictAlgByFun(dictAlg)
# Collect data
# Crafting effort correction: should we consider any?
CrEperAlg = {}
for alg in algorithms_with_data:
CrE = 0.0
if 1 < 3 and dictAlg[alg][0].algId == "GLOBAL":
tmp = dictAlg[alg].dictByNoise()
assert len(tmp.keys()) == 1
if tmp.keys()[0] == "noiselessall":
CrE = 0.5117
elif tmp.keys()[0] == "nzall":
CrE = 0.6572
CrEperAlg[alg] = CrE
if CrE != 0.0:
print "Crafting effort for", alg, "is", CrE
dictData = {} # list of (ert per function) per algorithm
dictMaxEvals = {} # list of (maxevals per function) per algorithm
bestERT = [] # best ert per function
# funcsolved = [set()] * len(targets) # number of functions solved per target
xbest2009 = []
maxevalsbest2009 = []
for f, dictAlgperFunc in dictFunc.iteritems():
if function_IDs and f not in function_IDs:
continue
# print target_values((f, dim))
for j, t in enumerate(target_values((f, dim))):
# for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)):
# funcsolved[j].add(f)
for alg in algorithms_with_data:
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
try:
entry = dictAlgperFunc[alg][0] # one element per fun and per dim.
evals = entry.detEvals([t])[0]
assert entry.dim == dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(
runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize
)[1]
except (KeyError, IndexError):
# set_trace()
warntxt = "Data for algorithm %s on function %d in %d-D " % (alg, f, dim) + "are missing.\n"
warnings.warn(warntxt)
dictData.setdefault(alg, []).extend(x)
dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc)
if displaybest2009:
# set_trace()
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgentry = bestalg.bestalgentries2009[(dim, f)]
bestalgevals = bestalgentry.detEvals(target_values((f, dim)))
# print bestalgevals
for j in range(len(bestalgevals[0])):
if bestalgevals[1][j]:
evals = bestalgevals[0][j]
# set_trace()
assert dim == bestalgentry.dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = bestalgentry.maxevals[bestalgevals[1][j]][np.isnan(evals)] / divisor
x = toolsstats.drawSP(
runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize
)[1]
else:
x = perfprofsamplesize * [np.inf]
runlengthunsucc = []
xbest2009.extend(x)
maxevalsbest2009.extend(runlengthunsucc)
if order is None:
order = dictData.keys()
# Display data
lines = []
if displaybest2009:
args = {
"ls": "-",
"linewidth": 6,
"marker": "D",
"markersize": 11.0,
"markeredgewidth": 1.5,
"markerfacecolor": refcolor,
"markeredgecolor": refcolor,
"color": refcolor,
"label": "best 2009",
"zorder": -1,
}
lines.append(plotdata(np.array(xbest2009), x_limit, maxevalsbest2009, CrE=0.0, **args))
def algname_to_label(algname, dirname=None):
"""to be extended to become generally useful"""
if isinstance(algname, (tuple, list)): # not sure this is needed
return " ".join([str(name) for name in algname])
return str(algname)
for i, alg in enumerate(order):
try:
data = dictData[alg]
maxevals = dictMaxEvals[alg]
except KeyError:
continue
args = styles[(i) % len(styles)]
args["linewidth"] = 1.5
args["markersize"] = 12.0
args["markeredgewidth"] = 1.5
args["markerfacecolor"] = "None"
args["markeredgecolor"] = args["color"]
args["label"] = algname_to_label(alg)
# args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib
# elif len(show_algorithms) > 0:
# args['color'] = 'wheat'
# args['ls'] = '-'
# args['zorder'] = -1
# plotdata calls pprldistr.plotECDF which calls ppfig.plotUnifLog... which does the work
lines.append(plotdata(np.array(data), x_limit, maxevals, CrE=CrEperAlg[alg], **args))
labels, handles = plotLegend(lines, x_limit)
if True: # isLateXLeg:
fileName = os.path.join(outputdir, "pprldmany_%s.tex" % (info))
with open(fileName, "w") as f:
f.write(r"\providecommand{\nperfprof}{7}")
algtocommand = {} # latex commands
for i, alg in enumerate(order):
tmp = r"\alg%sperfprof" % pptex.numtotext(i)
f.write(
r"\providecommand{%s}{\StrLeft{%s}{\nperfprof}}"
% (tmp, toolsdivers.str_to_latex(toolsdivers.strip_pathname2(algname_to_label(alg))))
)
algtocommand[algname_to_label(alg)] = tmp
if displaybest2009:
tmp = r"\algzeroperfprof"
f.write(r"\providecommand{%s}{best 2009}" % (tmp))
algtocommand["best 2009"] = tmp
commandnames = []
for label in labels:
commandnames.append(algtocommand[label])
# f.write(headleg)
if (
len(order) > 28
): # latex sidepanel won't work well for more than 25 algorithms, but original labels are also clipped
f.write(
r"\providecommand{\perfprofsidepanel}{\mbox{%s}\vfill\mbox{%s}}"
% (commandnames[0], commandnames[-1])
)
else:
fontsize_command = r"\tiny{}" if len(order) > 19 else ""
f.write(
r"\providecommand{\perfprofsidepanel}{{%s\mbox{%s}" % (fontsize_command, commandnames[0])
) # TODO: check len(labels) > 0
for i in range(1, len(labels)):
f.write("\n" + r"\vfill \mbox{%s}" % commandnames[i])
f.write("}}\n")
# f.write(footleg)
if verbose:
print "Wrote right-hand legend in %s" % fileName
figureName = os.path.join(outputdir, "pprldmany_%s" % (info))
# beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat)
beautify()
text = "f%s" % (ppfig.consecutiveNumbers(sorted(dictFunc.keys())))
text += ",%d-D" % dim # TODO: this is strange when different dimensions are plotted
plt.text(0.01, 0.98, text, horizontalalignment="left", verticalalignment="top", transform=plt.gca().transAxes)
if len(dictFunc) == 1:
plt.title(" ".join((str(dictFunc.keys()[0]), genericsettings.current_testbed.short_names[dictFunc.keys()[0]])))
a = plt.gca()
plt.xlim(xmin=1e-0, xmax=x_limit ** annotation_space_end_relative)
xticks, labels = plt.xticks()
tmp = []
for i in xticks:
tmp.append("%d" % round(np.log10(i)))
a.set_xticklabels(tmp)
if save_figure:
ppfig.saveFigure(figureName, verbose=verbose)
if len(dictFunc) == 1:
ppfig.save_single_functions_html(
os.path.join(outputdir, "pprldmany"),
"", # algorithms names are clearly visible in the figure
add_to_names="_%02dD" % (dim),
algorithmCount=ppfig.AlgorithmCount.NON_SPECIFIED,
)
if close_figure:
plt.close()
def main(dictAlg,
order=None,
outputdir='.',
info='default',
dimension=None,
verbose=True):
"""Generates a figure showing the performance of algorithms.
From a dictionary of :py:class:`DataSetList` sorted by algorithms,
generates the cumulative distribution function of the bootstrap
distribution of ERT for algorithms on multiple functions for
multiple targets altogether.
:param dict dictAlg: dictionary of :py:class:`DataSetList` instances
one instance is equivalent to one algorithm,
:param list targets: target function values
:param list order: sorted list of keys to dictAlg for plotting order
:param str outputdir: output directory
:param str info: output file name suffix
:param bool verbose: controls verbosity
"""
global x_limit # late assignment of default, because it can be set to None in config
global divide_by_dimension # not fully implemented/tested yet
if 'x_limit' not in globals() or x_limit is None:
x_limit = x_limit_default
tmp = pp.dictAlgByDim(dictAlg)
# tmp = pp.DictAlg(dictAlg).by_dim()
if len(tmp) != 1 and dimension is None:
raise ValueError('We never integrate over dimension.')
if dimension is not None:
if dimension not in tmp.keys():
raise ValueError('dimension %d not in dictAlg dimensions %s' %
(dimension, str(tmp.keys())))
tmp = {dimension: tmp[dimension]}
dim = tmp.keys()[0]
divisor = dim if divide_by_dimension else 1
algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []]
dictFunc = pp.dictAlgByFun(dictAlg)
# Collect data
# Crafting effort correction: should we consider any?
CrEperAlg = {}
for alg in algorithms_with_data:
CrE = 0.
if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL':
tmp = dictAlg[alg].dictByNoise()
assert len(tmp.keys()) == 1
if tmp.keys()[0] == 'noiselessall':
CrE = 0.5117
elif tmp.keys()[0] == 'nzall':
CrE = 0.6572
CrEperAlg[alg] = CrE
if CrE != 0.0:
print 'Crafting effort for', alg, 'is', CrE
dictData = {} # list of (ert per function) per algorithm
dictMaxEvals = {} # list of (maxevals per function) per algorithm
bestERT = [] # best ert per function
# funcsolved = [set()] * len(targets) # number of functions solved per target
xbest2009 = []
maxevalsbest2009 = []
for f, dictAlgperFunc in dictFunc.iteritems():
if function_IDs and f not in function_IDs:
continue
# print target_values((f, dim))
for j, t in enumerate(target_values((f, dim))):
# for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)):
# funcsolved[j].add(f)
for alg in algorithms_with_data:
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
try:
entry = dictAlgperFunc[alg][
0] # one element per fun and per dim.
evals = entry.detEvals([t])[0]
assert entry.dim == dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(runlengthsucc,
runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
except (KeyError, IndexError):
#set_trace()
warntxt = (
'Data for algorithm %s on function %d in %d-D ' %
(alg, f, dim) + 'are missing.\n')
warnings.warn(warntxt)
dictData.setdefault(alg, []).extend(x)
dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc)
if displaybest2009:
#set_trace()
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgentry = bestalg.bestalgentries2009[(dim, f)]
bestalgevals = bestalgentry.detEvals(target_values((f, dim)))
# print bestalgevals
for j in range(len(bestalgevals[0])):
if bestalgevals[1][j]:
evals = bestalgevals[0][j]
#set_trace()
assert dim == bestalgentry.dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = bestalgentry.maxevals[
bestalgevals[1][j]][np.isnan(evals)] / divisor
x = toolsstats.drawSP(runlengthsucc,
runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
else:
x = perfprofsamplesize * [np.inf]
runlengthunsucc = []
xbest2009.extend(x)
maxevalsbest2009.extend(runlengthunsucc)
if order is None:
order = dictData.keys()
# Display data
lines = []
if displaybest2009:
args = {
'ls': '-',
'linewidth': 6,
'marker': 'D',
'markersize': 11.,
'markeredgewidth': 1.5,
'markerfacecolor': refcolor,
'markeredgecolor': refcolor,
'color': refcolor,
'label': 'best 2009',
'zorder': -1
}
lines.append(
plotdata(np.array(xbest2009),
x_limit,
maxevalsbest2009,
CrE=0.,
**args))
def algname_to_label(algname, dirname=None):
"""to be extended to become generally useful"""
if isinstance(algname, (tuple, list)): # not sure this is needed
return ' '.join([str(name) for name in algname])
return str(algname)
for i, alg in enumerate(order):
try:
data = dictData[alg]
maxevals = dictMaxEvals[alg]
except KeyError:
continue
args = styles[(i) % len(styles)]
args['linewidth'] = 1.5
args['markersize'] = 12.
args['markeredgewidth'] = 1.5
args['markerfacecolor'] = 'None'
args['markeredgecolor'] = args['color']
args['label'] = algname_to_label(alg)
#args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib
#elif len(show_algorithms) > 0:
#args['color'] = 'wheat'
#args['ls'] = '-'
#args['zorder'] = -1
# plotdata calls pprldistr.plotECDF which calls ppfig.plotUnifLog... which does the work
lines.append(
plotdata(np.array(data),
x_limit,
maxevals,
CrE=CrEperAlg[alg],
**args))
labels, handles = plotLegend(lines, x_limit)
if True: # isLateXLeg:
fileName = os.path.join(outputdir, 'pprldmany_%s.tex' % (info))
with open(fileName, 'w') as f:
f.write(r'\providecommand{\nperfprof}{7}')
algtocommand = {} # latex commands
for i, alg in enumerate(order):
tmp = r'\alg%sperfprof' % pptex.numtotext(i)
f.write(
r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' %
(tmp,
toolsdivers.str_to_latex(
toolsdivers.strip_pathname2(algname_to_label(alg)))))
algtocommand[algname_to_label(alg)] = tmp
if displaybest2009:
tmp = r'\algzeroperfprof'
f.write(r'\providecommand{%s}{best 2009}' % (tmp))
algtocommand['best 2009'] = tmp
commandnames = []
for label in labels:
commandnames.append(algtocommand[label])
# f.write(headleg)
if len(
order
) > 28: # latex sidepanel won't work well for more than 25 algorithms, but original labels are also clipped
f.write(
r'\providecommand{\perfprofsidepanel}{\mbox{%s}\vfill\mbox{%s}}'
% (commandnames[0], commandnames[-1]))
else:
fontsize_command = r'\tiny{}' if len(order) > 19 else ''
f.write(r'\providecommand{\perfprofsidepanel}{{%s\mbox{%s}' %
(fontsize_command,
commandnames[0])) # TODO: check len(labels) > 0
for i in range(1, len(labels)):
f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i])
f.write('}}\n')
# f.write(footleg)
if verbose:
print 'Wrote right-hand legend in %s' % fileName
figureName = os.path.join(outputdir, 'pprldmany_%s' % (info))
#beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat)
beautify()
text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys())))
text += ',%d-D' % dim # TODO: this is strange when different dimensions are plotted
plt.text(0.01,
0.98,
text,
horizontalalignment="left",
verticalalignment="top",
transform=plt.gca().transAxes)
if len(dictFunc) == 1:
plt.title(' '.join(
(str(dictFunc.keys()[0]),
genericsettings.current_testbed.short_names[dictFunc.keys()[0]])))
a = plt.gca()
plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative)
xticks, labels = plt.xticks()
tmp = []
for i in xticks:
tmp.append('%d' % round(np.log10(i)))
a.set_xticklabels(tmp)
if save_figure:
ppfig.saveFigure(figureName, verbose=verbose)
if len(dictFunc) == 1:
ppfig.save_single_functions_html(
os.path.join(outputdir, 'pprldmany'),
'', # algorithms names are clearly visible in the figure
add_to_names='_%02dD' % (dim),
algorithmCount=ppfig.AlgorithmCount.NON_SPECIFIED)
if close_figure:
plt.close()
def main(dictAlg, sortedAlgs, target=1e-8, outputdir='ppdata', verbose=True):
"""From a DataSetList, returns figures showing the scaling: ERT/dim vs dim.
One function and one target per figure.
sortedAlgs is a list of string-identifies (folder names)
"""
dictFunc = pproc.dictAlgByFun(dictAlg)
for f in dictFunc:
filename = os.path.join(outputdir,'ppfigs_f%03d' % (f))
handles = []
fix_styles(len(sortedAlgs)) #
for i, alg in enumerate(sortedAlgs):
dictDim = dictFunc[f][alg].dictByDim()
#Collect data
dimert = []
ert = []
dimnbsucc = []
ynbsucc = []
nbsucc = []
dimmaxevals = []
maxevals = []
dimmedian = []
medianfes = []
for dim in sorted(dictDim):
assert len(dictDim[dim]) == 1
entry = dictDim[dim][0]
data = generateData(entry, target) # TODO: here we might want a different target for each function
if 1 < 3 or data[2] == 0: # No success
dimmaxevals.append(dim)
maxevals.append(float(data[3])/dim)
if data[2] > 0:
dimmedian.append(dim)
medianfes.append(data[4]/dim)
dimert.append(dim)
ert.append(float(data[0])/dim)
if data[1] < 1.:
dimnbsucc.append(dim)
ynbsucc.append(float(data[0])/dim)
nbsucc.append('%d' % data[2])
# Draw lines
tmp = plt.plot(dimert, ert, **styles[i]) #label=alg, )
plt.setp(tmp[0], markeredgecolor=plt.getp(tmp[0], 'color'))
# For legend
# tmp = plt.plot([], [], label=alg.replace('..' + os.sep, '').strip(os.sep), **styles[i])
tmp = plt.plot([], [], label=alg.split(os.sep)[-1], **styles[i])
plt.setp(tmp[0], markersize=12.,
markeredgecolor=plt.getp(tmp[0], 'color'))
if dimmaxevals:
tmp = plt.plot(dimmaxevals, maxevals, **styles[i])
plt.setp(tmp[0], markersize=20, #label=alg,
markeredgecolor=plt.getp(tmp[0], 'color'),
markeredgewidth=1,
markerfacecolor='None', linestyle='None')
handles.append(tmp)
#tmp2 = plt.plot(dimmedian, medianfes, ls='', marker='+',
# markersize=30, markeredgewidth=5,
# markeredgecolor=plt.getp(tmp, 'color'))[0]
#for i, n in enumerate(nbsucc):
# plt.text(dimnbsucc[i], numpy.array(ynbsucc[i])*1.85, n,
# verticalalignment='bottom',
# horizontalalignment='center')
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgdata = []
dimbestalg = list(df[0] for df in bestalg.bestalgentries2009 if df[1] == f)
dimbestalg.sort()
dimbestalg2 = []
for d in dimbestalg:
entry = bestalg.bestalgentries2009[(d, f)]
tmp = entry.detERT([target])[0]
if numpy.isfinite(tmp):
bestalgdata.append(float(tmp)/d)
dimbestalg2.append(d)
tmp = plt.plot(dimbestalg2, bestalgdata, color=refcolor, linewidth=10,
marker='d', markersize=25, markeredgecolor=refcolor, zorder=-1
#label='best 2009',
)
handles.append(tmp)
if show_significance: # plot significance-stars
xstar, ystar = [], []
dims = sorted(pproc.dictAlgByDim(dictFunc[f]))
for i, dim in enumerate(dims):
datasets = pproc.dictAlgByDim(dictFunc[f])[dim]
assert all([len(datasets[ialg]) == 1 for ialg in sortedAlgs if datasets[ialg]])
dsetlist = [datasets[ialg][0] for ialg in sortedAlgs if datasets[ialg]]
if len(dsetlist) > 1:
arzp, arialg = toolsstats.significance_all_best_vs_other(dsetlist, [target])
if arzp[0][1] * len(dims) < 0.05:
ert = dsetlist[arialg[0]].detERT([target])[0]
if ert < numpy.inf:
xstar.append(dim)
ystar.append(ert/dim)
plt.plot(xstar, ystar, 'k*', markerfacecolor=None, markeredgewidth=2, markersize=0.5*styles[0]['markersize'])
if funInfos:
plt.gca().set_title(funInfos[f])
isLegend = False
if legend:
plotLegend(handles)
elif 1 < 3:
if f in (1, 24, 101, 130) and len(sortedAlgs) < 6: # 6 elements at most in the boxed legend
isLegend = True
beautify(legend=isLegend, rightlegend=legend)
plt.text(plt.xlim()[0], plt.ylim()[0], 'ftarget=%.0e' % target)
saveFigure(filename, verbose=verbose)
plt.close()
# generate commands in tex file:
try:
abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
alg_definitions = []
for i in range(len(sortedAlgs)):
symb = r'{%s%s}' % (color_to_latex(styles[i]['color']),
marker_to_latex(styles[i]['marker']))
alg_definitions.append((', ' if i > 0 else '') + '%s:%s' % (symb, '\\algorithm' + abc[i % len(abc)]))
filename = os.path.join(outputdir, 'bbob_pproc_commands.tex')
toolsdivers.prepend_to_file(filename,
['\\providecommand{\\bbobppfigsftarget}{\\ensuremath{10^{%d}}}'
% int(numpy.round(numpy.log10(target))),
'\\providecommand{\\bbobppfigslegend}[1]{',
scaling_figure_legend,
'Legend: '] + alg_definitions + ['}']
)
if verbose:
print 'Wrote commands and legend to %s' % filename
# this is obsolete (however check templates)
filename = os.path.join(outputdir,'ppfigs.tex')
f = open(filename, 'w')
f.write('% Do not modify this file: calls to post-processing software'
+ ' will overwrite any modification.\n')
f.write('Legend: ')
for i in range(0, len(sortedAlgs)):
symb = r'{%s%s}' % (color_to_latex(styles[i]['color']),
marker_to_latex(styles[i]['marker']))
f.write((', ' if i > 0 else '') + '%s:%s' % (symb, writeLabels(sortedAlgs[i])))
f.close()
if verbose:
print '(obsolete) Wrote legend in %s' % filename
except IOError:
raise
handles.append(tmp)
if funInfos:
plt.gca().set_title(funInfos[f])
beautify(rightlegend=legend)
if legend:
plotLegend(handles)
else:
if f in (1, 24, 101, 130):
plt.legend()
saveFigure(filename, figFormat=genericsettings.fig_formats, verbose=verbose)
plt.close()