def main(dsList, dimsOfInterest, outputdir, info='', verbose=True):
"""Generate a table of ratio ERT/ERTbest vs target precision.
1 table per dimension will be generated.
Rank-sum tests table on "Final Data Points" for only one algorithm.
that is, for example, using 1/#fevals(ftarget) if ftarget was
reached and -f_final otherwise as input for the rank-sum test, where
obviously the larger the better.
"""
#TODO: check that it works for any reference algorithm?
#in the following the reference algorithm is the one given in
#bestalg.bestalgentries which is the virtual best of BBOB
dictDim = dsList.dictByDim()
targetf=1e-8
if info:
info = '_' + info
# insert a separator between the default file name and the additional
# information string.
dims = set(dictDim.keys())
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
header = [r'\#FEs/D']
for i in targetsOfInterest.labels():
header.append(r'\multicolumn{2}{@{}c@{}}{%s}'
% i)
else:
header = [r'$\Delta f$']
for i in targetsOfInterest.target_values:
header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}'
% (int(np.log10(i))))
header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}')
for d in dimsOfInterest:
table = [header]
extraeol = [r'\hline']
try:
dictFunc = dictDim[d].dictByFunc()
except KeyError:
continue
funcs = set(dictFunc.keys())
nbtests = float(len(funcs)) # #funcs tests times one algorithm
for f in sorted(funcs):
bestalgentry = bestalg.bestalgentries2009[(d, f)]
curline = [r'${\bf f_{%d}}$' % f]
bestalgdata = bestalgentry.detERT(targetsOfInterest((f,d)))
bestalgevals, bestalgalgs = bestalgentry.detEvals(targetsOfInterest((f,d)))
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
#write ftarget:fevals
for i in xrange(len(bestalgdata[:-1])):
temp="%.1e" %targetsOfInterest((f,d))[i]
if temp[-2]=="0":
temp=temp[:-2]+temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}'
% (temp,writeFEvalsMaxPrec(bestalgdata[i], 2)))
temp="%.1e" %targetsOfInterest((f,d))[-1]
if temp[-2]=="0":
temp=temp[:-2]+temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }'
% (temp,writeFEvalsMaxPrec(bestalgdata[-1], 2)))
#success
targetf=targetsOfInterest((f,d))[-1]
else:
# write #fevals of the reference alg
for i in bestalgdata[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}'
% writeFEvalsMaxPrec(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}'
% writeFEvalsMaxPrec(bestalgdata[-1], 2))
# write the success ratio for the reference alg
tmp = bestalgentry.detEvals([targetf])[0][0]
tmp2 = np.sum(np.isnan(tmp) == False) # count the nb of success
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(tmp))
table.append(curline[:])
extraeol.append('')
# generate all data for ranksum test
assert len(dictFunc[f]) == 1
entry = dictFunc[f][0] # take the first element
ertdata = entry.detERT(targetsOfInterest((f, d)))
testresbestvs1 = significancetest(bestalgentry, entry,
targetsOfInterest((f, d)))
#for nb, entry in enumerate(entries):
#curline = [r'\algshort\hspace*{\fill}']
curline = ['']
#data = entry.detERT(targetsOfInterest)
evals = entry.detEvals(targetsOfInterest((f,d)))
dispersion = []
data = []
for i in evals:
succ = (np.isnan(i) == False)
tmp = i.copy()
tmp[succ==False] = entry.maxevals[np.isnan(i)]
#set_trace()
# TODO: what is the difference between data and ertdata?
data.append(toolsstats.sp(tmp, issuccessful=succ)[0])
#if not any(succ):
#set_trace()
if any(succ):
tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ==False],
(10, 50, 90), samplesize)[0]
dispersion.append((tmp2[-1] - tmp2[0]) / 2.)
else:
dispersion.append(None)
assert data == ertdata
for i, ert in enumerate(data):
alignment = 'c'
if i == len(data) - 1: # last element
alignment = 'c|'
nbstars = 0
z, p = testresbestvs1[i]
if ert - bestalgdata[i] < 0. and not np.isinf(bestalgdata[i]):
evals = entry.detEvals([targetsOfInterest((f,d))[i]])[0]
evals[np.isnan(evals)] = entry.maxevals[np.isnan(evals)]
bestevals = bestalgentry.detEvals([targetsOfInterest((f,d))[i]])
bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0])
bestevals[np.isnan(bestevals)] = bestalgentry.maxevals[bestalgalg][np.isnan(bestevals)]
evals = np.array(sorted(evals))[0:min(len(evals), len(bestevals))]
bestevals = np.array(sorted(bestevals))[0:min(len(evals), len(bestevals))]
#The conditions for significance are now that ERT < ERT_best and
# all(sorted(FEvals_best) > sorted(FEvals_current)).
if ((nbtests * p) < 0.05 and ert - bestalgdata[i] < 0.
and z < 0.
and (np.isinf(bestalgdata[i])
or all(evals < bestevals))):
nbstars = -np.ceil(np.log10(nbtests * p))
isBold = False
if nbstars > 0:
isBold = True
if np.isinf(bestalgdata[i]): # if the best did not solve the problem
tmp = writeFEvalsMaxPrec(float(ert), 2)
if not np.isinf(ert):
tmp = r'\textit{%s}' % (tmp)
if isBold:
tmp = r'\textbf{%s}' % tmp
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tmp))
else:
# Formatting
tmp = float(ert) / bestalgdata[i]
assert not np.isnan(tmp)
tableentry = writeFEvalsMaxPrec(tmp, 2)
if np.isinf(tmp) and i == len(data)-1:
tableentry = (tableentry
+ r'\textit{%s}' % writeFEvals2(np.median(entry.maxevals), 2))
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3: # and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tableentry))
elif tableentry.find('e') > -1 or (np.isinf(tmp) and i != len(data) - 1):
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3: # and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tableentry))
else:
tmp = tableentry.split('.', 1)
if isBold:
tmp = list(r'\textbf{%s}' % i for i in tmp)
elif 11 < 3: # and significance0vs1 < 0:
tmp = list(r'\textit{%s}' % i for i in tmp)
tableentry = ' & .'.join(tmp)
if len(tmp) == 1:
tableentry += '&'
superscript = ''
if nbstars > 0:
#tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star'])
if z > 0:
superscript = r'\uparrow' #* nbstars
else:
superscript = r'\downarrow' #* nbstars
# print z, linebest[i], line1
if nbstars > 1:
superscript += str(int(min((9, nbstars))))
# superscript += str(int(nbstars))
#if superscript or significance0vs1:
#s = ''
#if significance0vs1 > 0:
#s = '\star'
#if significance0vs1 > 1:
#s += str(significance0vs1)
#s = r'$^{' + s + superscript + r'}$'
#if tableentry.endswith('}'):
#tableentry = tableentry[:-1] + s + r'}'
#else:
#tableentry += s
if dispersion[i]:
if not np.isinf(bestalgdata[i]):
tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
else:
tmp = writeFEvalsMaxPrec(dispersion[i], 1)
tableentry += (r'${\scriptscriptstyle(%s)}$' % tmp)
if superscript:
s = r'$^{' + superscript + r'}$'
if tableentry.endswith('}'):
tableentry = tableentry[:-1] + s + r'}'
else:
tableentry += s
curline.append(tableentry)
#curline.append(tableentry)
#if dispersion[i] is None or np.isinf(bestalgdata[i]):
#curline.append('')
#else:
#tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2)
#curline.append('(%s)' % tmp)
tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:]
try:
tmp = tmp[0]
curline.append('%d' % np.sum(np.isnan(tmp) == False))
except IndexError:
curline.append('%d' % 0)
curline.append('/%d' % entry.nbRuns())
table.append(curline[:])
extraeol.append(r'\hline')
extraeol[-1] = ''
outputfile = os.path.join(outputdir, 'pptable_%02dD%s.tex' % (d, info))
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
spec = r'@{}c@{}|' + '*{%d}{@{ }r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}'
else:
spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}'
#res = r'\providecommand{\algshort}{%s}' % alg1 + '\n'
#res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec,
#extraeol=extraeol)
res = tableLaTeX(table, spec=spec, extraeol=extraeol)
f = open(outputfile, 'w')
f.write(res)
f.close()
if verbose:
print "Table written in %s" % outputfile
def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True):
"""One table per dimension, modified to fit in 1 page per table."""
#TODO: method is long, split if possible
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3]
alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3]
open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a'
).write(r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' +
r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n')
if info:
info = '_' + info
dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys()))
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
header = [r'$\Delta f$']
for i in targetsOfInterest:
#header.append(r'\multicolumn{2}{@{}c@{}}{$10^{%d}$}' % (int(numpy.log10(i))))
header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' % (int(numpy.log10(i))))
header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}')
for d in dimsOfInterest: # TODO set as input arguments
table = [header]
extraeol = [r'\hline']
try:
dictFunc0 = dictDim0[d].dictByFunc()
dictFunc1 = dictDim1[d].dictByFunc()
except KeyError:
continue
funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys()))
nbtests = len(funcs) * 2. #len(dimsOfInterest)
for f in sorted(funcs):
bestalgentry = bestalg.bestalgentries2009[(d, f)]
curline = [r'${\bf f_{%d}}$' % f]
bestalgdata = bestalgentry.detERT(targetsOfInterest)
bestalgevals, bestalgalgs = bestalgentry.detEvals(targetsOfInterest)
for i in bestalgdata[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' % writeFEvalsMaxPrec(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2))
tmp = bestalgentry.detEvals([targetf])[0][0]
tmp2 = numpy.sum(numpy.isnan(tmp) == False)
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(tmp))
table.append(curline[:])
extraeol.append('')
rankdata0 = [] # never used
# generate all data from ranksum test
entries = []
ertdata = {}
for nb, dsList in enumerate((dictFunc0, dictFunc1)):
try:
entry = dsList[f][0] # take the first DataSet, there should be only one?
except KeyError:
warnings.warn('data missing for data set ' + str(nb) + ' and function ' + str(f))
print('*** Warning: data missing for data set ' + str(nb) + ' and function ' + str(f) + '***')
continue # TODO: problem here!
ertdata[nb] = entry.detERT(targetsOfInterest)
entries.append(entry)
for _t in ertdata.values():
for _tt in _t:
if _tt is None:
raise ValueError
if len(entries) < 2: # funcion not available for *both* algorithms
continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following
testres0vs1 = significancetest(entries[0], entries[1], targetsOfInterest)
testresbestvs1 = significancetest(bestalgentry, entries[1], targetsOfInterest)
testresbestvs0 = significancetest(bestalgentry, entries[0], targetsOfInterest)
for nb, entry in enumerate(entries):
if nb == 0:
curline = [r'1:\:\algorithmAshort\hspace*{\fill}']
else:
curline = [r'2:\:\algorithmBshort\hspace*{\fill}']
#data = entry.detERT(targetsOfInterest)
dispersion = []
data = []
evals = entry.detEvals(targetsOfInterest)
for i in evals:
succ = (numpy.isnan(i) == False)
tmp = i.copy()
tmp[succ==False] = entry.maxevals[numpy.isnan(i)]
#set_trace()
data.append(toolsstats.sp(tmp, issuccessful=succ)[0])
#if not any(succ):
#set_trace()
if any(succ):
tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ==False],
(10, 50, 90), samplesize)[0]
dispersion.append((tmp2[-1]-tmp2[0])/2.)
else:
dispersion.append(None)
if nb == 0:
assert not isinstance(data, numpy.ndarray)
data0 = data[:] # TODO: check if it is not an array, it's never used anyway?
for i, dati in enumerate(data):
z, p = testres0vs1[i] # TODO: there is something with the sign that I don't get
# assign significance flag, which is the -log10(p)
significance0vs1 = 0
if nb != 0:
z = -z # the test is symmetric
if nbtests * p < 0.05 and z > 0:
significance0vs1 = -int(numpy.ceil(numpy.log10(min([1.0, nbtests * p])))) # this is the larger the more significant
isBold = significance0vs1 > 0
alignment = 'c'
if i == len(data) - 1: # last element
alignment = 'c|'
if numpy.isinf(bestalgdata[i]): # if the 2009 best did not solve the problem
tmp = writeFEvalsMaxPrec(float(dati), 2)
if not numpy.isinf(dati):
tmp = r'\textit{%s}' % (tmp)
if isBold:
tmp = r'\textbf{%s}' % tmp
if dispersion[i] and numpy.isfinite(dispersion[i]):
tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tmp))
else:
# Formatting
tmp = float(dati)/bestalgdata[i]
assert not numpy.isnan(tmp)
isscientific = False
if tmp >= 1000:
isscientific = True
tableentry = writeFEvals2(tmp, 2, isscientific=isscientific)
tableentry = writeFEvalsMaxPrec(tmp, 2)
if numpy.isinf(tmp) and i == len(data)-1:
tableentry = (tableentry
+ r'\textit{%s}' % writeFEvals2(numpy.median(entry.maxevals), 2))
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore
tableentry = r'\textit{%s}' % tableentry
if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tableentry))
elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and i != len(data) - 1):
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3 and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tableentry))
else:
tmp = tableentry.split('.', 1)
if isBold:
tmp = list(r'\textbf{%s}' % i for i in tmp)
elif 11 < 3 and significance0vs1 < 0:
tmp = list(r'\textit{%s}' % i for i in tmp)
tableentry = ' & .'.join(tmp)
if len(tmp) == 1:
tableentry += '&'
if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
superscript = ''
if nb == 0:
z, p = testresbestvs0[i]
else:
z, p = testresbestvs1[i]
#The conditions are now that ERT < ERT_best
if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0.
and z < 0.):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
#tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star'])
if z > 0:
superscript = r'\uparrow' #* nbstars
else:
superscript = r'\downarrow' #* nbstars
# print z, linebest[i], line1
if nbstars > 1:
superscript += str(int(nbstars))
if superscript or significance0vs1:
s = ''
if significance0vs1 > 0:
s = '\star'
if significance0vs1 > 1:
s += str(significance0vs1)
s = r'$^{' + s + superscript + r'}$'
if tableentry.endswith('}'):
tableentry = tableentry[:-1] + s + r'}'
else:
tableentry += s
curline.append(tableentry)
#curline.append(tableentry)
#if dispersion[i] is None or numpy.isinf(bestalgdata[i]):
#curline.append('')
#else:
#tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2)
#curline.append('(%s)' % tmp)
tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:]
try:
tmp = tmp[0]
curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False))
except IndexError:
curline.append('%d' % 0)
curline.append('/%d' % entry.nbRuns())
table.append(curline[:])
extraeol.append('')
extraeol[-1] = r'\hline'
extraeol[-1] = ''
outputfile = os.path.join(outputdir, 'pptable2_%02dD%s.tex' % (d, info))
spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}'
res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n'
res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n'
# open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res)
#res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec,
#extraeol=extraeol)
res += tableLaTeX(table, spec=spec, extraeol=extraeol)
f = open(outputfile, 'w')
f.write(res)
f.close()
if verbose:
print "Table written in %s" % outputfile
def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True):
"""One table per dimension, modified to fit in 1 page per table."""
#TODO: method is long, split if possible
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3]
alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3]
open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(
r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' +
r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n')
if info:
info = '_' + info
dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys()))
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
header = []
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
header = [r'\#FEs/D']
for label in targetsOfInterest.labels():
header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % label)
else:
header = [r'$\Delta f_\mathrm{opt}$']
for label in targetsOfInterest.labels():
header.append(r'\multicolumn{2}{@{\,}c@{\,}}{%s}' % label)
header.append(r'\multicolumn{2}{@{}l@{}}{\#succ}')
for d in dimsOfInterest: # TODO set as input arguments
table = [header]
extraeol = [r'\hline']
try:
dictFunc0 = dictDim0[d].dictByFunc()
dictFunc1 = dictDim1[d].dictByFunc()
except KeyError:
continue
funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys()))
nbtests = len(funcs) * 2. #len(dimsOfInterest)
for f in sorted(funcs):
targets = targetsOfInterest((f, d))
targetf = targets[-1]
bestalgentry = bestalg.bestalgentries2009[(d, f)]
curline = [r'${\bf f_{%d}}$' % f]
bestalgdata = bestalgentry.detERT(targets)
bestalgevals, bestalgalgs = bestalgentry.detEvals(targets)
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
# write ftarget:fevals
for i in xrange(len(bestalgdata[:-1])):
temp = "%.1e" % targetsOfInterest((f, d))[i]
if temp[-2] == "0":
temp = temp[:-2] + temp[-1]
curline.append(
r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' %
(temp, writeFEvalsMaxPrec(bestalgdata[i], 2)))
temp = "%.1e" % targetsOfInterest((f, d))[-1]
if temp[-2] == "0":
temp = temp[:-2] + temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' %
(temp, writeFEvalsMaxPrec(bestalgdata[-1], 2)))
else:
# write #fevals of the reference alg
for i in bestalgdata[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' %
writeFEvalsMaxPrec(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' %
writeFEvalsMaxPrec(bestalgdata[-1], 2))
tmp = bestalgentry.detEvals([targetf])[0][0]
tmp2 = numpy.sum(numpy.isnan(tmp) == False)
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(tmp))
table.append(curline[:])
extraeol.append('')
rankdata0 = [] # never used
# generate all data from ranksum test
entries = []
ertdata = {}
for nb, dsList in enumerate((dictFunc0, dictFunc1)):
try:
entry = dsList[f][
0] # take the first DataSet, there should be only one?
except KeyError:
warnings.warn('data missing for data set ' + str(nb) +
' and function ' + str(f))
print('*** Warning: data missing for data set ' + str(nb) +
' and function ' + str(f) + '***')
continue # TODO: problem here!
ertdata[nb] = entry.detERT(targets)
entries.append(entry)
for _t in ertdata.values():
for _tt in _t:
if _tt is None:
raise ValueError
if len(entries) < 2: # funcion not available for *both* algorithms
continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following
testres0vs1 = significancetest(entries[0], entries[1], targets)
testresbestvs1 = significancetest(bestalgentry, entries[1],
targets)
testresbestvs0 = significancetest(bestalgentry, entries[0],
targets)
for nb, entry in enumerate(entries):
if nb == 0:
curline = [r'1:\:\algorithmAshort\hspace*{\fill}']
else:
curline = [r'2:\:\algorithmBshort\hspace*{\fill}']
#data = entry.detERT(targetsOfInterest)
dispersion = []
data = []
evals = entry.detEvals(targets)
for i in evals:
succ = (numpy.isnan(i) == False)
tmp = i.copy()
tmp[succ == False] = entry.maxevals[numpy.isnan(i)]
#set_trace()
data.append(toolsstats.sp(tmp, issuccessful=succ)[0])
#if not any(succ):
#set_trace()
if any(succ):
tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ == False],
(10, 50, 90), samplesize)[0]
dispersion.append((tmp2[-1] - tmp2[0]) / 2.)
else:
dispersion.append(None)
if nb == 0:
assert not isinstance(data, numpy.ndarray)
data0 = data[:] # TODO: check if it is not an array, it's never used anyway?
for i, dati in enumerate(data):
z, p = testres0vs1[
i] # TODO: there is something with the sign that I don't get
# assign significance flag, which is the -log10(p)
significance0vs1 = 0
if nb != 0:
z = -z # the test is symmetric
if nbtests * p < 0.05 and z > 0:
significance0vs1 = -int(
numpy.ceil(numpy.log10(min([
1.0, nbtests * p
])))) # this is the larger the more significant
isBold = significance0vs1 > 0
alignment = 'c'
if i == len(data) - 1: # last element
alignment = 'c|'
if numpy.isinf(
bestalgdata[i]
): # if the 2009 best did not solve the problem
tmp = writeFEvalsMaxPrec(float(dati), 2)
if not numpy.isinf(dati):
tmp = r'\textit{%s}' % (tmp)
if isBold:
tmp = r'\textbf{%s}' % tmp
if dispersion[i] and numpy.isfinite(dispersion[i]):
tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(
dispersion[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' %
(alignment, tmp))
else:
# Formatting
tmp = float(dati) / bestalgdata[i]
assert not numpy.isnan(tmp)
isscientific = False
if tmp >= 1000:
isscientific = True
tableentry = writeFEvals2(tmp,
2,
isscientific=isscientific)
tableentry = writeFEvalsMaxPrec(tmp, 2)
if numpy.isinf(tmp) and i == len(data) - 1:
tableentry = (
tableentry + r'\textit{%s}' %
writeFEvals2(numpy.median(entry.maxevals), 2))
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore
tableentry = r'\textit{%s}' % tableentry
if dispersion[i] and numpy.isfinite(
dispersion[i] / bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(
dispersion[i] / bestalgdata[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' %
(alignment, tableentry))
elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and
i != len(data) - 1):
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3 and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
if dispersion[i] and numpy.isfinite(
dispersion[i] / bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(
dispersion[i] / bestalgdata[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' %
(alignment, tableentry))
else:
tmp = tableentry.split('.', 1)
if isBold:
tmp = list(r'\textbf{%s}' % i for i in tmp)
elif 11 < 3 and significance0vs1 < 0:
tmp = list(r'\textit{%s}' % i for i in tmp)
tableentry = ' & .'.join(tmp)
if len(tmp) == 1:
tableentry += '&'
if dispersion[i] and numpy.isfinite(
dispersion[i] / bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(
dispersion[i] / bestalgdata[i], 1)
superscript = ''
if nb == 0:
z, p = testresbestvs0[i]
else:
z, p = testresbestvs1[i]
#The conditions are now that ERT < ERT_best
if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0.
and z < 0.):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
#tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star'])
if z > 0:
superscript = r'\uparrow' #* nbstars
else:
superscript = r'\downarrow' #* nbstars
# print z, linebest[i], line1
if nbstars > 1:
superscript += str(int(nbstars))
if superscript or significance0vs1:
s = ''
if significance0vs1 > 0:
s = '\star'
if significance0vs1 > 1:
s += str(significance0vs1)
s = r'$^{' + s + superscript + r'}$'
if tableentry.endswith('}'):
tableentry = tableentry[:-1] + s + r'}'
else:
tableentry += s
curline.append(tableentry)
#curline.append(tableentry)
#if dispersion[i] is None or numpy.isinf(bestalgdata[i]):
#curline.append('')
#else:
#tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2)
#curline.append('(%s)' % tmp)
tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:]
try:
tmp = tmp[0]
curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False))
except IndexError:
curline.append('%d' % 0)
curline.append('/%d' % entry.nbRuns())
table.append(curline[:])
extraeol.append('')
extraeol[-1] = r'\hline'
extraeol[-1] = ''
outputfile = os.path.join(outputdir,
'pptable2_%02dD%s.tex' % (d, info))
spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(
targetsOfInterest) + '|@{}r@{}@{}l@{}'
res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels(
alg0) + '\n'
res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels(
alg1) + '\n'
# open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res)
#res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec,
#extraeol=extraeol)
res += tableLaTeX(table, spec=spec, extraeol=extraeol)
f = open(outputfile, 'w')
f.write(res)
f.close()
if verbose:
print "Table written in %s" % outputfile
def mainnew(dsList0,
dsList1,
dimsOfInterest,
outputdir,
info='',
verbose=True):
"""Still in shambles: new cleaned up version.
One table per dimension...
"""
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3]
alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3]
if info:
info = '_' + info
dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys()))
if not bestalg.bestalgentries:
bestalg.loadBBOB2009()
header = [r'$\Delta f$']
for i in targetsOfInterest:
#header.append(r'\multicolumn{2}{@{}c@{}}{$10^{%d}$}' % (int(numpy.log10(i))))
header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' %
(int(numpy.log10(i))))
header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}')
for d in dimsOfInterest: # TODO set as input arguments
table = [header]
extraeol = [r'\hline']
dictFunc0 = dictDim0[d].dictByFunc()
dictFunc1 = dictDim1[d].dictByFunc()
funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys()))
nbtests = len(funcs) * 2. #len(dimsOfInterest)
for f in sorted(funcs):
bestalgentry = bestalg.bestalgentries[(d, f)]
curline = [r'${\bf f_{%d}}$' % f]
bestalgdata = bestalgentry.detERT(targetsOfInterest)
bestalgevals, bestalgalgs = bestalgentry.detEvals(
targetsOfInterest)
for i in bestalgdata[:-1]:
#if numpy.isnan(i):
#set_trace()
curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' %
writeFEvals2(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' %
writeFEvals2(bestalgdata[-1], 2))
tmp = bestalgentry.detEvals([targetf])[0][0]
tmp2 = numpy.sum(numpy.isnan(tmp) == False)
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(tmp))
table.append(curline[:])
extraeol.append('')
rankdata0 = []
# generate all data from ranksum test
entries = []
ertdata = {}
for nb, dsList in enumerate((dictFunc0, dictFunc1)):
try:
entry = dsList[f][0] # take the first element
except KeyError:
continue # TODO: problem here!
ertdata[nb] = entry.detERT(targetsOfInterest)
entries.append(entry)
testres0vs1 = significancetest(entries[0], entries[1],
targetsOfInterest)
testresbestvs1 = significancetest(bestalgentry, entries[1],
targetsOfInterest)
for nb, entry in enumerate(entries):
if nb == 0:
curline = [r'0:\:\algzeroshort\hspace*{\fill}']
else:
curline = [r'1:\:\algoneshort\hspace*{\fill}']
#curline = [r'\alg%sshort' % tmp]
#curline = [r'Alg%d' % nb]
#curline = [r'%.3s%d' % (entry.algId, nb)]
data = entry.detERT(targetsOfInterest)
if nb == 0:
assert not isinstance(data, numpy.ndarray)
data0 = data[:] # check if it is not an array
for i, j in enumerate(
data): # is j an appropriate identifier here?
#if numpy.isnan(float(j)/bestalgdata[i]):
# set_trace()
z, p = testres0vs1[
i] #TODO: there is something with the sign that I don't get
# assign significance flag
significance0vs1 = 0
if nb == 0:
istat0 = 0
istat1 = 1
else:
z = -z
istat0 = 1
istat1 = 0
# TODO: I don't understand the thing with the sign of significance0vs1
if (nbtests * p < 0.05 and z > 0
and not numpy.isinf(ertdata[istat0][i])
and z * (ertdata[istat1][i] - ertdata[istat0][i]) >
0): # z-value and ERT-ratio must agree
significance0vs1 = -int(
numpy.ceil(numpy.log10(nbtests * p)))
elif nbtests * p < 0.05 and z < 0 and z * (
ertdata[istat1][i] - ertdata[istat0][i]) > 0:
significance0vs1 = int(
numpy.ceil(numpy.log10(nbtests * p)))
alignment = 'c'
if i == len(data) - 1: # last element
alignment = 'c|'
if numpy.isinf(bestalgdata[i]):
tableentry = (
r'\multicolumn{2}{@{}%s@{}}{\textbf{\textit{%s}}}'
% (alignment, writeFEvals2(float(j), 2)))
# TODO: is this the desired behaviour?
else:
# Formatting
tmp = float(j) / bestalgdata[i]
assert not numpy.isnan(tmp)
isscientific = False
if tmp >= 1000:
isscientific = True
tableentry = writeFEvals2(tmp,
2,
isscientific=isscientific)
isBold = False
if significance0vs1 > 0:
isBold = True
if numpy.isinf(tmp) and i == len(data) - 1:
tableentry = (
tableentry + r'\textit{%s}' %
writeFEvals2(numpy.median(entry.maxevals), 2))
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3 and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' %
(alignment, tableentry))
elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and
i != len(data) - 1):
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif 11 < 3 and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' %
(alignment, tableentry))
else:
tmp = tableentry.split('.', 1)
if isBold:
tmp = list(r'\textbf{%s}' % i for i in tmp)
elif 11 < 3 and significance0vs1 < 0:
tmp = list(r'\textit{%s}' % i for i in tmp)
tableentry = ' & .'.join(tmp)
if len(tmp) == 1:
tableentry += '&'
superscript = ''
z, p = testresbestvs1[i]
#z, p = ranksums(rankdatabest[i], currankdata)
#if ((nbtests * p) < 0.05
# and ((numpy.isinf(bestalgdata[i]) and numpy.isinf(j))
# or z * (j - bestalgdata[i]) > 0)): # z-value and ERT-ratio must agree
#The conditions are now that ERT < ERT_best and
# all(sorted(FEvals_best) > sorted(FEvals_current)).
if j - bestalgdata[i] < 0. and not numpy.isinf(
bestalgdata[i]):
evals = entry.detEvals([targetsOfInterest[i]])[0]
evals[numpy.isnan(evals)] = entry.maxevals[numpy.isnan(
evals)]
bestevals = bestalgentry.detEvals(
[targetsOfInterest[i]])
bestevals, bestalgalg = (bestevals[0][0],
bestevals[1][0])
bestevals[numpy.isnan(
bestevals)] = bestalgentry.maxevals[bestalgalg][
numpy.isnan(bestevals)]
evals = numpy.array(
sorted(evals))[0:min(len(evals), len(bestevals))]
bestevals = numpy.array(sorted(
bestevals))[0:min(len(evals), len(bestevals))]
#The conditions are now that ERT < ERT_best and
# all(sorted(FEvals_best) > sorted(FEvals_current)).
if ((nbtests * p) < 0.05 and j - bestalgdata[i] < 0.
and z < 0. and (numpy.isinf(bestalgdata[i])
or all(evals < bestevals))):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
#tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star'])
if z > 0:
superscript = r'\uparrow' #* nbstars
else:
superscript = r'\downarrow' #* nbstars
# print z, linebest[i], line1
if nbstars > 1:
superscript += str(int(nbstars))
addition = ''
if superscript or significance0vs1:
s = ''
if significance0vs1 > 0:
s = '\star'
if significance0vs1 > 1:
s += str(significance0vs1)
s = r'$^{' + s + superscript + r'}$'
if tableentry.endswith('}'):
tableentry = tableentry[:-1] + s + r'}'
else:
tableentry += s
curline.append(tableentry)
# Two cases: both tabular give an overfull hbox
# AND generate a LaTeX Warning: Float too large for page by 16.9236pt on input line 421. (noisy)
# OR generate a LaTeX Warning: Float too large for page by 33.57658pt on input line 421. (noisy)
tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:]
try:
tmp = tmp[0]
curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False))
except IndexError:
curline.append('%d' % 0)
curline.append('/%d' % entry.nbRuns())
table.append(curline[:])
extraeol.append('')
extraeol[-1] = r'\hline'
extraeol[-1] = ''
outputfile = os.path.join(outputdir,
'cmptable_%02dD%s.tex' % (d, info))
spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(
targetsOfInterest) + '|@{}r@{}@{}l@{}'
res = r'\providecommand{\algzeroshort}{%s}' % alg0 + '\n'
res += r'\providecommand{\algoneshort}{%s}' % alg1 + '\n'
#res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec,
#extraeol=extraeol)
res += tableLaTeX(table, spec=spec, extraeol=extraeol)
f = open(outputfile, 'w')
f.write(res)
f.close()
if verbose:
print "Table written in %s" % outputfile
def main2(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True):
"""Generate comparison tables.
One table per dimension...
"""
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3]
alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3]
if info:
info = '_' + info
dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys()))
if not bestalg.bestalgentries:
bestalg.loadBBOB2009()
header = [r'$\Delta f$']
for i in targetsOfInterest:
#header.append(r'\multicolumn{2}{@{}c@{}}{$10^{%d}$}' % (int(numpy.log10(i))))
header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' %
(int(numpy.log10(i))))
header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}')
for d in dimsOfInterest: # TODO set as input arguments
table = [header]
extraeol = [r'\hline']
dictFunc0 = dictDim0[d].dictByFunc()
dictFunc1 = dictDim1[d].dictByFunc()
funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys()))
nbtests = len(funcs) * 2. #len(dimsOfInterest)
for f in sorted(funcs):
bestalgentry = bestalg.bestalgentries[(d, f)]
curline = [r'${\bf f_{%d}}$' % f]
bestalgdata = bestalgentry.detERT(targetsOfInterest)
bestalgevals, bestalgalgs = bestalgentry.detEvals(
targetsOfInterest)
for i in bestalgdata[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' %
writeFEvals2(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' %
writeFEvals2(bestalgdata[-1], 2))
rankdatabest = []
for i, j in enumerate(bestalgevals):
if bestalgalgs[i] is None:
tmp = -bestalgentry.finalfunvals[bestalgentry.algs[-1]]
else:
tmp = numpy.power(j, -1.)
tmp[numpy.isnan(tmp)] = -bestalgentry.finalfunvals[
bestalgalgs[i]][numpy.isnan(tmp)]
rankdatabest.append(tmp)
tmp = bestalgentry.detEvals([targetf])[0][0]
tmp2 = numpy.sum(numpy.isnan(tmp) == False)
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(tmp))
table.append(curline[:])
extraeol.append('')
rankdata0 = []
# generate all data for ranksum test
rankdata = {} # rankdata[nb +/- 1,i] is of interest later
ertdata = {}
for nb, entries in enumerate(
(dictFunc0, dictFunc1)): # copy paste of loop below
try:
entry = entries[f][0] # take the first element
except KeyError:
continue
ertdata[nb] = entry.detERT(targetsOfInterest)
evals = entry.detEvals(targetsOfInterest)
for i, tmp in enumerate(ertdata[nb]):
# print i
rankdata[nb, i] = numpy.power(evals[i], -1.)
rankdata[nb, i][numpy.isnan(
rankdata[nb, i])] = -entry.finalfunvals[numpy.isnan(
rankdata[nb, i])]
for nb, entries in enumerate((dictFunc0, dictFunc1)):
try:
entry = entries[f][0] # take the first element
except KeyError:
continue
if nb == 0:
curline = [r'0:\:\algzeroshort\hspace*{\fill}']
else:
curline = [r'1:\:\algoneshort\hspace*{\fill}']
#curline = [r'\alg%sshort' % tmp]
#curline = [r'Alg%d' % nb]
#curline = [r'%.3s%d' % (entry.algId, nb)]
data = entry.detERT(targetsOfInterest)
evals = entry.detEvals(targetsOfInterest)
if nb == 0:
assert not isinstance(data, numpy.ndarray)
data0 = data[:] # check if it is not an array
for i, j in enumerate(
data): # is j an appropriate identifier here?
#if numpy.isnan(float(j)/bestalgdata[i]):
# set_trace()
# assign significance flag
significance0vs1 = 0
if nb == 0:
istat0 = 0
istat1 = 1
else:
istat0 = 1
istat1 = 0
#set_trace()
z, p = ranksums(rankdata[istat0, i], rankdata[istat1, i])
if (nbtests * p < 0.05 and z > 0
and not numpy.isinf(ertdata[istat0][i])
and z * (ertdata[istat1][i] - ertdata[istat0][i]) >
0): # z-value and ERT-ratio must agree
significance0vs1 = -int(
numpy.ceil(numpy.log10(nbtests * p)))
if nbtests * p < 0.05 and z < 0 and z * (
ertdata[istat1][i] - ertdata[istat0][i]) > 0:
significance0vs1 = int(
numpy.ceil(numpy.log10(nbtests * p)))
alignment = 'c'
if i == len(data) - 1: # last element
alignment = 'c|'
if numpy.isinf(bestalgdata[i]):
tableentry = (
r'\multicolumn{2}{@{}%s@{}}{\textbf{\textit{%s}}}'
% (alignment, writeFEvals2(float(j), 2)))
# TODO: is this the desired behaviour?
else:
# Formatting
assert not numpy.isnan(float(j) / bestalgdata[i])
tmp = float(j) / bestalgdata[i]
tableentry = writeFEvals2(tmp, 2)
isBold = False
if 11 < 3 and tmp <= 3:
isBold = True
if significance0vs1 > 0:
isBold = True
if tableentry.find('e') > -1:
if isBold:
tableentry = r'\textbf{%s}' % tableentry
elif significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' %
(alignment, tableentry))
else:
tmp = tableentry.split('.', 1)
if isBold:
tmp = list(r'\textbf{%s}' % i for i in tmp)
elif significance0vs1 < 0:
tmp = list(r'\textit{%s}' % i for i in tmp)
tableentry = ' & .'.join(tmp)
if len(tmp) == 1:
tableentry += '&'
currankdata = numpy.power(evals[i], -1.)
currankdata[numpy.isnan(
currankdata
)] = -entry.finalfunvals[numpy.isnan(currankdata)]
if 11 < 3 and nb == 0:
rankdata0.append(currankdata.copy())
addition = ''
elif 11 < 3: #nb==1
z, p = ranksums(rankdata0[i], currankdata)
tmp = '.'
#addition = '' # Uncomment to gain space. Meanwhile it displays the worst case scenario
if ((nbtests * p) < 0.05 and
((numpy.isinf(data0[i]) and numpy.isinf(j)) or z *
(j - data0[i]) > 0)
): # z-value and ERT-ratio must agree
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
if z > 0:
tmp = r'\uparrow' #* nbstars
else:
tmp = r'\downarrow' #* nbstars
# print z, linebest[i], line1
if nbstars > 1:
tmp += str(int(nbstars))
#addition = tmp # uncomment to gain space.
addition = r'/' + tmp # Comment and uncomment line above to gain space
#addition = tmp # Comment and uncomment line above to gain space
superscript = '\uparrow'
superscript = ''
#if addition:
#superscript = '.'
z, p = ranksums(rankdatabest[i], currankdata)
if ((nbtests * p) < 0.05 and
((numpy.isinf(bestalgdata[i]) and numpy.isinf(j))
or z * (j - bestalgdata[i]) > 0)
): # z-value and ERT-ratio must agree
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
#tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star'])
#set_trace()
if z > 0:
superscript = r'\uparrow' #* nbstars
else:
superscript = r'\downarrow' #* nbstars
# print z, linebest[i], line1
if nbstars > 1:
superscript += str(int(nbstars))
addition = ''
if superscript or significance0vs1:
s = ''
if significance0vs1 > 0:
s = '\star'
if significance0vs1 > 1:
s += str(significance0vs1)
s = r'$^{' + s + superscript + r'}$'
if tableentry.endswith('}'):
tableentry = tableentry[:-1] + s + r'}'
else:
tableentry += s
if 11 < 3 and (superscript or addition):
isClosingBrace = False
if tableentry.endswith('}'):
isClosingBrace = True
tableentry = tableentry[:-1]
#tableentry += r'$^{' + superscript + '}_{' + addition + '}$'
tableentry += r'$^{' + superscript + addition + '}$'
if isClosingBrace:
tableentry += '}'
curline.append(tableentry)
# Two cases: both tabular give an overfull hbox
# AND generate a LaTeX Warning: Float too large for page by 16.9236pt on input line 421. (noisy)
# OR generate a LaTeX Warning: Float too large for page by 33.57658pt on input line 421. (noisy)
tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:]
try:
tmp = tmp[0]
curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False))
except IndexError:
curline.append('%d' % 0)
curline.append('/%d' % entry.nbRuns())
table.append(curline[:])
extraeol.append('')
extraeol[-1] = r'\hline'
extraeol[-1] = ''
outputfile = os.path.join(outputdir,
'cmptable_%02dD%s.tex' % (d, info))
spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(
targetsOfInterest) + '|@{}r@{}@{}l@{}'
res = r'\providecommand{\algzeroshort}{%s}' % alg0 + '\n'
res += r'\providecommand{\algoneshort}{%s}' % alg1 + '\n'
#res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec,
#extraeol=extraeol)
res += tableLaTeX(table, spec=spec, extraeol=extraeol)
f = open(outputfile, 'w')
f.write(res)
f.close()
if verbose:
print "Table written in %s" % outputfile
def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True):
"""One table per dimension, modified to fit in 1 page per table."""
#TODO: method is long, split if possible
dictDim0 = dsList0.dictByDim()
dictDim1 = dsList1.dictByDim()
alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop().replace(genericsettings.extraction_folder_prefix, '')[0:3]
alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop().replace(genericsettings.extraction_folder_prefix, '')[0:3]
open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a'
).write(r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' +
r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n')
if info:
info = '_' + info
dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys()))
bestalgentries = bestalg.loadBestAlgorithm(dsList0.isBiobjective())
header = []
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
header = [r'\#FEs/D']
headerHtml = ['<thead>\n<tr>\n<th>#FEs/D</th>\n']
for label in targetsOfInterest.labels():
header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % label)
headerHtml.append('<td>%s</td>\n' % label)
else:
header = [r'$\Delta f_\mathrm{opt}$']
headerHtml = ['<thead>\n<tr>\n<th>Δ f</th>\n']
for label in targetsOfInterest.labels():
header.append(r'\multicolumn{2}{@{\,}c@{\,}}{%s}' % label)
headerHtml.append('<td>%s</td>\n' % label)
header.append(r'\multicolumn{2}{@{}l@{}}{\#succ}')
headerHtml.append('<td>#succ</td>\n</tr>\n</thead>\n')
for d in dimsOfInterest: # TODO set as input arguments
table = [header]
tableHtml = headerHtml
extraeol = [r'\hline']
try:
dictFunc0 = dictDim0[d].dictByFunc()
dictFunc1 = dictDim1[d].dictByFunc()
except KeyError:
continue
funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys()))
nbtests = len(funcs) * 2. #len(dimsOfInterest)
tableHtml.append('<tbody>\n')
for f in sorted(funcs):
tableHtml.append('<tr>\n')
targets = targetsOfInterest((f, d))
targetf = targets[-1]
bestalgentry = bestalgentries[(d, f)]
curline = [r'${\bf f_{%d}}$' % f]
curlineHtml = ['<th><b>f<sub>%d</sub></b></th>\n' % f]
bestalgdata = bestalgentry.detERT(targets)
bestalgevals, bestalgalgs = bestalgentry.detEvals(targets)
if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues):
# write ftarget:fevals
for i in xrange(len(bestalgdata[:-1])):
temp = "%.1e" % targetsOfInterest((f, d))[i]
if temp[-2]=="0":
temp = temp[:-2]+temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}'
% (temp,writeFEvalsMaxPrec(bestalgdata[i], 2)))
curlineHtml.append('<td><i>%s</i>:%s</td>\n'
% (temp, writeFEvalsMaxPrec(bestalgdata[i], 2)))
temp = "%.1e" % targetsOfInterest((f, d))[-1]
if temp[-2]=="0":
temp = temp[:-2]+temp[-1]
curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }'
% (temp,writeFEvalsMaxPrec(bestalgdata[-1], 2)))
curlineHtml.append('<td><i>%s</i>:%s</td>\n'
% (temp, writeFEvalsMaxPrec(bestalgdata[-1], 2)))
else:
# write #fevals of the reference alg
for i in bestalgdata[:-1]:
curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}'
% writeFEvalsMaxPrec(i, 2))
curlineHtml.append('<td>%s</td>\n' % writeFEvalsMaxPrec(i, 2))
curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}'
% writeFEvalsMaxPrec(bestalgdata[-1], 2))
curlineHtml.append('<td>%s</td>\n' % writeFEvalsMaxPrec(bestalgdata[-1], 2))
tmp = bestalgentry.detEvals([targetf])[0][0]
tmp2 = numpy.sum(numpy.isnan(tmp) == False)
curline.append('%d' % (tmp2))
if tmp2 > 0:
curline.append('/%d' % len(tmp))
curlineHtml.append('<td>%d/%d</td>\n' % (tmp2, len(tmp)))
else:
curlineHtml.append('<td>%d</td>\n' % (tmp2))
table.append(curline[:])
tableHtml.extend(curlineHtml[:])
tableHtml.append('</tr>\n')
extraeol.append('')
rankdata0 = [] # never used
# generate all data from ranksum test
entries = []
ertdata = {}
for nb, dsList in enumerate((dictFunc0, dictFunc1)):
try:
entry = dsList[f][0] # take the first DataSet, there should be only one?
except KeyError:
warnings.warn('data missing for data set ' + str(nb) + ' and function ' + str(f))
print('*** Warning: data missing for data set ' + str(nb) + ' and function ' + str(f) + '***')
continue # TODO: problem here!
ertdata[nb] = entry.detERT(targets)
entries.append(entry)
for _t in ertdata.values():
for _tt in _t:
if _tt is None:
raise ValueError
if len(entries) < 2: # funcion not available for *both* algorithms
continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following
testres0vs1 = significancetest(entries[0], entries[1], targets)
testresbestvs1 = significancetest(bestalgentry, entries[1], targets)
testresbestvs0 = significancetest(bestalgentry, entries[0], targets)
for nb, entry in enumerate(entries):
tableHtml.append('<tr>\n')
if nb == 0:
curline = [r'1:\:\algorithmAshort\hspace*{\fill}']
curlineHtml = ['<th>1: %s</th>\n' % alg0]
else:
curline = [r'2:\:\algorithmBshort\hspace*{\fill}']
curlineHtml = ['<th>2: %s</th>\n' % alg1]
#data = entry.detERT(targetsOfInterest)
dispersion = []
data = []
evals = entry.detEvals(targets)
for i in evals:
succ = (numpy.isnan(i) == False)
tmp = i.copy()
tmp[succ==False] = entry.maxevals[numpy.isnan(i)]
#set_trace()
data.append(toolsstats.sp(tmp, issuccessful=succ)[0])
#if not any(succ):
#set_trace()
if any(succ):
tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ==False],
(10, 50, 90), samplesize)[0]
dispersion.append((tmp2[-1]-tmp2[0])/2.)
else:
dispersion.append(None)
if nb == 0:
assert not isinstance(data, numpy.ndarray)
data0 = data[:] # TODO: check if it is not an array, it's never used anyway?
for i, dati in enumerate(data):
z, p = testres0vs1[i] # TODO: there is something with the sign that I don't get
# assign significance flag, which is the -log10(p)
significance0vs1 = 0
if nb != 0:
z = -z # the test is symmetric
if nbtests * p < 0.05 and z > 0:
significance0vs1 = -int(numpy.ceil(numpy.log10(min([1.0, nbtests * p])))) # this is the larger the more significant
isBold = significance0vs1 > 0
alignment = 'c'
if i == len(data) - 1: # last element
alignment = 'c|'
if numpy.isinf(bestalgdata[i]): # if the 2009 best did not solve the problem
tmp = writeFEvalsMaxPrec(float(dati), 2)
if not numpy.isinf(dati):
tmpHtml = '<i>%s</i>' % (tmp)
tmp = r'\textit{%s}' % (tmp)
if isBold:
tmp = r'\textbf{%s}' % tmp
tmpHtml = '<b>%s</b>' % tmpHtml
if dispersion[i] and numpy.isfinite(dispersion[i]):
tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tmp))
tableentryHtml = (' (%s)' % tmp)
else:
# Formatting
tmp = float(dati)/bestalgdata[i]
assert not numpy.isnan(tmp)
isscientific = False
if tmp >= 1000:
isscientific = True
tableentry = writeFEvals2(tmp, 2, isscientific=isscientific)
tableentry = writeFEvalsMaxPrec(tmp, 2)
tableentryHtml = writeFEvalsMaxPrec(tmp, 2)
if numpy.isinf(tmp) and i == len(data)-1:
tableentry = (tableentry
+ r'\textit{%s}' % writeFEvals2(numpy.median(entry.maxevals), 2))
tableentryHtml = (tableentryHtml
+ ' <i>%s</i>' % writeFEvals2(numpy.median(entry.maxevals), 2))
if isBold:
tableentry = r'\textbf{%s}' % tableentry
tableentryHtml = '<b>%s</b>' % tableentryHtml
elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore
tableentry = r'\textit{%s}' % tableentry
tableentryHtml = '<i>%s</i>' % tableentryHtml
if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
tableentryHtml += ' (%s)' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tableentry))
elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and i != len(data) - 1):
if isBold:
tableentry = r'\textbf{%s}' % tableentry
tableentryHtml = '<b>%s</b>' % tableentryHtml
elif 11 < 3 and significance0vs1 < 0:
tableentry = r'\textit{%s}' % tableentry
tableentryHtml = '<i>%s</i>' % tableentryHtml
if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
tableentryHtml += ' (%s)' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}'
% (alignment, tableentry))
else:
tmp = tableentry.split('.', 1)
tmpHtml = tableentryHtml.split('.', 1)
if isBold:
tmp = list(r'\textbf{%s}' % i for i in tmp)
tmpHtml = list('<b>%s</b>' % i for i in tmpHtml)
elif 11 < 3 and significance0vs1 < 0:
tmp = list(r'\textit{%s}' % i for i in tmp)
tmpHtml = list('<i>%s</i>' % i for i in tmpHtml)
tableentry = ' & .'.join(tmp)
tableentryHtml = '.'.join(tmpHtml)
if len(tmp) == 1:
tableentry += '&'
if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]):
tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
tableentryHtml += ' (%s)' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1)
superscript = ''
superscriptHtml = ''
if nb == 0:
z, p = testresbestvs0[i]
else:
z, p = testresbestvs1[i]
#The conditions are now that ERT < ERT_best
if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0.
and z < 0.):
nbstars = -numpy.ceil(numpy.log10(nbtests * p))
#tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star'])
if z > 0:
superscript = r'\uparrow' #* nbstars
superscriptHtml = '↑'
else:
superscript = r'\downarrow' #* nbstars
superscriptHtml = '↓'
# print z, linebest[i], line1
if nbstars > 1:
superscript += str(int(nbstars))
superscriptHtml += str(int(nbstars))
if superscript or significance0vs1:
s = ''
shtml = ''
if significance0vs1 > 0:
s = '\star'
shtml = '★'
if significance0vs1 > 1:
s += str(significance0vs1)
shtml += str(significance0vs1)
s = r'$^{' + s + superscript + r'}$'
shtml = '<sup>' + shtml + superscriptHtml + '</sup>'
if tableentry.endswith('}'):
tableentry = tableentry[:-1] + s + r'}'
else:
tableentry += s
tableentryHtml += shtml
tableentryHtml = tableentryHtml.replace('$\infty$', '∞')
curlineHtml.append('<td>%s</td>\n' % tableentryHtml)
curline.append(tableentry)
#curline.append(tableentry)
#if dispersion[i] is None or numpy.isinf(bestalgdata[i]):
#curline.append('')
#else:
#tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2)
#curline.append('(%s)' % tmp)
tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:]
try:
tmp = tmp[0]
curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False))
curlineHtml.append('<td>%d' % numpy.sum(numpy.isnan(tmp) == False))
except IndexError:
curline.append('%d' % 0)
curlineHtml.append('<td>%d' % 0)
curline.append('/%d' % entry.nbRuns())
curlineHtml.append('/%d</td>\n' % entry.nbRuns())
table.append(curline[:])
tableHtml.extend(curlineHtml[:])
tableHtml.append('</tr>\n')
extraeol.append('')
extraeol[-1] = r'\hline'
extraeol[-1] = ''
outputfile = os.path.join(outputdir, 'pptable2_%02dD%s.tex' % (d, info))
spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}'
res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n'
res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n'
# open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res)
#res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec,
#extraeol=extraeol)
res += tableLaTeX(table, spec=spec, extraeol=extraeol)
f = open(outputfile, 'w')
f.write(res)
f.close()
res = ("").join(str(item) for item in tableHtml)
res = '<p><b>%d-D</b></p>\n<table>\n%s</table>\n' % (d, res)
filename = os.path.join(outputdir, genericsettings.two_algorithm_file_name + '.html')
lines = []
with open(filename) as infile:
for line in infile:
if '<!--pptable2Html-->' in line:
lines.append(res)
lines.append(line)
with open(filename, 'w') as outfile:
for line in lines:
outfile.write(line)
if verbose:
print "Table written in %s" % outputfile
