def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True): """One table per dimension, modified to fit in 1 page per table.""" #TODO: method is long, split if possible dictDim0 = dsList0.dictByDim() dictDim1 = dsList1.dictByDim() alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3] alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3] open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a' ).write(r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' + r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n') if info: info = '_' + info dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys())) if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() header = [r'$\Delta f$'] for i in targetsOfInterest: #header.append(r'\multicolumn{2}{@{}c@{}}{$10^{%d}$}' % (int(numpy.log10(i)))) header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' % (int(numpy.log10(i)))) header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}') for d in dimsOfInterest: # TODO set as input arguments table = [header] extraeol = [r'\hline'] try: dictFunc0 = dictDim0[d].dictByFunc() dictFunc1 = dictDim1[d].dictByFunc() except KeyError: continue funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys())) nbtests = len(funcs) * 2. #len(dimsOfInterest) for f in sorted(funcs): bestalgentry = bestalg.bestalgentries2009[(d, f)] curline = [r'${\bf f_{%d}}$' % f] bestalgdata = bestalgentry.detERT(targetsOfInterest) bestalgevals, bestalgalgs = bestalgentry.detEvals(targetsOfInterest) for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = numpy.sum(numpy.isnan(tmp) == False) curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) table.append(curline[:]) extraeol.append('') rankdata0 = [] # never used # generate all data from ranksum test entries = [] ertdata = {} for nb, dsList in enumerate((dictFunc0, dictFunc1)): try: entry = dsList[f][0] # take the first DataSet, there should be only one? except KeyError: warnings.warn('data missing for data set ' + str(nb) + ' and function ' + str(f)) print('*** Warning: data missing for data set ' + str(nb) + ' and function ' + str(f) + '***') continue # TODO: problem here! ertdata[nb] = entry.detERT(targetsOfInterest) entries.append(entry) for _t in ertdata.values(): for _tt in _t: if _tt is None: raise ValueError if len(entries) < 2: # funcion not available for *both* algorithms continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following testres0vs1 = significancetest(entries[0], entries[1], targetsOfInterest) testresbestvs1 = significancetest(bestalgentry, entries[1], targetsOfInterest) testresbestvs0 = significancetest(bestalgentry, entries[0], targetsOfInterest) for nb, entry in enumerate(entries): if nb == 0: curline = [r'1:\:\algorithmAshort\hspace*{\fill}'] else: curline = [r'2:\:\algorithmBshort\hspace*{\fill}'] #data = entry.detERT(targetsOfInterest) dispersion = [] data = [] evals = entry.detEvals(targetsOfInterest) for i in evals: succ = (numpy.isnan(i) == False) tmp = i.copy() tmp[succ==False] = entry.maxevals[numpy.isnan(i)] #set_trace() data.append(toolsstats.sp(tmp, issuccessful=succ)[0]) #if not any(succ): #set_trace() if any(succ): tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ==False], (10, 50, 90), samplesize)[0] dispersion.append((tmp2[-1]-tmp2[0])/2.) else: dispersion.append(None) if nb == 0: assert not isinstance(data, numpy.ndarray) data0 = data[:] # TODO: check if it is not an array, it's never used anyway? for i, dati in enumerate(data): z, p = testres0vs1[i] # TODO: there is something with the sign that I don't get # assign significance flag, which is the -log10(p) significance0vs1 = 0 if nb != 0: z = -z # the test is symmetric if nbtests * p < 0.05 and z > 0: significance0vs1 = -int(numpy.ceil(numpy.log10(min([1.0, nbtests * p])))) # this is the larger the more significant isBold = significance0vs1 > 0 alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' if numpy.isinf(bestalgdata[i]): # if the 2009 best did not solve the problem tmp = writeFEvalsMaxPrec(float(dati), 2) if not numpy.isinf(dati): tmp = r'\textit{%s}' % (tmp) if isBold: tmp = r'\textbf{%s}' % tmp if dispersion[i] and numpy.isfinite(dispersion[i]): tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tmp)) else: # Formatting tmp = float(dati)/bestalgdata[i] assert not numpy.isnan(tmp) isscientific = False if tmp >= 1000: isscientific = True tableentry = writeFEvals2(tmp, 2, isscientific=isscientific) tableentry = writeFEvalsMaxPrec(tmp, 2) if numpy.isinf(tmp) and i == len(data)-1: tableentry = (tableentry + r'\textit{%s}' % writeFEvals2(numpy.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore tableentry = r'\textit{%s}' % tableentry if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) elif 11 < 3 and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tableentry = ' & .'.join(tmp) if len(tmp) == 1: tableentry += '&' if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) superscript = '' if nb == 0: z, p = testresbestvs0[i] else: z, p = testresbestvs1[i] #The conditions are now that ERT < ERT_best if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0. and z < 0.): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars else: superscript = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(nbstars)) if superscript or significance0vs1: s = '' if significance0vs1 > 0: s = '\star' if significance0vs1 > 1: s += str(significance0vs1) s = r'$^{' + s + superscript + r'}$' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s curline.append(tableentry) #curline.append(tableentry) #if dispersion[i] is None or numpy.isinf(bestalgdata[i]): #curline.append('') #else: #tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2) #curline.append('(%s)' % tmp) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curline.append('/%d' % entry.nbRuns()) table.append(curline[:]) extraeol.append('') extraeol[-1] = r'\hline' extraeol[-1] = '' outputfile = os.path.join(outputdir, 'pptable2_%02dD%s.tex' % (d, info)) spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}' res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n' # open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res) #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res += tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() if verbose: print "Table written in %s" % outputfile
def main(dsList, dimsOfInterest, outputdir, info='', verbose=True): """Generate a table of ratio ERT/ERTbest vs target precision. 1 table per dimension will be generated. Rank-sum tests table on "Final Data Points" for only one algorithm. that is, for example, using 1/#fevals(ftarget) if ftarget was reached and -f_final otherwise as input for the rank-sum test, where obviously the larger the better. """ #TODO: check that it works for any reference algorithm? #in the following the reference algorithm is the one given in #bestalg.bestalgentries which is the virtual best of BBOB dictDim = dsList.dictByDim() targetf=1e-8 if info: info = '_' + info # insert a separator between the default file name and the additional # information string. dims = set(dictDim.keys()) if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): header = [r'\#FEs/D'] for i in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % i) else: header = [r'$\Delta f$'] for i in targetsOfInterest.target_values: header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' % (int(np.log10(i)))) header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}') for d in dimsOfInterest: table = [header] extraeol = [r'\hline'] try: dictFunc = dictDim[d].dictByFunc() except KeyError: continue funcs = set(dictFunc.keys()) nbtests = float(len(funcs)) # #funcs tests times one algorithm for f in sorted(funcs): bestalgentry = bestalg.bestalgentries2009[(d, f)] curline = [r'${\bf f_{%d}}$' % f] bestalgdata = bestalgentry.detERT(targetsOfInterest((f,d))) bestalgevals, bestalgalgs = bestalgentry.detEvals(targetsOfInterest((f,d))) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): #write ftarget:fevals for i in xrange(len(bestalgdata[:-1])): temp="%.1e" %targetsOfInterest((f,d))[i] if temp[-2]=="0": temp=temp[:-2]+temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp,writeFEvalsMaxPrec(bestalgdata[i], 2))) temp="%.1e" %targetsOfInterest((f,d))[-1] if temp[-2]=="0": temp=temp[:-2]+temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp,writeFEvalsMaxPrec(bestalgdata[-1], 2))) #success targetf=targetsOfInterest((f,d))[-1] else: # write #fevals of the reference alg for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) # write the success ratio for the reference alg tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = np.sum(np.isnan(tmp) == False) # count the nb of success curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) table.append(curline[:]) extraeol.append('') # generate all data for ranksum test assert len(dictFunc[f]) == 1 entry = dictFunc[f][0] # take the first element ertdata = entry.detERT(targetsOfInterest((f, d))) testresbestvs1 = significancetest(bestalgentry, entry, targetsOfInterest((f, d))) #for nb, entry in enumerate(entries): #curline = [r'\algshort\hspace*{\fill}'] curline = [''] #data = entry.detERT(targetsOfInterest) evals = entry.detEvals(targetsOfInterest((f,d))) dispersion = [] data = [] for i in evals: succ = (np.isnan(i) == False) tmp = i.copy() tmp[succ==False] = entry.maxevals[np.isnan(i)] #set_trace() # TODO: what is the difference between data and ertdata? data.append(toolsstats.sp(tmp, issuccessful=succ)[0]) #if not any(succ): #set_trace() if any(succ): tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ==False], (10, 50, 90), samplesize)[0] dispersion.append((tmp2[-1] - tmp2[0]) / 2.) else: dispersion.append(None) assert data == ertdata for i, ert in enumerate(data): alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' nbstars = 0 z, p = testresbestvs1[i] if ert - bestalgdata[i] < 0. and not np.isinf(bestalgdata[i]): evals = entry.detEvals([targetsOfInterest((f,d))[i]])[0] evals[np.isnan(evals)] = entry.maxevals[np.isnan(evals)] bestevals = bestalgentry.detEvals([targetsOfInterest((f,d))[i]]) bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0]) bestevals[np.isnan(bestevals)] = bestalgentry.maxevals[bestalgalg][np.isnan(bestevals)] evals = np.array(sorted(evals))[0:min(len(evals), len(bestevals))] bestevals = np.array(sorted(bestevals))[0:min(len(evals), len(bestevals))] #The conditions for significance are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if ((nbtests * p) < 0.05 and ert - bestalgdata[i] < 0. and z < 0. and (np.isinf(bestalgdata[i]) or all(evals < bestevals))): nbstars = -np.ceil(np.log10(nbtests * p)) isBold = False if nbstars > 0: isBold = True if np.isinf(bestalgdata[i]): # if the best did not solve the problem tmp = writeFEvalsMaxPrec(float(ert), 2) if not np.isinf(ert): tmp = r'\textit{%s}' % (tmp) if isBold: tmp = r'\textbf{%s}' % tmp tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tmp)) else: # Formatting tmp = float(ert) / bestalgdata[i] assert not np.isnan(tmp) tableentry = writeFEvalsMaxPrec(tmp, 2) if np.isinf(tmp) and i == len(data)-1: tableentry = (tableentry + r'\textit{%s}' % writeFEvals2(np.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3: # and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (np.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3: # and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) elif 11 < 3: # and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tableentry = ' & .'.join(tmp) if len(tmp) == 1: tableentry += '&' superscript = '' if nbstars > 0: #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars else: superscript = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(min((9, nbstars)))) # superscript += str(int(nbstars)) #if superscript or significance0vs1: #s = '' #if significance0vs1 > 0: #s = '\star' #if significance0vs1 > 1: #s += str(significance0vs1) #s = r'$^{' + s + superscript + r'}$' #if tableentry.endswith('}'): #tableentry = tableentry[:-1] + s + r'}' #else: #tableentry += s if dispersion[i]: if not np.isinf(bestalgdata[i]): tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) else: tmp = writeFEvalsMaxPrec(dispersion[i], 1) tableentry += (r'${\scriptscriptstyle(%s)}$' % tmp) if superscript: s = r'$^{' + superscript + r'}$' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s curline.append(tableentry) #curline.append(tableentry) #if dispersion[i] is None or np.isinf(bestalgdata[i]): #curline.append('') #else: #tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2) #curline.append('(%s)' % tmp) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % np.sum(np.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curline.append('/%d' % entry.nbRuns()) table.append(curline[:]) extraeol.append(r'\hline') extraeol[-1] = '' outputfile = os.path.join(outputdir, 'pptable_%02dD%s.tex' % (d, info)) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): spec = r'@{}c@{}|' + '*{%d}{@{ }r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}' else: spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}' #res = r'\providecommand{\algshort}{%s}' % alg1 + '\n' #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res = tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() if verbose: print "Table written in %s" % outputfile
def main(dictAlg, sortedAlgs, outputdir='.', verbose=True, function_targets_line=True): # [1, 13, 101] """Generate one table per func with results of multiple algorithms.""" """Difference with the first version: * numbers aligned using the decimal separator * premices for dispersion measure * significance test against best algorithm * table width... Takes ``targetsOfInterest`` from this file as "input argument" to compute the desired target values. ``targetsOfInterest`` might be configured via config. """ # TODO: method is long, terrible to read, split if possible if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() # Sort data per dimension and function dictData = {} dsListperAlg = list(dictAlg[i] for i in sortedAlgs) for n, entries in enumerate(dsListperAlg): tmpdictdim = entries.dictByDim() for d in tmpdictdim: tmpdictfun = tmpdictdim[d].dictByFunc() for f in tmpdictfun: dictData.setdefault((d, f), {})[n] = tmpdictfun[f] nbtests = len(dictData) for df in dictData: # Generate one table per df # first update targets for each dimension-function pair if needed: targets = targetsOfInterest((df[1], df[0])) targetf = targets[-1] # best 2009 refalgentry = bestalg.bestalgentries2009[df] refalgert = refalgentry.detERT(targets) refalgevals = (refalgentry.detEvals((targetf, ))[0][0]) refalgnbruns = len(refalgevals) refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False) # Process the data # The following variables will be lists of elements each corresponding # to an algorithm algnames = [] #algdata = [] algerts = [] algevals = [] algdisp = [] algnbsucc = [] algnbruns = [] algmedmaxevals = [] algmedfinalfunvals = [] algtestres = [] algentries = [] for n in sorted(dictData[df].keys()): entries = dictData[df][n] # the number of datasets for a given dimension and function (df) # should be strictly 1. TODO: find a way to warn # TODO: do this checking before... why wasn't it triggered by ppperprof? if len(entries) > 1: txt = ("There is more than a single entry associated with " "folder %s on %d-D f%d." % (sortedAlgs[n], df[0], df[1])) raise Exception(txt) entry = entries[0] algentries.append(entry) algnames.append(sortedAlgs[n]) evals = entry.detEvals(targets) #tmpdata = [] tmpdisp = [] tmpert = [] for i, e in enumerate(evals): succ = (numpy.isnan(e) == False) ec = e.copy() # note: here was the previous bug (changes made in e also appeared in evals !) ec[succ == False] = entry.maxevals[succ == False] ert = toolsstats.sp(ec, issuccessful=succ)[0] #tmpdata.append(ert/refalgert[i]) if succ.any(): tmp = toolsstats.drawSP(ec[succ], entry.maxevals[succ == False], [10, 50, 90], samplesize=samplesize)[0] tmpdisp.append((tmp[-1] - tmp[0])/2.) else: tmpdisp.append(numpy.nan) tmpert.append(ert) algerts.append(tmpert) algevals.append(evals) #algdata.append(tmpdata) algdisp.append(tmpdisp) algmedmaxevals.append(numpy.median(entry.maxevals)) algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) #algmedmaxevals.append(numpy.median(entry.maxevals)/df[0]) #algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) algtestres.append(significancetest(refalgentry, entry, targets)) # determine success probability for Df = 1e-8 e = entry.detEvals((targetf ,))[0] algnbsucc.append(numpy.sum(numpy.isnan(e) == False)) algnbruns.append(len(e)) # Process over all data # find best values... nalgs = len(dictData[df]) maxRank = 1 + numpy.floor(0.14 * nalgs) # number of algs to be displayed in bold isBoldArray = [] # Point out the best values algfinaldata = [] # Store median function values/median number of function evaluations tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank) for i, erts in enumerate(algerts): tmp = [] for j, ert in enumerate(erts): # algi targetj tmp.append(i in tmptop[j] or (nalgs > 7 and algerts[i][j] <= 3. * refalgert[j])) isBoldArray.append(tmp) algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i])) # significance test of best given algorithm against all others best_alg_idx = numpy.array(algerts).argsort(0)[0, :] # indexed by target index significance_versus_others = significance_all_best_vs_other(algentries, targets, best_alg_idx)[0] # Create the table table = [] spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % (len(targets)) # in case StrLeft not working: replaced c@{} with l@{ } spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % (len(targets)) # in case StrLeft not working: replaced c@{} with l@{ } extraeol = [] # Generate header lines if with_table_heading: header = funInfos[df[1]] if funInfos else 'f%d' % df[1] table.append([r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}' % (2 * len(targets) + 2, header)]) extraeol.append('') if function_targets_line is True or (function_targets_line and df[1] in function_targets_line): if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): curline = [r'\#FEs/D'] for i in targetsOfInterest.labels(): curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' % i) else: curline = [r'$\Delta f_\mathrm{opt}$'] for t in targets: curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}' % writeFEvals2(t, precision=1, isscientific=True)) # curline.append(r'\multicolumn{2}{@{\,}X@{}|}{%s}' # % writeFEvals2(targets[-1], precision=1, isscientific=True)) curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') table.append(curline) extraeol.append(r'\hline') # extraeol.append(r'\hline\arrayrulecolor{tableShade}') curline = [r'ERT$_{\text{best}}$'] if with_table_heading else [r'\textbf{f%d}' % df[1]] if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): # write ftarget:fevals for i in xrange(len(refalgert[:-1])): temp="%.1e" %targetsOfInterest((df[1], df[0]))[i] if temp[-2]=="0": temp=temp[:-2]+temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp,writeFEvalsMaxPrec(refalgert[i], 2))) temp="%.1e" %targetsOfInterest((df[1], df[0]))[-1] if temp[-2]=="0": temp=temp[:-2]+temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp,writeFEvalsMaxPrec(refalgert[-1], 2))) else: # write #fevals of the reference alg for i in refalgert[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(refalgert[-1], 2)) # write the success ratio for the reference alg tmp2 = numpy.sum(numpy.isnan(refalgevals) == False) # count the nb of success curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(refalgevals)) table.append(curline[:]) extraeol.append('') #for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))): #print i, gna, gno #set_trace() # Format data #if df == (5, 17): #set_trace() header = r'\providecommand{\ntables}{7}' for i, alg in enumerate(algnames): #algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames, #data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres): commandname = r'\alg%stables' % numtotext(i) # header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg))) header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % (commandname, str_to_latex(strip_pathname2(alg))) curline = [commandname + r'\hspace*{\fill}'] # each list element becomes a &-separated table entry? for j, tmp in enumerate(zip(algerts[i], algdisp[i], # j is target index isBoldArray[i], algtestres[i])): ert, dispersion, isBold, testres = tmp alignment = '@{\,}X@{\,}' if j == len(algerts[i]) - 1: alignment = '@{\,}X@{\,}|' data = ert/refalgert[j] # write star for significance against all other algorithms str_significance_subsup = '' if (len(best_alg_idx) > 0 and len(significance_versus_others) > 0 and i == best_alg_idx[j] and nbtests * significance_versus_others[j][1] < 0.05): logp = -numpy.ceil(numpy.log10(nbtests * significance_versus_others[j][1])) str_significance_subsup = r"^{%s%s}" % (significance_vs_others_symbol, str(int(logp)) if logp > 1 else '') # moved out of the above else: this was a bug!? z, p = testres if (nbtests * p) < 0.05 and data < 1. and z < 0.: if not numpy.isinf(refalgert[j]): tmpevals = algevals[i][j].copy() tmpevals[numpy.isnan(tmpevals)] = algentries[i].maxevals[numpy.isnan(tmpevals)] bestevals = refalgentry.detEvals(targets) bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0]) bestevals[numpy.isnan(bestevals)] = refalgentry.maxevals[bestalgalg][numpy.isnan(bestevals)] tmpevals = numpy.array(sorted(tmpevals))[0:min(len(tmpevals), len(bestevals))] bestevals = numpy.array(sorted(bestevals))[0:min(len(tmpevals), len(bestevals))] #The conditions are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) # tmp2[-1] += r'$^{%s}$' % superscript str_significance_subsup += r'_{%s%s}' % (significance_vs_ref_symbol, str(int(nbstars)) if nbstars > 1 else '') if str_significance_subsup: str_significance_subsup = '$%s$' % str_significance_subsup # format number in variable data if numpy.isnan(data): curline.append(r'\multicolumn{2}{%s}{.}' % alignment) else: if numpy.isinf(refalgert[j]): curline.append(r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}' % (alignment, writeFEvalsMaxPrec(algerts[i][j], 2), writeFEvalsMaxPrec(dispersion, precdispersion), str_significance_subsup)) continue tmp = writeFEvalsMaxPrec(data, precfloat, maxfloatrepr=maxfloatrepr) if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation if numpy.isinf(data) and j == len(algerts[i]) - 1: tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec(algfinaldata[i][1], 0, maxfloatrepr=maxfloatrepr) else: tmp = writeFEvalsMaxPrec(data, precscien, maxfloatrepr=data) if isBold: tmp = r'\textbf{%s}' % tmp if not numpy.isnan(dispersion): tmpdisp = dispersion/refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp += r'\mbox{\tiny (%s)}' % tmpdisp curline.append(r'\multicolumn{2}{%s}{%s%s}' % (alignment, tmp, str_significance_subsup)) else: tmp2 = tmp.split('.', 1) if len(tmp2) < 2: tmp2.append('') else: tmp2[-1] = '.' + tmp2[-1] if isBold: tmp3 = [] for k in tmp2: tmp3.append(r'\textbf{%s}' % k) tmp2 = tmp3 if not numpy.isnan(dispersion): tmpdisp = dispersion/refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.01: tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec(tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp)) tmp2[-1] += str_significance_subsup curline.extend(tmp2) curline.append('%d' % algnbsucc[i]) curline.append('/%d' % algnbruns[i]) table.append(curline) extraeol.append('') # Write table res = tableXLaTeX(table, spec=spec, extraeol=extraeol) try: filename = os.path.join(outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0])) f = open(filename, 'w') f.write(header + '\n') f.write(res) if verbose: print 'Wrote table in %s' % filename except: raise else: f.close()
def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True): """One table per dimension, modified to fit in 1 page per table.""" #TODO: method is long, split if possible dictDim0 = dsList0.dictByDim() dictDim1 = dsList1.dictByDim() alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3] alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3] open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write( r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' + r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n') if info: info = '_' + info dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys())) if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() header = [] if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): header = [r'\#FEs/D'] for label in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % label) else: header = [r'$\Delta f_\mathrm{opt}$'] for label in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{\,}c@{\,}}{%s}' % label) header.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') for d in dimsOfInterest: # TODO set as input arguments table = [header] extraeol = [r'\hline'] try: dictFunc0 = dictDim0[d].dictByFunc() dictFunc1 = dictDim1[d].dictByFunc() except KeyError: continue funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys())) nbtests = len(funcs) * 2. #len(dimsOfInterest) for f in sorted(funcs): targets = targetsOfInterest((f, d)) targetf = targets[-1] bestalgentry = bestalg.bestalgentries2009[(d, f)] curline = [r'${\bf f_{%d}}$' % f] bestalgdata = bestalgentry.detERT(targets) bestalgevals, bestalgalgs = bestalgentry.detEvals(targets) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): # write ftarget:fevals for i in xrange(len(bestalgdata[:-1])): temp = "%.1e" % targetsOfInterest((f, d))[i] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append( r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp, writeFEvalsMaxPrec(bestalgdata[i], 2))) temp = "%.1e" % targetsOfInterest((f, d))[-1] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp, writeFEvalsMaxPrec(bestalgdata[-1], 2))) else: # write #fevals of the reference alg for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = numpy.sum(numpy.isnan(tmp) == False) curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) table.append(curline[:]) extraeol.append('') rankdata0 = [] # never used # generate all data from ranksum test entries = [] ertdata = {} for nb, dsList in enumerate((dictFunc0, dictFunc1)): try: entry = dsList[f][ 0] # take the first DataSet, there should be only one? except KeyError: warnings.warn('data missing for data set ' + str(nb) + ' and function ' + str(f)) print('*** Warning: data missing for data set ' + str(nb) + ' and function ' + str(f) + '***') continue # TODO: problem here! ertdata[nb] = entry.detERT(targets) entries.append(entry) for _t in ertdata.values(): for _tt in _t: if _tt is None: raise ValueError if len(entries) < 2: # funcion not available for *both* algorithms continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following testres0vs1 = significancetest(entries[0], entries[1], targets) testresbestvs1 = significancetest(bestalgentry, entries[1], targets) testresbestvs0 = significancetest(bestalgentry, entries[0], targets) for nb, entry in enumerate(entries): if nb == 0: curline = [r'1:\:\algorithmAshort\hspace*{\fill}'] else: curline = [r'2:\:\algorithmBshort\hspace*{\fill}'] #data = entry.detERT(targetsOfInterest) dispersion = [] data = [] evals = entry.detEvals(targets) for i in evals: succ = (numpy.isnan(i) == False) tmp = i.copy() tmp[succ == False] = entry.maxevals[numpy.isnan(i)] #set_trace() data.append(toolsstats.sp(tmp, issuccessful=succ)[0]) #if not any(succ): #set_trace() if any(succ): tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ == False], (10, 50, 90), samplesize)[0] dispersion.append((tmp2[-1] - tmp2[0]) / 2.) else: dispersion.append(None) if nb == 0: assert not isinstance(data, numpy.ndarray) data0 = data[:] # TODO: check if it is not an array, it's never used anyway? for i, dati in enumerate(data): z, p = testres0vs1[ i] # TODO: there is something with the sign that I don't get # assign significance flag, which is the -log10(p) significance0vs1 = 0 if nb != 0: z = -z # the test is symmetric if nbtests * p < 0.05 and z > 0: significance0vs1 = -int( numpy.ceil(numpy.log10(min([ 1.0, nbtests * p ])))) # this is the larger the more significant isBold = significance0vs1 > 0 alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' if numpy.isinf( bestalgdata[i] ): # if the 2009 best did not solve the problem tmp = writeFEvalsMaxPrec(float(dati), 2) if not numpy.isinf(dati): tmp = r'\textit{%s}' % (tmp) if isBold: tmp = r'\textbf{%s}' % tmp if dispersion[i] and numpy.isfinite(dispersion[i]): tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tmp)) else: # Formatting tmp = float(dati) / bestalgdata[i] assert not numpy.isnan(tmp) isscientific = False if tmp >= 1000: isscientific = True tableentry = writeFEvals2(tmp, 2, isscientific=isscientific) tableentry = writeFEvalsMaxPrec(tmp, 2) if numpy.isinf(tmp) and i == len(data) - 1: tableentry = ( tableentry + r'\textit{%s}' % writeFEvals2(numpy.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore tableentry = r'\textit{%s}' % tableentry if dispersion[i] and numpy.isfinite( dispersion[i] / bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry if dispersion[i] and numpy.isfinite( dispersion[i] / bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) elif 11 < 3 and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tableentry = ' & .'.join(tmp) if len(tmp) == 1: tableentry += '&' if dispersion[i] and numpy.isfinite( dispersion[i] / bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) superscript = '' if nb == 0: z, p = testresbestvs0[i] else: z, p = testresbestvs1[i] #The conditions are now that ERT < ERT_best if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0. and z < 0.): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars else: superscript = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(nbstars)) if superscript or significance0vs1: s = '' if significance0vs1 > 0: s = '\star' if significance0vs1 > 1: s += str(significance0vs1) s = r'$^{' + s + superscript + r'}$' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s curline.append(tableentry) #curline.append(tableentry) #if dispersion[i] is None or numpy.isinf(bestalgdata[i]): #curline.append('') #else: #tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2) #curline.append('(%s)' % tmp) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curline.append('/%d' % entry.nbRuns()) table.append(curline[:]) extraeol.append('') extraeol[-1] = r'\hline' extraeol[-1] = '' outputfile = os.path.join(outputdir, 'pptable2_%02dD%s.tex' % (d, info)) spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len( targetsOfInterest) + '|@{}r@{}@{}l@{}' res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels( alg0) + '\n' res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels( alg1) + '\n' # open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res) #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res += tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() if verbose: print "Table written in %s" % outputfile
def main(dictAlg, sortedAlgs, targets, outputdir='.', verbose=True, function_targets_line=True): # [1, 13, 101] """Generate one table per func with results of multiple algorithms.""" """Difference with the first version: * numbers aligned using the decimal separator * premices for dispersion measure * significance test against best algorithm * table width... """ # TODO: method is long, terrible to read, split if possible if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() # Sort data per dimension and function dictData = {} dsListperAlg = list(dictAlg[i] for i in sortedAlgs) for n, entries in enumerate(dsListperAlg): tmpdictdim = entries.dictByDim() for d in tmpdictdim: tmpdictfun = tmpdictdim[d].dictByFunc() for f in tmpdictfun: dictData.setdefault((d, f), {})[n] = tmpdictfun[f] nbtests = len(dictData) for df in dictData: # Generate one table per df # best 2009 refalgentry = bestalg.bestalgentries2009[df] refalgert = refalgentry.detERT(targets) refalgevals = (refalgentry.detEvals((targetf, ))[0][0]) refalgnbruns = len(refalgevals) refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False) # Process the data # The following variables will be lists of elements each corresponding # to an algorithm algnames = [] #algdata = [] algerts = [] algevals = [] algdisp = [] algnbsucc = [] algnbruns = [] algmedmaxevals = [] algmedfinalfunvals = [] algtestres = [] algentries = [] for n in sorted(dictData[df].keys()): entries = dictData[df][n] # the number of datasets for a given dimension and function (df) # should be strictly 1. TODO: find a way to warn # TODO: do this checking before... why wasn't it triggered by ppperprof? if len(entries) > 1: txt = ("There is more than a single entry associated with " "folder %s on %d-D f%d." % (sortedAlgs[n], df[0], df[1])) raise Exception(txt) entry = entries[0] algentries.append(entry) algnames.append(sortedAlgs[n]) evals = entry.detEvals(targets) #tmpdata = [] tmpdisp = [] tmpert = [] for i, e in enumerate(evals): succ = (numpy.isnan(e) == False) e[succ == False] = entry.maxevals[succ == False] ert = toolsstats.sp(e, issuccessful=succ)[0] #tmpdata.append(ert/refalgert[i]) if succ.any(): tmp = toolsstats.drawSP(e[succ], entry.maxevals[succ == False], [10, 50, 90], samplesize=samplesize)[0] tmpdisp.append((tmp[-1] - tmp[0]) / 2.) else: tmpdisp.append(numpy.nan) tmpert.append(ert) algerts.append(tmpert) algevals.append(evals) #algdata.append(tmpdata) algdisp.append(tmpdisp) algmedmaxevals.append(numpy.median(entry.maxevals)) algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) #algmedmaxevals.append(numpy.median(entry.maxevals)/df[0]) #algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) algtestres.append(significancetest(refalgentry, entry, targets)) # determine success probability for Df = 1e-8 e = entry.detEvals((targetf, ))[0] algnbsucc.append(numpy.sum(numpy.isnan(e) == False)) algnbruns.append(len(e)) # Process over all data # find best values... nalgs = len(dictData[df]) maxRank = 1 + numpy.floor( 0.14 * nalgs) # number of algs to be displayed in bold isBoldArray = [] # Point out the best values algfinaldata = [ ] # Store median function values/median number of function evaluations tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank) for i, erts in enumerate(algerts): tmp = [] for j, ert in enumerate(erts): # algi targetj tmp.append(i in tmptop[j] or (nalgs > 7 and algerts[i][j] <= 3. * refalgert[j])) isBoldArray.append(tmp) algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i])) # significance test of best given algorithm against all others best_alg_idx = numpy.array(algerts).argsort(0)[ 0, :] # indexed by target index significance_versus_others = significance_all_best_vs_other( algentries, targets, best_alg_idx)[0] # Create the table table = [] spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % ( len(targets) ) # in case StrLeft not working: replaced c@{} with l@{ } spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % ( len(targets) ) # in case StrLeft not working: replaced c@{} with l@{ } extraeol = [] # Generate header lines if with_table_heading: header = funInfos[df[1]] if funInfos else 'f%d' % df[1] table.append([ r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}' % (2 * len(targets) + 2, header) ]) extraeol.append('') if function_targets_line is True or (function_targets_line and df[1] in function_targets_line): curline = [r'$\Delta f_\mathrm{opt}$'] for t in targets[0:-1]: curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}' % writeFEvals2(t, precision=1, isscientific=True)) curline.append( r'\multicolumn{2}{@{\,}X@{}|}{%s}' % writeFEvals2(targets[-1], precision=1, isscientific=True)) curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') table.append(curline) extraeol.append(r'\hline') # extraeol.append(r'\hline\arrayrulecolor{tableShade}') curline = [r'ERT$_{\text{best}}$' ] if with_table_heading else [r'\textbf{f%d}' % df[1]] for i in refalgert[0:-1]: curline.append(r'\multicolumn{2}{@{\,}X@{\,}}{%s}' % writeFEvalsMaxPrec(float(i), 2)) curline.append(r'\multicolumn{2}{@{\,}X@{\,}|}{%s}' % writeFEvalsMaxPrec(float(refalgert[-1]), 2)) curline.append('%d' % refalgnbsucc) if refalgnbsucc: curline.append('/%d' % refalgnbruns) #curline.append(curline[0]) table.append(curline) extraeol.append('') #for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))): #print i, gna, gno #set_trace() # Format data #if df == (5, 17): #set_trace() header = r'\providecommand{\ntables}{7}' for i, alg in enumerate(algnames): #algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames, #data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres): commandname = r'\alg%stables' % numtotext(i) # header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg))) header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % ( commandname, str_to_latex(strip_pathname(alg))) curline = [ commandname + r'\hspace*{\fill}' ] # each list element becomes a &-separated table entry? for j, tmp in enumerate( zip( algerts[i], algdisp[i], # j is target index isBoldArray[i], algtestres[i])): ert, dispersion, isBold, testres = tmp alignment = '@{\,}X@{\,}' if j == len(algerts[i]) - 1: alignment = '@{\,}X@{\,}|' data = ert / refalgert[j] # write star for significance against all other algorithms str_significance_subsup = '' if (len(best_alg_idx) > 0 and len(significance_versus_others) > 0 and i == best_alg_idx[j] and nbtests * significance_versus_others[j][1] < 0.05): logp = -numpy.ceil( numpy.log10( nbtests * significance_versus_others[j][1])) str_significance_subsup = r"^{%s%s}" % ( significance_vs_others_symbol, str(int(logp)) if logp > 1 else '') # moved out of the above else: this was a bug!? z, p = testres if (nbtests * p) < 0.05 and data < 1. and z < 0.: if not numpy.isinf(refalgert[j]): tmpevals = algevals[i][j].copy() tmpevals[numpy.isnan(tmpevals)] = algentries[ i].maxevals[numpy.isnan(tmpevals)] bestevals = refalgentry.detEvals([targets[j]]) bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0]) bestevals[numpy.isnan( bestevals)] = refalgentry.maxevals[bestalgalg][ numpy.isnan(bestevals)] tmpevals = numpy.array(sorted( tmpevals))[0:min(len(tmpevals), len(bestevals))] bestevals = numpy.array(sorted( bestevals))[0:min(len(tmpevals), len(bestevals))] #The conditions are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) # tmp2[-1] += r'$^{%s}$' % superscript str_significance_subsup += r'_{%s%s}' % ( significance_vs_ref_symbol, str(int(nbstars)) if nbstars > 1 else '') if str_significance_subsup: str_significance_subsup = '$%s$' % str_significance_subsup # format number in variable data if numpy.isnan(data): curline.append(r'\multicolumn{2}{%s}{.}' % alignment) else: if numpy.isinf(refalgert[j]): curline.append( r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}' % (alignment, writeFEvalsMaxPrec(algerts[i][j], 2), writeFEvalsMaxPrec(dispersion, precdispersion), str_significance_subsup)) continue tmp = writeFEvalsMaxPrec(data, precfloat, maxfloatrepr=maxfloatrepr) if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation if numpy.isinf(data) and j == len(algerts[i]) - 1: tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec( algfinaldata[i][1], 0, maxfloatrepr=maxfloatrepr) else: tmp = writeFEvalsMaxPrec(data, precscien, maxfloatrepr=data) if isBold: tmp = r'\textbf{%s}' % tmp if not numpy.isnan(dispersion): tmpdisp = dispersion / refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp += r'\mbox{\tiny (%s)}' % tmpdisp curline.append( r'\multicolumn{2}{%s}{%s%s}' % (alignment, tmp, str_significance_subsup)) else: tmp2 = tmp.split('.', 1) if len(tmp2) < 2: tmp2.append('') else: tmp2[-1] = '.' + tmp2[-1] if isBold: tmp3 = [] for k in tmp2: tmp3.append(r'\textbf{%s}' % k) tmp2 = tmp3 if not numpy.isnan(dispersion): tmpdisp = dispersion / refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.01: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp)) tmp2[-1] += str_significance_subsup curline.extend(tmp2) curline.append('%d' % algnbsucc[i]) curline.append('/%d' % algnbruns[i]) table.append(curline) extraeol.append('') # Write table res = tableXLaTeX(table, spec=spec, extraeol=extraeol) try: filename = os.path.join( outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0])) f = open(filename, 'w') f.write(header + '\n') f.write(res) if verbose: print 'Wrote table in %s' % filename except: raise else: f.close()
def main(dictAlg, sortedAlgs, outputdir='.', verbose=True, function_targets_line=True): # [1, 13, 101] """Generate one table per func with results of multiple algorithms.""" """Difference with the first version: * numbers aligned using the decimal separator * premices for dispersion measure * significance test against best algorithm * table width... Takes ``targetsOfInterest`` from this file as "input argument" to compute the desired target values. ``targetsOfInterest`` might be configured via config. """ # TODO: method is long, terrible to read, split if possible if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() # Sort data per dimension and function dictData = {} dsListperAlg = list(dictAlg[i] for i in sortedAlgs) for n, entries in enumerate(dsListperAlg): tmpdictdim = entries.dictByDim() for d in tmpdictdim: tmpdictfun = tmpdictdim[d].dictByFunc() for f in tmpdictfun: dictData.setdefault((d, f), {})[n] = tmpdictfun[f] nbtests = len(dictData) for df in dictData: # Generate one table per df # first update targets for each dimension-function pair if needed: targets = targetsOfInterest((df[1], df[0])) targetf = targets[-1] # best 2009 refalgentry = bestalg.bestalgentries2009[df] refalgert = refalgentry.detERT(targets) refalgevals = (refalgentry.detEvals((targetf, ))[0][0]) refalgnbruns = len(refalgevals) refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False) # Process the data # The following variables will be lists of elements each corresponding # to an algorithm algnames = [] #algdata = [] algerts = [] algevals = [] algdisp = [] algnbsucc = [] algnbruns = [] algmedmaxevals = [] algmedfinalfunvals = [] algtestres = [] algentries = [] for n in sorted(dictData[df].keys()): entries = dictData[df][n] # the number of datasets for a given dimension and function (df) # should be strictly 1. TODO: find a way to warn # TODO: do this checking before... why wasn't it triggered by ppperprof? if len(entries) > 1: print entries txt = ("There is more than a single entry associated with " "folder %s on %d-D f%d." % (sortedAlgs[n], df[0], df[1])) raise Exception(txt) entry = entries[0] algentries.append(entry) algnames.append(sortedAlgs[n]) evals = entry.detEvals(targets) #tmpdata = [] tmpdisp = [] tmpert = [] for i, e in enumerate(evals): succ = (numpy.isnan(e) == False) ec = e.copy( ) # note: here was the previous bug (changes made in e also appeared in evals !) ec[succ == False] = entry.maxevals[succ == False] ert = toolsstats.sp(ec, issuccessful=succ)[0] #tmpdata.append(ert/refalgert[i]) if succ.any(): tmp = toolsstats.drawSP(ec[succ], entry.maxevals[succ == False], [10, 50, 90], samplesize=samplesize)[0] tmpdisp.append((tmp[-1] - tmp[0]) / 2.) else: tmpdisp.append(numpy.nan) tmpert.append(ert) algerts.append(tmpert) algevals.append(evals) #algdata.append(tmpdata) algdisp.append(tmpdisp) algmedmaxevals.append(numpy.median(entry.maxevals)) algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) #algmedmaxevals.append(numpy.median(entry.maxevals)/df[0]) #algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) algtestres.append(significancetest(refalgentry, entry, targets)) # determine success probability for Df = 1e-8 e = entry.detEvals((targetf, ))[0] algnbsucc.append(numpy.sum(numpy.isnan(e) == False)) algnbruns.append(len(e)) # Process over all data # find best values... nalgs = len(dictData[df]) maxRank = 1 + numpy.floor( 0.14 * nalgs) # number of algs to be displayed in bold isBoldArray = [] # Point out the best values algfinaldata = [ ] # Store median function values/median number of function evaluations tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank) for i, erts in enumerate(algerts): tmp = [] for j, ert in enumerate(erts): # algi targetj tmp.append(i in tmptop[j] or (nalgs > 7 and algerts[i][j] <= 3. * refalgert[j])) isBoldArray.append(tmp) algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i])) # significance test of best given algorithm against all others best_alg_idx = numpy.array(algerts).argsort(0)[ 0, :] # indexed by target index significance_versus_others = significance_all_best_vs_other( algentries, targets, best_alg_idx)[0] # Create the table table = [] tableHtml = [] spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % ( len(targets) ) # in case StrLeft not working: replaced c@{} with l@{ } spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % ( len(targets) ) # in case StrLeft not working: replaced c@{} with l@{ } extraeol = [] # Generate header lines if with_table_heading: header = funInfos[df[1]] if funInfos else 'f%d' % df[1] table.append([ r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}' % (2 * len(targets) + 2, header) ]) extraeol.append('') if function_targets_line is True or (function_targets_line and df[1] in function_targets_line): if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): curline = [r'\#FEs/D'] curlineHtml = ['<thead>\n<tr>\n<th>#FEs/D<br>REPLACEH</th>\n'] counter = 1 for i in targetsOfInterest.labels(): curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' % i) curlineHtml.append('<td>%s<br>REPLACE%d</td>\n' % (i, counter)) counter += 1 else: curline = [r'$\Delta f_\mathrm{opt}$'] curlineHtml = [ '<thead>\n<tr>\n<th>Δ f<sub>opt</sub><br>REPLACEH</th>\n' ] counter = 1 for t in targets: curline.append( r'\multicolumn{2}{@{\,}X@{\,}}{%s}' % writeFEvals2(t, precision=1, isscientific=True)) curlineHtml.append( '<td>%s<br>REPLACE%d</td>\n' % (writeFEvals2( t, precision=1, isscientific=True), counter)) counter += 1 # curline.append(r'\multicolumn{2}{@{\,}X@{}|}{%s}' # % writeFEvals2(targets[-1], precision=1, isscientific=True)) curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') curlineHtml.append('<td>#succ<br>REPLACEF</td>\n</tr>\n</thead>\n') table.append(curline) extraeol.append(r'\hline') # extraeol.append(r'\hline\arrayrulecolor{tableShade}') curline = [r'ERT$_{\text{best}}$' ] if with_table_heading else [r'\textbf{f%d}' % df[1]] replaceValue = 'ERT<sub>best</sub>' if with_table_heading else ( '<b>f%d</b>' % df[1]) curlineHtml = [ item.replace('REPLACEH', replaceValue) for item in curlineHtml ] if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): # write ftarget:fevals counter = 1 for i in xrange(len(refalgert[:-1])): temp = "%.1e" % targetsOfInterest((df[1], df[0]))[i] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append( r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp, writeFEvalsMaxPrec(refalgert[i], 2))) replaceValue = '<i>%s</i>:%s' % ( temp, writeFEvalsMaxPrec(refalgert[i], 2)) curlineHtml = [ item.replace('REPLACE%d' % counter, replaceValue) for item in curlineHtml ] counter += 1 temp = "%.1e" % targetsOfInterest((df[1], df[0]))[-1] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp, writeFEvalsMaxPrec(refalgert[-1], 2))) replaceValue = '<i>%s</i>:%s' % ( temp, writeFEvalsMaxPrec(refalgert[-1], 2)) curlineHtml = [ item.replace('REPLACE%d' % counter, replaceValue) for item in curlineHtml ] else: # write #fevals of the reference alg counter = 1 for i in refalgert[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curlineHtml = [ item.replace('REPLACE%d' % counter, writeFEvalsMaxPrec(i, 2)) for item in curlineHtml ] counter += 1 curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(refalgert[-1], 2)) curlineHtml = [ item.replace('REPLACE%d' % counter, writeFEvalsMaxPrec(refalgert[-1], 2)) for item in curlineHtml ] # write the success ratio for the reference alg tmp2 = numpy.sum( numpy.isnan(refalgevals) == False) # count the nb of success curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(refalgevals)) replaceValue = '%d/%d' % (tmp2, len(refalgevals)) else: replaceValue = '%d' % tmp2 curlineHtml = [ item.replace('REPLACEF', replaceValue) for item in curlineHtml ] table.append(curline[:]) tableHtml.extend(curlineHtml[:]) tableHtml.append('<tbody>\n') extraeol.append('') #for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))): #print i, gna, gno #set_trace() # Format data #if df == (5, 17): #set_trace() header = r'\providecommand{\ntables}{7}' for i, alg in enumerate(algnames): tableHtml.append('<tr>\n') #algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames, #data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres): commandname = r'\alg%stables' % numtotext(i) # header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg))) header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % ( commandname, str_to_latex(strip_pathname1(alg))) curline = [ commandname + r'\hspace*{\fill}' ] # each list element becomes a &-separated table entry? curlineHtml = [ '<th>%s</th>\n' % str_to_latex(strip_pathname1(alg)) ] for j, tmp in enumerate( zip( algerts[i], algdisp[i], # j is target index isBoldArray[i], algtestres[i])): ert, dispersion, isBold, testres = tmp alignment = '@{\,}X@{\,}' if j == len(algerts[i]) - 1: alignment = '@{\,}X@{\,}|' data = ert / refalgert[j] # write star for significance against all other algorithms str_significance_subsup = '' str_significance_subsup_html = '' if (len(best_alg_idx) > 0 and len(significance_versus_others) > 0 and i == best_alg_idx[j] and nbtests * significance_versus_others[j][1] < 0.05): logp = -numpy.ceil( numpy.log10( nbtests * significance_versus_others[j][1])) logp = numpy.min( (9, logp)) # not messing up the format and handling inf str_significance_subsup = r"^{%s%s}" % ( significance_vs_others_symbol, str(int(logp)) if logp > 1 else '') str_significance_subsup_html = '<sup>%s%s</sup>' % ( significance_vs_others_symbol_html, str(int(logp)) if logp > 1 else '') # moved out of the above else: this was a bug!? z, p = testres if (nbtests * p) < 0.05 and data < 1. and z < 0.: if not numpy.isinf(refalgert[j]): tmpevals = algevals[i][j].copy() tmpevals[numpy.isnan(tmpevals)] = algentries[ i].maxevals[numpy.isnan(tmpevals)] bestevals = refalgentry.detEvals(targets) bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0]) bestevals[numpy.isnan( bestevals)] = refalgentry.maxevals[bestalgalg][ numpy.isnan(bestevals)] tmpevals = numpy.array(sorted( tmpevals))[0:min(len(tmpevals), len(bestevals))] bestevals = numpy.array(sorted( bestevals))[0:min(len(tmpevals), len(bestevals))] #The conditions are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) # tmp2[-1] += r'$^{%s}$' % superscript str_significance_subsup += r'_{%s%s}' % ( significance_vs_ref_symbol, str(int(nbstars)) if nbstars > 1 else '') str_significance_subsup_html = '<sub>%s%s</sub>' % ( significance_vs_ref_symbol_html, str(int(nbstars)) if nbstars > 1 else '') if str_significance_subsup: str_significance_subsup = '$%s$' % str_significance_subsup # format number in variable data if numpy.isnan(data): curline.append(r'\multicolumn{2}{%s}{.}' % alignment) else: if numpy.isinf(refalgert[j]): curline.append( r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}' % (alignment, writeFEvalsMaxPrec(algerts[i][j], 2), writeFEvalsMaxPrec(dispersion, precdispersion), str_significance_subsup)) curlineHtml.append( '<td sorttable_customkey=\"%f\"><b>%s</b> (%s)%s</td>\n' % (algerts[i][j], writeFEvalsMaxPrec(algerts[i][j], 2), writeFEvalsMaxPrec(dispersion, precdispersion), str_significance_subsup_html)) continue tmp = writeFEvalsMaxPrec(data, precfloat, maxfloatrepr=maxfloatrepr) tmpHtml = writeFEvalsMaxPrec(data, precfloat, maxfloatrepr=maxfloatrepr) sortKey = data if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation if numpy.isinf(data) and j == len(algerts[i]) - 1: tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec( algfinaldata[i][1], 0, maxfloatrepr=maxfloatrepr) tmpHtml += '<i>%s</i>' % writeFEvalsMaxPrec( algfinaldata[i][1], 0, maxfloatrepr=maxfloatrepr) sortKey = algfinaldata[i][1] else: tmp = writeFEvalsMaxPrec(data, precscien, maxfloatrepr=data) if isBold: tmpHtml = '<b>%s</b>' % tmp tmp = r'\textbf{%s}' % tmp if not numpy.isnan(dispersion): tmpdisp = dispersion / refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp += r'\mbox{\tiny (%s)}' % tmpdisp tmpHtml += ' (%s)' % tmpdisp curline.append( r'\multicolumn{2}{%s}{%s%s}' % (alignment, tmp, str_significance_subsup)) tmpHtml = tmpHtml.replace('$\infty$', '∞') if (numpy.isinf(sortKey)): sortKey = sys.maxint curlineHtml.append( '<td sorttable_customkey=\"%f\">%s%s</td>' % (sortKey, tmpHtml, str_significance_subsup_html)) else: tmp2 = tmp.split('.', 1) if len(tmp2) < 2: tmp2.append('') else: tmp2[-1] = '.' + tmp2[-1] if isBold: tmp3 = [] tmp3html = [] for k in tmp2: tmp3.append(r'\textbf{%s}' % k) tmp3html.append('<b>%s</b>' % k) tmp2 = tmp3 tmp2html = tmp3html else: tmp2html = [] tmp2html.extend(tmp2) if not numpy.isnan(dispersion): tmpdisp = dispersion / refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.01: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp)) tmp2html[-1] += ' (%s)' % tmpdisp tmp2[-1] += str_significance_subsup tmp2html[-1] += str_significance_subsup_html curline.extend(tmp2) tmp2html = ("").join(str(item) for item in tmp2html) tmp2html = tmp2html.replace('$\infty$', '∞') curlineHtml.append( '<td sorttable_customkey=\"%f\">%s</td>' % (data, tmp2html)) curline.append('%d' % algnbsucc[i]) curline.append('/%d' % algnbruns[i]) table.append(curline) curlineHtml.append('<td sorttable_customkey=\"%d\">%d/%d</td>\n' % (algnbsucc[i], algnbsucc[i], algnbruns[i])) tableHtml.extend(curlineHtml[:]) extraeol.append('') # Write table res = tableXLaTeX(table, spec=spec, extraeol=extraeol) try: filename = os.path.join( outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0])) f = open(filename, 'w') f.write(header + '\n') f.write(res) res = ("").join(str(item) for item in tableHtml) res = '\n<table class=\"sortable\" style=\"width:800px \">\n%s</table>\n<p/>\n' % res if df[0] in (5, 20): filename = os.path.join( outputdir, genericsettings.many_algorithm_file_name + '.html') lines = [] with open(filename) as infile: for line in infile: if '<!--' + 'pptablesf%03d%02dDHtml' % ( df[1], df[0]) + '-->' in line: lines.append(res) lines.append(line) with open(filename, 'w') as outfile: for line in lines: outfile.write(line) if verbose: print 'Wrote table in %s' % filename except: raise else: f.close()
def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True): """One table per dimension, modified to fit in 1 page per table.""" #TODO: method is long, split if possible dictDim0 = dsList0.dictByDim() dictDim1 = dsList1.dictByDim() alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop().replace(genericsettings.extraction_folder_prefix, '')[0:3] alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop().replace(genericsettings.extraction_folder_prefix, '')[0:3] open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a' ).write(r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' + r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n') if info: info = '_' + info dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys())) bestalgentries = bestalg.loadBestAlgorithm(dsList0.isBiobjective()) header = [] if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): header = [r'\#FEs/D'] headerHtml = ['<thead>\n<tr>\n<th>#FEs/D</th>\n'] for label in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % label) headerHtml.append('<td>%s</td>\n' % label) else: header = [r'$\Delta f_\mathrm{opt}$'] headerHtml = ['<thead>\n<tr>\n<th>Δ f</th>\n'] for label in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{\,}c@{\,}}{%s}' % label) headerHtml.append('<td>%s</td>\n' % label) header.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') headerHtml.append('<td>#succ</td>\n</tr>\n</thead>\n') for d in dimsOfInterest: # TODO set as input arguments table = [header] tableHtml = headerHtml extraeol = [r'\hline'] try: dictFunc0 = dictDim0[d].dictByFunc() dictFunc1 = dictDim1[d].dictByFunc() except KeyError: continue funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys())) nbtests = len(funcs) * 2. #len(dimsOfInterest) tableHtml.append('<tbody>\n') for f in sorted(funcs): tableHtml.append('<tr>\n') targets = targetsOfInterest((f, d)) targetf = targets[-1] bestalgentry = bestalgentries[(d, f)] curline = [r'${\bf f_{%d}}$' % f] curlineHtml = ['<th><b>f<sub>%d</sub></b></th>\n' % f] bestalgdata = bestalgentry.detERT(targets) bestalgevals, bestalgalgs = bestalgentry.detEvals(targets) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): # write ftarget:fevals for i in xrange(len(bestalgdata[:-1])): temp = "%.1e" % targetsOfInterest((f, d))[i] if temp[-2]=="0": temp = temp[:-2]+temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp,writeFEvalsMaxPrec(bestalgdata[i], 2))) curlineHtml.append('<td><i>%s</i>:%s</td>\n' % (temp, writeFEvalsMaxPrec(bestalgdata[i], 2))) temp = "%.1e" % targetsOfInterest((f, d))[-1] if temp[-2]=="0": temp = temp[:-2]+temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp,writeFEvalsMaxPrec(bestalgdata[-1], 2))) curlineHtml.append('<td><i>%s</i>:%s</td>\n' % (temp, writeFEvalsMaxPrec(bestalgdata[-1], 2))) else: # write #fevals of the reference alg for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curlineHtml.append('<td>%s</td>\n' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) curlineHtml.append('<td>%s</td>\n' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = numpy.sum(numpy.isnan(tmp) == False) curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) curlineHtml.append('<td>%d/%d</td>\n' % (tmp2, len(tmp))) else: curlineHtml.append('<td>%d</td>\n' % (tmp2)) table.append(curline[:]) tableHtml.extend(curlineHtml[:]) tableHtml.append('</tr>\n') extraeol.append('') rankdata0 = [] # never used # generate all data from ranksum test entries = [] ertdata = {} for nb, dsList in enumerate((dictFunc0, dictFunc1)): try: entry = dsList[f][0] # take the first DataSet, there should be only one? except KeyError: warnings.warn('data missing for data set ' + str(nb) + ' and function ' + str(f)) print('*** Warning: data missing for data set ' + str(nb) + ' and function ' + str(f) + '***') continue # TODO: problem here! ertdata[nb] = entry.detERT(targets) entries.append(entry) for _t in ertdata.values(): for _tt in _t: if _tt is None: raise ValueError if len(entries) < 2: # funcion not available for *both* algorithms continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following testres0vs1 = significancetest(entries[0], entries[1], targets) testresbestvs1 = significancetest(bestalgentry, entries[1], targets) testresbestvs0 = significancetest(bestalgentry, entries[0], targets) for nb, entry in enumerate(entries): tableHtml.append('<tr>\n') if nb == 0: curline = [r'1:\:\algorithmAshort\hspace*{\fill}'] curlineHtml = ['<th>1: %s</th>\n' % alg0] else: curline = [r'2:\:\algorithmBshort\hspace*{\fill}'] curlineHtml = ['<th>2: %s</th>\n' % alg1] #data = entry.detERT(targetsOfInterest) dispersion = [] data = [] evals = entry.detEvals(targets) for i in evals: succ = (numpy.isnan(i) == False) tmp = i.copy() tmp[succ==False] = entry.maxevals[numpy.isnan(i)] #set_trace() data.append(toolsstats.sp(tmp, issuccessful=succ)[0]) #if not any(succ): #set_trace() if any(succ): tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ==False], (10, 50, 90), samplesize)[0] dispersion.append((tmp2[-1]-tmp2[0])/2.) else: dispersion.append(None) if nb == 0: assert not isinstance(data, numpy.ndarray) data0 = data[:] # TODO: check if it is not an array, it's never used anyway? for i, dati in enumerate(data): z, p = testres0vs1[i] # TODO: there is something with the sign that I don't get # assign significance flag, which is the -log10(p) significance0vs1 = 0 if nb != 0: z = -z # the test is symmetric if nbtests * p < 0.05 and z > 0: significance0vs1 = -int(numpy.ceil(numpy.log10(min([1.0, nbtests * p])))) # this is the larger the more significant isBold = significance0vs1 > 0 alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' if numpy.isinf(bestalgdata[i]): # if the 2009 best did not solve the problem tmp = writeFEvalsMaxPrec(float(dati), 2) if not numpy.isinf(dati): tmpHtml = '<i>%s</i>' % (tmp) tmp = r'\textit{%s}' % (tmp) if isBold: tmp = r'\textbf{%s}' % tmp tmpHtml = '<b>%s</b>' % tmpHtml if dispersion[i] and numpy.isfinite(dispersion[i]): tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tmp)) tableentryHtml = (' (%s)' % tmp) else: # Formatting tmp = float(dati)/bestalgdata[i] assert not numpy.isnan(tmp) isscientific = False if tmp >= 1000: isscientific = True tableentry = writeFEvals2(tmp, 2, isscientific=isscientific) tableentry = writeFEvalsMaxPrec(tmp, 2) tableentryHtml = writeFEvalsMaxPrec(tmp, 2) if numpy.isinf(tmp) and i == len(data)-1: tableentry = (tableentry + r'\textit{%s}' % writeFEvals2(numpy.median(entry.maxevals), 2)) tableentryHtml = (tableentryHtml + ' <i>%s</i>' % writeFEvals2(numpy.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry tableentryHtml = '<b>%s</b>' % tableentryHtml elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore tableentry = r'\textit{%s}' % tableentry tableentryHtml = '<i>%s</i>' % tableentryHtml if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentryHtml += ' (%s)' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry tableentryHtml = '<b>%s</b>' % tableentryHtml elif 11 < 3 and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentryHtml = '<i>%s</i>' % tableentryHtml if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentryHtml += ' (%s)' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) tmpHtml = tableentryHtml.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) tmpHtml = list('<b>%s</b>' % i for i in tmpHtml) elif 11 < 3 and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tmpHtml = list('<i>%s</i>' % i for i in tmpHtml) tableentry = ' & .'.join(tmp) tableentryHtml = '.'.join(tmpHtml) if len(tmp) == 1: tableentry += '&' if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentryHtml += ' (%s)' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) superscript = '' superscriptHtml = '' if nb == 0: z, p = testresbestvs0[i] else: z, p = testresbestvs1[i] #The conditions are now that ERT < ERT_best if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0. and z < 0.): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars superscriptHtml = '↑' else: superscript = r'\downarrow' #* nbstars superscriptHtml = '↓' # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(nbstars)) superscriptHtml += str(int(nbstars)) if superscript or significance0vs1: s = '' shtml = '' if significance0vs1 > 0: s = '\star' shtml = '★' if significance0vs1 > 1: s += str(significance0vs1) shtml += str(significance0vs1) s = r'$^{' + s + superscript + r'}$' shtml = '<sup>' + shtml + superscriptHtml + '</sup>' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s tableentryHtml += shtml tableentryHtml = tableentryHtml.replace('$\infty$', '∞') curlineHtml.append('<td>%s</td>\n' % tableentryHtml) curline.append(tableentry) #curline.append(tableentry) #if dispersion[i] is None or numpy.isinf(bestalgdata[i]): #curline.append('') #else: #tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2) #curline.append('(%s)' % tmp) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False)) curlineHtml.append('<td>%d' % numpy.sum(numpy.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curlineHtml.append('<td>%d' % 0) curline.append('/%d' % entry.nbRuns()) curlineHtml.append('/%d</td>\n' % entry.nbRuns()) table.append(curline[:]) tableHtml.extend(curlineHtml[:]) tableHtml.append('</tr>\n') extraeol.append('') extraeol[-1] = r'\hline' extraeol[-1] = '' outputfile = os.path.join(outputdir, 'pptable2_%02dD%s.tex' % (d, info)) spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}' res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n' # open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res) #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res += tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() res = ("").join(str(item) for item in tableHtml) res = '<p><b>%d-D</b></p>\n<table>\n%s</table>\n' % (d, res) filename = os.path.join(outputdir, genericsettings.two_algorithm_file_name + '.html') lines = [] with open(filename) as infile: for line in infile: if '<!--pptable2Html-->' in line: lines.append(res) lines.append(line) with open(filename, 'w') as outfile: for line in lines: outfile.write(line) if verbose: print "Table written in %s" % outputfile