def main(dsList, dimsOfInterest, outputdir, info='', verbose=True): """Generate a table of ratio ERT/ERTbest vs target precision. 1 table per dimension will be generated. Rank-sum tests table on "Final Data Points" for only one algorithm. that is, for example, using 1/#fevals(ftarget) if ftarget was reached and -f_final otherwise as input for the rank-sum test, where obviously the larger the better. """ #TODO: check that it works for any reference algorithm? #in the following the reference algorithm is the one given in #bestalg.bestalgentries which is the virtual best of BBOB dictDim = dsList.dictByDim() targetf=1e-8 if info: info = '_' + info # insert a separator between the default file name and the additional # information string. dims = set(dictDim.keys()) if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): header = [r'\#FEs/D'] for i in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % i) else: header = [r'$\Delta f$'] for i in targetsOfInterest.target_values: header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' % (int(np.log10(i)))) header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}') for d in dimsOfInterest: table = [header] extraeol = [r'\hline'] try: dictFunc = dictDim[d].dictByFunc() except KeyError: continue funcs = set(dictFunc.keys()) nbtests = float(len(funcs)) # #funcs tests times one algorithm for f in sorted(funcs): bestalgentry = bestalg.bestalgentries2009[(d, f)] curline = [r'${\bf f_{%d}}$' % f] bestalgdata = bestalgentry.detERT(targetsOfInterest((f,d))) bestalgevals, bestalgalgs = bestalgentry.detEvals(targetsOfInterest((f,d))) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): #write ftarget:fevals for i in xrange(len(bestalgdata[:-1])): temp="%.1e" %targetsOfInterest((f,d))[i] if temp[-2]=="0": temp=temp[:-2]+temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp,writeFEvalsMaxPrec(bestalgdata[i], 2))) temp="%.1e" %targetsOfInterest((f,d))[-1] if temp[-2]=="0": temp=temp[:-2]+temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp,writeFEvalsMaxPrec(bestalgdata[-1], 2))) #success targetf=targetsOfInterest((f,d))[-1] else: # write #fevals of the reference alg for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) # write the success ratio for the reference alg tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = np.sum(np.isnan(tmp) == False) # count the nb of success curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) table.append(curline[:]) extraeol.append('') # generate all data for ranksum test assert len(dictFunc[f]) == 1 entry = dictFunc[f][0] # take the first element ertdata = entry.detERT(targetsOfInterest((f, d))) testresbestvs1 = significancetest(bestalgentry, entry, targetsOfInterest((f, d))) #for nb, entry in enumerate(entries): #curline = [r'\algshort\hspace*{\fill}'] curline = [''] #data = entry.detERT(targetsOfInterest) evals = entry.detEvals(targetsOfInterest((f,d))) dispersion = [] data = [] for i in evals: succ = (np.isnan(i) == False) tmp = i.copy() tmp[succ==False] = entry.maxevals[np.isnan(i)] #set_trace() # TODO: what is the difference between data and ertdata? data.append(toolsstats.sp(tmp, issuccessful=succ)[0]) #if not any(succ): #set_trace() if any(succ): tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ==False], (10, 50, 90), samplesize)[0] dispersion.append((tmp2[-1] - tmp2[0]) / 2.) else: dispersion.append(None) assert data == ertdata for i, ert in enumerate(data): alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' nbstars = 0 z, p = testresbestvs1[i] if ert - bestalgdata[i] < 0. and not np.isinf(bestalgdata[i]): evals = entry.detEvals([targetsOfInterest((f,d))[i]])[0] evals[np.isnan(evals)] = entry.maxevals[np.isnan(evals)] bestevals = bestalgentry.detEvals([targetsOfInterest((f,d))[i]]) bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0]) bestevals[np.isnan(bestevals)] = bestalgentry.maxevals[bestalgalg][np.isnan(bestevals)] evals = np.array(sorted(evals))[0:min(len(evals), len(bestevals))] bestevals = np.array(sorted(bestevals))[0:min(len(evals), len(bestevals))] #The conditions for significance are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if ((nbtests * p) < 0.05 and ert - bestalgdata[i] < 0. and z < 0. and (np.isinf(bestalgdata[i]) or all(evals < bestevals))): nbstars = -np.ceil(np.log10(nbtests * p)) isBold = False if nbstars > 0: isBold = True if np.isinf(bestalgdata[i]): # if the best did not solve the problem tmp = writeFEvalsMaxPrec(float(ert), 2) if not np.isinf(ert): tmp = r'\textit{%s}' % (tmp) if isBold: tmp = r'\textbf{%s}' % tmp tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tmp)) else: # Formatting tmp = float(ert) / bestalgdata[i] assert not np.isnan(tmp) tableentry = writeFEvalsMaxPrec(tmp, 2) if np.isinf(tmp) and i == len(data)-1: tableentry = (tableentry + r'\textit{%s}' % writeFEvals2(np.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3: # and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (np.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3: # and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) elif 11 < 3: # and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tableentry = ' & .'.join(tmp) if len(tmp) == 1: tableentry += '&' superscript = '' if nbstars > 0: #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars else: superscript = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(min((9, nbstars)))) # superscript += str(int(nbstars)) #if superscript or significance0vs1: #s = '' #if significance0vs1 > 0: #s = '\star' #if significance0vs1 > 1: #s += str(significance0vs1) #s = r'$^{' + s + superscript + r'}$' #if tableentry.endswith('}'): #tableentry = tableentry[:-1] + s + r'}' #else: #tableentry += s if dispersion[i]: if not np.isinf(bestalgdata[i]): tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) else: tmp = writeFEvalsMaxPrec(dispersion[i], 1) tableentry += (r'${\scriptscriptstyle(%s)}$' % tmp) if superscript: s = r'$^{' + superscript + r'}$' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s curline.append(tableentry) #curline.append(tableentry) #if dispersion[i] is None or np.isinf(bestalgdata[i]): #curline.append('') #else: #tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2) #curline.append('(%s)' % tmp) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % np.sum(np.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curline.append('/%d' % entry.nbRuns()) table.append(curline[:]) extraeol.append(r'\hline') extraeol[-1] = '' outputfile = os.path.join(outputdir, 'pptable_%02dD%s.tex' % (d, info)) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): spec = r'@{}c@{}|' + '*{%d}{@{ }r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}' else: spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}' #res = r'\providecommand{\algshort}{%s}' % alg1 + '\n' #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res = tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() if verbose: print "Table written in %s" % outputfile
def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True): """One table per dimension, modified to fit in 1 page per table.""" #TODO: method is long, split if possible dictDim0 = dsList0.dictByDim() dictDim1 = dsList1.dictByDim() alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3] alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3] open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a' ).write(r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' + r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n') if info: info = '_' + info dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys())) if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() header = [r'$\Delta f$'] for i in targetsOfInterest: #header.append(r'\multicolumn{2}{@{}c@{}}{$10^{%d}$}' % (int(numpy.log10(i)))) header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' % (int(numpy.log10(i)))) header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}') for d in dimsOfInterest: # TODO set as input arguments table = [header] extraeol = [r'\hline'] try: dictFunc0 = dictDim0[d].dictByFunc() dictFunc1 = dictDim1[d].dictByFunc() except KeyError: continue funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys())) nbtests = len(funcs) * 2. #len(dimsOfInterest) for f in sorted(funcs): bestalgentry = bestalg.bestalgentries2009[(d, f)] curline = [r'${\bf f_{%d}}$' % f] bestalgdata = bestalgentry.detERT(targetsOfInterest) bestalgevals, bestalgalgs = bestalgentry.detEvals(targetsOfInterest) for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = numpy.sum(numpy.isnan(tmp) == False) curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) table.append(curline[:]) extraeol.append('') rankdata0 = [] # never used # generate all data from ranksum test entries = [] ertdata = {} for nb, dsList in enumerate((dictFunc0, dictFunc1)): try: entry = dsList[f][0] # take the first DataSet, there should be only one? except KeyError: warnings.warn('data missing for data set ' + str(nb) + ' and function ' + str(f)) print('*** Warning: data missing for data set ' + str(nb) + ' and function ' + str(f) + '***') continue # TODO: problem here! ertdata[nb] = entry.detERT(targetsOfInterest) entries.append(entry) for _t in ertdata.values(): for _tt in _t: if _tt is None: raise ValueError if len(entries) < 2: # funcion not available for *both* algorithms continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following testres0vs1 = significancetest(entries[0], entries[1], targetsOfInterest) testresbestvs1 = significancetest(bestalgentry, entries[1], targetsOfInterest) testresbestvs0 = significancetest(bestalgentry, entries[0], targetsOfInterest) for nb, entry in enumerate(entries): if nb == 0: curline = [r'1:\:\algorithmAshort\hspace*{\fill}'] else: curline = [r'2:\:\algorithmBshort\hspace*{\fill}'] #data = entry.detERT(targetsOfInterest) dispersion = [] data = [] evals = entry.detEvals(targetsOfInterest) for i in evals: succ = (numpy.isnan(i) == False) tmp = i.copy() tmp[succ==False] = entry.maxevals[numpy.isnan(i)] #set_trace() data.append(toolsstats.sp(tmp, issuccessful=succ)[0]) #if not any(succ): #set_trace() if any(succ): tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ==False], (10, 50, 90), samplesize)[0] dispersion.append((tmp2[-1]-tmp2[0])/2.) else: dispersion.append(None) if nb == 0: assert not isinstance(data, numpy.ndarray) data0 = data[:] # TODO: check if it is not an array, it's never used anyway? for i, dati in enumerate(data): z, p = testres0vs1[i] # TODO: there is something with the sign that I don't get # assign significance flag, which is the -log10(p) significance0vs1 = 0 if nb != 0: z = -z # the test is symmetric if nbtests * p < 0.05 and z > 0: significance0vs1 = -int(numpy.ceil(numpy.log10(min([1.0, nbtests * p])))) # this is the larger the more significant isBold = significance0vs1 > 0 alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' if numpy.isinf(bestalgdata[i]): # if the 2009 best did not solve the problem tmp = writeFEvalsMaxPrec(float(dati), 2) if not numpy.isinf(dati): tmp = r'\textit{%s}' % (tmp) if isBold: tmp = r'\textbf{%s}' % tmp if dispersion[i] and numpy.isfinite(dispersion[i]): tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tmp)) else: # Formatting tmp = float(dati)/bestalgdata[i] assert not numpy.isnan(tmp) isscientific = False if tmp >= 1000: isscientific = True tableentry = writeFEvals2(tmp, 2, isscientific=isscientific) tableentry = writeFEvalsMaxPrec(tmp, 2) if numpy.isinf(tmp) and i == len(data)-1: tableentry = (tableentry + r'\textit{%s}' % writeFEvals2(numpy.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore tableentry = r'\textit{%s}' % tableentry if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) elif 11 < 3 and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tableentry = ' & .'.join(tmp) if len(tmp) == 1: tableentry += '&' if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) superscript = '' if nb == 0: z, p = testresbestvs0[i] else: z, p = testresbestvs1[i] #The conditions are now that ERT < ERT_best if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0. and z < 0.): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars else: superscript = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(nbstars)) if superscript or significance0vs1: s = '' if significance0vs1 > 0: s = '\star' if significance0vs1 > 1: s += str(significance0vs1) s = r'$^{' + s + superscript + r'}$' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s curline.append(tableentry) #curline.append(tableentry) #if dispersion[i] is None or numpy.isinf(bestalgdata[i]): #curline.append('') #else: #tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2) #curline.append('(%s)' % tmp) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curline.append('/%d' % entry.nbRuns()) table.append(curline[:]) extraeol.append('') extraeol[-1] = r'\hline' extraeol[-1] = '' outputfile = os.path.join(outputdir, 'pptable2_%02dD%s.tex' % (d, info)) spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}' res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n' # open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res) #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res += tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() if verbose: print "Table written in %s" % outputfile
def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True): """One table per dimension, modified to fit in 1 page per table.""" #TODO: method is long, split if possible dictDim0 = dsList0.dictByDim() dictDim1 = dsList1.dictByDim() alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3] alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3] open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write( r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' + r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n') if info: info = '_' + info dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys())) if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() header = [] if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): header = [r'\#FEs/D'] for label in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % label) else: header = [r'$\Delta f_\mathrm{opt}$'] for label in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{\,}c@{\,}}{%s}' % label) header.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') for d in dimsOfInterest: # TODO set as input arguments table = [header] extraeol = [r'\hline'] try: dictFunc0 = dictDim0[d].dictByFunc() dictFunc1 = dictDim1[d].dictByFunc() except KeyError: continue funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys())) nbtests = len(funcs) * 2. #len(dimsOfInterest) for f in sorted(funcs): targets = targetsOfInterest((f, d)) targetf = targets[-1] bestalgentry = bestalg.bestalgentries2009[(d, f)] curline = [r'${\bf f_{%d}}$' % f] bestalgdata = bestalgentry.detERT(targets) bestalgevals, bestalgalgs = bestalgentry.detEvals(targets) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): # write ftarget:fevals for i in xrange(len(bestalgdata[:-1])): temp = "%.1e" % targetsOfInterest((f, d))[i] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append( r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp, writeFEvalsMaxPrec(bestalgdata[i], 2))) temp = "%.1e" % targetsOfInterest((f, d))[-1] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp, writeFEvalsMaxPrec(bestalgdata[-1], 2))) else: # write #fevals of the reference alg for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = numpy.sum(numpy.isnan(tmp) == False) curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) table.append(curline[:]) extraeol.append('') rankdata0 = [] # never used # generate all data from ranksum test entries = [] ertdata = {} for nb, dsList in enumerate((dictFunc0, dictFunc1)): try: entry = dsList[f][ 0] # take the first DataSet, there should be only one? except KeyError: warnings.warn('data missing for data set ' + str(nb) + ' and function ' + str(f)) print('*** Warning: data missing for data set ' + str(nb) + ' and function ' + str(f) + '***') continue # TODO: problem here! ertdata[nb] = entry.detERT(targets) entries.append(entry) for _t in ertdata.values(): for _tt in _t: if _tt is None: raise ValueError if len(entries) < 2: # funcion not available for *both* algorithms continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following testres0vs1 = significancetest(entries[0], entries[1], targets) testresbestvs1 = significancetest(bestalgentry, entries[1], targets) testresbestvs0 = significancetest(bestalgentry, entries[0], targets) for nb, entry in enumerate(entries): if nb == 0: curline = [r'1:\:\algorithmAshort\hspace*{\fill}'] else: curline = [r'2:\:\algorithmBshort\hspace*{\fill}'] #data = entry.detERT(targetsOfInterest) dispersion = [] data = [] evals = entry.detEvals(targets) for i in evals: succ = (numpy.isnan(i) == False) tmp = i.copy() tmp[succ == False] = entry.maxevals[numpy.isnan(i)] #set_trace() data.append(toolsstats.sp(tmp, issuccessful=succ)[0]) #if not any(succ): #set_trace() if any(succ): tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ == False], (10, 50, 90), samplesize)[0] dispersion.append((tmp2[-1] - tmp2[0]) / 2.) else: dispersion.append(None) if nb == 0: assert not isinstance(data, numpy.ndarray) data0 = data[:] # TODO: check if it is not an array, it's never used anyway? for i, dati in enumerate(data): z, p = testres0vs1[ i] # TODO: there is something with the sign that I don't get # assign significance flag, which is the -log10(p) significance0vs1 = 0 if nb != 0: z = -z # the test is symmetric if nbtests * p < 0.05 and z > 0: significance0vs1 = -int( numpy.ceil(numpy.log10(min([ 1.0, nbtests * p ])))) # this is the larger the more significant isBold = significance0vs1 > 0 alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' if numpy.isinf( bestalgdata[i] ): # if the 2009 best did not solve the problem tmp = writeFEvalsMaxPrec(float(dati), 2) if not numpy.isinf(dati): tmp = r'\textit{%s}' % (tmp) if isBold: tmp = r'\textbf{%s}' % tmp if dispersion[i] and numpy.isfinite(dispersion[i]): tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tmp)) else: # Formatting tmp = float(dati) / bestalgdata[i] assert not numpy.isnan(tmp) isscientific = False if tmp >= 1000: isscientific = True tableentry = writeFEvals2(tmp, 2, isscientific=isscientific) tableentry = writeFEvalsMaxPrec(tmp, 2) if numpy.isinf(tmp) and i == len(data) - 1: tableentry = ( tableentry + r'\textit{%s}' % writeFEvals2(numpy.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore tableentry = r'\textit{%s}' % tableentry if dispersion[i] and numpy.isfinite( dispersion[i] / bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry if dispersion[i] and numpy.isfinite( dispersion[i] / bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) elif 11 < 3 and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tableentry = ' & .'.join(tmp) if len(tmp) == 1: tableentry += '&' if dispersion[i] and numpy.isfinite( dispersion[i] / bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) superscript = '' if nb == 0: z, p = testresbestvs0[i] else: z, p = testresbestvs1[i] #The conditions are now that ERT < ERT_best if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0. and z < 0.): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars else: superscript = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(nbstars)) if superscript or significance0vs1: s = '' if significance0vs1 > 0: s = '\star' if significance0vs1 > 1: s += str(significance0vs1) s = r'$^{' + s + superscript + r'}$' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s curline.append(tableentry) #curline.append(tableentry) #if dispersion[i] is None or numpy.isinf(bestalgdata[i]): #curline.append('') #else: #tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2) #curline.append('(%s)' % tmp) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curline.append('/%d' % entry.nbRuns()) table.append(curline[:]) extraeol.append('') extraeol[-1] = r'\hline' extraeol[-1] = '' outputfile = os.path.join(outputdir, 'pptable2_%02dD%s.tex' % (d, info)) spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len( targetsOfInterest) + '|@{}r@{}@{}l@{}' res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels( alg0) + '\n' res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels( alg1) + '\n' # open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res) #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res += tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() if verbose: print "Table written in %s" % outputfile
def mainnew(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True): """Still in shambles: new cleaned up version. One table per dimension... """ dictDim0 = dsList0.dictByDim() dictDim1 = dsList1.dictByDim() alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3] alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3] if info: info = '_' + info dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys())) if not bestalg.bestalgentries: bestalg.loadBBOB2009() header = [r'$\Delta f$'] for i in targetsOfInterest: #header.append(r'\multicolumn{2}{@{}c@{}}{$10^{%d}$}' % (int(numpy.log10(i)))) header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' % (int(numpy.log10(i)))) header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}') for d in dimsOfInterest: # TODO set as input arguments table = [header] extraeol = [r'\hline'] dictFunc0 = dictDim0[d].dictByFunc() dictFunc1 = dictDim1[d].dictByFunc() funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys())) nbtests = len(funcs) * 2. #len(dimsOfInterest) for f in sorted(funcs): bestalgentry = bestalg.bestalgentries[(d, f)] curline = [r'${\bf f_{%d}}$' % f] bestalgdata = bestalgentry.detERT(targetsOfInterest) bestalgevals, bestalgalgs = bestalgentry.detEvals( targetsOfInterest) for i in bestalgdata[:-1]: #if numpy.isnan(i): #set_trace() curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' % writeFEvals2(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvals2(bestalgdata[-1], 2)) tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = numpy.sum(numpy.isnan(tmp) == False) curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) table.append(curline[:]) extraeol.append('') rankdata0 = [] # generate all data from ranksum test entries = [] ertdata = {} for nb, dsList in enumerate((dictFunc0, dictFunc1)): try: entry = dsList[f][0] # take the first element except KeyError: continue # TODO: problem here! ertdata[nb] = entry.detERT(targetsOfInterest) entries.append(entry) testres0vs1 = significancetest(entries[0], entries[1], targetsOfInterest) testresbestvs1 = significancetest(bestalgentry, entries[1], targetsOfInterest) for nb, entry in enumerate(entries): if nb == 0: curline = [r'0:\:\algzeroshort\hspace*{\fill}'] else: curline = [r'1:\:\algoneshort\hspace*{\fill}'] #curline = [r'\alg%sshort' % tmp] #curline = [r'Alg%d' % nb] #curline = [r'%.3s%d' % (entry.algId, nb)] data = entry.detERT(targetsOfInterest) if nb == 0: assert not isinstance(data, numpy.ndarray) data0 = data[:] # check if it is not an array for i, j in enumerate( data): # is j an appropriate identifier here? #if numpy.isnan(float(j)/bestalgdata[i]): # set_trace() z, p = testres0vs1[ i] #TODO: there is something with the sign that I don't get # assign significance flag significance0vs1 = 0 if nb == 0: istat0 = 0 istat1 = 1 else: z = -z istat0 = 1 istat1 = 0 # TODO: I don't understand the thing with the sign of significance0vs1 if (nbtests * p < 0.05 and z > 0 and not numpy.isinf(ertdata[istat0][i]) and z * (ertdata[istat1][i] - ertdata[istat0][i]) > 0): # z-value and ERT-ratio must agree significance0vs1 = -int( numpy.ceil(numpy.log10(nbtests * p))) elif nbtests * p < 0.05 and z < 0 and z * ( ertdata[istat1][i] - ertdata[istat0][i]) > 0: significance0vs1 = int( numpy.ceil(numpy.log10(nbtests * p))) alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' if numpy.isinf(bestalgdata[i]): tableentry = ( r'\multicolumn{2}{@{}%s@{}}{\textbf{\textit{%s}}}' % (alignment, writeFEvals2(float(j), 2))) # TODO: is this the desired behaviour? else: # Formatting tmp = float(j) / bestalgdata[i] assert not numpy.isnan(tmp) isscientific = False if tmp >= 1000: isscientific = True tableentry = writeFEvals2(tmp, 2, isscientific=isscientific) isBold = False if significance0vs1 > 0: isBold = True if numpy.isinf(tmp) and i == len(data) - 1: tableentry = ( tableentry + r'\textit{%s}' % writeFEvals2(numpy.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) elif 11 < 3 and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tableentry = ' & .'.join(tmp) if len(tmp) == 1: tableentry += '&' superscript = '' z, p = testresbestvs1[i] #z, p = ranksums(rankdatabest[i], currankdata) #if ((nbtests * p) < 0.05 # and ((numpy.isinf(bestalgdata[i]) and numpy.isinf(j)) # or z * (j - bestalgdata[i]) > 0)): # z-value and ERT-ratio must agree #The conditions are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if j - bestalgdata[i] < 0. and not numpy.isinf( bestalgdata[i]): evals = entry.detEvals([targetsOfInterest[i]])[0] evals[numpy.isnan(evals)] = entry.maxevals[numpy.isnan( evals)] bestevals = bestalgentry.detEvals( [targetsOfInterest[i]]) bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0]) bestevals[numpy.isnan( bestevals)] = bestalgentry.maxevals[bestalgalg][ numpy.isnan(bestevals)] evals = numpy.array( sorted(evals))[0:min(len(evals), len(bestevals))] bestevals = numpy.array(sorted( bestevals))[0:min(len(evals), len(bestevals))] #The conditions are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if ((nbtests * p) < 0.05 and j - bestalgdata[i] < 0. and z < 0. and (numpy.isinf(bestalgdata[i]) or all(evals < bestevals))): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars else: superscript = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(nbstars)) addition = '' if superscript or significance0vs1: s = '' if significance0vs1 > 0: s = '\star' if significance0vs1 > 1: s += str(significance0vs1) s = r'$^{' + s + superscript + r'}$' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s curline.append(tableentry) # Two cases: both tabular give an overfull hbox # AND generate a LaTeX Warning: Float too large for page by 16.9236pt on input line 421. (noisy) # OR generate a LaTeX Warning: Float too large for page by 33.57658pt on input line 421. (noisy) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curline.append('/%d' % entry.nbRuns()) table.append(curline[:]) extraeol.append('') extraeol[-1] = r'\hline' extraeol[-1] = '' outputfile = os.path.join(outputdir, 'cmptable_%02dD%s.tex' % (d, info)) spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len( targetsOfInterest) + '|@{}r@{}@{}l@{}' res = r'\providecommand{\algzeroshort}{%s}' % alg0 + '\n' res += r'\providecommand{\algoneshort}{%s}' % alg1 + '\n' #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res += tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() if verbose: print "Table written in %s" % outputfile
def main2(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True): """Generate comparison tables. One table per dimension... """ dictDim0 = dsList0.dictByDim() dictDim1 = dsList1.dictByDim() alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3] alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3] if info: info = '_' + info dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys())) if not bestalg.bestalgentries: bestalg.loadBBOB2009() header = [r'$\Delta f$'] for i in targetsOfInterest: #header.append(r'\multicolumn{2}{@{}c@{}}{$10^{%d}$}' % (int(numpy.log10(i)))) header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' % (int(numpy.log10(i)))) header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}') for d in dimsOfInterest: # TODO set as input arguments table = [header] extraeol = [r'\hline'] dictFunc0 = dictDim0[d].dictByFunc() dictFunc1 = dictDim1[d].dictByFunc() funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys())) nbtests = len(funcs) * 2. #len(dimsOfInterest) for f in sorted(funcs): bestalgentry = bestalg.bestalgentries[(d, f)] curline = [r'${\bf f_{%d}}$' % f] bestalgdata = bestalgentry.detERT(targetsOfInterest) bestalgevals, bestalgalgs = bestalgentry.detEvals( targetsOfInterest) for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' % writeFEvals2(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvals2(bestalgdata[-1], 2)) rankdatabest = [] for i, j in enumerate(bestalgevals): if bestalgalgs[i] is None: tmp = -bestalgentry.finalfunvals[bestalgentry.algs[-1]] else: tmp = numpy.power(j, -1.) tmp[numpy.isnan(tmp)] = -bestalgentry.finalfunvals[ bestalgalgs[i]][numpy.isnan(tmp)] rankdatabest.append(tmp) tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = numpy.sum(numpy.isnan(tmp) == False) curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) table.append(curline[:]) extraeol.append('') rankdata0 = [] # generate all data for ranksum test rankdata = {} # rankdata[nb +/- 1,i] is of interest later ertdata = {} for nb, entries in enumerate( (dictFunc0, dictFunc1)): # copy paste of loop below try: entry = entries[f][0] # take the first element except KeyError: continue ertdata[nb] = entry.detERT(targetsOfInterest) evals = entry.detEvals(targetsOfInterest) for i, tmp in enumerate(ertdata[nb]): # print i rankdata[nb, i] = numpy.power(evals[i], -1.) rankdata[nb, i][numpy.isnan( rankdata[nb, i])] = -entry.finalfunvals[numpy.isnan( rankdata[nb, i])] for nb, entries in enumerate((dictFunc0, dictFunc1)): try: entry = entries[f][0] # take the first element except KeyError: continue if nb == 0: curline = [r'0:\:\algzeroshort\hspace*{\fill}'] else: curline = [r'1:\:\algoneshort\hspace*{\fill}'] #curline = [r'\alg%sshort' % tmp] #curline = [r'Alg%d' % nb] #curline = [r'%.3s%d' % (entry.algId, nb)] data = entry.detERT(targetsOfInterest) evals = entry.detEvals(targetsOfInterest) if nb == 0: assert not isinstance(data, numpy.ndarray) data0 = data[:] # check if it is not an array for i, j in enumerate( data): # is j an appropriate identifier here? #if numpy.isnan(float(j)/bestalgdata[i]): # set_trace() # assign significance flag significance0vs1 = 0 if nb == 0: istat0 = 0 istat1 = 1 else: istat0 = 1 istat1 = 0 #set_trace() z, p = ranksums(rankdata[istat0, i], rankdata[istat1, i]) if (nbtests * p < 0.05 and z > 0 and not numpy.isinf(ertdata[istat0][i]) and z * (ertdata[istat1][i] - ertdata[istat0][i]) > 0): # z-value and ERT-ratio must agree significance0vs1 = -int( numpy.ceil(numpy.log10(nbtests * p))) if nbtests * p < 0.05 and z < 0 and z * ( ertdata[istat1][i] - ertdata[istat0][i]) > 0: significance0vs1 = int( numpy.ceil(numpy.log10(nbtests * p))) alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' if numpy.isinf(bestalgdata[i]): tableentry = ( r'\multicolumn{2}{@{}%s@{}}{\textbf{\textit{%s}}}' % (alignment, writeFEvals2(float(j), 2))) # TODO: is this the desired behaviour? else: # Formatting assert not numpy.isnan(float(j) / bestalgdata[i]) tmp = float(j) / bestalgdata[i] tableentry = writeFEvals2(tmp, 2) isBold = False if 11 < 3 and tmp <= 3: isBold = True if significance0vs1 > 0: isBold = True if tableentry.find('e') > -1: if isBold: tableentry = r'\textbf{%s}' % tableentry elif significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) elif significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tableentry = ' & .'.join(tmp) if len(tmp) == 1: tableentry += '&' currankdata = numpy.power(evals[i], -1.) currankdata[numpy.isnan( currankdata )] = -entry.finalfunvals[numpy.isnan(currankdata)] if 11 < 3 and nb == 0: rankdata0.append(currankdata.copy()) addition = '' elif 11 < 3: #nb==1 z, p = ranksums(rankdata0[i], currankdata) tmp = '.' #addition = '' # Uncomment to gain space. Meanwhile it displays the worst case scenario if ((nbtests * p) < 0.05 and ((numpy.isinf(data0[i]) and numpy.isinf(j)) or z * (j - data0[i]) > 0) ): # z-value and ERT-ratio must agree nbstars = -numpy.ceil(numpy.log10(nbtests * p)) if z > 0: tmp = r'\uparrow' #* nbstars else: tmp = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: tmp += str(int(nbstars)) #addition = tmp # uncomment to gain space. addition = r'/' + tmp # Comment and uncomment line above to gain space #addition = tmp # Comment and uncomment line above to gain space superscript = '\uparrow' superscript = '' #if addition: #superscript = '.' z, p = ranksums(rankdatabest[i], currankdata) if ((nbtests * p) < 0.05 and ((numpy.isinf(bestalgdata[i]) and numpy.isinf(j)) or z * (j - bestalgdata[i]) > 0) ): # z-value and ERT-ratio must agree nbstars = -numpy.ceil(numpy.log10(nbtests * p)) #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) #set_trace() if z > 0: superscript = r'\uparrow' #* nbstars else: superscript = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(nbstars)) addition = '' if superscript or significance0vs1: s = '' if significance0vs1 > 0: s = '\star' if significance0vs1 > 1: s += str(significance0vs1) s = r'$^{' + s + superscript + r'}$' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s if 11 < 3 and (superscript or addition): isClosingBrace = False if tableentry.endswith('}'): isClosingBrace = True tableentry = tableentry[:-1] #tableentry += r'$^{' + superscript + '}_{' + addition + '}$' tableentry += r'$^{' + superscript + addition + '}$' if isClosingBrace: tableentry += '}' curline.append(tableentry) # Two cases: both tabular give an overfull hbox # AND generate a LaTeX Warning: Float too large for page by 16.9236pt on input line 421. (noisy) # OR generate a LaTeX Warning: Float too large for page by 33.57658pt on input line 421. (noisy) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curline.append('/%d' % entry.nbRuns()) table.append(curline[:]) extraeol.append('') extraeol[-1] = r'\hline' extraeol[-1] = '' outputfile = os.path.join(outputdir, 'cmptable_%02dD%s.tex' % (d, info)) spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len( targetsOfInterest) + '|@{}r@{}@{}l@{}' res = r'\providecommand{\algzeroshort}{%s}' % alg0 + '\n' res += r'\providecommand{\algoneshort}{%s}' % alg1 + '\n' #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res += tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() if verbose: print "Table written in %s" % outputfile
def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True): """One table per dimension, modified to fit in 1 page per table.""" #TODO: method is long, split if possible dictDim0 = dsList0.dictByDim() dictDim1 = dsList1.dictByDim() alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop().replace(genericsettings.extraction_folder_prefix, '')[0:3] alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop().replace(genericsettings.extraction_folder_prefix, '')[0:3] open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a' ).write(r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' + r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n') if info: info = '_' + info dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys())) bestalgentries = bestalg.loadBestAlgorithm(dsList0.isBiobjective()) header = [] if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): header = [r'\#FEs/D'] headerHtml = ['<thead>\n<tr>\n<th>#FEs/D</th>\n'] for label in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % label) headerHtml.append('<td>%s</td>\n' % label) else: header = [r'$\Delta f_\mathrm{opt}$'] headerHtml = ['<thead>\n<tr>\n<th>Δ f</th>\n'] for label in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{\,}c@{\,}}{%s}' % label) headerHtml.append('<td>%s</td>\n' % label) header.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') headerHtml.append('<td>#succ</td>\n</tr>\n</thead>\n') for d in dimsOfInterest: # TODO set as input arguments table = [header] tableHtml = headerHtml extraeol = [r'\hline'] try: dictFunc0 = dictDim0[d].dictByFunc() dictFunc1 = dictDim1[d].dictByFunc() except KeyError: continue funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys())) nbtests = len(funcs) * 2. #len(dimsOfInterest) tableHtml.append('<tbody>\n') for f in sorted(funcs): tableHtml.append('<tr>\n') targets = targetsOfInterest((f, d)) targetf = targets[-1] bestalgentry = bestalgentries[(d, f)] curline = [r'${\bf f_{%d}}$' % f] curlineHtml = ['<th><b>f<sub>%d</sub></b></th>\n' % f] bestalgdata = bestalgentry.detERT(targets) bestalgevals, bestalgalgs = bestalgentry.detEvals(targets) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): # write ftarget:fevals for i in xrange(len(bestalgdata[:-1])): temp = "%.1e" % targetsOfInterest((f, d))[i] if temp[-2]=="0": temp = temp[:-2]+temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp,writeFEvalsMaxPrec(bestalgdata[i], 2))) curlineHtml.append('<td><i>%s</i>:%s</td>\n' % (temp, writeFEvalsMaxPrec(bestalgdata[i], 2))) temp = "%.1e" % targetsOfInterest((f, d))[-1] if temp[-2]=="0": temp = temp[:-2]+temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp,writeFEvalsMaxPrec(bestalgdata[-1], 2))) curlineHtml.append('<td><i>%s</i>:%s</td>\n' % (temp, writeFEvalsMaxPrec(bestalgdata[-1], 2))) else: # write #fevals of the reference alg for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curlineHtml.append('<td>%s</td>\n' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) curlineHtml.append('<td>%s</td>\n' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = numpy.sum(numpy.isnan(tmp) == False) curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) curlineHtml.append('<td>%d/%d</td>\n' % (tmp2, len(tmp))) else: curlineHtml.append('<td>%d</td>\n' % (tmp2)) table.append(curline[:]) tableHtml.extend(curlineHtml[:]) tableHtml.append('</tr>\n') extraeol.append('') rankdata0 = [] # never used # generate all data from ranksum test entries = [] ertdata = {} for nb, dsList in enumerate((dictFunc0, dictFunc1)): try: entry = dsList[f][0] # take the first DataSet, there should be only one? except KeyError: warnings.warn('data missing for data set ' + str(nb) + ' and function ' + str(f)) print('*** Warning: data missing for data set ' + str(nb) + ' and function ' + str(f) + '***') continue # TODO: problem here! ertdata[nb] = entry.detERT(targets) entries.append(entry) for _t in ertdata.values(): for _tt in _t: if _tt is None: raise ValueError if len(entries) < 2: # funcion not available for *both* algorithms continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following testres0vs1 = significancetest(entries[0], entries[1], targets) testresbestvs1 = significancetest(bestalgentry, entries[1], targets) testresbestvs0 = significancetest(bestalgentry, entries[0], targets) for nb, entry in enumerate(entries): tableHtml.append('<tr>\n') if nb == 0: curline = [r'1:\:\algorithmAshort\hspace*{\fill}'] curlineHtml = ['<th>1: %s</th>\n' % alg0] else: curline = [r'2:\:\algorithmBshort\hspace*{\fill}'] curlineHtml = ['<th>2: %s</th>\n' % alg1] #data = entry.detERT(targetsOfInterest) dispersion = [] data = [] evals = entry.detEvals(targets) for i in evals: succ = (numpy.isnan(i) == False) tmp = i.copy() tmp[succ==False] = entry.maxevals[numpy.isnan(i)] #set_trace() data.append(toolsstats.sp(tmp, issuccessful=succ)[0]) #if not any(succ): #set_trace() if any(succ): tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ==False], (10, 50, 90), samplesize)[0] dispersion.append((tmp2[-1]-tmp2[0])/2.) else: dispersion.append(None) if nb == 0: assert not isinstance(data, numpy.ndarray) data0 = data[:] # TODO: check if it is not an array, it's never used anyway? for i, dati in enumerate(data): z, p = testres0vs1[i] # TODO: there is something with the sign that I don't get # assign significance flag, which is the -log10(p) significance0vs1 = 0 if nb != 0: z = -z # the test is symmetric if nbtests * p < 0.05 and z > 0: significance0vs1 = -int(numpy.ceil(numpy.log10(min([1.0, nbtests * p])))) # this is the larger the more significant isBold = significance0vs1 > 0 alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' if numpy.isinf(bestalgdata[i]): # if the 2009 best did not solve the problem tmp = writeFEvalsMaxPrec(float(dati), 2) if not numpy.isinf(dati): tmpHtml = '<i>%s</i>' % (tmp) tmp = r'\textit{%s}' % (tmp) if isBold: tmp = r'\textbf{%s}' % tmp tmpHtml = '<b>%s</b>' % tmpHtml if dispersion[i] and numpy.isfinite(dispersion[i]): tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tmp)) tableentryHtml = (' (%s)' % tmp) else: # Formatting tmp = float(dati)/bestalgdata[i] assert not numpy.isnan(tmp) isscientific = False if tmp >= 1000: isscientific = True tableentry = writeFEvals2(tmp, 2, isscientific=isscientific) tableentry = writeFEvalsMaxPrec(tmp, 2) tableentryHtml = writeFEvalsMaxPrec(tmp, 2) if numpy.isinf(tmp) and i == len(data)-1: tableentry = (tableentry + r'\textit{%s}' % writeFEvals2(numpy.median(entry.maxevals), 2)) tableentryHtml = (tableentryHtml + ' <i>%s</i>' % writeFEvals2(numpy.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry tableentryHtml = '<b>%s</b>' % tableentryHtml elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore tableentry = r'\textit{%s}' % tableentry tableentryHtml = '<i>%s</i>' % tableentryHtml if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentryHtml += ' (%s)' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry tableentryHtml = '<b>%s</b>' % tableentryHtml elif 11 < 3 and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentryHtml = '<i>%s</i>' % tableentryHtml if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentryHtml += ' (%s)' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) tmpHtml = tableentryHtml.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) tmpHtml = list('<b>%s</b>' % i for i in tmpHtml) elif 11 < 3 and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tmpHtml = list('<i>%s</i>' % i for i in tmpHtml) tableentry = ' & .'.join(tmp) tableentryHtml = '.'.join(tmpHtml) if len(tmp) == 1: tableentry += '&' if dispersion[i] and numpy.isfinite(dispersion[i]/bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) tableentryHtml += ' (%s)' % writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 1) superscript = '' superscriptHtml = '' if nb == 0: z, p = testresbestvs0[i] else: z, p = testresbestvs1[i] #The conditions are now that ERT < ERT_best if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0. and z < 0.): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars superscriptHtml = '↑' else: superscript = r'\downarrow' #* nbstars superscriptHtml = '↓' # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(nbstars)) superscriptHtml += str(int(nbstars)) if superscript or significance0vs1: s = '' shtml = '' if significance0vs1 > 0: s = '\star' shtml = '★' if significance0vs1 > 1: s += str(significance0vs1) shtml += str(significance0vs1) s = r'$^{' + s + superscript + r'}$' shtml = '<sup>' + shtml + superscriptHtml + '</sup>' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s tableentryHtml += shtml tableentryHtml = tableentryHtml.replace('$\infty$', '∞') curlineHtml.append('<td>%s</td>\n' % tableentryHtml) curline.append(tableentry) #curline.append(tableentry) #if dispersion[i] is None or numpy.isinf(bestalgdata[i]): #curline.append('') #else: #tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2) #curline.append('(%s)' % tmp) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False)) curlineHtml.append('<td>%d' % numpy.sum(numpy.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curlineHtml.append('<td>%d' % 0) curline.append('/%d' % entry.nbRuns()) curlineHtml.append('/%d</td>\n' % entry.nbRuns()) table.append(curline[:]) tableHtml.extend(curlineHtml[:]) tableHtml.append('</tr>\n') extraeol.append('') extraeol[-1] = r'\hline' extraeol[-1] = '' outputfile = os.path.join(outputdir, 'pptable2_%02dD%s.tex' % (d, info)) spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len(targetsOfInterest) + '|@{}r@{}@{}l@{}' res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n' # open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res) #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res += tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() res = ("").join(str(item) for item in tableHtml) res = '<p><b>%d-D</b></p>\n<table>\n%s</table>\n' % (d, res) filename = os.path.join(outputdir, genericsettings.two_algorithm_file_name + '.html') lines = [] with open(filename) as infile: for line in infile: if '<!--pptable2Html-->' in line: lines.append(res) lines.append(line) with open(filename, 'w') as outfile: for line in lines: outfile.write(line) if verbose: print "Table written in %s" % outputfile