def computeERT(fevals, maxevals): data = fevals.copy() success = (numpy.isnan(data) == False) if any(numpy.isnan(data)): data[numpy.isnan(data)] = maxevals[numpy.isnan(data)] res = toolsstats.sp(data, issuccessful=success) return res[0]
def computeERT(hdata, maxevals): res = [] for i in hdata: data = i.copy() data = data[1:] succ = (np.isnan(data) == False) if any(np.isnan(data)): data[np.isnan(data)] = maxevals[np.isnan(data)] tmp = [i[0]] tmp.extend(toolsstats.sp(data, issuccessful=succ)) res.append(tmp) return np.vstack(res)
def generateData(dataSet, target): """Returns an array of results to be plotted. Oth column is ert, 1st is the success rate, 2nd the number of successes, 3rd the mean of the number of function evaluations, and 4th the median of number of function evaluations of successful runs or numpy.nan. """ res = [] data = dataSet.detEvals([target])[0] succ = (numpy.isnan(data) == False) data[numpy.isnan(data)] = dataSet.maxevals[numpy.isnan(data)] res.extend(toolsstats.sp(data, issuccessful=succ, allowinf=False)) res.append(numpy.mean(data)) if res[2] > 0: res.append(toolsstats.prctile(data[succ], 50)[0]) else: res.append(numpy.nan) res[3] = numpy.max(dataSet.maxevals) return res
def generateData(dataSet, targetFuncValue): """Returns an array of results to be plotted. 1st column is ert, 2nd is the number of success, 3rd the success rate, 4th the sum of the number of function evaluations, and finally the median on successful runs. """ res = [] data = [] it = iter(reversed(dataSet.evals)) i = it.next() prev = np.array([np.nan] * len(i)) while i[0] <= targetFuncValue: prev = i try: i = it.next() except StopIteration: break data = prev[1:].copy() # keep only the number of function evaluations. succ = (np.isnan(data) == False) if succ.any(): med = toolsstats.prctile(data[succ], 50)[0] #Line above was modified at rev 3050 to make sure that we consider only #successful trials in the median else: med = np.nan data[np.isnan(data)] = dataSet.maxevals[np.isnan(data)] res = [] res.extend(toolsstats.sp(data, issuccessful=succ, allowinf=False)) res.append(np.mean(data)) #mean(FE) res.append(med) return np.array(res)
def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True): """One table per dimension, modified to fit in 1 page per table.""" #TODO: method is long, split if possible dictDim0 = dsList0.dictByDim() dictDim1 = dsList1.dictByDim() alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3] alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3] open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write( r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' + r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n') if info: info = '_' + info dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys())) if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() header = [] if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): header = [r'\#FEs/D'] for label in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % label) else: header = [r'$\Delta f_\mathrm{opt}$'] for label in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{\,}c@{\,}}{%s}' % label) header.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') for d in dimsOfInterest: # TODO set as input arguments table = [header] extraeol = [r'\hline'] try: dictFunc0 = dictDim0[d].dictByFunc() dictFunc1 = dictDim1[d].dictByFunc() except KeyError: continue funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys())) nbtests = len(funcs) * 2. #len(dimsOfInterest) for f in sorted(funcs): targets = targetsOfInterest((f, d)) targetf = targets[-1] bestalgentry = bestalg.bestalgentries2009[(d, f)] curline = [r'${\bf f_{%d}}$' % f] bestalgdata = bestalgentry.detERT(targets) bestalgevals, bestalgalgs = bestalgentry.detEvals(targets) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): # write ftarget:fevals for i in range(len(bestalgdata[:-1])): temp = "%.1e" % targetsOfInterest((f, d))[i] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append( r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp, writeFEvalsMaxPrec(bestalgdata[i], 2))) temp = "%.1e" % targetsOfInterest((f, d))[-1] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp, writeFEvalsMaxPrec(bestalgdata[-1], 2))) else: # write #fevals of the reference alg for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = numpy.sum(numpy.isnan(tmp) == False) curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) table.append(curline[:]) extraeol.append('') rankdata0 = [] # never used # generate all data from ranksum test entries = [] ertdata = {} for nb, dsList in enumerate((dictFunc0, dictFunc1)): try: entry = dsList[f][ 0] # take the first DataSet, there should be only one? except KeyError: warnings.warn('data missing for data set ' + str(nb) + ' and function ' + str(f)) print('*** Warning: data missing for data set ' + str(nb) + ' and function ' + str(f) + '***') continue # TODO: problem here! ertdata[nb] = entry.detERT(targets) entries.append(entry) for _t in ertdata.values(): for _tt in _t: if _tt is None: raise ValueError if len(entries) < 2: # funcion not available for *both* algorithms continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following testres0vs1 = significancetest(entries[0], entries[1], targets) testresbestvs1 = significancetest(bestalgentry, entries[1], targets) testresbestvs0 = significancetest(bestalgentry, entries[0], targets) for nb, entry in enumerate(entries): if nb == 0: curline = [r'1:\:\algorithmAshort\hspace*{\fill}'] else: curline = [r'2:\:\algorithmBshort\hspace*{\fill}'] #data = entry.detERT(targetsOfInterest) dispersion = [] data = [] evals = entry.detEvals(targets) for i in evals: succ = (numpy.isnan(i) == False) tmp = i.copy() tmp[succ == False] = entry.maxevals[numpy.isnan(i)] #set_trace() data.append(toolsstats.sp(tmp, issuccessful=succ)[0]) #if not any(succ): #set_trace() if any(succ): tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ == False], (10, 50, 90), samplesize)[0] dispersion.append((tmp2[-1] - tmp2[0]) / 2.) else: dispersion.append(None) if nb == 0: assert not isinstance(data, numpy.ndarray) data0 = data[:] # TODO: check if it is not an array, it's never used anyway? for i, dati in enumerate(data): z, p = testres0vs1[ i] # TODO: there is something with the sign that I don't get # assign significance flag, which is the -log10(p) significance0vs1 = 0 if nb != 0: z = -z # the test is symmetric if nbtests * p < 0.05 and z > 0: significance0vs1 = -int( numpy.ceil(numpy.log10(min([ 1.0, nbtests * p ])))) # this is the larger the more significant isBold = significance0vs1 > 0 alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' if numpy.isinf( bestalgdata[i] ): # if the 2009 best did not solve the problem tmp = writeFEvalsMaxPrec(float(dati), 2) if not numpy.isinf(dati): tmp = r'\textit{%s}' % (tmp) if isBold: tmp = r'\textbf{%s}' % tmp if dispersion[i] and numpy.isfinite(dispersion[i]): tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tmp)) else: # Formatting tmp = float(dati) / bestalgdata[i] assert not numpy.isnan(tmp) isscientific = False if tmp >= 1000: isscientific = True tableentry = writeFEvals2(tmp, 2, isscientific=isscientific) tableentry = writeFEvalsMaxPrec(tmp, 2) if numpy.isinf(tmp) and i == len(data) - 1: tableentry = ( tableentry + r'\textit{%s}' % writeFEvals2(numpy.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore tableentry = r'\textit{%s}' % tableentry if dispersion[i] and numpy.isfinite( dispersion[i] / bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry if dispersion[i] and numpy.isfinite( dispersion[i] / bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) elif 11 < 3 and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tableentry = ' & .'.join(tmp) if len(tmp) == 1: tableentry += '&' if dispersion[i] and numpy.isfinite( dispersion[i] / bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) superscript = '' if nb == 0: z, p = testresbestvs0[i] else: z, p = testresbestvs1[i] #The conditions are now that ERT < ERT_best if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0. and z < 0.): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars else: superscript = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(nbstars)) if superscript or significance0vs1: s = '' if significance0vs1 > 0: s = '\star' if significance0vs1 > 1: s += str(significance0vs1) s = r'$^{' + s + superscript + r'}$' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s curline.append(tableentry) #curline.append(tableentry) #if dispersion[i] is None or numpy.isinf(bestalgdata[i]): #curline.append('') #else: #tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2) #curline.append('(%s)' % tmp) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curline.append('/%d' % entry.nbRuns()) table.append(curline[:]) extraeol.append('') extraeol[-1] = r'\hline' extraeol[-1] = '' outputfile = os.path.join(outputdir, 'pptable2_%02dD%s.tex' % (d, info)) spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len( targetsOfInterest) + '|@{}r@{}@{}l@{}' res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels( alg0) + '\n' res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels( alg1) + '\n' # open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res) #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res += tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() if verbose: print("Table written in %s" % outputfile)
def main(dictAlg, sortedAlgs, outputdir='.', verbose=True, function_targets_line=True): # [1, 13, 101] """Generate one table per func with results of multiple algorithms.""" """Difference with the first version: * numbers aligned using the decimal separator * premices for dispersion measure * significance test against best algorithm * table width... Takes ``targetsOfInterest`` from this file as "input argument" to compute the desired target values. ``targetsOfInterest`` might be configured via config. """ # TODO: method is long, terrible to read, split if possible if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() # Sort data per dimension and function dictData = {} dsListperAlg = list(dictAlg[i] for i in sortedAlgs) for n, entries in enumerate(dsListperAlg): tmpdictdim = entries.dictByDim() for d in tmpdictdim: tmpdictfun = tmpdictdim[d].dictByFunc() for f in tmpdictfun: dictData.setdefault((d, f), {})[n] = tmpdictfun[f] nbtests = len(dictData) for df in dictData: # Generate one table per df # first update targets for each dimension-function pair if needed: targets = targetsOfInterest((df[1], df[0])) targetf = targets[-1] # best 2009 refalgentry = bestalg.bestalgentries2009[df] refalgert = refalgentry.detERT(targets) refalgevals = (refalgentry.detEvals((targetf, ))[0][0]) refalgnbruns = len(refalgevals) refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False) # Process the data # The following variables will be lists of elements each corresponding # to an algorithm algnames = [] #algdata = [] algerts = [] algevals = [] algdisp = [] algnbsucc = [] algnbruns = [] algmedmaxevals = [] algmedfinalfunvals = [] algtestres = [] algentries = [] for n in sorted(dictData[df].keys()): entries = dictData[df][n] # the number of datasets for a given dimension and function (df) # should be strictly 1. TODO: find a way to warn # TODO: do this checking before... why wasn't it triggered by ppperprof? if len(entries) > 1: txt = ("There is more than a single entry associated with " "folder %s on %d-D f%d." % (sortedAlgs[n], df[0], df[1])) raise Exception(txt) entry = entries[0] algentries.append(entry) algnames.append(sortedAlgs[n]) evals = entry.detEvals(targets) #tmpdata = [] tmpdisp = [] tmpert = [] for i, e in enumerate(evals): succ = (numpy.isnan(e) == False) ec = e.copy( ) # note: here was the previous bug (changes made in e also appeared in evals !) ec[succ == False] = entry.maxevals[succ == False] ert = toolsstats.sp(ec, issuccessful=succ)[0] #tmpdata.append(ert/refalgert[i]) if succ.any(): tmp = toolsstats.drawSP(ec[succ], entry.maxevals[succ == False], [10, 50, 90], samplesize=samplesize)[0] tmpdisp.append((tmp[-1] - tmp[0]) / 2.) else: tmpdisp.append(numpy.nan) tmpert.append(ert) algerts.append(tmpert) algevals.append(evals) #algdata.append(tmpdata) algdisp.append(tmpdisp) algmedmaxevals.append(numpy.median(entry.maxevals)) algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) #algmedmaxevals.append(numpy.median(entry.maxevals)/df[0]) #algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) algtestres.append(significancetest(refalgentry, entry, targets)) # determine success probability for Df = 1e-8 e = entry.detEvals((targetf, ))[0] algnbsucc.append(numpy.sum(numpy.isnan(e) == False)) algnbruns.append(len(e)) # Process over all data # find best values... nalgs = len(dictData[df]) maxRank = 1 + numpy.floor( 0.14 * nalgs) # number of algs to be displayed in bold isBoldArray = [] # Point out the best values algfinaldata = [ ] # Store median function values/median number of function evaluations tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank) for i, erts in enumerate(algerts): tmp = [] for j, ert in enumerate(erts): # algi targetj tmp.append(i in tmptop[j] or (nalgs > 7 and algerts[i][j] <= 3. * refalgert[j])) isBoldArray.append(tmp) algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i])) # significance test of best given algorithm against all others best_alg_idx = numpy.array(algerts).argsort(0)[ 0, :] # indexed by target index significance_versus_others = significance_all_best_vs_other( algentries, targets, best_alg_idx)[0] # Create the table table = [] spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % ( len(targets) ) # in case StrLeft not working: replaced c@{} with l@{ } spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % ( len(targets) ) # in case StrLeft not working: replaced c@{} with l@{ } extraeol = [] # Generate header lines if with_table_heading: header = funInfos[df[1]] if funInfos else 'f%d' % df[1] table.append([ r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}' % (2 * len(targets) + 2, header) ]) extraeol.append('') if function_targets_line is True or (function_targets_line and df[1] in function_targets_line): if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): curline = [r'\#FEs/D'] for i in targetsOfInterest.labels(): curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' % i) else: curline = [r'$\Delta f_\mathrm{opt}$'] for t in targets: curline.append( r'\multicolumn{2}{@{\,}X@{\,}}{%s}' % writeFEvals2(t, precision=1, isscientific=True)) # curline.append(r'\multicolumn{2}{@{\,}X@{}|}{%s}' # % writeFEvals2(targets[-1], precision=1, isscientific=True)) curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') table.append(curline) extraeol.append(r'\hline') # extraeol.append(r'\hline\arrayrulecolor{tableShade}') curline = [r'ERT$_{\text{best}}$' ] if with_table_heading else [r'\textbf{f%d}' % df[1]] if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): # write ftarget:fevals for i in range(len(refalgert[:-1])): temp = "%.1e" % targetsOfInterest((df[1], df[0]))[i] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append( r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp, writeFEvalsMaxPrec(refalgert[i], 2))) temp = "%.1e" % targetsOfInterest((df[1], df[0]))[-1] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp, writeFEvalsMaxPrec(refalgert[-1], 2))) else: # write #fevals of the reference alg for i in refalgert[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(refalgert[-1], 2)) # write the success ratio for the reference alg tmp2 = numpy.sum( numpy.isnan(refalgevals) == False) # count the nb of success curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(refalgevals)) table.append(curline[:]) extraeol.append('') #for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))): #print i, gna, gno #set_trace() # Format data #if df == (5, 17): #set_trace() header = r'\providecommand{\ntables}{7}' for i, alg in enumerate(algnames): #algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames, #data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres): commandname = r'\alg%stables' % numtotext(i) # header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg))) header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % ( commandname, str_to_latex(strip_pathname2(alg))) curline = [ commandname + r'\hspace*{\fill}' ] # each list element becomes a &-separated table entry? for j, tmp in enumerate( zip( algerts[i], algdisp[i], # j is target index isBoldArray[i], algtestres[i])): ert, dispersion, isBold, testres = tmp alignment = '@{\,}X@{\,}' if j == len(algerts[i]) - 1: alignment = '@{\,}X@{\,}|' data = ert / refalgert[j] # write star for significance against all other algorithms str_significance_subsup = '' if (len(best_alg_idx) > 0 and len(significance_versus_others) > 0 and i == best_alg_idx[j] and nbtests * significance_versus_others[j][1] < 0.05): logp = -numpy.ceil( numpy.log10( nbtests * significance_versus_others[j][1])) str_significance_subsup = r"^{%s%s}" % ( significance_vs_others_symbol, str(int(logp)) if logp > 1 else '') # moved out of the above else: this was a bug!? z, p = testres if (nbtests * p) < 0.05 and data < 1. and z < 0.: if not numpy.isinf(refalgert[j]): tmpevals = algevals[i][j].copy() tmpevals[numpy.isnan(tmpevals)] = algentries[ i].maxevals[numpy.isnan(tmpevals)] bestevals = refalgentry.detEvals(targets) bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0]) bestevals[numpy.isnan( bestevals)] = refalgentry.maxevals[bestalgalg][ numpy.isnan(bestevals)] tmpevals = numpy.array(sorted( tmpevals))[0:min(len(tmpevals), len(bestevals))] bestevals = numpy.array(sorted( bestevals))[0:min(len(tmpevals), len(bestevals))] #The conditions are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) # tmp2[-1] += r'$^{%s}$' % superscript str_significance_subsup += r'_{%s%s}' % ( significance_vs_ref_symbol, str(int(nbstars)) if nbstars > 1 else '') if str_significance_subsup: str_significance_subsup = '$%s$' % str_significance_subsup # format number in variable data if numpy.isnan(data): curline.append(r'\multicolumn{2}{%s}{.}' % alignment) else: if numpy.isinf(refalgert[j]): curline.append( r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}' % (alignment, writeFEvalsMaxPrec(algerts[i][j], 2), writeFEvalsMaxPrec(dispersion, precdispersion), str_significance_subsup)) continue tmp = writeFEvalsMaxPrec(data, precfloat, maxfloatrepr=maxfloatrepr) if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation if numpy.isinf(data) and j == len(algerts[i]) - 1: tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec( algfinaldata[i][1], 0, maxfloatrepr=maxfloatrepr) else: tmp = writeFEvalsMaxPrec(data, precscien, maxfloatrepr=data) if isBold: tmp = r'\textbf{%s}' % tmp if not numpy.isnan(dispersion): tmpdisp = dispersion / refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp += r'\mbox{\tiny (%s)}' % tmpdisp curline.append( r'\multicolumn{2}{%s}{%s%s}' % (alignment, tmp, str_significance_subsup)) else: tmp2 = tmp.split('.', 1) if len(tmp2) < 2: tmp2.append('') else: tmp2[-1] = '.' + tmp2[-1] if isBold: tmp3 = [] for k in tmp2: tmp3.append(r'\textbf{%s}' % k) tmp2 = tmp3 if not numpy.isnan(dispersion): tmpdisp = dispersion / refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.01: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp)) tmp2[-1] += str_significance_subsup curline.extend(tmp2) curline.append('%d' % algnbsucc[i]) curline.append('/%d' % algnbruns[i]) table.append(curline) extraeol.append('') # Write table res = tableXLaTeX(table, spec=spec, extraeol=extraeol) try: filename = os.path.join( outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0])) f = open(filename, 'w') f.write(header + '\n') f.write(res) if verbose: print('Wrote table in %s' % filename) except: raise else: f.close()
def main(dsList, dimsOfInterest, outputdir, info='', verbose=True): """Generate a table of ratio ERT/ERTbest vs target precision. 1 table per dimension will be generated. Rank-sum tests table on "Final Data Points" for only one algorithm. that is, for example, using 1/#fevals(ftarget) if ftarget was reached and -f_final otherwise as input for the rank-sum test, where obviously the larger the better. """ #TODO: check that it works for any reference algorithm? #in the following the reference algorithm is the one given in #bestalg.bestalgentries which is the virtual best of BBOB dictDim = dsList.dictByDim() targetf = 1e-8 if info: info = '_' + info # insert a separator between the default file name and the additional # information string. dims = set(dictDim.keys()) if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): header = [r'\#FEs/D'] for i in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % i) else: header = [r'$\Delta f$'] for i in targetsOfInterest.target_values: header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' % (int(np.log10(i)))) header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}') for d in dimsOfInterest: table = [header] extraeol = [r'\hline'] try: dictFunc = dictDim[d].dictByFunc() except KeyError: continue funcs = set(dictFunc.keys()) nbtests = float(len(funcs)) # #funcs tests times one algorithm for f in sorted(funcs): bestalgentry = bestalg.bestalgentries2009[(d, f)] curline = [r'${\bf f_{%d}}$' % f] bestalgdata = bestalgentry.detERT(targetsOfInterest((f, d))) bestalgevals, bestalgalgs = bestalgentry.detEvals( targetsOfInterest((f, d))) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): #write ftarget:fevals for i in range(len(bestalgdata[:-1])): temp = "%.1e" % targetsOfInterest((f, d))[i] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append( r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp, writeFEvalsMaxPrec(bestalgdata[i], 2))) temp = "%.1e" % targetsOfInterest((f, d))[-1] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp, writeFEvalsMaxPrec(bestalgdata[-1], 2))) #success targetf = targetsOfInterest((f, d))[-1] else: # write #fevals of the reference alg for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) # write the success ratio for the reference alg tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = np.sum(np.isnan(tmp) == False) # count the nb of success curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) table.append(curline[:]) extraeol.append('') # generate all data for ranksum test assert len(dictFunc[f]) == 1 entry = dictFunc[f][0] # take the first element ertdata = entry.detERT(targetsOfInterest((f, d))) testresbestvs1 = significancetest(bestalgentry, entry, targetsOfInterest((f, d))) #for nb, entry in enumerate(entries): #curline = [r'\algshort\hspace*{\fill}'] curline = [''] #data = entry.detERT(targetsOfInterest) evals = entry.detEvals(targetsOfInterest((f, d))) dispersion = [] data = [] for i in evals: succ = (np.isnan(i) == False) tmp = i.copy() tmp[succ == False] = entry.maxevals[np.isnan(i)] #set_trace() # TODO: what is the difference between data and ertdata? data.append(toolsstats.sp(tmp, issuccessful=succ)[0]) #if not any(succ): #set_trace() if any(succ): tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ == False], (10, 50, 90), samplesize)[0] dispersion.append((tmp2[-1] - tmp2[0]) / 2.) else: dispersion.append(None) assert data == ertdata for i, ert in enumerate(data): alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' nbstars = 0 z, p = testresbestvs1[i] if ert - bestalgdata[i] < 0. and not np.isinf(bestalgdata[i]): evals = entry.detEvals([targetsOfInterest((f, d))[i]])[0] evals[np.isnan(evals)] = entry.maxevals[np.isnan(evals)] bestevals = bestalgentry.detEvals( [targetsOfInterest((f, d))[i]]) bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0]) bestevals[np.isnan(bestevals)] = bestalgentry.maxevals[ bestalgalg][np.isnan(bestevals)] evals = np.array( sorted(evals))[0:min(len(evals), len(bestevals))] bestevals = np.array( sorted(bestevals))[0:min(len(evals), len(bestevals))] #The conditions for significance are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if ((nbtests * p) < 0.05 and ert - bestalgdata[i] < 0. and z < 0. and (np.isinf(bestalgdata[i]) or all(evals < bestevals))): nbstars = -np.ceil(np.log10(nbtests * p)) isBold = False if nbstars > 0: isBold = True if np.isinf(bestalgdata[i] ): # if the best did not solve the problem tmp = writeFEvalsMaxPrec(float(ert), 2) if not np.isinf(ert): tmp = r'\textit{%s}' % (tmp) if isBold: tmp = r'\textbf{%s}' % tmp tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tmp)) else: # Formatting tmp = float(ert) / bestalgdata[i] assert not np.isnan(tmp) tableentry = writeFEvalsMaxPrec(tmp, 2) if np.isinf(tmp) and i == len(data) - 1: tableentry = ( tableentry + r'\textit{%s}' % writeFEvals2(np.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3: # and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (np.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3: # and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) elif 11 < 3: # and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tableentry = ' & .'.join(tmp) if len(tmp) == 1: tableentry += '&' superscript = '' if nbstars > 0: #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars else: superscript = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(min((9, nbstars)))) # superscript += str(int(nbstars)) #if superscript or significance0vs1: #s = '' #if significance0vs1 > 0: #s = '\star' #if significance0vs1 > 1: #s += str(significance0vs1) #s = r'$^{' + s + superscript + r'}$' #if tableentry.endswith('}'): #tableentry = tableentry[:-1] + s + r'}' #else: #tableentry += s if dispersion[i]: if not np.isinf(bestalgdata[i]): tmp = writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) else: tmp = writeFEvalsMaxPrec(dispersion[i], 1) tableentry += (r'${\scriptscriptstyle(%s)}$' % tmp) if superscript: s = r'$^{' + superscript + r'}$' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s curline.append(tableentry) #curline.append(tableentry) #if dispersion[i] is None or np.isinf(bestalgdata[i]): #curline.append('') #else: #tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2) #curline.append('(%s)' % tmp) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % np.sum(np.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curline.append('/%d' % entry.nbRuns()) table.append(curline[:]) extraeol.append(r'\hline') extraeol[-1] = '' outputfile = os.path.join(outputdir, 'pptable_%02dD%s.tex' % (d, info)) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): spec = r'@{}c@{}|' + '*{%d}{@{ }r@{}@{}l@{}}' % len( targetsOfInterest) + '|@{}r@{}@{}l@{}' else: spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len( targetsOfInterest) + '|@{}r@{}@{}l@{}' #res = r'\providecommand{\algshort}{%s}' % alg1 + '\n' #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res = tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() if verbose: print("Table written in %s" % outputfile)
def annotate(entry0, entry1, dim, minfvalue=1e-8, nbtests=1): """Display some annotations associated to the graphs generated.""" isEarlyStop = False ha = 'left' va = 'center' lastfvalue = min(entry0.evals[-1][0], entry1.evals[-1][0]) if minfvalue < lastfvalue: isEarlyStop = True #ha = 'center' #va = 'top' if not minfvalue or minfvalue < lastfvalue: minfvalue = lastfvalue line = [] data0 = entry0.detEvals([minfvalue])[0] evals0 = data0.copy() succ = (np.isnan(evals0) == False) evals0[np.isnan(evals0)] = entry0.maxevals[np.isnan(evals0)] line.append(toolsstats.sp(evals0, issuccessful=succ)) data1 = entry1.detEvals([minfvalue])[0] evals1 = data1.copy() succ = (np.isnan(evals1) == False) evals1[np.isnan(evals1)] = entry1.maxevals[np.isnan(evals1)] line.append(toolsstats.sp(evals1, issuccessful=succ)) # What's the situation? txt = '%dD' % dim if (line[0][2] > 0 and line[1][2] > 0 and line[1][2] < 10): tmp = str(int(line[1][2])) tmp2 = str(int(line[0][2])) txt = tmp + '/' + tmp2 dims = dimension_index ax = plt.gca() assert line[0][2] > 0 or line[1][2] > 0 signdata = line[1][0] - line[0][0] if line[0][2] > 0 and line[1][2] > 0: trans = ax.transData annotcoord = [minfvalue, line[1][0] / line[0][0]] elif line[0][2] == 0: trans = blend(ax.transData, ax.transAxes) annotcoord = [ minfvalue, -line[1][1] / 2 + 0.5 + offset * (5 - dims[dim]) ] #if va == 'top': # va = 'bottom' elif line[1][2] == 0: trans = blend(ax.transData, ax.transAxes) annotcoord = [ minfvalue, line[0][1] / 2 + 0.5 - offset * (5 - dims[dim]) ] plt.text(annotcoord[0], annotcoord[1], txt, horizontalalignment=ha, verticalalignment=va, transform=trans) #ranksum test line0 = np.power(data0, -1.) line0[np.isnan(line0)] = -entry0.finalfunvals[np.isnan(line0)] line1 = np.power(data1, -1.) line1[np.isnan(line1)] = -entry1.finalfunvals[np.isnan(line1)] # one-tailed statistics: scipy.stats.mannwhitneyu, two-tailed statistics: scipy.stats.ranksumtest z, p = ranksumtest(line0, line1) # Set the correct line in data0 and data1 nbstars = 0 # sign of z-value and data must agree if ((nbtests * p) < 0.05 and (z * signdata) > 0): nbstars = np.min([5, -np.ceil(np.log10(nbtests * p + 1e-99))]) if nbstars > 0: xstars = annotcoord[0] * np.power(incrstars, np.arange( 1., 1. + nbstars)) # the additional slicing [0:int(nbstars)] is due to # np.arange(1., 1. - 0.1 * nbstars, -0.1) not having the right number # of elements due to numerical error ystars = [annotcoord[1]] * nbstars try: h = plt.plot(xstars, ystars, marker='*', ls='', color='w', markersize=5 * linewidth, markeredgecolor='k', markerfacecolor='None', zorder=20, markeredgewidth=0.4 * linewidth, transform=trans, clip_on=False) except KeyError: #Version problem h = plt.plot(xstars, ystars, marker='+', ls='', color='w', markersize=2.5 * linewidth, markeredgecolor='k', zorder=20, markeredgewidth=0.2 * linewidth, transform=trans, clip_on=False)
def plot(dsList, param='dim', targets=(10., 1., 1e-1, 1e-2, 1e-3, 1e-5, 1e-8)): """Generate plot of ERT vs param.""" dictparam = dsList.dictByParam(param) params = sorted(dictparam) # sorted because we draw lines # generate plot from dsList res = [] # collect data rawdata = {} for p in params: assert len(dictparam[p]) == 1 rawdata[p] = dictparam[p][0].detEvals(targets) # expect dictparam[p] to have only one element # plot lines for ERT xpltdata = params for i, t in enumerate(targets): ypltdata = [] for p in params: data = rawdata[p][i] unsucc = np.isnan(data) assert len(dictparam[p]) == 1 data[unsucc] = dictparam[p][0].maxevals # compute ERT ert, srate, succ = toolsstats.sp(data, issuccessful=(unsucc == False)) ypltdata.append(ert) res.extend(plt.plot(xpltdata, ypltdata, markersize=20, zorder=len(targets) - i, **styles[i])) # for the legend plt.plot([], [], markersize=10, label=' %+d' % (np.log10(targets[i])), **styles[i]) # plot median of successful runs for hardest target with a success for p in params: for i, t in enumerate(reversed(targets)): # targets has to be from hardest to easiest data = rawdata[p][i] data = data[np.isnan(data) == False] if len(data) > 0: median = toolsstats.prctile(data, 50.)[0] res.extend(plt.plot(p, median, styles[i]['color'], **medmarker)) break # plot average number of function evaluations for the hardest target xpltdata = [] ypltdata = [] for p in params: data = rawdata[p][0] # first target xpltdata.append(p) if (np.isnan(data) == False).all(): tmpdata = data.copy() assert len(dictparam[p]) == 1 tmpdata[np.isnan(data)] = dictparam[p][0].maxevals[np.isnan(data)] tmp = np.mean(tmpdata) else: tmp = np.nan # Check what happens when plotting NaN ypltdata.append(tmp) res.extend(plt.plot(xpltdata, ypltdata, **avgstyle)) # display numbers of successes for hardest target where there is still one success for p in params: for i, t in enumerate(targets): # targets has to be from hardest to easiest data = rawdata[p][i] unsucc = np.isnan(data) assert len(dictparam[p]) == 1 data[unsucc] = dictparam[p][0].maxevals # compute ERT ert, srate, succ = toolsstats.sp(data, issuccessful=(unsucc == False)) if srate == 1.: break elif succ > 0: res.append(plt.text(p, ert * 1.85, "%d" % succ, axes=plt.gca(), horizontalalignment="center", verticalalignment="bottom")) break return res