def _plotERTDistr(dsList, target, **plotArgs): """This method is obsolete, should be removed? The replacement for simulated runlengths is in pprldmany? Creates simulated run time distributions (it is not an ERT distribution) from a DataSetList. :keyword DataSet dsList: Input data sets :keyword dict target: target precision :keyword plotArgs: keyword arguments to pass to plot command :return: resulting plot. Details: calls ``plotECDF``. """ x = [] nn = 0 samplesize = genericsettings.simulated_runlength_bootstrap_sample_size # samplesize should be at least 1000 percentiles = 0.5 # could be anything... for i in dsList: # funcs.add(i.funcId) for j in i.evals: if j[0] <= target[i.funcId]: runlengthsucc = j[1:][np.isfinite(j[1:])] runlengthunsucc = i.maxevals[np.isnan(j[1:])] tmp = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=percentiles, samplesize=samplesize) x.extend(tmp[1]) break nn += samplesize res = plotECDF(x, nn, **plotArgs) return res
def _treat(ds): # Rec array: http://docs.scipy.org/doc/numpy/user/basics.rec.html bestentry = bestalg.bestalgentries2009[(ds.dim, ds.funcId)] bestert = bestentry.detERT(targets) bestevals, bestalgs = bestentry.detEvals(targets) bestfinaldata = bestentry.detEvals([finaltarget])[0][0] ert = ds.detERT(targets) evals = ds.detEvals(targets) finaldata = ds.detEvals([finaltarget])[0] dtype = [] bestdtype = [] for i, t in enumerate(targets): dtype.append((('ERT ratio (iq 10-90), df=%e' % t, 'df=%e' % t), '2f')) bestdtype.append((('best ERT df=%e' % t, 'df=%e' % t), 'f')) dtype.append( (('nb success final target=%e' % t, 'finaltarget=%e' % t), 'i8')) dtype.append(('nbruns', 'i8')) bestdtype.append( (('nb success finaltarget=%e' % t, 'finaltarget=%e' % t), 'i8')) bestdtype.append(('nbruns', 'i8')) besttable = np.zeros(1, dtype=bestdtype) wholetable = np.zeros(1, dtype=dtype) table = wholetable[0] bestdata = list() bestdata.extend(bestert) bestdata.append(np.sum(np.isnan(bestfinaldata) == False)) bestdata.append(len(bestfinaldata)) besttable[0] = tuple(bestdata) data = list() for i, e in enumerate(evals): # loop over targets unsucc = np.isnan(e) bt = toolsstats.drawSP(e[unsucc == False], ds.maxevals[unsucc], (10, 90), samplesize)[0] data.append((ert[i] / bestert[i], (bt[-1] - bt[0]) / 2. / bestert[i])) data.append(np.sum(np.isnan(finaldata) == False)) data.append(ds.nbRuns()) table = tuple(data) # fill with tuple not list nor array! # TODO: do the significance test thing here. return besttable, wholetable
def main(dsList0, dsList1, dimsOfInterest, outputdir, info='', verbose=True): """One table per dimension, modified to fit in 1 page per table.""" #TODO: method is long, split if possible dictDim0 = dsList0.dictByDim() dictDim1 = dsList1.dictByDim() alg0 = set(i[0] for i in dsList0.dictByAlg().keys()).pop()[0:3] alg1 = set(i[0] for i in dsList1.dictByAlg().keys()).pop()[0:3] open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write( r'\providecommand{\algorithmAshort}{%s}' % writeLabels(alg0) + '\n' + r'\providecommand{\algorithmBshort}{%s}' % writeLabels(alg1) + '\n') if info: info = '_' + info dims = set.intersection(set(dictDim0.keys()), set(dictDim1.keys())) if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() header = [] if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): header = [r'\#FEs/D'] for label in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % label) else: header = [r'$\Delta f_\mathrm{opt}$'] for label in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{\,}c@{\,}}{%s}' % label) header.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') for d in dimsOfInterest: # TODO set as input arguments table = [header] extraeol = [r'\hline'] try: dictFunc0 = dictDim0[d].dictByFunc() dictFunc1 = dictDim1[d].dictByFunc() except KeyError: continue funcs = set.union(set(dictFunc0.keys()), set(dictFunc1.keys())) nbtests = len(funcs) * 2. #len(dimsOfInterest) for f in sorted(funcs): targets = targetsOfInterest((f, d)) targetf = targets[-1] bestalgentry = bestalg.bestalgentries2009[(d, f)] curline = [r'${\bf f_{%d}}$' % f] bestalgdata = bestalgentry.detERT(targets) bestalgevals, bestalgalgs = bestalgentry.detEvals(targets) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): # write ftarget:fevals for i in range(len(bestalgdata[:-1])): temp = "%.1e" % targetsOfInterest((f, d))[i] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append( r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp, writeFEvalsMaxPrec(bestalgdata[i], 2))) temp = "%.1e" % targetsOfInterest((f, d))[-1] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp, writeFEvalsMaxPrec(bestalgdata[-1], 2))) else: # write #fevals of the reference alg for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = numpy.sum(numpy.isnan(tmp) == False) curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) table.append(curline[:]) extraeol.append('') rankdata0 = [] # never used # generate all data from ranksum test entries = [] ertdata = {} for nb, dsList in enumerate((dictFunc0, dictFunc1)): try: entry = dsList[f][ 0] # take the first DataSet, there should be only one? except KeyError: warnings.warn('data missing for data set ' + str(nb) + ' and function ' + str(f)) print('*** Warning: data missing for data set ' + str(nb) + ' and function ' + str(f) + '***') continue # TODO: problem here! ertdata[nb] = entry.detERT(targets) entries.append(entry) for _t in ertdata.values(): for _tt in _t: if _tt is None: raise ValueError if len(entries) < 2: # funcion not available for *both* algorithms continue # TODO: check which one is missing and make sure that what is there is displayed properly in the following testres0vs1 = significancetest(entries[0], entries[1], targets) testresbestvs1 = significancetest(bestalgentry, entries[1], targets) testresbestvs0 = significancetest(bestalgentry, entries[0], targets) for nb, entry in enumerate(entries): if nb == 0: curline = [r'1:\:\algorithmAshort\hspace*{\fill}'] else: curline = [r'2:\:\algorithmBshort\hspace*{\fill}'] #data = entry.detERT(targetsOfInterest) dispersion = [] data = [] evals = entry.detEvals(targets) for i in evals: succ = (numpy.isnan(i) == False) tmp = i.copy() tmp[succ == False] = entry.maxevals[numpy.isnan(i)] #set_trace() data.append(toolsstats.sp(tmp, issuccessful=succ)[0]) #if not any(succ): #set_trace() if any(succ): tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ == False], (10, 50, 90), samplesize)[0] dispersion.append((tmp2[-1] - tmp2[0]) / 2.) else: dispersion.append(None) if nb == 0: assert not isinstance(data, numpy.ndarray) data0 = data[:] # TODO: check if it is not an array, it's never used anyway? for i, dati in enumerate(data): z, p = testres0vs1[ i] # TODO: there is something with the sign that I don't get # assign significance flag, which is the -log10(p) significance0vs1 = 0 if nb != 0: z = -z # the test is symmetric if nbtests * p < 0.05 and z > 0: significance0vs1 = -int( numpy.ceil(numpy.log10(min([ 1.0, nbtests * p ])))) # this is the larger the more significant isBold = significance0vs1 > 0 alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' if numpy.isinf( bestalgdata[i] ): # if the 2009 best did not solve the problem tmp = writeFEvalsMaxPrec(float(dati), 2) if not numpy.isinf(dati): tmp = r'\textit{%s}' % (tmp) if isBold: tmp = r'\textbf{%s}' % tmp if dispersion[i] and numpy.isfinite(dispersion[i]): tmp += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tmp)) else: # Formatting tmp = float(dati) / bestalgdata[i] assert not numpy.isnan(tmp) isscientific = False if tmp >= 1000: isscientific = True tableentry = writeFEvals2(tmp, 2, isscientific=isscientific) tableentry = writeFEvalsMaxPrec(tmp, 2) if numpy.isinf(tmp) and i == len(data) - 1: tableentry = ( tableentry + r'\textit{%s}' % writeFEvals2(numpy.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: # cave: negative significance has no meaning anymore tableentry = r'\textit{%s}' % tableentry if dispersion[i] and numpy.isfinite( dispersion[i] / bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (numpy.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3 and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry if dispersion[i] and numpy.isfinite( dispersion[i] / bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) elif 11 < 3 and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tableentry = ' & .'.join(tmp) if len(tmp) == 1: tableentry += '&' if dispersion[i] and numpy.isfinite( dispersion[i] / bestalgdata[i]): tableentry += r'${\scriptscriptstyle (%s)}$' % writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) superscript = '' if nb == 0: z, p = testresbestvs0[i] else: z, p = testresbestvs1[i] #The conditions are now that ERT < ERT_best if ((nbtests * p) < 0.05 and dati - bestalgdata[i] < 0. and z < 0.): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars else: superscript = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(nbstars)) if superscript or significance0vs1: s = '' if significance0vs1 > 0: s = '\star' if significance0vs1 > 1: s += str(significance0vs1) s = r'$^{' + s + superscript + r'}$' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s curline.append(tableentry) #curline.append(tableentry) #if dispersion[i] is None or numpy.isinf(bestalgdata[i]): #curline.append('') #else: #tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2) #curline.append('(%s)' % tmp) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % numpy.sum(numpy.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curline.append('/%d' % entry.nbRuns()) table.append(curline[:]) extraeol.append('') extraeol[-1] = r'\hline' extraeol[-1] = '' outputfile = os.path.join(outputdir, 'pptable2_%02dD%s.tex' % (d, info)) spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len( targetsOfInterest) + '|@{}r@{}@{}l@{}' res = r'\providecommand{\algorithmAshort}{%s}' % writeLabels( alg0) + '\n' res += r'\providecommand{\algorithmBshort}{%s}' % writeLabels( alg1) + '\n' # open(os.path.join(outputdir, 'bbob_pproc_commands.tex'), 'a').write(res) #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res += tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() if verbose: print("Table written in %s" % outputfile)
def main(dictAlg, sortedAlgs, outputdir='.', verbose=True, function_targets_line=True): # [1, 13, 101] """Generate one table per func with results of multiple algorithms.""" """Difference with the first version: * numbers aligned using the decimal separator * premices for dispersion measure * significance test against best algorithm * table width... Takes ``targetsOfInterest`` from this file as "input argument" to compute the desired target values. ``targetsOfInterest`` might be configured via config. """ # TODO: method is long, terrible to read, split if possible if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() # Sort data per dimension and function dictData = {} dsListperAlg = list(dictAlg[i] for i in sortedAlgs) for n, entries in enumerate(dsListperAlg): tmpdictdim = entries.dictByDim() for d in tmpdictdim: tmpdictfun = tmpdictdim[d].dictByFunc() for f in tmpdictfun: dictData.setdefault((d, f), {})[n] = tmpdictfun[f] nbtests = len(dictData) for df in dictData: # Generate one table per df # first update targets for each dimension-function pair if needed: targets = targetsOfInterest((df[1], df[0])) targetf = targets[-1] # best 2009 refalgentry = bestalg.bestalgentries2009[df] refalgert = refalgentry.detERT(targets) refalgevals = (refalgentry.detEvals((targetf, ))[0][0]) refalgnbruns = len(refalgevals) refalgnbsucc = numpy.sum(numpy.isnan(refalgevals) == False) # Process the data # The following variables will be lists of elements each corresponding # to an algorithm algnames = [] #algdata = [] algerts = [] algevals = [] algdisp = [] algnbsucc = [] algnbruns = [] algmedmaxevals = [] algmedfinalfunvals = [] algtestres = [] algentries = [] for n in sorted(dictData[df].keys()): entries = dictData[df][n] # the number of datasets for a given dimension and function (df) # should be strictly 1. TODO: find a way to warn # TODO: do this checking before... why wasn't it triggered by ppperprof? if len(entries) > 1: txt = ("There is more than a single entry associated with " "folder %s on %d-D f%d." % (sortedAlgs[n], df[0], df[1])) raise Exception(txt) entry = entries[0] algentries.append(entry) algnames.append(sortedAlgs[n]) evals = entry.detEvals(targets) #tmpdata = [] tmpdisp = [] tmpert = [] for i, e in enumerate(evals): succ = (numpy.isnan(e) == False) ec = e.copy( ) # note: here was the previous bug (changes made in e also appeared in evals !) ec[succ == False] = entry.maxevals[succ == False] ert = toolsstats.sp(ec, issuccessful=succ)[0] #tmpdata.append(ert/refalgert[i]) if succ.any(): tmp = toolsstats.drawSP(ec[succ], entry.maxevals[succ == False], [10, 50, 90], samplesize=samplesize)[0] tmpdisp.append((tmp[-1] - tmp[0]) / 2.) else: tmpdisp.append(numpy.nan) tmpert.append(ert) algerts.append(tmpert) algevals.append(evals) #algdata.append(tmpdata) algdisp.append(tmpdisp) algmedmaxevals.append(numpy.median(entry.maxevals)) algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) #algmedmaxevals.append(numpy.median(entry.maxevals)/df[0]) #algmedfinalfunvals.append(numpy.median(entry.finalfunvals)) algtestres.append(significancetest(refalgentry, entry, targets)) # determine success probability for Df = 1e-8 e = entry.detEvals((targetf, ))[0] algnbsucc.append(numpy.sum(numpy.isnan(e) == False)) algnbruns.append(len(e)) # Process over all data # find best values... nalgs = len(dictData[df]) maxRank = 1 + numpy.floor( 0.14 * nalgs) # number of algs to be displayed in bold isBoldArray = [] # Point out the best values algfinaldata = [ ] # Store median function values/median number of function evaluations tmptop = getTopIndicesOfColumns(algerts, maxRank=maxRank) for i, erts in enumerate(algerts): tmp = [] for j, ert in enumerate(erts): # algi targetj tmp.append(i in tmptop[j] or (nalgs > 7 and algerts[i][j] <= 3. * refalgert[j])) isBoldArray.append(tmp) algfinaldata.append((algmedfinalfunvals[i], algmedmaxevals[i])) # significance test of best given algorithm against all others best_alg_idx = numpy.array(algerts).argsort(0)[ 0, :] # indexed by target index significance_versus_others = significance_all_best_vs_other( algentries, targets, best_alg_idx)[0] # Create the table table = [] spec = r'@{}c@{}|*{%d}{@{\,}r@{}X@{\,}}|@{}r@{}@{}l@{}' % ( len(targets) ) # in case StrLeft not working: replaced c@{} with l@{ } spec = r'@{}c@{}|*{%d}{@{}r@{}X@{}}|@{}r@{}@{}l@{}' % ( len(targets) ) # in case StrLeft not working: replaced c@{} with l@{ } extraeol = [] # Generate header lines if with_table_heading: header = funInfos[df[1]] if funInfos else 'f%d' % df[1] table.append([ r'\multicolumn{%d}{@{\,}c@{\,}}{{\textbf{%s}}}' % (2 * len(targets) + 2, header) ]) extraeol.append('') if function_targets_line is True or (function_targets_line and df[1] in function_targets_line): if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): curline = [r'\#FEs/D'] for i in targetsOfInterest.labels(): curline.append(r'\multicolumn{2}{@{}c@{}}{%s}' % i) else: curline = [r'$\Delta f_\mathrm{opt}$'] for t in targets: curline.append( r'\multicolumn{2}{@{\,}X@{\,}}{%s}' % writeFEvals2(t, precision=1, isscientific=True)) # curline.append(r'\multicolumn{2}{@{\,}X@{}|}{%s}' # % writeFEvals2(targets[-1], precision=1, isscientific=True)) curline.append(r'\multicolumn{2}{@{}l@{}}{\#succ}') table.append(curline) extraeol.append(r'\hline') # extraeol.append(r'\hline\arrayrulecolor{tableShade}') curline = [r'ERT$_{\text{best}}$' ] if with_table_heading else [r'\textbf{f%d}' % df[1]] if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): # write ftarget:fevals for i in range(len(refalgert[:-1])): temp = "%.1e" % targetsOfInterest((df[1], df[0]))[i] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append( r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp, writeFEvalsMaxPrec(refalgert[i], 2))) temp = "%.1e" % targetsOfInterest((df[1], df[0]))[-1] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp, writeFEvalsMaxPrec(refalgert[-1], 2))) else: # write #fevals of the reference alg for i in refalgert[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(refalgert[-1], 2)) # write the success ratio for the reference alg tmp2 = numpy.sum( numpy.isnan(refalgevals) == False) # count the nb of success curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(refalgevals)) table.append(curline[:]) extraeol.append('') #for i, gna in enumerate(zip((1, 2, 3), ('bla', 'blo', 'bli'))): #print i, gna, gno #set_trace() # Format data #if df == (5, 17): #set_trace() header = r'\providecommand{\ntables}{7}' for i, alg in enumerate(algnames): #algname, entries, irs, line, line2, succ, runs, testres1alg in zip(algnames, #data, dispersion, isBoldArray, isItalArray, nbsucc, nbruns, testres): commandname = r'\alg%stables' % numtotext(i) # header += r'\providecommand{%s}{{%s}{}}' % (commandname, str_to_latex(strip_pathname(alg))) header += r'\providecommand{%s}{\StrLeft{%s}{\ntables}}' % ( commandname, str_to_latex(strip_pathname2(alg))) curline = [ commandname + r'\hspace*{\fill}' ] # each list element becomes a &-separated table entry? for j, tmp in enumerate( zip( algerts[i], algdisp[i], # j is target index isBoldArray[i], algtestres[i])): ert, dispersion, isBold, testres = tmp alignment = '@{\,}X@{\,}' if j == len(algerts[i]) - 1: alignment = '@{\,}X@{\,}|' data = ert / refalgert[j] # write star for significance against all other algorithms str_significance_subsup = '' if (len(best_alg_idx) > 0 and len(significance_versus_others) > 0 and i == best_alg_idx[j] and nbtests * significance_versus_others[j][1] < 0.05): logp = -numpy.ceil( numpy.log10( nbtests * significance_versus_others[j][1])) str_significance_subsup = r"^{%s%s}" % ( significance_vs_others_symbol, str(int(logp)) if logp > 1 else '') # moved out of the above else: this was a bug!? z, p = testres if (nbtests * p) < 0.05 and data < 1. and z < 0.: if not numpy.isinf(refalgert[j]): tmpevals = algevals[i][j].copy() tmpevals[numpy.isnan(tmpevals)] = algentries[ i].maxevals[numpy.isnan(tmpevals)] bestevals = refalgentry.detEvals(targets) bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0]) bestevals[numpy.isnan( bestevals)] = refalgentry.maxevals[bestalgalg][ numpy.isnan(bestevals)] tmpevals = numpy.array(sorted( tmpevals))[0:min(len(tmpevals), len(bestevals))] bestevals = numpy.array(sorted( bestevals))[0:min(len(tmpevals), len(bestevals))] #The conditions are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if numpy.isinf(refalgert[j]) or all(tmpevals < bestevals): nbstars = -numpy.ceil(numpy.log10(nbtests * p)) # tmp2[-1] += r'$^{%s}$' % superscript str_significance_subsup += r'_{%s%s}' % ( significance_vs_ref_symbol, str(int(nbstars)) if nbstars > 1 else '') if str_significance_subsup: str_significance_subsup = '$%s$' % str_significance_subsup # format number in variable data if numpy.isnan(data): curline.append(r'\multicolumn{2}{%s}{.}' % alignment) else: if numpy.isinf(refalgert[j]): curline.append( r'\multicolumn{2}{%s}{\textbf{%s}\mbox{\tiny (%s)}%s}' % (alignment, writeFEvalsMaxPrec(algerts[i][j], 2), writeFEvalsMaxPrec(dispersion, precdispersion), str_significance_subsup)) continue tmp = writeFEvalsMaxPrec(data, precfloat, maxfloatrepr=maxfloatrepr) if data >= maxfloatrepr or data < 0.01: # either inf or scientific notation if numpy.isinf(data) and j == len(algerts[i]) - 1: tmp += r'\,\textit{%s}' % writeFEvalsMaxPrec( algfinaldata[i][1], 0, maxfloatrepr=maxfloatrepr) else: tmp = writeFEvalsMaxPrec(data, precscien, maxfloatrepr=data) if isBold: tmp = r'\textbf{%s}' % tmp if not numpy.isnan(dispersion): tmpdisp = dispersion / refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.005: # TODO: hack tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp += r'\mbox{\tiny (%s)}' % tmpdisp curline.append( r'\multicolumn{2}{%s}{%s%s}' % (alignment, tmp, str_significance_subsup)) else: tmp2 = tmp.split('.', 1) if len(tmp2) < 2: tmp2.append('') else: tmp2[-1] = '.' + tmp2[-1] if isBold: tmp3 = [] for k in tmp2: tmp3.append(r'\textbf{%s}' % k) tmp2 = tmp3 if not numpy.isnan(dispersion): tmpdisp = dispersion / refalgert[j] if tmpdisp >= maxfloatrepr or tmpdisp < 0.01: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=tmpdisp) else: tmpdisp = writeFEvalsMaxPrec( tmpdisp, precdispersion, maxfloatrepr=maxfloatrepr) tmp2[-1] += (r'\mbox{\tiny (%s)}' % (tmpdisp)) tmp2[-1] += str_significance_subsup curline.extend(tmp2) curline.append('%d' % algnbsucc[i]) curline.append('/%d' % algnbruns[i]) table.append(curline) extraeol.append('') # Write table res = tableXLaTeX(table, spec=spec, extraeol=extraeol) try: filename = os.path.join( outputdir, 'pptables_f%03d_%02dD.tex' % (df[1], df[0])) f = open(filename, 'w') f.write(header + '\n') f.write(res) if verbose: print('Wrote table in %s' % filename) except: raise else: f.close()
def main(dsList, dimsOfInterest, outputdir, info='', verbose=True): """Generate a table of ratio ERT/ERTbest vs target precision. 1 table per dimension will be generated. Rank-sum tests table on "Final Data Points" for only one algorithm. that is, for example, using 1/#fevals(ftarget) if ftarget was reached and -f_final otherwise as input for the rank-sum test, where obviously the larger the better. """ #TODO: check that it works for any reference algorithm? #in the following the reference algorithm is the one given in #bestalg.bestalgentries which is the virtual best of BBOB dictDim = dsList.dictByDim() targetf = 1e-8 if info: info = '_' + info # insert a separator between the default file name and the additional # information string. dims = set(dictDim.keys()) if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): header = [r'\#FEs/D'] for i in targetsOfInterest.labels(): header.append(r'\multicolumn{2}{@{}c@{}}{%s}' % i) else: header = [r'$\Delta f$'] for i in targetsOfInterest.target_values: header.append(r'\multicolumn{2}{@{}c@{}}{1e%+d}' % (int(np.log10(i)))) header.append(r'\multicolumn{2}{|@{}r@{}}{\#succ}') for d in dimsOfInterest: table = [header] extraeol = [r'\hline'] try: dictFunc = dictDim[d].dictByFunc() except KeyError: continue funcs = set(dictFunc.keys()) nbtests = float(len(funcs)) # #funcs tests times one algorithm for f in sorted(funcs): bestalgentry = bestalg.bestalgentries2009[(d, f)] curline = [r'${\bf f_{%d}}$' % f] bestalgdata = bestalgentry.detERT(targetsOfInterest((f, d))) bestalgevals, bestalgalgs = bestalgentry.detEvals( targetsOfInterest((f, d))) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): #write ftarget:fevals for i in range(len(bestalgdata[:-1])): temp = "%.1e" % targetsOfInterest((f, d))[i] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append( r'\multicolumn{2}{@{}c@{}}{\textit{%s}:%s \quad}' % (temp, writeFEvalsMaxPrec(bestalgdata[i], 2))) temp = "%.1e" % targetsOfInterest((f, d))[-1] if temp[-2] == "0": temp = temp[:-2] + temp[-1] curline.append(r'\multicolumn{2}{@{}c@{}|}{\textit{%s}:%s }' % (temp, writeFEvalsMaxPrec(bestalgdata[-1], 2))) #success targetf = targetsOfInterest((f, d))[-1] else: # write #fevals of the reference alg for i in bestalgdata[:-1]: curline.append(r'\multicolumn{2}{@{}c@{}}{%s \quad}' % writeFEvalsMaxPrec(i, 2)) curline.append(r'\multicolumn{2}{@{}c@{}|}{%s}' % writeFEvalsMaxPrec(bestalgdata[-1], 2)) # write the success ratio for the reference alg tmp = bestalgentry.detEvals([targetf])[0][0] tmp2 = np.sum(np.isnan(tmp) == False) # count the nb of success curline.append('%d' % (tmp2)) if tmp2 > 0: curline.append('/%d' % len(tmp)) table.append(curline[:]) extraeol.append('') # generate all data for ranksum test assert len(dictFunc[f]) == 1 entry = dictFunc[f][0] # take the first element ertdata = entry.detERT(targetsOfInterest((f, d))) testresbestvs1 = significancetest(bestalgentry, entry, targetsOfInterest((f, d))) #for nb, entry in enumerate(entries): #curline = [r'\algshort\hspace*{\fill}'] curline = [''] #data = entry.detERT(targetsOfInterest) evals = entry.detEvals(targetsOfInterest((f, d))) dispersion = [] data = [] for i in evals: succ = (np.isnan(i) == False) tmp = i.copy() tmp[succ == False] = entry.maxevals[np.isnan(i)] #set_trace() # TODO: what is the difference between data and ertdata? data.append(toolsstats.sp(tmp, issuccessful=succ)[0]) #if not any(succ): #set_trace() if any(succ): tmp2 = toolsstats.drawSP(tmp[succ], tmp[succ == False], (10, 50, 90), samplesize)[0] dispersion.append((tmp2[-1] - tmp2[0]) / 2.) else: dispersion.append(None) assert data == ertdata for i, ert in enumerate(data): alignment = 'c' if i == len(data) - 1: # last element alignment = 'c|' nbstars = 0 z, p = testresbestvs1[i] if ert - bestalgdata[i] < 0. and not np.isinf(bestalgdata[i]): evals = entry.detEvals([targetsOfInterest((f, d))[i]])[0] evals[np.isnan(evals)] = entry.maxevals[np.isnan(evals)] bestevals = bestalgentry.detEvals( [targetsOfInterest((f, d))[i]]) bestevals, bestalgalg = (bestevals[0][0], bestevals[1][0]) bestevals[np.isnan(bestevals)] = bestalgentry.maxevals[ bestalgalg][np.isnan(bestevals)] evals = np.array( sorted(evals))[0:min(len(evals), len(bestevals))] bestevals = np.array( sorted(bestevals))[0:min(len(evals), len(bestevals))] #The conditions for significance are now that ERT < ERT_best and # all(sorted(FEvals_best) > sorted(FEvals_current)). if ((nbtests * p) < 0.05 and ert - bestalgdata[i] < 0. and z < 0. and (np.isinf(bestalgdata[i]) or all(evals < bestevals))): nbstars = -np.ceil(np.log10(nbtests * p)) isBold = False if nbstars > 0: isBold = True if np.isinf(bestalgdata[i] ): # if the best did not solve the problem tmp = writeFEvalsMaxPrec(float(ert), 2) if not np.isinf(ert): tmp = r'\textit{%s}' % (tmp) if isBold: tmp = r'\textbf{%s}' % tmp tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tmp)) else: # Formatting tmp = float(ert) / bestalgdata[i] assert not np.isnan(tmp) tableentry = writeFEvalsMaxPrec(tmp, 2) if np.isinf(tmp) and i == len(data) - 1: tableentry = ( tableentry + r'\textit{%s}' % writeFEvals2(np.median(entry.maxevals), 2)) if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3: # and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) elif tableentry.find('e') > -1 or (np.isinf(tmp) and i != len(data) - 1): if isBold: tableentry = r'\textbf{%s}' % tableentry elif 11 < 3: # and significance0vs1 < 0: tableentry = r'\textit{%s}' % tableentry tableentry = (r'\multicolumn{2}{@{}%s@{}}{%s}' % (alignment, tableentry)) else: tmp = tableentry.split('.', 1) if isBold: tmp = list(r'\textbf{%s}' % i for i in tmp) elif 11 < 3: # and significance0vs1 < 0: tmp = list(r'\textit{%s}' % i for i in tmp) tableentry = ' & .'.join(tmp) if len(tmp) == 1: tableentry += '&' superscript = '' if nbstars > 0: #tmp = '\hspace{-.5ex}'.join(nbstars * [r'\star']) if z > 0: superscript = r'\uparrow' #* nbstars else: superscript = r'\downarrow' #* nbstars # print z, linebest[i], line1 if nbstars > 1: superscript += str(int(min((9, nbstars)))) # superscript += str(int(nbstars)) #if superscript or significance0vs1: #s = '' #if significance0vs1 > 0: #s = '\star' #if significance0vs1 > 1: #s += str(significance0vs1) #s = r'$^{' + s + superscript + r'}$' #if tableentry.endswith('}'): #tableentry = tableentry[:-1] + s + r'}' #else: #tableentry += s if dispersion[i]: if not np.isinf(bestalgdata[i]): tmp = writeFEvalsMaxPrec( dispersion[i] / bestalgdata[i], 1) else: tmp = writeFEvalsMaxPrec(dispersion[i], 1) tableentry += (r'${\scriptscriptstyle(%s)}$' % tmp) if superscript: s = r'$^{' + superscript + r'}$' if tableentry.endswith('}'): tableentry = tableentry[:-1] + s + r'}' else: tableentry += s curline.append(tableentry) #curline.append(tableentry) #if dispersion[i] is None or np.isinf(bestalgdata[i]): #curline.append('') #else: #tmp = writeFEvalsMaxPrec(dispersion[i]/bestalgdata[i], 2) #curline.append('(%s)' % tmp) tmp = entry.evals[entry.evals[:, 0] <= targetf, 1:] try: tmp = tmp[0] curline.append('%d' % np.sum(np.isnan(tmp) == False)) except IndexError: curline.append('%d' % 0) curline.append('/%d' % entry.nbRuns()) table.append(curline[:]) extraeol.append(r'\hline') extraeol[-1] = '' outputfile = os.path.join(outputdir, 'pptable_%02dD%s.tex' % (d, info)) if isinstance(targetsOfInterest, pproc.RunlengthBasedTargetValues): spec = r'@{}c@{}|' + '*{%d}{@{ }r@{}@{}l@{}}' % len( targetsOfInterest) + '|@{}r@{}@{}l@{}' else: spec = r'@{}c@{}|' + '*{%d}{@{}r@{}@{}l@{}}' % len( targetsOfInterest) + '|@{}r@{}@{}l@{}' #res = r'\providecommand{\algshort}{%s}' % alg1 + '\n' #res += tableLaTeXStar(table, width=r'0.45\textwidth', spec=spec, #extraeol=extraeol) res = tableLaTeX(table, spec=spec, extraeol=extraeol) f = open(outputfile, 'w') f.write(res) f.close() if verbose: print("Table written in %s" % outputfile)
def plot(dsList, dsref, targets=defaulttargets, istoolsstats=False, **kwargs): """Generates a graph showing the performance profile of an algorithm. We display the empirical cumulative distribution function ECDF of the bootstrapped distribution of the expected running time (ERT) for an algorithm to reach the function value :py:data:`targets` normalized by the ERT of the reference algorithm for these targets. :param DataSetList dsList: data set for one algorithm :param DataSetList dsref: reference data set for normalization :param seq targets: target function values :param dict kwargs: additional parameters provided to plot function. :returns: handles """ res = [] assert len( dsList.dictByDim()) == 1 # We never integrate over dimensions... data = [] for entry in dsList: for t in targets: # TODO: alternative: min(dsref[(entry.dim, entry.funcId)].detEvals((t,))[0]) # is the min from the alg with the best ERT flg_ert = 1 if flg_ert: normalizer = dsref[(entry.dim, entry.funcId)].detERT((t, ))[0] else: pass if np.isinf(normalizer): continue if istoolsstats: x = [np.inf] * perfprofsamplesize runlengthunsucc = [] evals = entry.detEvals([t])[0] runlengthsucc = evals[np.isnan(evals) == False] runlengthunsucc = entry.maxevals[np.isnan(evals)] if len(runlengthsucc) > 0: x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] data.extend(i / normalizer for i in x) else: x = entry.detERT([t])[0] data.append(x / normalizer) #if (np.array(tmp) < 1e-1).any(): # set_trace() # Display data data = np.array(data) data = data[np.isnan(data) == False] # Take away the nans n = len(data) data = data[np.isinf(data) == False] # Take away the infs # data = data[data <= maxval] # Take away rightmost data #data = np.exp(craftingeffort) * data # correction by crafting effort CrE #set_trace() if len(data) == 0: # data is empty. res = plotECDF(np.array((1., )), n=np.inf, **kwargs) else: plt.plot((min(data), ), (0, ), **kwargs) res = plotECDF(np.array(data), n=n, **kwargs) h = plt.plot((max(data), ), (float(len(data)) / n, ), **kwargs) plt.setp(h, 'marker', 'x') #plotdata(np.array(data), x_limit, maxevals, # CrE=0., kwargs=kwargs) # if maxevals: # Should cover the case where maxevals is None or empty # x3 = np.median(maxevals) # if np.any(data > x3): # y3 = float(np.sum(data <= x3)) / n # h = plt.plot((x3,), (y3,), marker='x', markersize=30, # markeredgecolor=plt.getp(res[0], 'color'), # ls='', color=plt.getp(res[0], 'color')) # h.extend(res) # res = h # so the last element in res still has the label. return res
def main(dictAlg, order=None, outputdir='.', info='default', dimension=None, verbose=True): """Generates a figure showing the performance of algorithms. From a dictionary of :py:class:`DataSetList` sorted by algorithms, generates the cumulative distribution function of the bootstrap distribution of ERT for algorithms on multiple functions for multiple targets altogether. :param dict dictAlg: dictionary of :py:class:`DataSetList` instances one instance is equivalent to one algorithm, :param list targets: target function values :param list order: sorted list of keys to dictAlg for plotting order :param str outputdir: output directory :param str info: output file name suffix :param bool verbose: controls verbosity """ global x_limit # late assignment of default, because it can be set to None in config global divide_by_dimension # not fully implemented/tested yet if 'x_limit' not in globals() or x_limit is None: x_limit = x_limit_default tmp = pp.dictAlgByDim(dictAlg) # tmp = pp.DictAlg(dictAlg).by_dim() if len(tmp) != 1 and dimension is None: raise ValueError('We never integrate over dimension.') if dimension is not None: if dimension not in tmp.keys(): raise ValueError('dimension %d not in dictAlg dimensions %s' % (dimension, str(tmp.keys()))) tmp = {dimension: tmp[dimension]} dim = tmp.keys()[0] divisor = dim if divide_by_dimension else 1 algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []] dictFunc = pp.dictAlgByFun(dictAlg) # Collect data # Crafting effort correction: should we consider any? CrEperAlg = {} for alg in algorithms_with_data: CrE = 0. if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL': tmp = dictAlg[alg].dictByNoise() assert len(tmp.keys()) == 1 if tmp.keys()[0] == 'noiselessall': CrE = 0.5117 elif tmp.keys()[0] == 'nzall': CrE = 0.6572 CrEperAlg[alg] = CrE if CrE != 0.0: print('Crafting effort for', alg, 'is', CrE) dictData = {} # list of (ert per function) per algorithm dictMaxEvals = {} # list of (maxevals per function) per algorithm bestERT = [] # best ert per function # funcsolved = [set()] * len(targets) # number of functions solved per target xbest2009 = [] maxevalsbest2009 = [] for f, dictAlgperFunc in dictFunc.iteritems(): if function_IDs and f not in function_IDs: continue # print target_values((f, dim)) for j, t in enumerate(target_values((f, dim))): # for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)): # funcsolved[j].add(f) for alg in algorithms_with_data: x = [np.inf] * perfprofsamplesize runlengthunsucc = [] try: entry = dictAlgperFunc[alg][ 0] # one element per fun and per dim. evals = entry.detEvals([t])[0] assert entry.dim == dim runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor if len(runlengthsucc) > 0: x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] except (KeyError, IndexError): #set_trace() warntxt = ( 'Data for algorithm %s on function %d in %d-D ' % (alg, f, dim) + 'are missing.\n') warnings.warn(warntxt) dictData.setdefault(alg, []).extend(x) dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc) if displaybest2009: #set_trace() if not bestalg.bestalgentries2009: bestalg.loadBBOB2009() bestalgentry = bestalg.bestalgentries2009[(dim, f)] bestalgevals = bestalgentry.detEvals(target_values((f, dim))) # print bestalgevals for j in range(len(bestalgevals[0])): if bestalgevals[1][j]: evals = bestalgevals[0][j] #set_trace() assert dim == bestalgentry.dim runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = bestalgentry.maxevals[ bestalgevals[1][j]][np.isnan(evals)] / divisor x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] else: x = perfprofsamplesize * [np.inf] runlengthunsucc = [] xbest2009.extend(x) maxevalsbest2009.extend(runlengthunsucc) if order is None: order = dictData.keys() # Display data lines = [] if displaybest2009: args = { 'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11., 'markeredgewidth': 1.5, 'markerfacecolor': refcolor, 'markeredgecolor': refcolor, 'color': refcolor, 'label': 'best 2009', 'zorder': -1 } lines.append( plotdata(np.array(xbest2009), x_limit, maxevalsbest2009, CrE=0., **args)) for i, alg in enumerate(order): try: data = dictData[alg] maxevals = dictMaxEvals[alg] except KeyError: continue args = styles[(i) % len(styles)] args['linewidth'] = 1.5 args['markersize'] = 12. args['markeredgewidth'] = 1.5 args['markerfacecolor'] = 'None' args['markeredgecolor'] = args['color'] args['label'] = alg #args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib #elif len(show_algorithms) > 0: #args['color'] = 'wheat' #args['ls'] = '-' #args['zorder'] = -1 lines.append( plotdata(np.array(data), x_limit, maxevals, CrE=CrEperAlg[alg], **args)) labels, handles = plotLegend(lines, x_limit) if True: # isLateXLeg: fileName = os.path.join(outputdir, 'pprldmany_%s.tex' % (info)) try: f = open(fileName, 'w') f.write(r'\providecommand{\nperfprof}{7}') algtocommand = {} for i, alg in enumerate(order): tmp = r'\alg%sperfprof' % pptex.numtotext(i) f.write(r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' % (tmp, toolsdivers.str_to_latex( toolsdivers.strip_pathname2(alg)))) algtocommand[alg] = tmp commandnames = [] if displaybest2009: tmp = r'\algzeroperfprof' f.write(r'\providecommand{%s}{best 2009}' % (tmp)) algtocommand['best 2009'] = tmp for l in labels: commandnames.append(algtocommand[l]) # f.write(headleg) f.write(r'\providecommand{\perfprofsidepanel}{\mbox{%s}' % commandnames[0]) # TODO: check len(labels) > 0 for i in range(1, len(labels)): f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i]) f.write('}\n') # f.write(footleg) if verbose: print('Wrote right-hand legend in %s' % fileName) except: raise # TODO: Does this make sense? else: f.close() figureName = os.path.join(outputdir, 'pprldmany_%s' % (info)) #beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat) beautify() text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys()))) text += ',%d-D' % dim # TODO: this is strange when different dimensions are plotted plt.text(0.01, 0.98, text, horizontalalignment="left", verticalalignment="top", transform=plt.gca().transAxes) if len(dictFunc) == 1: plt.title(' '.join( (str(dictFunc.keys()[0]), genericsettings.current_testbed.short_names[dictFunc.keys()[0]]))) a = plt.gca() plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative) xticks, labels = plt.xticks() tmp = [] for i in xticks: tmp.append('%d' % round(np.log10(i))) a.set_xticklabels(tmp) # print 'in_a_hurry ==', genericsettings.in_a_hurry if 1 < 3: ppfig.saveFigure(figureName, verbose=verbose) plt.close()
def plot(dsList, targets=None, craftingeffort=0., **kwargs): """This function is obsolete? Generates a graph of the run length distribution of an algorithm. We display the empirical cumulative distribution function ECDF of the bootstrapped distribution of the runlength for an algorithm (in number of function evaluations) to reach the target functions value :py:data:`targets`. :param DataSetList dsList: data set for one algorithm :param seq targets: target function values :param float crafting effort: the data will be multiplied by the exponential of this value :param dict kwargs: additional parameters provided to plot function. :returns: handles """ if targets is None: targets = target_values # set above or in config.py try: if np.min(targets) >= 1: ValueError( 'smallest target f-value is not smaller than one, use ``pproc.TargetValues(targets)`` to prevent this error' ) targets = pp.TargetValues(targets) except TypeError: pass res = [] assert len(pp.DataSetList( dsList).dictByDim()) == 1 # We never integrate over dimensions... data = [] maxevals = [] for entry in dsList: for t in targets((entry.funcId, entry.dim)): divisor = entry.dim if divide_by_dimension else 1 x = [np.inf] * perfprofsamplesize runlengthunsucc = [] evals = entry.detEvals([t])[0] runlengthsucc = evals[np.isnan(evals) == False] / divisor runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor if len(runlengthsucc) > 0: x = toolsstats.drawSP(runlengthsucc, runlengthunsucc, percentiles=[50], samplesize=perfprofsamplesize)[1] data.extend(x) maxevals.extend(runlengthunsucc) # Display data data = np.array(data) data = data[np.isnan(data) == False] # Take away the nans n = len(data) data = data[np.isinf(data) == False] # Take away the infs # data = data[data <= maxval] # Take away rightmost data data = np.exp(craftingeffort) * data # correction by crafting effort CrE if len(data) == 0: # data is empty. res = pprldistr.plotECDF(np.array((1., )), n=np.inf, **kwargs) else: res = pprldistr.plotECDF(np.array(data), n=n, **kwargs) #plotdata(np.array(data), x_limit, maxevals, # CrE=0., **kwargs) if maxevals: # Should cover the case where maxevals is None or empty x3 = np.median(maxevals) if np.any(data > x3): y3 = float(np.sum(data <= x3)) / n h = plt_plot((x3, ), (y3, ), marker='x', markersize=24, markeredgewidth=3, markeredgecolor=plt.getp(res[0], 'color'), ls='', color=plt.getp(res[0], 'color')) h.extend(res) res = h # so the last element in res still has the label. return res