def build(dictAlg, sortedAlg=None):
"""Merge datasets in an algorithm portfolio.
:param dict dictAlg: dictionary of data sets with algorithm name for
keys
:param seq sortedAlgs: sequence for sorting the entries of
:py:data:`dictAlg`, if not provided,
dictAlg.keys() will be instead
:returns: an instance of :py:class:`DataSetList` with the porfolio
data sets
"""
if not sortedAlg:
sortedAlg = dictAlg.keys()
tmpres = []
for f, i in pp.dictAlgByFun(dictAlg).iteritems():
for d, j in pp.dictAlgByDim(i).iteritems():
tmp = []
if sortedAlg:
tmplist = list(j[k] for k in sortedAlg)
else:
tmplist = j.values()
for k in tmplist:
assert len(k) == 1 # one element list
tmp.append(k[0])
try:
tmpres.append(DataSet(tmp))
except Usage, err:
print >>sys.stderr, err.msg
def generate(dictalg):
"""Generates dictionary of best algorithm data set.
"""
# dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)
res = {}
for f, i in pproc.dictAlgByFun(dictalg).iteritems():
for d, j in pproc.dictAlgByDim(i).iteritems():
tmp = BestAlgSet(j)
res[(d, f)] = tmp
return res
def generate(dictalg):
"""Generates dictionary of best algorithm data set.
"""
#dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)
res = {}
for f, i in pproc.dictAlgByFun(dictalg).iteritems():
for d, j in pproc.dictAlgByDim(i).iteritems():
tmp = BestAlgSet(j)
res[(d, f)] = tmp
return res
def main(dictAlg, outputdir='.', verbose=True):
"""Main routine for generating convergence plots
"""
global warned # bind variable warned into this scope
dictFun = pproc.dictAlgByFun(dictAlg)
for l in dictFun: # l appears to be the function id!?
for i in dictFun[
l]: # please, what is i??? appears to be the algorithm-key
plt.figure()
if type(i) in (list, tuple):
figurename = "ppconv_plot_" + i[0] + "_f" + str(l)
else:
try:
figurename = "ppconv_plot_" + dictFun[l][
i].algId + "_f" + str(l)
except AttributeError: # this is a (rather desperate) bug-fix attempt that works for the unit test
figurename = "ppconv_plot_" + dictFun[l][i][
0].algId + "_f" + str(l)
plt.xlabel('number of function evaluations / dimension')
plt.ylabel('Median of fitness')
plt.grid()
ax = plt.gca()
ax.set_yscale("log")
ax.set_xscale("log")
for j in dictFun[l][i]: # please, what is j??? a dataset
dimList_b = []
dimList_f = []
dimList_b.append(j.funvals[:, 0])
dimList_f.append(j.funvals[:, 1:])
bs, fs = rearrange(dimList_b, dimList_f)
labeltext = str(j.dim) + "D"
try:
if 11 < 3:
plt.errorbar(bs[0] / j.dim,
fs[0][0],
yerr=[fs[0][1], fs[0][2]],
label=labeltext)
else:
plt.errorbar(bs[0] / j.dim, fs[0][0], label=labeltext)
except FloatingPointError: # that's a bit of a hack
if not warned:
print(
'Warning: floating point error when plotting errorbars, ignored'
)
warned = True
plt.legend(loc=3)
saveFigure(os.path.join(outputdir, figurename.replace(' ', '')),
genericsettings.fig_formats,
verbose=verbose)
plt.close()
print("Convergence plots done.")
def main(dictAlg, outputdir='.', verbose=True):
"""Main routine for generating convergence plots
"""
global warned # bind variable warned into this scope
dictFun = pproc.dictAlgByFun(dictAlg)
for l in dictFun: # l appears to be the function id!?
for i in dictFun[l]: # please, what is i??? appears to be the algorithm-key
plt.figure()
if 1 < 3: # no algorithm name in filename, as everywhere else
figurename = "ppconv_" + "f%03d" % l
else: # previous version with algorithm name, but this is not very practical later
if type(i) in (list, tuple):
figurename = "ppconv_plot_" + i[0] + "_f" + str(l)
else:
try:
figurename = "ppconv_plot_" + dictFun[l][i].algId + "_f" + str(l)
except AttributeError: # this is a (rather desperate) bug-fix attempt that works for the unit test
figurename = "ppconv_plot_" + dictFun[l][i][0].algId + "_f" + str(l)
plt.xlabel('number of function evaluations / dimension')
plt.ylabel('Median of fitness')
plt.grid()
ax = plt.gca()
ax.set_yscale("log")
ax.set_xscale("log")
for j in dictFun[l][i]: # please, what is j??? a dataset
dimList_b = []
dimList_f = []
dimList_b.append(j.funvals[:,0])
dimList_f.append(j.funvals[:,1:])
bs, fs= rearrange(dimList_b, dimList_f)
labeltext=str(j.dim)+"D"
try:
if 11 < 3:
plt.errorbar(bs[0] / j.dim, fs[0][0], yerr = [fs[0][1], fs[0][2]], label = labeltext)
else:
plt.errorbar(bs[0] / j.dim, fs[0][0], label = labeltext)
except FloatingPointError: # that's a bit of a hack
if 1 < 3 or not warned:
print('Warning: floating point error when plotting errorbars, ignored')
warned = True
beautify()
saveFigure(os.path.join(outputdir, figurename.replace(' ','')),
genericsettings.getFigFormats(), verbose=verbose)
plt.close()
try:
algname = str(dictFun[l].keys()[0][0])
except KeyError:
algname = str(dictFun[l].keys()[0])
save_single_functions_html(os.path.join(outputdir, 'ppconv'),
algname) # first try
print("Convergence plots done.")
def main(argv=None):
"""
"""
if argv is None:
argv = sys.argv[1:]
try:
try:
opts, args = getopt.getopt(argv, "h", ["help"])
except getopt.error, msg:
raise Usage(msg)
args = algs
# if not (args):
# usage()
# sys.exit()
outputdir = 'bestAlg'
verbose = True
#Process options
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
else:
assert False, "unhandled option"
dsList, sortedAlgs, dictAlg = processInputArgs(args, verbose=verbose)
#set_trace()
if not os.path.exists(outputdir):
os.mkdir(outputdir)
if verbose:
print 'Folder %s was created.' % (outputdir)
res = {}
for f, i in dictAlgByFun(dictAlg).iteritems():
for d, j in dictAlgByDim(i).iteritems():
tmp = BestAlgSet(j)
picklefilename = os.path.join(
outputdir, 'bestalg_f%03d_%02d.pickle' % (f, d))
fid = open(picklefilename, 'w')
pickle.dump(tmp, fid, 2)
fid.close()
res[(d, f)] = tmp
picklefilename = os.path.join(outputdir, 'bestalg.pickle')
fid = open(picklefilename, 'w')
pickle.dump(res, fid, 2)
fid.close()
def all_single_functions(dictAlg, sortedAlgs=None, outputdir='.',
verbose=0):
dictFG = pp.dictAlgByFun(dictAlg)
for fg, tmpdictAlg in dictFG.iteritems():
dictDim = pp.dictAlgByDim(tmpdictAlg)
for d, entries in dictDim.iteritems():
single_fct_output_dir = (outputdir.rstrip(os.sep) + os.sep +
'pprldmany-single-functions'
# + os.sep + ('f%03d' % fg)
)
if not os.path.exists(single_fct_output_dir):
os.makedirs(single_fct_output_dir)
main(entries,
order=sortedAlgs,
outputdir=single_fct_output_dir,
info=('f%03d_%02dD' % (fg, d)),
verbose=verbose)
def all_single_functions(dictAlg, sortedAlgs=None, outputdir='.', verbose=0):
dictFG = pp.dictAlgByFun(dictAlg)
for fg, tmpdictAlg in dictFG.iteritems():
dictDim = pp.dictAlgByDim(tmpdictAlg)
for d, entries in dictDim.iteritems():
single_fct_output_dir = (
outputdir.rstrip(os.sep) + os.sep +
'pprldmany-single-functions'
# + os.sep + ('f%03d' % fg)
)
if not os.path.exists(single_fct_output_dir):
os.makedirs(single_fct_output_dir)
main(entries,
order=sortedAlgs,
outputdir=single_fct_output_dir,
info=('f%03d_%02dD' % (fg, d)),
verbose=verbose)
def main(dictAlg, outputdir=".", verbose=True):
"""Main routine for generating convergence plots
"""
global warned # bind variable warned into this scope
dictFun = pproc.dictAlgByFun(dictAlg)
for l in dictFun: # l appears to be the function id!?
for i in dictFun[l]: # please, what is i??? appears to be the algorithm-key
plt.figure()
if type(i) in (list, tuple):
figurename = "ppconv_plot_" + i[0] + "_f" + str(l)
else:
try:
figurename = "ppconv_plot_" + dictFun[l][i].algId + "_f" + str(l)
except AttributeError: # this is a (rather desperate) bug-fix attempt that works for the unit test
figurename = "ppconv_plot_" + dictFun[l][i][0].algId + "_f" + str(l)
plt.xlabel("number of function evaluations / dimension")
plt.ylabel("Median of fitness")
plt.grid()
ax = plt.gca()
ax.set_yscale("log")
ax.set_xscale("log")
for j in dictFun[l][i]: # please, what is j??? a dataset
dimList_b = []
dimList_f = []
dimList_b.append(j.funvals[:, 0])
dimList_f.append(j.funvals[:, 1:])
bs, fs = rearrange(dimList_b, dimList_f)
labeltext = str(j.dim) + "D"
try:
if 11 < 3:
plt.errorbar(bs[0] / j.dim, fs[0][0], yerr=[fs[0][1], fs[0][2]], label=labeltext)
else:
plt.errorbar(bs[0] / j.dim, fs[0][0], label=labeltext)
except FloatingPointError: # that's a bit of a hack
if not warned:
print("Warning: floating point error when plotting errorbars, ignored")
warned = True
plt.legend(loc=3)
saveFigure(
os.path.join(outputdir, figurename.replace(" ", "")), genericsettings.fig_formats, verbose=verbose
)
plt.close()
print("Convergence plots done.")
outputdir=outputdir,
info=('%02dD_%s' % (d, ng)),
verbose=verbose)
# ECDFs per function groups
dictFG = pproc.dictAlgByFuncGroup(dictAlg)
for fg, tmpdictAlg in dictFG.iteritems():
dictDim = pproc.dictAlgByDim(tmpdictAlg)
for d, entries in dictDim.iteritems():
pprldmany.main(entries,
order=sortedAlgs,
outputdir=outputdir,
info=('%02dD_%s' % (d, fg)),
verbose=verbose)
if isRldOnSingleFcts: # copy-paste from above, here for each function instead of function groups
# ECDFs for each function
dictFG = pproc.dictAlgByFun(dictAlg)
for fg, tmpdictAlg in dictFG.iteritems():
dictDim = pproc.dictAlgByDim(tmpdictAlg)
for d, entries in dictDim.iteritems():
single_fct_output_dir = (outputdir.rstrip(os.sep) + os.sep +
'pprldmany-single-functions'
# + os.sep + ('f%03d' % fg)
)
if not os.path.exists(single_fct_output_dir):
os.makedirs(single_fct_output_dir)
pprldmany.main(entries,
order=sortedAlgs,
outputdir=single_fct_output_dir,
info=('f%03d_%02dD' % (fg, d)),
verbose=verbose)
print "ECDFs of run lengths figures done."
def main(dictAlg, isBiobjective, order=None, outputdir='.', info='default',
dimension=None, verbose=True):
"""Generates a figure showing the performance of algorithms.
From a dictionary of :py:class:`DataSetList` sorted by algorithms,
generates the cumulative distribution function of the bootstrap
distribution of ERT for algorithms on multiple functions for
multiple targets altogether.
:param dict dictAlg: dictionary of :py:class:`DataSetList` instances
one instance is equivalent to one algorithm,
:param list targets: target function values
:param list order: sorted list of keys to dictAlg for plotting order
:param str outputdir: output directory
:param str info: output file name suffix
:param bool verbose: controls verbosity
"""
global x_limit # late assignment of default, because it can be set to None in config
global divide_by_dimension # not fully implemented/tested yet
if 'x_limit' not in globals() or x_limit is None:
x_limit = x_limit_default
tmp = pp.dictAlgByDim(dictAlg)
# tmp = pp.DictAlg(dictAlg).by_dim()
if len(tmp) != 1 and dimension is None:
raise ValueError('We never integrate over dimension.')
if dimension is not None:
if dimension not in tmp.keys():
raise ValueError('dimension %d not in dictAlg dimensions %s'
% (dimension, str(tmp.keys())))
tmp = {dimension: tmp[dimension]}
dim = tmp.keys()[0]
divisor = dim if divide_by_dimension else 1
algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []]
dictFunc = pp.dictAlgByFun(dictAlg)
# Collect data
# Crafting effort correction: should we consider any?
CrEperAlg = {}
for alg in algorithms_with_data:
CrE = 0.
if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL':
tmp = dictAlg[alg].dictByNoise()
assert len(tmp.keys()) == 1
if tmp.keys()[0] == 'noiselessall':
CrE = 0.5117
elif tmp.keys()[0] == 'nzall':
CrE = 0.6572
CrEperAlg[alg] = CrE
if CrE != 0.0:
print 'Crafting effort for', alg, 'is', CrE
dictData = {} # list of (ert per function) per algorithm
dictMaxEvals = {} # list of (maxevals per function) per algorithm
bestERT = [] # best ert per function
# funcsolved = [set()] * len(targets) # number of functions solved per target
xbest2009 = []
maxevalsbest2009 = []
for f, dictAlgperFunc in dictFunc.iteritems():
if function_IDs and f not in function_IDs:
continue
# print target_values((f, dim))
for j, t in enumerate(target_values((f, dim))):
# for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)):
# funcsolved[j].add(f)
for alg in algorithms_with_data:
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
try:
entry = dictAlgperFunc[alg][0] # one element per fun and per dim.
evals = entry.detEvals([t])[0]
assert entry.dim == dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
except (KeyError, IndexError):
#set_trace()
warntxt = ('Data for algorithm %s on function %d in %d-D '
% (alg, f, dim)
+ 'are missing.\n')
warnings.warn(warntxt)
dictData.setdefault(alg, []).extend(x)
dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc)
displaybest2009 = not isBiobjective #disabled until we find the bug
if displaybest2009:
#set_trace()
bestalgentries = bestalg.loadBestAlgorithm(isBiobjective)
bestalgentry = bestalgentries[(dim, f)]
bestalgevals = bestalgentry.detEvals(target_values((f, dim)))
# print bestalgevals
for j in range(len(bestalgevals[0])):
if bestalgevals[1][j]:
evals = bestalgevals[0][j]
#set_trace()
assert dim == bestalgentry.dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = bestalgentry.maxevals[bestalgevals[1][j]][np.isnan(evals)] / divisor
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
else:
x = perfprofsamplesize * [np.inf]
runlengthunsucc = []
xbest2009.extend(x)
maxevalsbest2009.extend(runlengthunsucc)
if order is None:
order = dictData.keys()
# Display data
lines = []
if displaybest2009:
args = {'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11.,
'markeredgewidth': 1.5, 'markerfacecolor': refcolor,
'markeredgecolor': refcolor, 'color': refcolor,
'label': 'best 2009', 'zorder': -1}
lines.append(plotdata(np.array(xbest2009), x_limit, maxevalsbest2009,
CrE = 0., **args))
def algname_to_label(algname, dirname=None):
"""to be extended to become generally useful"""
if isinstance(algname, (tuple, list)): # not sure this is needed
return ' '.join([str(name) for name in algname])
return str(algname)
for i, alg in enumerate(order):
try:
data = dictData[alg]
maxevals = dictMaxEvals[alg]
except KeyError:
continue
args = styles[(i) % len(styles)]
args['linewidth'] = 1.5
args['markersize'] = 12.
args['markeredgewidth'] = 1.5
args['markerfacecolor'] = 'None'
args['markeredgecolor'] = args['color']
args['label'] = algname_to_label(alg)
#args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib
#elif len(show_algorithms) > 0:
#args['color'] = 'wheat'
#args['ls'] = '-'
#args['zorder'] = -1
# plotdata calls pprldistr.plotECDF which calls ppfig.plotUnifLog... which does the work
lines.append(plotdata(np.array(data), x_limit, maxevals,
CrE=CrEperAlg[alg], **args))
labels, handles = plotLegend(lines, x_limit)
if True: # isLateXLeg:
fileName = os.path.join(outputdir,'pprldmany_%s.tex' % (info))
with open(fileName, 'w') as f:
f.write(r'\providecommand{\nperfprof}{7}')
algtocommand = {} # latex commands
for i, alg in enumerate(order):
tmp = r'\alg%sperfprof' % pptex.numtotext(i)
f.write(r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' %
(tmp, toolsdivers.str_to_latex(
toolsdivers.strip_pathname2(algname_to_label(alg)))))
algtocommand[algname_to_label(alg)] = tmp
if displaybest2009:
tmp = r'\algzeroperfprof'
f.write(r'\providecommand{%s}{best 2009}' % (tmp))
algtocommand['best 2009'] = tmp
commandnames = []
for label in labels:
commandnames.append(algtocommand[label])
# f.write(headleg)
if len(order) > 28: # latex sidepanel won't work well for more than 25 algorithms, but original labels are also clipped
f.write(r'\providecommand{\perfprofsidepanel}{\mbox{%s}\vfill\mbox{%s}}'
% (commandnames[0], commandnames[-1]))
else:
fontsize_command = r'\tiny{}' if len(order) > 19 else ''
f.write(r'\providecommand{\perfprofsidepanel}{{%s\mbox{%s}' %
(fontsize_command, commandnames[0])) # TODO: check len(labels) > 0
for i in range(1, len(labels)):
f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i])
f.write('}}\n')
# f.write(footleg)
if verbose:
print 'Wrote right-hand legend in %s' % fileName
figureName = os.path.join(outputdir,'pprldmany_%s' % (info))
#beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat)
beautify()
text = ppfig.consecutiveNumbers(sorted(dictFunc.keys()), 'f')
text += ',%d-D' % dim # TODO: this is strange when different dimensions are plotted
plt.text(0.01, 0.98, text, horizontalalignment="left",
verticalalignment="top", transform=plt.gca().transAxes)
if len(dictFunc) == 1:
plt.title(' '.join((str(dictFunc.keys()[0]),
genericsettings.current_testbed.short_names[dictFunc.keys()[0]])))
a = plt.gca()
plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative)
xticks, labels = plt.xticks()
tmp = []
for i in xticks:
tmp.append('%d' % round(np.log10(i)))
a.set_xticklabels(tmp)
if save_figure:
ppfig.saveFigure(figureName, verbose=verbose)
if len(dictFunc) == 1:
ppfig.save_single_functions_html(
os.path.join(outputdir, 'pprldmany'),
'', # algorithms names are clearly visible in the figure
add_to_names='_%02dD' %(dim),
algorithmCount=ppfig.AlgorithmCount.NON_SPECIFIED
)
if close_figure:
plt.close()
def main(dictAlg, order=None, outputdir='.', info='default',
verbose=True):
"""Generates a figure showing the performance of algorithms.
From a dictionary of :py:class:`DataSetList` sorted by algorithms,
generates the cumulative distribution function of the bootstrap
distribution of ERT for algorithms on multiple functions for
multiple targets altogether.
:param dict dictAlg: dictionary of :py:class:`DataSetList` instances
one instance is equivalent to one algorithm,
:param list targets: target function values
:param list order: sorted list of keys to dictAlg for plotting order
:param str outputdir: output directory
:param str info: output file name suffix
:param bool verbose: controls verbosity
"""
global x_limit # late assignment of default, because it can be set to None in config
if 'x_limit' not in globals() or x_limit is None:
x_limit = x_limit_default
tmp = pp.dictAlgByDim(dictAlg)
# tmp = pp.DictAlg(dictAlg).by_dim()
if len(tmp) != 1:
raise Exception('We never integrate over dimension.')
dim = tmp.keys()[0]
algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []]
dictFunc = pp.dictAlgByFun(dictAlg)
# Collect data
# Crafting effort correction: should we consider any?
CrEperAlg = {}
for alg in algorithms_with_data:
CrE = 0.
if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL':
tmp = dictAlg[alg].dictByNoise()
assert len(tmp.keys()) == 1
if tmp.keys()[0] == 'noiselessall':
CrE = 0.5117
elif tmp.keys()[0] == 'nzall':
CrE = 0.6572
CrEperAlg[alg] = CrE
if CrE != 0.0:
print 'Crafting effort for', alg, 'is', CrE
dictData = {} # list of (ert per function) per algorithm
dictMaxEvals = {} # list of (maxevals per function) per algorithm
bestERT = [] # best ert per function
# funcsolved = [set()] * len(targets) # number of functions solved per target
xbest2009 = []
maxevalsbest2009 = []
for f, dictAlgperFunc in dictFunc.iteritems():
if function_IDs and f not in function_IDs:
continue
# print target_values((f, dim))
for j, t in enumerate(target_values((f, dim))):
# for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)):
# funcsolved[j].add(f)
for alg in algorithms_with_data:
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
try:
entry = dictAlgperFunc[alg][0] # one element per fun and per dim.
evals = entry.detEvals([t])[0]
runlengthsucc = evals[np.isnan(evals) == False] / entry.dim
runlengthunsucc = entry.maxevals[np.isnan(evals)] / entry.dim
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
except (KeyError, IndexError):
#set_trace()
warntxt = ('Data for algorithm %s on function %d in %d-D '
% (alg, f, dim)
+ 'are missing.\n')
warnings.warn(warntxt)
dictData.setdefault(alg, []).extend(x)
dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc)
if displaybest2009:
#set_trace()
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgentry = bestalg.bestalgentries2009[(dim, f)]
bestalgevals = bestalgentry.detEvals(target_values((f, dim)))
# print bestalgevals
for j in range(len(bestalgevals[0])):
if bestalgevals[1][j]:
evals = bestalgevals[0][j]
#set_trace()
runlengthsucc = evals[np.isnan(evals) == False] / bestalgentry.dim
runlengthunsucc = bestalgentry.maxevals[bestalgevals[1][j]][np.isnan(evals)] / bestalgentry.dim
x = toolsstats.drawSP(runlengthsucc, runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
else:
x = perfprofsamplesize * [np.inf]
runlengthunsucc = []
xbest2009.extend(x)
maxevalsbest2009.extend(runlengthunsucc)
if order is None:
order = dictData.keys()
# Display data
lines = []
if displaybest2009:
args = {'ls': '-', 'linewidth': 6, 'marker': 'D', 'markersize': 11.,
'markeredgewidth': 1.5, 'markerfacecolor': refcolor,
'markeredgecolor': refcolor, 'color': refcolor,
'label': 'best 2009', 'zorder': -1}
lines.append(plotdata(np.array(xbest2009), x_limit, maxevalsbest2009,
CrE = 0., **args))
for i, alg in enumerate(order):
try:
data = dictData[alg]
maxevals = dictMaxEvals[alg]
except KeyError:
continue
args = styles[(i) % len(styles)]
args['linewidth'] = 1.5
args['markersize'] = 12.
args['markeredgewidth'] = 1.5
args['markerfacecolor'] = 'None'
args['markeredgecolor'] = args['color']
args['label'] = alg
#args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib
#elif len(show_algorithms) > 0:
#args['color'] = 'wheat'
#args['ls'] = '-'
#args['zorder'] = -1
lines.append(plotdata(np.array(data), x_limit, maxevals,
CrE=CrEperAlg[alg], **args))
labels, handles = plotLegend(lines, x_limit)
if True: #isLateXLeg:
fileName = os.path.join(outputdir,'pprldmany_%s.tex' % (info))
try:
f = open(fileName, 'w')
f.write(r'\providecommand{\nperfprof}{7}')
algtocommand = {}
for i, alg in enumerate(order):
tmp = r'\alg%sperfprof' % pptex.numtotext(i)
f.write(r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' % (tmp, toolsdivers.str_to_latex(toolsdivers.strip_pathname2(alg))))
algtocommand[alg] = tmp
commandnames = []
if displaybest2009:
tmp = r'\algzeroperfprof'
f.write(r'\providecommand{%s}{best 2009}' % (tmp))
algtocommand['best 2009'] = tmp
for l in labels:
commandnames.append(algtocommand[l])
# f.write(headleg)
f.write(r'\providecommand{\perfprofsidepanel}{\mbox{%s}' % commandnames[0]) # TODO: check len(labels) > 0
for i in range(1, len(labels)):
f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i])
f.write('}\n')
# f.write(footleg)
if verbose:
print 'Wrote right-hand legend in %s' % fileName
except:
raise # TODO: Does this make sense?
else:
f.close()
figureName = os.path.join(outputdir,'pprldmany_%s' % (info))
#beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat)
beautify()
text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys())))
text += ',%d-D' % dim
plt.text(0.01, 0.98, text, horizontalalignment="left",
verticalalignment="top", transform=plt.gca().transAxes)
a = plt.gca()
plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative)
xticks, labels = plt.xticks()
tmp = []
for i in xticks:
tmp.append('%d' % round(np.log10(i)))
a.set_xticklabels(tmp)
ppfig.saveFigure(figureName, verbose=verbose)
plt.close()
def main(dictAlg, sortedAlgs, target=1e-8, outputdir='ppdata', verbose=True):
"""From a DataSetList, returns figures showing the scaling: ERT/dim vs dim.
One function and one target per figure.
sortedAlgs is a list of string-identifies (folder names)
"""
dictFunc = pproc.dictAlgByFun(dictAlg)
for f in dictFunc:
filename = os.path.join(outputdir,'ppfigs_f%03d' % (f))
handles = []
fix_styles(len(sortedAlgs)) #
for i, alg in enumerate(sortedAlgs):
dictDim = dictFunc[f][alg].dictByDim()
#Collect data
dimert = []
ert = []
dimnbsucc = []
ynbsucc = []
nbsucc = []
dimmaxevals = []
maxevals = []
dimmedian = []
medianfes = []
for dim in sorted(dictDim):
assert len(dictDim[dim]) == 1
entry = dictDim[dim][0]
data = generateData(entry, target) # TODO: here we might want a different target for each function
if 1 < 3 or data[2] == 0: # No success
dimmaxevals.append(dim)
maxevals.append(float(data[3])/dim)
if data[2] > 0:
dimmedian.append(dim)
medianfes.append(data[4]/dim)
dimert.append(dim)
ert.append(float(data[0])/dim)
if data[1] < 1.:
dimnbsucc.append(dim)
ynbsucc.append(float(data[0])/dim)
nbsucc.append('%d' % data[2])
# Draw lines
tmp = plt.plot(dimert, ert, **styles[i]) #label=alg, )
plt.setp(tmp[0], markeredgecolor=plt.getp(tmp[0], 'color'))
# For legend
# tmp = plt.plot([], [], label=alg.replace('..' + os.sep, '').strip(os.sep), **styles[i])
tmp = plt.plot([], [], label=alg.split(os.sep)[-1], **styles[i])
plt.setp(tmp[0], markersize=12.,
markeredgecolor=plt.getp(tmp[0], 'color'))
if dimmaxevals:
tmp = plt.plot(dimmaxevals, maxevals, **styles[i])
plt.setp(tmp[0], markersize=20, #label=alg,
markeredgecolor=plt.getp(tmp[0], 'color'),
markeredgewidth=1,
markerfacecolor='None', linestyle='None')
handles.append(tmp)
#tmp2 = plt.plot(dimmedian, medianfes, ls='', marker='+',
# markersize=30, markeredgewidth=5,
# markeredgecolor=plt.getp(tmp, 'color'))[0]
#for i, n in enumerate(nbsucc):
# plt.text(dimnbsucc[i], numpy.array(ynbsucc[i])*1.85, n,
# verticalalignment='bottom',
# horizontalalignment='center')
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgdata = []
dimbestalg = list(df[0] for df in bestalg.bestalgentries2009 if df[1] == f)
dimbestalg.sort()
dimbestalg2 = []
for d in dimbestalg:
entry = bestalg.bestalgentries2009[(d, f)]
tmp = entry.detERT([target])[0]
if numpy.isfinite(tmp):
bestalgdata.append(float(tmp)/d)
dimbestalg2.append(d)
tmp = plt.plot(dimbestalg2, bestalgdata, color=refcolor, linewidth=10,
marker='d', markersize=25, markeredgecolor=refcolor, zorder=-1
#label='best 2009',
)
handles.append(tmp)
if show_significance: # plot significance-stars
xstar, ystar = [], []
dims = sorted(pproc.dictAlgByDim(dictFunc[f]))
for i, dim in enumerate(dims):
datasets = pproc.dictAlgByDim(dictFunc[f])[dim]
assert all([len(datasets[ialg]) == 1 for ialg in sortedAlgs if datasets[ialg]])
dsetlist = [datasets[ialg][0] for ialg in sortedAlgs if datasets[ialg]]
if len(dsetlist) > 1:
arzp, arialg = toolsstats.significance_all_best_vs_other(dsetlist, [target])
if arzp[0][1] * len(dims) < 0.05:
ert = dsetlist[arialg[0]].detERT([target])[0]
if ert < numpy.inf:
xstar.append(dim)
ystar.append(ert/dim)
plt.plot(xstar, ystar, 'k*', markerfacecolor=None, markeredgewidth=2, markersize=0.5*styles[0]['markersize'])
if funInfos:
plt.gca().set_title(funInfos[f])
isLegend = False
if legend:
plotLegend(handles)
elif 1 < 3:
if f in (1, 24, 101, 130) and len(sortedAlgs) < 6: # 6 elements at most in the boxed legend
isLegend = True
beautify(legend=isLegend, rightlegend=legend)
plt.text(plt.xlim()[0], plt.ylim()[0], 'ftarget=%.0e' % target)
saveFigure(filename, verbose=verbose)
plt.close()
# generate commands in tex file:
try:
abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
alg_definitions = []
for i in range(len(sortedAlgs)):
symb = r'{%s%s}' % (color_to_latex(styles[i]['color']),
marker_to_latex(styles[i]['marker']))
alg_definitions.append((', ' if i > 0 else '') + '%s:%s' % (symb, '\\algorithm' + abc[i % len(abc)]))
filename = os.path.join(outputdir, 'bbob_pproc_commands.tex')
toolsdivers.prepend_to_file(filename,
['\\providecommand{\\bbobppfigsftarget}{\\ensuremath{10^{%d}}}'
% int(numpy.round(numpy.log10(target))),
'\\providecommand{\\bbobppfigslegend}[1]{',
scaling_figure_legend,
'Legend: '] + alg_definitions + ['}']
)
if verbose:
print 'Wrote commands and legend to %s' % filename
# this is obsolete (however check templates)
filename = os.path.join(outputdir,'ppfigs.tex')
f = open(filename, 'w')
f.write('% Do not modify this file: calls to post-processing software'
+ ' will overwrite any modification.\n')
f.write('Legend: ')
for i in range(0, len(sortedAlgs)):
symb = r'{%s%s}' % (color_to_latex(styles[i]['color']),
marker_to_latex(styles[i]['marker']))
f.write((', ' if i > 0 else '') + '%s:%s' % (symb, writeLabels(sortedAlgs[i])))
f.close()
if verbose:
print '(obsolete) Wrote legend in %s' % filename
except IOError:
raise
handles.append(tmp)
if funInfos:
plt.gca().set_title(funInfos[f])
beautify(rightlegend=legend)
if legend:
plotLegend(handles)
else:
if f in (1, 24, 101, 130):
plt.legend()
saveFigure(filename, figFormat=genericsettings.fig_formats, verbose=verbose)
plt.close()
def extractBestAlgorithms(args = algs2009, f_factor=2,
target_lb=1e-8, target_ub=1e22):
"""Returns (and prints) per dimension a list of algorithms within
algorithm list args that contains an algorithm if for any
dimension/target/function pair this algorithm:
- is the best algorithm wrt ERT
- its own ERT lies within a factor f_factor of the best ERT
- there is no algorithm within a factor of f_factor of the best ERT
and the current algorithm is the second best.
"""
# TODO: use pproc.TargetValues class as input target values
# default target values:
targets = pproc.TargetValues(
10**np.arange(np.log10(max((1e-8, target_lb))),
np.log10(target_ub) + 1e-9, 0.2))
# there should be a simpler way to express this to become the
# interface of this function
print 'Loading algorithm data from given algorithm list...\n'
verbose = True
dsList, sortedAlgs, dictAlg = pproc.processInputArgs(args, verbose=verbose)
print 'This may take a while (depending on the number of algorithms)'
selectedAlgsPerProblem = {}
for f, i in pproc.dictAlgByFun(dictAlg).iteritems():
for d, j in pproc.dictAlgByDim(i).iteritems():
selectedAlgsPerProblemDF = []
best = BestAlgSet(j)
for i in range(0, len(best.target)):
t = best.target[i]
# if ((t <= target_ub) and (t >= target_lb)):
if toolsstats.in_approximately(t,
targets((f, d), discretize=True)):
# add best for this target:
selectedAlgsPerProblemDF.append(best.algs[i])
# add second best or all algorithms that have an ERT
# within a factor of f_factor of the best:
secondbest_ERT = np.infty
secondbest_str = ''
secondbest_included = False
for astring in j:
currdictalg = dictAlg[astring].dictByDim()
if currdictalg.has_key(d):
curralgdata = currdictalg[d][f-1]
currERT = curralgdata.detERT([t])[0]
if (astring != best.algs[i]):
if (currERT < secondbest_ERT):
secondbest_ERT = currERT
secondbest_str = astring
if (currERT <= best.detERT([t])[0] * f_factor):
selectedAlgsPerProblemDF.append(astring)
secondbest_included = True
if not (secondbest_included) and (secondbest_str != ''):
selectedAlgsPerProblemDF.append(secondbest_str)
if len(selectedAlgsPerProblemDF) > 0:
selectedAlgsPerProblem[(d, f)] = selectedAlgsPerProblemDF
print 'pre-processing of function', f, 'done.'
print 'loading of best algorithm(s) data done.'
countsperalgorithm = {}
for (d, f) in selectedAlgsPerProblem:
print 'dimension:', d, ', function:', f
setofalgs = set(selectedAlgsPerProblem[d,f])
# now count how often algorithm a is best for the extracted targets
for a in setofalgs:
# use setdefault to initialize with zero if a entry not existant:
countsperalgorithm.setdefault((d, a), 0)
countsperalgorithm[(d,a)] += selectedAlgsPerProblem[d,f].count(a)
selectedalgsperdimension = {}
for (d,a) in sorted(countsperalgorithm):
if not selectedalgsperdimension.has_key(d):
selectedalgsperdimension[d] = []
selectedalgsperdimension[d].append((countsperalgorithm[(d,a)], a))
for d in sorted(selectedalgsperdimension):
print d, 'D:'
for (count, alg) in sorted(selectedalgsperdimension[d], reverse=True):
print count, alg
print '\n'
print " done."
return selectedalgsperdimension
def extractBestAlgorithms(args=algs2009,
f_factor=2,
target_lb=1e-8,
target_ub=1e22):
"""Returns (and prints) per dimension a list of algorithms within
algorithm list args that contains an algorithm if for any
dimension/target/function pair this algorithm:
- is the best algorithm wrt ERT
- its own ERT lies within a factor f_factor of the best ERT
- there is no algorithm within a factor of f_factor of the best ERT
and the current algorithm is the second best.
"""
# TODO: use pproc.TargetValues class as input target values
# default target values:
targets = pproc.TargetValues(10**np.arange(
np.log10(max((1e-8, target_lb))),
np.log10(target_ub) + 1e-9, 0.2))
# there should be a simpler way to express this to become the
# interface of this function
print 'Loading algorithm data from given algorithm list...\n'
verbose = True
dsList, sortedAlgs, dictAlg = pproc.processInputArgs(args, verbose=verbose)
print 'This may take a while (depending on the number of algorithms)'
selectedAlgsPerProblem = {}
for f, i in pproc.dictAlgByFun(dictAlg).iteritems():
for d, j in pproc.dictAlgByDim(i).iteritems():
selectedAlgsPerProblemDF = []
best = BestAlgSet(j)
for i in range(0, len(best.target)):
t = best.target[i]
# if ((t <= target_ub) and (t >= target_lb)):
if toolsstats.in_approximately(
t, targets((f, d), discretize=True)):
# add best for this target:
selectedAlgsPerProblemDF.append(best.algs[i])
# add second best or all algorithms that have an ERT
# within a factor of f_factor of the best:
secondbest_ERT = np.infty
secondbest_str = ''
secondbest_included = False
for astring in j:
currdictalg = dictAlg[astring].dictByDim()
if currdictalg.has_key(d):
curralgdata = currdictalg[d][f - 1]
currERT = curralgdata.detERT([t])[0]
if (astring != best.algs[i]):
if (currERT < secondbest_ERT):
secondbest_ERT = currERT
secondbest_str = astring
if (currERT <= best.detERT([t])[0] * f_factor):
selectedAlgsPerProblemDF.append(astring)
secondbest_included = True
if not (secondbest_included) and (secondbest_str != ''):
selectedAlgsPerProblemDF.append(secondbest_str)
if len(selectedAlgsPerProblemDF) > 0:
selectedAlgsPerProblem[(d, f)] = selectedAlgsPerProblemDF
print 'pre-processing of function', f, 'done.'
print 'loading of best algorithm(s) data done.'
countsperalgorithm = {}
for (d, f) in selectedAlgsPerProblem:
print 'dimension:', d, ', function:', f
setofalgs = set(selectedAlgsPerProblem[d, f])
# now count how often algorithm a is best for the extracted targets
for a in setofalgs:
# use setdefault to initialize with zero if a entry not existant:
countsperalgorithm.setdefault((d, a), 0)
countsperalgorithm[(d, a)] += selectedAlgsPerProblem[d, f].count(a)
selectedalgsperdimension = {}
for (d, a) in sorted(countsperalgorithm):
if not selectedalgsperdimension.has_key(d):
selectedalgsperdimension[d] = []
selectedalgsperdimension[d].append((countsperalgorithm[(d, a)], a))
for d in sorted(selectedalgsperdimension):
print d, 'D:'
for (count, alg) in sorted(selectedalgsperdimension[d], reverse=True):
print count, alg
print '\n'
print " done."
return selectedalgsperdimension
def main(dictAlg, sortedAlgs=None, target=ftarget_default, outputdir='ppdata', verbose=True):
"""From a DataSetList, returns figures showing the scaling: ERT/dim vs dim.
One function and one target per figure.
``target`` can be a scalar, a list with one element or a
``pproc.TargetValues`` instance with one target.
``sortedAlgs`` is a list of string-identifies (folder names)
"""
# target becomes a TargetValues "list" with one element
target = pproc.TargetValues.cast([target] if numpy.isscalar(target) else target)
latex_commands_filename = os.path.join(outputdir, 'bbob_pproc_commands.tex')
assert isinstance(target, pproc.TargetValues)
if len(target) != 1:
raise ValueError('only a single target can be managed in ppfigs, ' + str(len(target)) + ' targets were given')
dictFunc = pproc.dictAlgByFun(dictAlg)
if sortedAlgs is None:
sortedAlgs = sorted(dictAlg.keys())
if not os.path.isdir(outputdir):
os.mkdir(outputdir)
for f in dictFunc:
filename = os.path.join(outputdir,'ppfigs_f%03d' % (f))
handles = []
fix_styles(len(sortedAlgs)) #
for i, alg in enumerate(sortedAlgs):
dictDim = dictFunc[f][alg].dictByDim() # this does not look like the most obvious solution
#Collect data
dimert = []
ert = []
dimnbsucc = []
ynbsucc = []
nbsucc = []
dimmaxevals = []
maxevals = []
dimmedian = []
medianfes = []
for dim in sorted(dictDim):
assert len(dictDim[dim]) == 1
entry = dictDim[dim][0]
data = generateData(entry, target((f, dim))[0]) # TODO: here we might want a different target for each function
if 1 < 3 or data[2] == 0: # No success
dimmaxevals.append(dim)
maxevals.append(float(data[3])/dim)
if data[2] > 0:
dimmedian.append(dim)
medianfes.append(data[4]/dim)
dimert.append(dim)
ert.append(float(data[0])/dim)
if data[1] < 1.:
dimnbsucc.append(dim)
ynbsucc.append(float(data[0])/dim)
nbsucc.append('%d' % data[2])
# Draw lines
tmp = plt.plot(dimert, ert, **styles[i]) #label=alg, )
plt.setp(tmp[0], markeredgecolor=plt.getp(tmp[0], 'color'))
# For legend
# tmp = plt.plot([], [], label=alg.replace('..' + os.sep, '').strip(os.sep), **styles[i])
tmp = plt.plot([], [], label=alg.split(os.sep)[-1], **styles[i])
plt.setp(tmp[0], markersize=12.,
markeredgecolor=plt.getp(tmp[0], 'color'))
if dimmaxevals:
tmp = plt.plot(dimmaxevals, maxevals, **styles[i])
plt.setp(tmp[0], markersize=20, #label=alg,
markeredgecolor=plt.getp(tmp[0], 'color'),
markeredgewidth=1,
markerfacecolor='None', linestyle='None')
handles.append(tmp)
#tmp2 = plt.plot(dimmedian, medianfes, ls='', marker='+',
# markersize=30, markeredgewidth=5,
# markeredgecolor=plt.getp(tmp, 'color'))[0]
#for i, n in enumerate(nbsucc):
# plt.text(dimnbsucc[i], numpy.array(ynbsucc[i])*1.85, n,
# verticalalignment='bottom',
# horizontalalignment='center')
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgdata = []
dimbestalg = list(df[0] for df in bestalg.bestalgentries2009 if df[1] == f)
dimbestalg.sort()
dimbestalg2 = []
for d in dimbestalg:
entry = bestalg.bestalgentries2009[(d, f)]
tmp = entry.detERT(target((f, d)))[0]
if numpy.isfinite(tmp):
bestalgdata.append(float(tmp)/d)
dimbestalg2.append(d)
tmp = plt.plot(dimbestalg2, bestalgdata, color=refcolor, linewidth=10,
marker='d', markersize=25, markeredgecolor=refcolor, zorder=-1
#label='best 2009',
)
handles.append(tmp)
if show_significance: # plot significance-stars
xstar, ystar = [], []
dims = sorted(pproc.dictAlgByDim(dictFunc[f]))
for i, dim in enumerate(dims):
datasets = pproc.dictAlgByDim(dictFunc[f])[dim]
assert all([len(datasets[ialg]) == 1 for ialg in sortedAlgs if datasets[ialg]])
dsetlist = [datasets[ialg][0] for ialg in sortedAlgs if datasets[ialg]]
if len(dsetlist) > 1:
arzp, arialg = toolsstats.significance_all_best_vs_other(dsetlist, target((f, dim)))
if arzp[0][1] * len(dims) < show_significance:
ert = dsetlist[arialg[0]].detERT(target((f, dim)))[0]
if ert < numpy.inf:
xstar.append(dim)
ystar.append(ert/dim)
plt.plot(xstar, ystar, 'k*', markerfacecolor=None, markeredgewidth=2, markersize=0.5*styles[0]['markersize'])
if funInfos:
plt.gca().set_title(funInfos[f])
isLegend = False
if legend:
plotLegend(handles)
elif 1 < 3:
if f in (1, 24, 101, 130) and len(sortedAlgs) < 6: # 6 elements at most in the boxed legend
isLegend = True
beautify(legend=isLegend, rightlegend=legend)
plt.text(plt.xlim()[0], plt.ylim()[0], 'target ' + target.label_name() + ': ' + target.label(0)) # TODO: check
saveFigure(filename, verbose=verbose)
plt.close()
# generate commands in tex file:
try:
abc = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
alg_definitions = []
for i in range(len(sortedAlgs)):
symb = r'{%s%s}' % (color_to_latex(styles[i]['color']),
marker_to_latex(styles[i]['marker']))
alg_definitions.append((', ' if i > 0 else '') + '%s:%s' % (symb, '\\algorithm' + abc[i % len(abc)]))
toolsdivers.prepend_to_file(latex_commands_filename,
[#'\\providecommand{\\bbobppfigsftarget}{\\ensuremath{10^{%s}}}'
# % target.loglabel(0), # int(numpy.round(numpy.log10(target))),
'\\providecommand{\\bbobppfigslegend}[1]{',
scaling_figure_caption(target),
'Legend: '] + alg_definitions + ['}']
)
toolsdivers.prepend_to_file(latex_commands_filename,
['\\providecommand{\\bbobECDFslegend}[1]{',
ecdfs_figure_caption(target), '}']
)
if verbose:
print 'Wrote commands and legend to %s' % filename
# this is obsolete (however check templates)
filename = os.path.join(outputdir,'ppfigs.tex')
f = open(filename, 'w')
f.write('% Do not modify this file: calls to post-processing software'
+ ' will overwrite any modification.\n')
f.write('Legend: ')
for i in range(0, len(sortedAlgs)):
symb = r'{%s%s}' % (color_to_latex(styles[i]['color']),
marker_to_latex(styles[i]['marker']))
f.write((', ' if i > 0 else '') + '%s:%s' % (symb, writeLabels(sortedAlgs[i])))
f.close()
if verbose:
print '(obsolete) Wrote legend in %s' % filename
except IOError:
raise
handles.append(tmp)
if funInfos:
plt.gca().set_title(funInfos[f])
beautify(rightlegend=legend)
if legend:
plotLegend(handles)
else:
if f in (1, 24, 101, 130):
plt.legend()
saveFigure(filename, figFormat=genericsettings.fig_formats, verbose=verbose)
plt.close()
def main(dictAlg,
order=None,
outputdir='.',
info='default',
dimension=None,
verbose=True):
"""Generates a figure showing the performance of algorithms.
From a dictionary of :py:class:`DataSetList` sorted by algorithms,
generates the cumulative distribution function of the bootstrap
distribution of ERT for algorithms on multiple functions for
multiple targets altogether.
:param dict dictAlg: dictionary of :py:class:`DataSetList` instances
one instance is equivalent to one algorithm,
:param list targets: target function values
:param list order: sorted list of keys to dictAlg for plotting order
:param str outputdir: output directory
:param str info: output file name suffix
:param bool verbose: controls verbosity
"""
global x_limit # late assignment of default, because it can be set to None in config
global divide_by_dimension # not fully implemented/tested yet
if 'x_limit' not in globals() or x_limit is None:
x_limit = x_limit_default
tmp = pp.dictAlgByDim(dictAlg)
# tmp = pp.DictAlg(dictAlg).by_dim()
if len(tmp) != 1 and dimension is None:
raise ValueError('We never integrate over dimension.')
if dimension is not None:
if dimension not in tmp.keys():
raise ValueError('dimension %d not in dictAlg dimensions %s' %
(dimension, str(tmp.keys())))
tmp = {dimension: tmp[dimension]}
dim = tmp.keys()[0]
divisor = dim if divide_by_dimension else 1
algorithms_with_data = [a for a in dictAlg.keys() if dictAlg[a] != []]
dictFunc = pp.dictAlgByFun(dictAlg)
# Collect data
# Crafting effort correction: should we consider any?
CrEperAlg = {}
for alg in algorithms_with_data:
CrE = 0.
if 1 < 3 and dictAlg[alg][0].algId == 'GLOBAL':
tmp = dictAlg[alg].dictByNoise()
assert len(tmp.keys()) == 1
if tmp.keys()[0] == 'noiselessall':
CrE = 0.5117
elif tmp.keys()[0] == 'nzall':
CrE = 0.6572
CrEperAlg[alg] = CrE
if CrE != 0.0:
print 'Crafting effort for', alg, 'is', CrE
dictData = {} # list of (ert per function) per algorithm
dictMaxEvals = {} # list of (maxevals per function) per algorithm
bestERT = [] # best ert per function
# funcsolved = [set()] * len(targets) # number of functions solved per target
xbest2009 = []
maxevalsbest2009 = []
for f, dictAlgperFunc in dictFunc.iteritems():
if function_IDs and f not in function_IDs:
continue
# print target_values((f, dim))
for j, t in enumerate(target_values((f, dim))):
# for j, t in enumerate(genericsettings.current_testbed.ecdf_target_values(1e2, f)):
# funcsolved[j].add(f)
for alg in algorithms_with_data:
x = [np.inf] * perfprofsamplesize
runlengthunsucc = []
try:
entry = dictAlgperFunc[alg][
0] # one element per fun and per dim.
evals = entry.detEvals([t])[0]
assert entry.dim == dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = entry.maxevals[np.isnan(evals)] / divisor
if len(runlengthsucc) > 0:
x = toolsstats.drawSP(runlengthsucc,
runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
except (KeyError, IndexError):
#set_trace()
warntxt = (
'Data for algorithm %s on function %d in %d-D ' %
(alg, f, dim) + 'are missing.\n')
warnings.warn(warntxt)
dictData.setdefault(alg, []).extend(x)
dictMaxEvals.setdefault(alg, []).extend(runlengthunsucc)
if displaybest2009:
#set_trace()
if not bestalg.bestalgentries2009:
bestalg.loadBBOB2009()
bestalgentry = bestalg.bestalgentries2009[(dim, f)]
bestalgevals = bestalgentry.detEvals(target_values((f, dim)))
# print bestalgevals
for j in range(len(bestalgevals[0])):
if bestalgevals[1][j]:
evals = bestalgevals[0][j]
#set_trace()
assert dim == bestalgentry.dim
runlengthsucc = evals[np.isnan(evals) == False] / divisor
runlengthunsucc = bestalgentry.maxevals[
bestalgevals[1][j]][np.isnan(evals)] / divisor
x = toolsstats.drawSP(runlengthsucc,
runlengthunsucc,
percentiles=[50],
samplesize=perfprofsamplesize)[1]
else:
x = perfprofsamplesize * [np.inf]
runlengthunsucc = []
xbest2009.extend(x)
maxevalsbest2009.extend(runlengthunsucc)
if order is None:
order = dictData.keys()
# Display data
lines = []
if displaybest2009:
args = {
'ls': '-',
'linewidth': 6,
'marker': 'D',
'markersize': 11.,
'markeredgewidth': 1.5,
'markerfacecolor': refcolor,
'markeredgecolor': refcolor,
'color': refcolor,
'label': 'best 2009',
'zorder': -1
}
lines.append(
plotdata(np.array(xbest2009),
x_limit,
maxevalsbest2009,
CrE=0.,
**args))
def algname_to_label(algname, dirname=None):
"""to be extended to become generally useful"""
if isinstance(algname, (tuple, list)): # not sure this is needed
return ' '.join([str(name) for name in algname])
return str(algname)
for i, alg in enumerate(order):
try:
data = dictData[alg]
maxevals = dictMaxEvals[alg]
except KeyError:
continue
args = styles[(i) % len(styles)]
args['linewidth'] = 1.5
args['markersize'] = 12.
args['markeredgewidth'] = 1.5
args['markerfacecolor'] = 'None'
args['markeredgecolor'] = args['color']
args['label'] = algname_to_label(alg)
#args['markevery'] = perfprofsamplesize # option available in latest version of matplotlib
#elif len(show_algorithms) > 0:
#args['color'] = 'wheat'
#args['ls'] = '-'
#args['zorder'] = -1
# plotdata calls pprldistr.plotECDF which calls ppfig.plotUnifLog... which does the work
lines.append(
plotdata(np.array(data),
x_limit,
maxevals,
CrE=CrEperAlg[alg],
**args))
labels, handles = plotLegend(lines, x_limit)
if True: # isLateXLeg:
fileName = os.path.join(outputdir, 'pprldmany_%s.tex' % (info))
with open(fileName, 'w') as f:
f.write(r'\providecommand{\nperfprof}{7}')
algtocommand = {} # latex commands
for i, alg in enumerate(order):
tmp = r'\alg%sperfprof' % pptex.numtotext(i)
f.write(
r'\providecommand{%s}{\StrLeft{%s}{\nperfprof}}' %
(tmp,
toolsdivers.str_to_latex(
toolsdivers.strip_pathname2(algname_to_label(alg)))))
algtocommand[algname_to_label(alg)] = tmp
if displaybest2009:
tmp = r'\algzeroperfprof'
f.write(r'\providecommand{%s}{best 2009}' % (tmp))
algtocommand['best 2009'] = tmp
commandnames = []
for label in labels:
commandnames.append(algtocommand[label])
# f.write(headleg)
if len(
order
) > 28: # latex sidepanel won't work well for more than 25 algorithms, but original labels are also clipped
f.write(
r'\providecommand{\perfprofsidepanel}{\mbox{%s}\vfill\mbox{%s}}'
% (commandnames[0], commandnames[-1]))
else:
fontsize_command = r'\tiny{}' if len(order) > 19 else ''
f.write(r'\providecommand{\perfprofsidepanel}{{%s\mbox{%s}' %
(fontsize_command,
commandnames[0])) # TODO: check len(labels) > 0
for i in range(1, len(labels)):
f.write('\n' + r'\vfill \mbox{%s}' % commandnames[i])
f.write('}}\n')
# f.write(footleg)
if verbose:
print 'Wrote right-hand legend in %s' % fileName
figureName = os.path.join(outputdir, 'pprldmany_%s' % (info))
#beautify(figureName, funcsolved, x_limit*x_annote_factor, False, fileFormat=figformat)
beautify()
text = 'f%s' % (ppfig.consecutiveNumbers(sorted(dictFunc.keys())))
text += ',%d-D' % dim # TODO: this is strange when different dimensions are plotted
plt.text(0.01,
0.98,
text,
horizontalalignment="left",
verticalalignment="top",
transform=plt.gca().transAxes)
if len(dictFunc) == 1:
plt.title(' '.join(
(str(dictFunc.keys()[0]),
genericsettings.current_testbed.short_names[dictFunc.keys()[0]])))
a = plt.gca()
plt.xlim(xmin=1e-0, xmax=x_limit**annotation_space_end_relative)
xticks, labels = plt.xticks()
tmp = []
for i in xticks:
tmp.append('%d' % round(np.log10(i)))
a.set_xticklabels(tmp)
if save_figure:
ppfig.saveFigure(figureName, verbose=verbose)
if len(dictFunc) == 1:
ppfig.save_single_functions_html(
os.path.join(outputdir, 'pprldmany'),
'', # algorithms names are clearly visible in the figure
add_to_names='_%02dD' % (dim),
algorithmCount=ppfig.AlgorithmCount.NON_SPECIFIED)
if close_figure:
plt.close()
outputdir=outputdir,
info=('%02dD_%s' % (d, ng)),
verbose=verbose)
# ECDFs per function groups
dictFG = pproc.dictAlgByFuncGroup(dictAlg)
for fg, tmpdictAlg in dictFG.iteritems():
dictDim = pproc.dictAlgByDim(tmpdictAlg)
for d, entries in dictDim.iteritems():
pprldmany.main(entries,
order=sortedAlgs,
outputdir=outputdir,
info=('%02dD_%s' % (d, fg)),
verbose=verbose)
if isRldOnSingleFcts: # copy-paste from above, here for each function instead of function groups
# ECDFs for each function
dictFG = pproc.dictAlgByFun(dictAlg)
for fg, tmpdictAlg in dictFG.iteritems():
dictDim = pproc.dictAlgByDim(tmpdictAlg)
for d, entries in dictDim.iteritems():
single_fct_output_dir = (outputdir.rstrip(os.sep) + os.sep +
'pprldmany-single-functions'
# + os.sep + ('f%03d' % fg)
)
if not os.path.exists(single_fct_output_dir):
os.makedirs(single_fct_output_dir)
pprldmany.main(entries,
order=sortedAlgs,
outputdir=single_fct_output_dir,
info=('f%03d_%02dD' % (fg, d)),
verbose=verbose)
print "ECDFs of run lengths figures done."
