def add_layer(self, html_str: str, layer_name, data_dict: dict):
'''
add a further layer of top data_dict keys
'''
if data_dict.get("tooltip"):
tooltip = "<div class=\"help-tip\"><p>{}</p></div>".format(
data_dict.get("tooltip"))
elif get_tooltip(
layer_name): # if no tooltip is parsed, try to look it up
tooltip = "<div class=\"help-tip\"><p>{}</p></div>".format(
get_tooltip(layer_name))
else:
tooltip = ""
html_str += "<div class=\"accordion\">{0} {1}</div>\n".format(
layer_name, tooltip)
html_str += "<div class=\"panel\">\n"
for k, v in data_dict.items():
if isinstance(v, dict):
html_str = self.add_layer(html_str, k, v)
elif k == "figure":
html_str += "<div align=\"center\">\n"
if isinstance(v, str):
html_str += "<a href=\"{0}\" data-lightbox=\"{0}\" data-title=\"{0}\"><img src=\"{0}\" alt=\"Plot\" width=\"600px\"></a>\n".format(
v[len(self.output_dn):].lstrip("/"))
else:
# List with multiple figures size relative, put next to each other
width = (100 - len(v)) / len(v)
for fig in v:
html_str += "<a href=\"{0}\" data-lightbox=\"{1}\" data-title=\"{0}\"><img src=\"{0}\" alt=\"Plot\" style=\"float: left; width: {2}%; margin-right: 1%; margin-bottom: 0.5em;\"></a>\n".format(
fig[len(self.output_dn):].lstrip("/"), str(v),
int(width))
html_str += "<p style=\"clear: both;\">"
html_str += "</div>\n"
elif k == "table":
html_str += "<div align=\"center\">\n{}\n</div>\n".format(v)
elif k == "html":
html_str += "<div align=\"center\">\n<a href='{}'>Interactive Plot</a>\n</div>\n".format(
v[len(self.output_dn):].lstrip("/"))
#html_str += "<div align=\"center\"><iframe src='{}' frameborder='0' scrolling='no' width='700px' height='500px'></iframe></div>\n".format(v[len(self.output_dn):].lstrip("/"))
html_str += "</div>"
return html_str
def add_layer(self,
layer_name,
data_dict: OrderedDict,
is_tab: bool = False):
'''
add a further layer of top data_dict keys
Parameters
----------
layer_name: str
name of the layer
data_dict : OrderedDict
see constructor
is_tab: bool
if True, don't use accordion but tab-structure to wrap content
Returns
-------
(script, div): (str, str)
script goes into header, div goes into body
'''
script, div = "", ""
if layer_name is None:
layer_name = ""
unique_layer_name = layer_name + self.get_unique_id()
# Add tooltip, if possible
tooltip = data_dict.get("tooltip", None)
if tooltip is not None:
tooltip = "<div class=\"help-tip\"><p>{}</p></div>".format(tooltip)
# TODO elif is obsolete / can be merged into first option (simplify!)
elif get_tooltip(
layer_name): # if no tooltip is parsed, try to look it up
tooltip = "<div class=\"help-tip\"><p>{}</p></div>".format(
get_tooltip(layer_name))
else:
tooltip = ""
# Start accordion-panel
if not is_tab:
div += "<div class=\"accordion\">{0} {1}</div>\n".format(
layer_name, tooltip)
div += "<div class=\"panel\">\n"
# If this layer represents budgets, add tabs for this layer, add tabs-code
sublayer_names = [
k for k, v in data_dict.items() if isinstance(v, dict)
]
use_tabs = False
if len(sublayer_names) >= 1 and all(
[sn.lower().startswith('budget') for sn in sublayer_names]):
use_tabs = True
if use_tabs:
div += "<div class=\"tab\">\n"
tabs_names = [
k.replace('_', ' ') for k, v in data_dict.items()
if isinstance(v, dict)
]
default_open_id = "defaultOpen" + self.get_unique_id()
div += " <button class=\"tablinks\" onclick=\"openTab(event, '{0}', '{1}')\" "\
"id=\"{2}\">{1}</button>\n".format(unique_layer_name, tabs_names[0], default_open_id)
for name in tabs_names[1:]:
div += " <button class=\"tablinks\" onclick=\"openTab(event, '{0}', '{1}')\">{1}</button>\n".format(
unique_layer_name, name)
div += "</div>\n"
for k, v in data_dict.items():
if k == "tooltip":
continue
if k.startswith('budget'):
self.budget = k[7:]
if not v:
if isinstance(v, dict):
continue
else:
return '', ''
elif isinstance(v, dict):
if use_tabs:
div += "<div id=\"{0}\" class=\"tabcontent\">\n".format(
unique_layer_name + k.replace('_', ' '))
div += "<div class=\"pane\">\n"
add_script, add_div = self.add_layer(k, v, is_tab=use_tabs)
script += add_script
div += add_div
if use_tabs: # close div
div += "</div>\n"
div += "</div>\n"
elif k == "figure":
div += figure_to_html(v, prefix=self.output_dn)
elif k == "figure_x2":
div += figure_to_html(v, prefix=self.output_dn, max_in_a_row=2)
elif k == "table":
div += "<div style=\"overflow-x: auto\" align=\"center\">\n{}\n</div>\n".format(
v)
elif k == "html":
div += ("<div align=\"center\">\n<a href='{}'>Interactive "
"Plot</a>\n</div>\n".format(
v[len(self.output_dn):].lstrip("/")))
elif k == "bokeh":
# Escape path for URL (remove spaces, slashes and single quotes)
path_script = os.path.join(
self.relative_content_js, '_'.join([
layer_name, self.budget,
self.get_unique_id(), 'script.js'
]))
path_script = path_script.translate(
{ord(c): None
for c in ' \''})
# Write script to file
if self.output_dn:
with open(os.path.join(self.output_dn, path_script),
'w') as fn:
js_code = re.sub('<.*?>', '',
v[0].strip()) # Remove script-tags
fn.write(js_code)
script += "<script src=\"" + path_script + "\"></script>\n"
else:
script += v[0]
div += "<div align=\"center\">\n{}\n</div>\n".format(v[1])
else:
try:
div += v
except Exception as err:
self.logger.warning(
"Failed on interpreting: %s, %s, %s (Error: %s)",
str(layer_name),
str(k),
str(v),
err,
exc_info=1)
if use_tabs: # close tab with selecting first element by default
div += "<script> \n"
div += "// Get the element with id=\"{}\" and click on it \n".format(
default_open_id)
div += "document.getElementById(\"{}\").click(); \n".format(
default_open_id)
div += "</script> \n"
if not is_tab:
div += "</div>"
return script, div
def add_layer(self, layer_name, data_dict: OrderedDict):
'''
add a further layer of top data_dict keys
Parameters
----------
layer_name: str
name of the layer
data_dict : OrderedDict
{"top1" : {
"tooltip": str|None,
"subtop1": { # generates a further bottom if it is dictionary
"tooltip": str|None,
...
}
"table": str|None (html table)
"figure" : str|None (file name)
"bokeh" : ( str,str)|None # (script, div)
}
"top2": { ... }
}
Returns
-------
(script, div): (str, str)
script goes into header, div goes into body
'''
script, div = "", ""
if data_dict.get("tooltip"):
tooltip = "<div class=\"help-tip\"><p>{}</p></div>".format(
data_dict.get("tooltip"))
elif get_tooltip(
layer_name): # if no tooltip is parsed, try to look it up
tooltip = "<div class=\"help-tip\"><p>{}</p></div>".format(
get_tooltip(layer_name))
else:
tooltip = ""
div += "<div class=\"accordion\">{0} {1}</div>\n".format(
layer_name, tooltip)
div += "<div class=\"panel\">\n"
for k, v in data_dict.items():
if k.startswith('budget'):
self.budget = k[7:]
if not v:
return '', ''
elif isinstance(v, dict):
add_script, add_div = self.add_layer(k, v)
script += add_script
div += add_div
elif k == "figure":
div += "<div align=\"center\">\n"
if isinstance(v, str):
div += ("<a href=\"{0}\" data-lightbox=\"{0}\" "
"data-title=\"{0}\"><img src=\"{0}\" alt=\"Plot\" "
"width=\"600px\"></a>\n".format(
v[len(self.output_dn):].lstrip("/")))
else:
# List with multiple figures size relative, put next to each other
width = (100 - len(v)) / len(v)
for fig in v:
div += "<a href=\"{0}\" data-lightbox=\"{1}\" data-title=\"{0}\"><img src=\"{0}\"".format(
fig[len(self.output_dn):].lstrip("/"), str(v))
div += " alt=\"Plot\" style=\"float: left; width: {}%; margin-right: "\
"1%; margin-bottom: 0.5em;\"></a>\n".format(int(width))
div += "<p style=\"clear: both;\">"
div += "</div>\n"
elif k == "figure_x2":
# four figures in a grid
div += "<div align=\"center\">\n"
for fig in v:
path = fig[len(self.output_dn):].lstrip("/")
div += "<a href=\"{}\" ".format(path)
div += "data-lightbox=\"{}\" ".format(str(v))
div += "data-title=\"{0}\"><img src=\"{0}\" alt=\"Plot\" ".format(
path)
div += "style=\"float: left; width: 49%; margin-right: 1%; margin-bottom: 0.5em;\"></a>\n"
if v.index(fig) % 2 == 1:
div += " <br> "
div += "<p style=\"clear: both;\">"
div += "</div>\n"
elif k == "table":
div += "<div align=\"center\">\n{}\n</div>\n".format(v)
elif k == "html":
div += ("<div align=\"center\">\n<a href='{}'>Interactive "
"Plot</a>\n</div>\n".format(
v[len(self.output_dn):].lstrip("/")))
elif k == "bokeh":
# Escape path for URL (replace and ' with . ;)
path_script = os.path.join(
self.relative_content_js,
layer_name + self.budget + '_script.js')
path_script = path_script.translate(
{ord(c): None
for c in ' \''})
# Write script to file
with open(os.path.join(self.output_dn, path_script),
'w') as fn:
js_code = re.sub('<.*?>', '',
v[0].strip()) # Remove script-tags
fn.write(js_code)
script += "<script src=\"" + path_script + "\"></script>\n"
div += "<div align=\"center\">\n{}\n</div>\n".format(v[1])
else:
div += v
div += "</div>"
return script, div
def analyze(self,
performance=True,
cdf=True,
scatter=True,
confviz=True,
param_importance=['forward_selection', 'ablation', 'fanova'],
feature_analysis=[
"box_violin", "correlation", "feat_importance",
"clustering", "feature_cdf"
],
parallel_coordinates=True,
cost_over_time=True,
algo_footprint=True):
"""Analyze the available data and build HTML-webpage as dict.
Save webpage in 'self.output/CAVE/report.html'.
Analyzing is performed with the analyzer-instance that is initialized in
the __init__
Parameters
----------
performance: bool
whether to calculate par10-values
cdf: bool
whether to plot cdf
scatter: bool
whether to plot scatter
confviz: bool
whether to perform configuration visualization
param_importance: List[str]
containing methods for parameter importance
feature_analysis: List[str]
containing methods for feature analysis
parallel_coordinates: bool
whether to plot parallel coordinates
cost_over_time: bool
whether to plot cost over time
algo_footprint: bool
whether to plot algorithm footprints
"""
# Check arguments
for p in param_importance:
if p not in [
'forward_selection', 'ablation', 'fanova', 'incneighbor'
]:
raise ValueError(
"%s not a valid option for parameter "
"importance!", p)
for f in feature_analysis:
if f not in [
"box_violin", "correlation", "importance", "clustering",
"feature_cdf"
]:
raise ValueError("%s not a valid option for feature analysis!",
f)
# Start analysis
overview = self.analyzer.create_overview_table(self.best_run.folder)
self.website["Meta Data"] = {"table": overview}
compare_config = self.analyzer.config_to_html(self.default,
self.incumbent)
self.website["Best configuration"] = {"table": compare_config}
########## PERFORMANCE ANALYSIS
self.website["Performance Analysis"] = OrderedDict()
if performance:
performance_table = self.analyzer.create_performance_table(
self.default, self.incumbent)
self.website["Performance Analysis"]["Performance Table"] = {
"table": performance_table
}
if cdf:
cdf_path = self.analyzer.plot_cdf()
self.website["Performance Analysis"][
"empirical Cumulative Distribution Function (eCDF)"] = {
"figure": cdf_path
}
if scatter and (self.scenario.train_insts != [[None]]):
scatter_path = self.analyzer.plot_scatter()
self.website["Performance Analysis"]["Scatterplot"] = {
"figure": scatter_path
}
elif scatter:
self.logger.info(
"Scatter plot desired, but no instances available.")
# Build report before time-consuming analysis
self.build_website()
if algo_footprint and self.scenario.feature_dict:
algorithms = {self.default: "default", self.incumbent: "incumbent"}
# Add all available incumbents to test portfolio strategy
#for r in self.runs:
# if not r.get_incumbent() in algorithms:
# algorithms[r.get_incumbent()] = str(self.runs.index(r))
algo_footprint_plots = self.analyzer.plot_algorithm_footprint(
algorithms)
self.website["Performance Analysis"][
"Algorithm Footprints"] = OrderedDict()
for p in algo_footprint_plots:
header = os.path.splitext(os.path.split(p)[1])[0] # algo name
self.website["Performance Analysis"]["Algorithm Footprints"][
header] = {
"figure": p,
"tooltip":
get_tooltip("Algorithm Footprints") + ": " + header
}
self.build_website()
########### Configurator's behavior
self.website["Configurator's behavior"] = OrderedDict()
if confviz:
if self.scenario.feature_array is None:
self.scenario.feature_array = np.array([[]])
# Sort runhistories and incs wrt cost
incumbents = [r.solver.incumbent for r in self.runs]
trajectories = [r.traj for r in self.runs]
runhistories = [r.runhistory for r in self.runs]
costs = [self.validated_rh.get_cost(i) for i in incumbents]
costs, incumbents, runhistories, trajectories = (
list(t) for t in zip(
*sorted(zip(costs, incumbents, runhistories, trajectories),
key=lambda x: x[0])))
incumbents = list(map(lambda x: x['incumbent'], trajectories[0]))
confviz_script = self.analyzer.plot_confviz(
incumbents, runhistories)
self.website["Configurator's behavior"][
"Configurator Footprint"] = {
"table": confviz_script
}
elif confviz:
self.logger.info("Configuration visualization desired, but no "
"instance-features available.")
self.build_website()
if cost_over_time:
cost_over_time_path = self.analyzer.plot_cost_over_time(
self.best_run.traj, self.validator)
self.website["Configurator's behavior"]["Cost over time"] = {
"figure": cost_over_time_path
}
self.build_website()
self.parameter_importance(ablation='ablation' in param_importance,
fanova='fanova' in param_importance,
forward_selection='forward_selection'
in param_importance,
incneighbor='incneighbor'
in param_importance)
self.build_website()
if parallel_coordinates:
# Should be after parameter importance, if performed.
n_params = 6
parallel_path = self.analyzer.plot_parallel_coordinates(n_params)
self.website["Configurator's behavior"]["Parallel Coordinates"] = {
"figure": parallel_path
}
self.build_website()
if self.scenario.feature_dict:
self.feature_analysis(box_violin='box_violin' in feature_analysis,
correlation='correlation'
in feature_analysis,
clustering='clustering' in feature_analysis,
importance='importance' in feature_analysis)
else:
self.logger.info('No feature analysis possible')
self.logger.info("CAVE finished. Report is located in %s",
os.path.join(self.output, 'report.html'))
self.build_website()