def _plot_get_source(self, conf_list, runs, X, inc_list, hp_names):
"""
Create ColumnDataSource with all the necessary data
Contains for each configuration evaluated on any run:
- all parameters and values
- origin (if conflicting, origin from best run counts)
- type (default, incumbent or candidate)
- # of runs
- size
- color
Parameters
----------
conf_list: list[Configuration]
configurations
runs: list[int]
runs per configuration (same order as conf_list)
X: np.array
configuration-parameters as 2-dimensional array
inc_list: list[Configuration]
incumbents for this conf-run
hp_names: list[str]
names of hyperparameters
Returns
-------
source: ColumnDataSource
source with attributes as requested
"""
# Remove all configurations without any runs
keep = [i for i in range(len(runs)) if runs[i] > 0]
runs = np.array(runs)[keep]
conf_list = np.array(conf_list)[keep]
X = X[keep]
source = ColumnDataSource(data=dict(x=X[:, 0], y=X[:, 1]))
for k in hp_names: # Add parameters for each config
source.add([c[k] if c[k] else "None" for c in conf_list],
escape_parameter_name(k))
default = conf_list[0].configuration_space.get_default_configuration()
conf_types = [
"Default" if c == default else "Final Incumbent" if c
== inc_list[-1] else "Incumbent" if c in inc_list else "Candidate"
for c in conf_list
]
# We group "Local Search" and "Random Search (sorted)" both into local
origins = [self._get_config_origin(c) for c in conf_list]
source.add(conf_types, 'type')
source.add(origins, 'origin')
sizes = self._get_size(runs)
sizes = [
s * 3 if conf_types[idx] == "Default" else s
for idx, s in enumerate(sizes)
]
source.add(sizes, 'size')
source.add(self._get_color(source.data['type']), 'color')
source.add(runs, 'runs')
# To enforce zorder, we categorize all entries according to their size
# Since we plot all different zorder-levels sequentially, we use a
# manually defined level of influence
num_bins = 20 # How fine-grained the size-ordering should be
min_size, max_size = min(source.data['size']), max(source.data['size'])
step_size = (max_size - min_size) / num_bins
if step_size == 0:
step_size = 1
zorder = [
str(int((s - min_size) / step_size)) for s in source.data['size']
]
source.add(zorder, 'zorder') # string, so we can apply group filter
return source
def plot(self,
X,
conf_list: list,
runs_per_quantile,
inc_list: list = None,
contour_data=None,
time_slider=False):
"""
plots sampled configuration in 2d-space;
uses bokeh for interactive plot
saves results in self.output, if set
Parameters
----------
X: np.array
np.array with 2-d coordinates for each configuration
conf_list: list
list of ALL configurations in the same order as X
runs_per_quantile: list[np.array]
configurator-run to be analyzed, as a np.array with
the number of target-algorithm-runs per config per quantile.
inc_list: list
list of incumbents (Configuration)
contour_data: list
contour data (xx,yy,Z)
time_slider: bool
whether or not to have a time_slider-widget on cfp-plot
INCREASES FILE-SIZE DRAMATICALLY
Returns
-------
(script, div): str
script and div of the bokeh-figure
over_time_paths: List[str]
list with paths to the different quantiled timesteps of the
configurator run (for static evaluation)
"""
if not inc_list:
inc_list = []
over_time_paths = [] # development of the search space over time
hp_names = [
k.name for k in # Hyperparameter names
conf_list[0].configuration_space.get_hyperparameters()
]
# Get individual sources for quantiles
sources = [
self._plot_get_source(conf_list, quantiled_run, X, inc_list,
hp_names)
for quantiled_run in runs_per_quantile
]
# Define what appears in tooltips
# TODO add only important parameters (needs to change order of exec pimp before conf-footprints)
hover = HoverTool(
tooltips=[('type', '@type'), ('origin',
'@origin'), ('runs', '@runs')] +
[(k, '@' + escape_parameter_name(k)) for k in hp_names])
# bokeh-figure
x_range = [min(X[:, 0]) - 1, max(X[:, 0]) + 1]
y_range = [min(X[:, 1]) - 1, max(X[:, 1]) + 1]
scatter_glyph_render_groups = []
for idx, source in enumerate(sources):
if not time_slider or idx == 0:
# Only plot all quantiles in one plot if timeslider is on
p = figure(plot_height=500,
plot_width=600,
tools=[hover, 'save'],
x_range=x_range,
y_range=y_range)
if contour_data is not None:
p = self._plot_contour(p, contour_data, x_range, y_range)
views, markers = self._plot_create_views(source)
self.logger.debug("Plotting quantile %d!", idx)
scatter_glyph_render_groups.append(
self._plot_scatter(p, source, views, markers))
if self.output_dir:
file_path = "cfp_over_time/configurator_footprint" + str(
idx) + ".png"
over_time_paths.append(os.path.join(self.output_dir,
file_path))
self.logger.debug("Saving plot to %s", over_time_paths[-1])
export_bokeh(p, over_time_paths[-1], self.logger)
if time_slider:
self.logger.debug("Adding timeslider")
slider = self._plot_get_timeslider(scatter_glyph_render_groups)
layout = column(p, widgetbox(slider))
else:
self.logger.debug("Not adding timeslider")
layout = column(p)
script, div = components(layout)
if self.output_dir:
path = os.path.join(self.output_dir,
"content/images/configurator_footprint.png")
export_bokeh(p, path, self.logger)
return (script, div), over_time_paths
def plot(self,
X,
conf_list: list,
runs_per_quantile,
inc_list: list=None,
contour_data=None,
use_timeslider=False,
use_checkbox=True,
timeslider_labels=None):
"""
plots sampled configuration in 2d-space;
uses bokeh for interactive plot
saves results in self.output, if set
Parameters
----------
X: np.array
np.array with 2-d coordinates for each configuration
conf_list: list
list of ALL configurations in the same order as X
runs_per_quantile: list[np.array]
configurator-run to be analyzed, as a np.array with
the number of target-algorithm-runs per config per quantile.
inc_list: list
list of incumbents (Configuration)
contour_data: list
contour data (xx,yy,Z)
use_timeslider: bool
whether or not to have a time_slider-widget on cfp-plot
INCREASES FILE-SIZE DRAMATICALLY
use_checkbox: bool
have checkboxes to toggle individual runs
Returns
-------
(script, div): str
script and div of the bokeh-figure
over_time_paths: List[str]
list with paths to the different quantiled timesteps of the
configurator run (for static evaluation)
"""
if not inc_list:
inc_list = []
over_time_paths = [] # development of the search space over time
hp_names = [k.name for k in # Hyperparameter names
conf_list[0].configuration_space.get_hyperparameters()]
# bokeh-figure
x_range = [min(X[:, 0]) - 1, max(X[:, 0]) + 1]
y_range = [min(X[:, 1]) - 1, max(X[:, 1]) + 1]
# Get individual sources for quantiles
sources, used_configs = zip(*[self._plot_get_source(conf_list, quantiled_run, X, inc_list, hp_names)
for quantiled_run in runs_per_quantile])
# We collect all glyphs in one list
# Then we have to dicts to identify groups of glyphs (for interactivity)
# They map the name of the group to a list of indices (of the respective glyphs that are in the group)
# Those indices refer to the main list of all glyphs
# This is necessary to enable interactivity for two inputs at the same time
all_glyphs = []
overtime_groups = {}
run_groups = {run : [] for run in self.configs_in_run.keys()}
# Iterate over quantiles (this updates overtime_groups)
for idx, source, u_cfgs in zip(range(len(sources)), sources, used_configs):
# Create new plot if necessary (only plot all quantiles in one single plot if timeslider is on)
if not use_timeslider or idx == 0:
p = self._create_figure(x_range, y_range)
if contour_data is not None: # TODO
contour_handles, color_mapper = self._plot_contour(p, contour_data, x_range, y_range)
# Create views and scatter
views, views_by_run, markers = self._create_views(source, u_cfgs)
scatter_handles = self._scatter(p, source, views, markers)
self.logger.debug("Quantile %d: %d scatter-handles", idx, len(scatter_handles))
if len(scatter_handles) == 0:
self.logger.debug("No configs in quantile %d (?!)", idx)
continue
# Add to groups
start = len(all_glyphs)
all_glyphs.extend(scatter_handles)
overtime_groups[str(idx)] = [str(i) for i in range(start, len(all_glyphs))]
for run, indices in views_by_run.items():
run_groups[run].extend([str(start + i) for i in indices])
# Write to file
if self.output_dir:
file_path = "cfp_over_time/configurator_footprint" + str(idx) + ".png"
over_time_paths.append(os.path.join(self.output_dir, file_path))
self.logger.debug("Saving plot to %s", over_time_paths[-1])
export_bokeh(p, over_time_paths[-1], self.logger)
# Add hovertool (define what appears in tooltips)
# TODO add only important parameters (needs to change order of exec pimp before conf-footprints)
hover = HoverTool(tooltips=[('type', '@type'), ('origin', '@origin'), ('runs', '@runs')] +
[(k, '@' + escape_parameter_name(k)) for k in hp_names],
renderers=all_glyphs)
p.add_tools(hover)
# Build dashboard
timeslider, checkbox, select_all, select_none, checkbox_title = self._get_widgets(all_glyphs, overtime_groups, run_groups,
slider_labels=timeslider_labels)
contour_checkbox, contour_title = self._contour_radiobuttongroup(contour_handles, color_mapper)
layout = p
if use_timeslider:
self.logger.debug("Adding timeslider")
layout = column(layout, widgetbox(timeslider))
if use_checkbox:
self.logger.debug("Adding checkboxes")
layout = row(layout,
column(widgetbox(checkbox_title),
widgetbox(checkbox),
row(widgetbox(select_all, width=100),
widgetbox(select_none, width=100)),
widgetbox(contour_title),
widgetbox(contour_checkbox)))
if self.output_dir:
path = os.path.join(self.output_dir, "content/images/configurator_footprint.png")
export_bokeh(p, path, self.logger)
return layout, over_time_paths