def plot(self): """ Plot performance over time, using all trajectory entries. max_time denotes max(wallclock_limit, highest recorded time). """ rh, runs, output_fn, validator = self.rh, self.runs, self.output_fn, self.validator # Add lines to be plotted to lines (key-values must be zippable) lines = [] # Get plotting data and create CDS if self.bohb_results: lines.append(self._get_bohb_line(validator, runs, rh)) for b in self.bohb_results[0].HB_config['budgets']: lines.append(self._get_bohb_line(validator, runs, rh, b)) else: lines.append(self._get_avg(validator, runs, rh)) lines.extend(self._get_all_runs(validator, runs, rh)) data = {'name': [], 'time': [], 'mean': [], 'upper': [], 'lower': []} hp_names = self.scenario.cs.get_hyperparameter_names() for p in hp_names: data[p] = [] for line in lines: for t, m, u, l, c in zip(line.time, line.mean, line.upper, line.lower, line.config): if not (np.isfinite(m) and np.isfinite(u) and np.isfinite(l)): self.logger.debug("Why is there a NaN? (%s)", str(line)) raise ValueError( "There is a NaN value in your data, this should be filtered out. " "Please report this to github.com/automl/CAVE/issues and provide the " "debug/debug.log and the output of `pip freeze`, if you can." ) data['name'].append(line.name) data['time'].append(t) data['mean'].append(m) data['upper'].append(u) data['lower'].append(l) for p in hp_names: data[p].append(c[p] if (c and p in c) else 'inactive') source = ColumnDataSource(data=data) # Create plot x_range = Range1d(min(source.data['time']), max(source.data['time'])) y_label = 'estimated {}'.format(self.scenario.run_obj if self.scenario. run_obj != 'quality' else 'cost') p = figure(plot_width=700, plot_height=500, tools=['save', 'pan', 'box_zoom', 'wheel_zoom', 'reset'], x_range=x_range, x_axis_type='log', y_axis_type='log' if self.scenario.run_obj == 'runtime' else 'linear', x_axis_label='time (sec)', y_axis_label=y_label, title="Cost over time") colors = itertools.cycle(Dark2_5) renderers = [] legend_it = [] for line, color in zip(lines, colors): # CDSview w GroupFilter name = line.name view = CDSView( source=source, filters=[GroupFilter(column_name='name', group=str(name))]) renderers.append([ p.line('time', 'mean', source=source, view=view, line_color=color, visible=True if line.name in ['average', 'all budgets'] else False) ]) # Add to legend legend_it.append((name, renderers[-1])) if name in ['average', 'all budgets'] or 'budget' in name: # Fill area (uncertainty) # Defined as sequence of coordinates, so for step-effect double and arange accordingly ([(t0, v0), (t1, v0), (t1, v1), ... (tn, vn-1)]) band_x = np.append(line.time, line.time[::-1]) band_y = np.append(line.lower, line.upper[::-1]) renderers[-1].extend([ p.patch(band_x, band_y, color='#7570B3', fill_alpha=0.2, visible=True if line.name in ['average', 'all budgets'] else False) ]) # Tooltips tooltips = [("estimated performance", "@mean"), ("at-time", "@time")] p.add_tools( HoverTool( renderers=[i for s in renderers for i in s], tooltips=tooltips, )) # MAKE hovertips stay fixed in position # callback=CustomJS(code=""" # var tooltips = document.getElementsByClassName("bk-tooltip"); # for (var i = 0, len = tooltips.length; i < len; i ++) { # tooltips[i].style.top = ""; // unset what bokeh.js sets # tooltips[i].style.left = ""; # tooltips[i].style.bottom = "0px"; # tooltips[i].style.left = "0px"; # } # """))) # TODO optional: activate different tooltips for different renderers, doesn't work properly #tooltips_configs = tooltips[:] + [(p, '@'+p) for p in hp_names] #if 'average' in [l.name for l in lines]: # p.add_tools(HoverTool(renderers=[renderers[0]], tooltips=tooltips_avg ))#, mode='vline')) # Wrap renderers in nested lists for checkbox-code checkbox, select_all, select_none = get_checkbox( renderers, [l[0] for l in legend_it]) checkbox.active = [0] # Tilt tick labels and configure axis labels p.xaxis.major_label_orientation = 3 / 4 p.xaxis.axis_label_text_font_size = p.yaxis.axis_label_text_font_size = "15pt" p.xaxis.major_label_text_font_size = p.yaxis.major_label_text_font_size = "12pt" p.title.text_font_size = "15pt" legend = Legend( items=legend_it, location='bottom_left', #(0, -60), label_text_font_size="8pt") legend.click_policy = "hide" p.add_layout(legend, 'right') # Assign objects and save png's layout = row( p, column( widgetbox(checkbox, width=100), row(widgetbox(select_all, width=50), widgetbox(select_none, width=50)))) output_path = os.path.join(self.output_dir, output_fn) export_bokeh(p, output_path, self.logger) self.plots.append(output_path) return layout
def plot(self, X, conf_list: list, runs_per_quantile, inc_list: list = None, contour_data=None, time_slider=False): """ plots sampled configuration in 2d-space; uses bokeh for interactive plot saves results in self.output, if set Parameters ---------- X: np.array np.array with 2-d coordinates for each configuration conf_list: list list of ALL configurations in the same order as X runs_per_quantile: list[np.array] configurator-run to be analyzed, as a np.array with the number of target-algorithm-runs per config per quantile. inc_list: list list of incumbents (Configuration) contour_data: list contour data (xx,yy,Z) time_slider: bool whether or not to have a time_slider-widget on cfp-plot INCREASES FILE-SIZE DRAMATICALLY Returns ------- (script, div): str script and div of the bokeh-figure over_time_paths: List[str] list with paths to the different quantiled timesteps of the configurator run (for static evaluation) """ if not inc_list: inc_list = [] over_time_paths = [] # development of the search space over time hp_names = [ k.name for k in # Hyperparameter names conf_list[0].configuration_space.get_hyperparameters() ] # Get individual sources for quantiles sources = [ self._plot_get_source(conf_list, quantiled_run, X, inc_list, hp_names) for quantiled_run in runs_per_quantile ] # Define what appears in tooltips # TODO add only important parameters (needs to change order of exec pimp before conf-footprints) hover = HoverTool( tooltips=[('type', '@type'), ('origin', '@origin'), ('runs', '@runs')] + [(k, '@' + escape_parameter_name(k)) for k in hp_names]) # bokeh-figure x_range = [min(X[:, 0]) - 1, max(X[:, 0]) + 1] y_range = [min(X[:, 1]) - 1, max(X[:, 1]) + 1] scatter_glyph_render_groups = [] for idx, source in enumerate(sources): if not time_slider or idx == 0: # Only plot all quantiles in one plot if timeslider is on p = figure(plot_height=500, plot_width=600, tools=[hover, 'save'], x_range=x_range, y_range=y_range) if contour_data is not None: p = self._plot_contour(p, contour_data, x_range, y_range) views, markers = self._plot_create_views(source) self.logger.debug("Plotting quantile %d!", idx) scatter_glyph_render_groups.append( self._plot_scatter(p, source, views, markers)) if self.output_dir: file_path = "cfp_over_time/configurator_footprint" + str( idx) + ".png" over_time_paths.append(os.path.join(self.output_dir, file_path)) self.logger.debug("Saving plot to %s", over_time_paths[-1]) export_bokeh(p, over_time_paths[-1], self.logger) if time_slider: self.logger.debug("Adding timeslider") slider = self._plot_get_timeslider(scatter_glyph_render_groups) layout = column(p, widgetbox(slider)) else: self.logger.debug("Not adding timeslider") layout = column(p) script, div = components(layout) if self.output_dir: path = os.path.join(self.output_dir, "content/images/configurator_footprint.png") export_bokeh(p, path, self.logger) return (script, div), over_time_paths
def plot(self, X, conf_list: list, runs_per_quantile, inc_list: list=None, contour_data=None, use_timeslider=False, use_checkbox=True, timeslider_labels=None): """ plots sampled configuration in 2d-space; uses bokeh for interactive plot saves results in self.output, if set Parameters ---------- X: np.array np.array with 2-d coordinates for each configuration conf_list: list list of ALL configurations in the same order as X runs_per_quantile: list[np.array] configurator-run to be analyzed, as a np.array with the number of target-algorithm-runs per config per quantile. inc_list: list list of incumbents (Configuration) contour_data: list contour data (xx,yy,Z) use_timeslider: bool whether or not to have a time_slider-widget on cfp-plot INCREASES FILE-SIZE DRAMATICALLY use_checkbox: bool have checkboxes to toggle individual runs Returns ------- (script, div): str script and div of the bokeh-figure over_time_paths: List[str] list with paths to the different quantiled timesteps of the configurator run (for static evaluation) """ if not inc_list: inc_list = [] over_time_paths = [] # development of the search space over time hp_names = [k.name for k in # Hyperparameter names conf_list[0].configuration_space.get_hyperparameters()] # bokeh-figure x_range = [min(X[:, 0]) - 1, max(X[:, 0]) + 1] y_range = [min(X[:, 1]) - 1, max(X[:, 1]) + 1] # Get individual sources for quantiles sources, used_configs = zip(*[self._plot_get_source(conf_list, quantiled_run, X, inc_list, hp_names) for quantiled_run in runs_per_quantile]) # We collect all glyphs in one list # Then we have to dicts to identify groups of glyphs (for interactivity) # They map the name of the group to a list of indices (of the respective glyphs that are in the group) # Those indices refer to the main list of all glyphs # This is necessary to enable interactivity for two inputs at the same time all_glyphs = [] overtime_groups = {} run_groups = {run : [] for run in self.configs_in_run.keys()} # Iterate over quantiles (this updates overtime_groups) for idx, source, u_cfgs in zip(range(len(sources)), sources, used_configs): # Create new plot if necessary (only plot all quantiles in one single plot if timeslider is on) if not use_timeslider or idx == 0: p = self._create_figure(x_range, y_range) if contour_data is not None: # TODO contour_handles, color_mapper = self._plot_contour(p, contour_data, x_range, y_range) # Create views and scatter views, views_by_run, markers = self._create_views(source, u_cfgs) scatter_handles = self._scatter(p, source, views, markers) self.logger.debug("Quantile %d: %d scatter-handles", idx, len(scatter_handles)) if len(scatter_handles) == 0: self.logger.debug("No configs in quantile %d (?!)", idx) continue # Add to groups start = len(all_glyphs) all_glyphs.extend(scatter_handles) overtime_groups[str(idx)] = [str(i) for i in range(start, len(all_glyphs))] for run, indices in views_by_run.items(): run_groups[run].extend([str(start + i) for i in indices]) # Write to file if self.output_dir: file_path = "cfp_over_time/configurator_footprint" + str(idx) + ".png" over_time_paths.append(os.path.join(self.output_dir, file_path)) self.logger.debug("Saving plot to %s", over_time_paths[-1]) export_bokeh(p, over_time_paths[-1], self.logger) # Add hovertool (define what appears in tooltips) # TODO add only important parameters (needs to change order of exec pimp before conf-footprints) hover = HoverTool(tooltips=[('type', '@type'), ('origin', '@origin'), ('runs', '@runs')] + [(k, '@' + escape_parameter_name(k)) for k in hp_names], renderers=all_glyphs) p.add_tools(hover) # Build dashboard timeslider, checkbox, select_all, select_none, checkbox_title = self._get_widgets(all_glyphs, overtime_groups, run_groups, slider_labels=timeslider_labels) contour_checkbox, contour_title = self._contour_radiobuttongroup(contour_handles, color_mapper) layout = p if use_timeslider: self.logger.debug("Adding timeslider") layout = column(layout, widgetbox(timeslider)) if use_checkbox: self.logger.debug("Adding checkboxes") layout = row(layout, column(widgetbox(checkbox_title), widgetbox(checkbox), row(widgetbox(select_all, width=100), widgetbox(select_none, width=100)), widgetbox(contour_title), widgetbox(contour_checkbox))) if self.output_dir: path = os.path.join(self.output_dir, "content/images/configurator_footprint.png") export_bokeh(p, path, self.logger) return layout, over_time_paths
def plot_interactive_footprint(self): """Use bokeh to create an interactive algorithm footprint with zoom and hover tooltips. Should avoid problems with overplotting (since we can zoom) and provide better information about instances.""" features = np.array(self.features_2d) instances = self.insts runhistory = self.rh algo = {v: k for k, v in self.algo_name.items()} incumbent = algo['incumbent'] default = algo['default'] source = ColumnDataSource(data=dict(x=features[:, 0], y=features[:, 1])) # Add all necessary information for incumbent and default source.add(instances, 'instance_name') instance_set = [ 'train' if i in self.train_feats.keys() else 'test' for i in instances ] source.add(instance_set, 'instance_set') # train or test for config, name in [(incumbent, 'incumbent'), (default, 'default')]: cost = get_cost_dict_for_config(runhistory, config) source.add([cost[i] for i in instances], '{}_cost'.format(name)) # TODO should be in function good, bad = self._get_good_bad(config) color = [ 1 if idx in good else 0 for idx, i in enumerate(instances) ] # TODO end color = ['blue' if c else 'red' for c in color] self.logger.debug("%s colors: %s", name, str(color)) source.add(color, '{}_color'.format(name)) source.add(source.data['default_color'], 'color') # Define what appears in tooltips hover = HoverTool(tooltips=[ ('instance name', '@instance_name'), ('def cost', '@default_cost'), ('inc_cost', '@incumbent_cost'), ('set', '@instance_set'), ]) # Add radio-button def_inc_callback = CustomJS(args=dict(source=source), code=""" var data = source.data; if (cb_obj.active == 0) { data['color'] = data['default_color']; } else { data['color'] = data['incumbent_color']; } source.change.emit(); """) def_inc_radio_button = RadioButtonGroup( labels=["default", "incumbent"], active=0, callback=def_inc_callback) # Plot x_range = DataRange1d(bounds='auto', start=min(features[:, 0]) - 1, end=max(features[:, 0]) + 1) y_range = DataRange1d(bounds='auto', start=min(features[:, 1]) - 1, end=max(features[:, 1]) + 1) p = figure( plot_height=500, plot_width=600, tools=[hover, 'save', 'wheel_zoom', 'box_zoom', 'pan', 'reset'], active_drag='box_zoom', x_range=x_range, y_range=y_range) # Scatter train and test individually to toggle them train_view = CDSView( source=source, filters=[GroupFilter(column_name='instance_set', group='train')]) test_view = CDSView( source=source, filters=[GroupFilter(column_name='instance_set', group='test')]) train = p.scatter(x='x', y='y', source=source, view=train_view, color='color') test = p.scatter(x='x', y='y', source=source, view=test_view, color='color') p.xaxis.axis_label, p.yaxis.axis_label = 'principal component 1', 'principal component 2' p.xaxis.axis_label_text_font_size = p.yaxis.axis_label_text_font_size = "15pt" train_test_callback = CustomJS(args=dict(source=source, train_view=train, test_view=test), code=""" var data = source.data; if (cb_obj.active == 0) { train_view.visible = true; test_view.visible = true; } else if (cb_obj.active == 1) { train_view.visible = true; test_view.visible = false; } else { train_view.visible = false; test_view.visible = true; } """) train_test_radio_button = RadioButtonGroup( labels=["all", "train", "test"], active=0, callback=train_test_callback) # Export and return if self.output_dir: path = os.path.join(self.output_dir, "content/images/algorithm_footprint.png") export_bokeh(p, path, self.logger) layout = column( p, row(widgetbox(def_inc_radio_button), widgetbox(train_test_radio_button))) return layout
def plot(self, rh: RunHistory, runs: List[ConfiguratorRun], output_fn: str = "performance_over_time.png", validator: Union[None, Validator] = None): """ Plot performance over time, using all trajectory entries with max_time = wallclock_limit or (if inf) the highest recorded time Parameters ---------- rh: RunHistory runhistory to use runs: List[SMACrun] list of configurator-runs output_fn: str path to output-png validator: TODO description """ self.logger.debug( "Estimating costs over time for %d runs, save png in %s.", len(runs), output_fn) validated = True # TODO ? if len(runs) > 1: # If there is more than one run, we average over the runs means, times = [], [] all_times = [] for run in runs: # Ignore variances as we plot variance over runs validated = validated and run.traj mean, _, time = self._get_mean_var_time( validator, run.traj, not run.validated_runhistory, rh) means.append(mean.flatten()) all_times.extend(time) times.append(time) means = np.array(means) times = np.array(times) all_times = np.array(sorted(all_times)) at = [0 for _ in runs ] # keep track at which timestep each trajectory is m = [np.nan for _ in runs] # used to compute the mean over the timesteps mean = np.ones((len(all_times), 1)) * -1 var = np.ones((len(all_times), 1)) * -1 upper = np.ones((len(all_times), 1)) * -1 lower = np.ones((len(all_times), 1)) * -1 for time_idx, t in enumerate(all_times): for traj_idx, entry_idx in enumerate(at): try: if t == times[traj_idx][entry_idx]: m[traj_idx] = means[traj_idx][entry_idx] at[traj_idx] += 1 except IndexError: pass # Reached the end of one trajectory. No need to check it further # var[time_idx][0] = np.nanvar(m) u, l, m_ = np.nanpercentile(m, 75), np.nanpercentile( m, 25), np.nanpercentile(m, 50) # self.logger.debug((mean[time_idx][0] + np.sqrt(var[time_idx][0]), mean[time_idx][0], # mean[time_idx][0] - np.sqrt(var[time_idx][0]))) # self.logger.debug((l, m_, u)) upper[time_idx][0] = u mean[time_idx][0] = m_ lower[time_idx][0] = l time = all_times else: # no new statistics computation necessary validated = True if runs[0].validated_runhistory else False mean, var, time = self._get_mean_var_time(validator, runs[0].traj, not validated, rh) upper = lower = mean mean = mean[:, 0] upper = upper[:, 0] lower = lower[:, 0] uncertainty_upper = upper # mean + np.sqrt(var) uncertainty_lower = lower # mean - np.sqrt(var) clip_y_lower = False if self.scenario.run_obj == 'runtime': # y-axis on log -> clip plot # Determine clipping point from lowest legal value clip_y_lower = min( list(uncertainty_lower[uncertainty_lower > 0]) + list(mean)) * 0.8 uncertainty_lower[uncertainty_lower <= 0] = clip_y_lower * 0.9 time_double = [t for sub in zip(time, time) for t in sub][1:-1] mean_double = [t for sub in zip(mean, mean) for t in sub][:-2] source = ColumnDataSource( data=dict(x=time_double, y=mean_double, epm_perf=mean_double)) hover = HoverTool(tooltips=[("performance", "@epm_perf"), ("at-time", "@x")]) p = figure(plot_width=700, plot_height=500, tools=[hover, 'save'], x_range=Range1d(max(min(time), 1), max(time)), x_axis_type='log', y_axis_type='log' if self.scenario.run_obj == 'runtime' else 'linear', title="Cost over time") if clip_y_lower: p.y_range = Range1d(clip_y_lower, 1.2 * max(uncertainty_upper)) # start after 1% of the configuration budget # p.x_range = Range1d(min(time) + (max(time) - min(time)) * 0.01, max(time)) # Plot label = self.scenario.run_obj label = '{}{}'.format('validated ' if validated else 'estimated ', label) p.line('x', 'y', source=source, legend=label) # Fill area (uncertainty) # Defined as sequence of coordinates, so for step-effect double and # arange accordingly ([(t0, v0), (t1, v0), (t1, v1), ... (tn, vn-1)]) time_double = [t for sub in zip(time, time) for t in sub][1:-1] uncertainty_lower_double = [ u for sub in zip(uncertainty_lower, uncertainty_lower) for u in sub ][:-2] uncertainty_upper_double = [ u for sub in zip(uncertainty_upper, uncertainty_upper) for u in sub ][:-2] band_x = np.append(time_double, time_double[::-1]) band_y = np.append(uncertainty_lower_double, uncertainty_upper_double[::-1]) p.patch(band_x, band_y, color='#7570B3', fill_alpha=0.2) # Tilt tick labels p.xaxis.major_label_orientation = 3 / 4 p.legend.location = "top_right" p.xaxis.axis_label = "time (sec)" p.yaxis.axis_label = label p.xaxis.axis_label_text_font_size = "15pt" p.yaxis.axis_label_text_font_size = "15pt" p.xaxis.major_label_text_font_size = "12pt" p.yaxis.major_label_text_font_size = "12pt" p.title.text_font_size = "15pt" p.legend.label_text_font_size = "15pt" script, div = components(p) export_bokeh(p, os.path.join(self.output_dir, output_fn), self.logger) return script, div