def get_optimization_trace(self, objective_optimum: Optional[float] = None ) -> AxPlotConfig: """Retrieves the plot configuration for optimization trace, which shows the evolution of the objective mean over iterations. Args: objective_optimum: Optimal objective, if known, for display in the visualization. """ if not self.experiment.trials: raise ValueError("Cannot generate plot as there are no trials.") objective_name = self.experiment.optimization_config.objective.metric.name best_objectives = np.array([[ checked_cast(Trial, trial).objective_mean for trial in self.experiment.trials.values() ]]) hover_labels = [ _format_dict(not_none(checked_cast(Trial, trial).arm).parameters) for trial in self.experiment.trials.values() ] return optimization_trace_single_method( y=(np.minimum.accumulate(best_objectives, axis=1) if self.experiment.optimization_config.objective.minimize else np.maximum.accumulate(best_objectives, axis=1)), optimum=objective_optimum, title="Model performance vs. # of iterations", ylabel=objective_name.capitalize(), hover_labels=hover_labels, )
def _error_scatter_trace( arms: List[Union[PlotInSampleArm, PlotOutOfSampleArm]], y_axis_var: PlotMetric, x_axis_var: Optional[PlotMetric] = None, y_axis_label: Optional[str] = None, x_axis_label: Optional[str] = None, status_quo_arm: Optional[PlotInSampleArm] = None, show_CI: bool = True, name: str = "In-sample", color: Tuple[int] = COLORS.STEELBLUE.value, visible: bool = True, legendgroup: Optional[str] = None, showlegend: bool = True, hoverinfo: str = "text", show_arm_details_on_hover: bool = True, show_context: bool = False, arm_noun: str = "arm", ) -> Dict[str, Any]: """Plot scatterplot with error bars. Args: arms (List[Union[PlotInSampleArm, PlotOutOfSampleArm]]): a list of in-sample or out-of-sample arms. In-sample arms have observed data, while out-of-sample arms just have predicted data. As a result, when passing out-of-sample arms, pred must be True. y_axis_var: name of metric for y-axis, along with whether it is observed or predicted. x_axis_var: name of metric for x-axis, along with whether it is observed or predicted. If None, arm names are automatically used. y_axis_label: custom label to use for y axis. If None, use metric name from `y_axis_var`. x_axis_label: custom label to use for x axis. If None, use metric name from `x_axis_var` if that is not None. status_quo_arm: the status quo arm. Necessary for relative metrics. show_CI: if True, plot confidence intervals. name: name of trace. Default is "In-sample". color: color as rgb tuple. Default is (128, 177, 211), which corresponds to COLORS.STEELBLUE. visible: if True, trace is visible (default). legendgroup: group for legends. showlegend: if True, legend if rendered. hoverinfo: information to show on hover. Default is custom text. show_arm_details_on_hover: if True, display parameterizations of arms on hover. Default is True. show_context: if True and show_arm_details_on_hover, context will be included in the hover. arm_noun: noun to use instead of "arm" (e.g. group) """ x, x_se, y, y_se = _error_scatter_data( arms=arms, y_axis_var=y_axis_var, x_axis_var=x_axis_var, status_quo_arm=status_quo_arm, ) labels = [] arm_names = [a.name for a in arms] # No relativization if no x variable. rel_x = x_axis_var.rel if x_axis_var else False rel_y = y_axis_var.rel for i in range(len(arm_names)): heading = f"<b>{arm_noun.title()} {arm_names[i]}</b><br>" x_lab = ("{name}: {estimate}{perc} {ci}<br>".format( name=x_axis_var.metric if x_axis_label is None else x_axis_label, estimate=(round(x[i], DECIMALS) if isinstance( x[i], numbers.Number) else x[i]), ci="" if x_se is None else _format_CI(x[i], x_se[i], rel_x), perc="%" if rel_x else "", ) if x_axis_var is not None else "") y_lab = "{name}: {estimate}{perc} {ci}<br>".format( name=y_axis_var.metric if y_axis_label is None else y_axis_label, estimate=(round(y[i], DECIMALS) if isinstance( y[i], numbers.Number) else y[i]), ci="" if y_se is None else _format_CI(y[i], y_se[i], rel_y), perc="%" if rel_y else "", ) parameterization = (_format_dict(arms[i].parameters, "Parameterization") if show_arm_details_on_hover else "") context = ( # Expected `Dict[str, Optional[Union[bool, float, str]]]` for 1st anonymous # parameter to call `ax.plot.helper._format_dict` but got # `Optional[Dict[str, Union[float, str]]]`. # pyre-fixme[6]: _format_dict(arms[i].context_stratum, "Context") if show_arm_details_on_hover and show_context # noqa W503 and arms[i].context_stratum # noqa W503 else "") labels.append("{arm_name}<br>{xlab}{ylab}{param_blob}{context}".format( arm_name=heading, xlab=x_lab, ylab=y_lab, param_blob=parameterization, context=context, )) i += 1 trace = go.Scatter( x=x, y=y, marker={"color": rgba(color)}, mode="markers", name=name, text=labels, hoverinfo=hoverinfo, ) if show_CI: if x_se is not None: trace.update( error_x={ "type": "data", "array": np.multiply(x_se, Z), "color": rgba(color, CI_OPACITY), }) if y_se is not None: trace.update( error_y={ "type": "data", "array": np.multiply(y_se, Z), "color": rgba(color, CI_OPACITY), }) if visible is not None: trace.update(visible=visible) if legendgroup is not None: trace.update(legendgroup=legendgroup) if showlegend is not None: trace.update(showlegend=showlegend) return trace
def _get_single_pareto_trace( frontier: ParetoFrontierResults, CI_level: float, legend_label: str = "mean", trace_color: Tuple[int] = COLORS.STEELBLUE.value, show_parameterization_on_hover: bool = True, ) -> go.Scatter: primary_means = frontier.means[frontier.primary_metric] primary_sems = frontier.sems[frontier.primary_metric] secondary_means = frontier.means[frontier.secondary_metric] secondary_sems = frontier.sems[frontier.secondary_metric] absolute_metrics = frontier.absolute_metrics all_metrics = frontier.means.keys() if frontier.arm_names is None: arm_names = [ f"Parameterization {i}" for i in range(len(frontier.param_dicts)) ] else: arm_names = [f"Arm {name}" for name in frontier.arm_names] if CI_level is not None: Z = 0.5 * norm.ppf(1 - (1 - CI_level) / 2) else: Z = None labels = [] for i, param_dict in enumerate(frontier.param_dicts): label = f"<b>{arm_names[i]}</b><br>" for metric in all_metrics: metric_lab = _make_label( mean=frontier.means[metric][i], sem=frontier.sems[metric][i], name=metric, is_relative=metric not in absolute_metrics, Z=Z, ) label += metric_lab parameterization = (_format_dict(param_dict, "Parameterization") if show_parameterization_on_hover else "") label += parameterization labels.append(label) return go.Scatter( name=legend_label, legendgroup=legend_label, x=secondary_means, y=primary_means, error_x={ "type": "data", "array": Z * np.array(secondary_sems), "thickness": 2, "color": rgba(trace_color, CI_OPACITY), }, error_y={ "type": "data", "array": Z * np.array(primary_sems), "thickness": 2, "color": rgba(trace_color, CI_OPACITY), }, mode="markers", text=labels, hoverinfo="text", marker={"color": rgba(trace_color)}, )
def plot_pareto_frontier( frontier: ParetoFrontierResults, CI_level: float = DEFAULT_CI_LEVEL, show_parameterization_on_hover: bool = True, ) -> AxPlotConfig: """Plot a Pareto frontier from a ParetoFrontierResults object. Args: frontier (ParetoFrontierResults): The results of the Pareto frontier computation. CI_level (float, optional): The confidence level, i.e. 0.95 (95%) show_parameterization_on_hover (bool, optional): If True, show the parameterization of the points on the frontier on hover. Returns: AEPlotConfig: The resulting Plotly plot definition. """ primary_means = frontier.means[frontier.primary_metric] primary_sems = frontier.sems[frontier.primary_metric] secondary_means = frontier.means[frontier.secondary_metric] secondary_sems = frontier.sems[frontier.secondary_metric] absolute_metrics = frontier.absolute_metrics all_metrics = frontier.means.keys() if frontier.arm_names is None: arm_names = [ f"Parameterization {i}" for i in range(len(frontier.param_dicts)) ] else: arm_names = [f"Arm {name}" for name in frontier.arm_names] if CI_level is not None: Z = 0.5 * norm.ppf(1 - (1 - CI_level) / 2) else: Z = None primary_threshold = None secondary_threshold = None if frontier.objective_thresholds is not None: primary_threshold = frontier.objective_thresholds.get( frontier.primary_metric, None) secondary_threshold = frontier.objective_thresholds.get( frontier.secondary_metric, None) labels = [] rel_x = frontier.secondary_metric not in absolute_metrics rel_y = frontier.primary_metric not in absolute_metrics for i, param_dict in enumerate(frontier.param_dicts): label = f"<b>{arm_names[i]}</b><br>" for metric in all_metrics: metric_lab = _make_label( mean=frontier.means[metric][i], sem=frontier.sems[metric][i], name=metric, is_relative=metric not in absolute_metrics, Z=Z, ) label += metric_lab parameterization = (_format_dict(param_dict, "Parameterization") if show_parameterization_on_hover else "") label += parameterization labels.append(label) traces = [ go.Scatter( x=secondary_means, y=primary_means, error_x={ "type": "data", "array": Z * np.array(secondary_sems), "thickness": 2, "color": rgba(COLORS.STEELBLUE.value, CI_OPACITY), }, error_y={ "type": "data", "array": Z * np.array(primary_sems), "thickness": 2, "color": rgba(COLORS.STEELBLUE.value, CI_OPACITY), }, mode="markers", text=labels, hoverinfo="text", ) ] shapes = [] if primary_threshold is not None: shapes.append({ "type": "line", "xref": "paper", "x0": 0.0, "x1": 1.0, "yref": "y", "y0": primary_threshold, "y1": primary_threshold, "line": { "color": rgba(COLORS.CORAL.value), "width": 3 }, }) if secondary_threshold is not None: shapes.append({ "type": "line", "yref": "paper", "y0": 0.0, "y1": 1.0, "xref": "x", "x0": secondary_threshold, "x1": secondary_threshold, "line": { "color": rgba(COLORS.CORAL.value), "width": 3 }, }) layout = go.Layout( title="Pareto Frontier", xaxis={ "title": frontier.secondary_metric, "ticksuffix": "%" if rel_x else "", "zeroline": True, }, yaxis={ "title": frontier.primary_metric, "ticksuffix": "%" if rel_y else "", "zeroline": True, }, hovermode="closest", legend={"orientation": "h"}, width=750, height=500, margin=go.layout.Margin(pad=4, l=225, b=75, t=75), # noqa E741 shapes=shapes, ) fig = go.Figure(data=traces, layout=layout) return AxPlotConfig(data=fig, plot_type=AxPlotTypes.GENERIC)
def plot_pareto_frontier( frontier: ParetoFrontierResults, CI_level: float = DEFAULT_CI_LEVEL, show_parameterization_on_hover: bool = True, ) -> AxPlotConfig: """Plot a Pareto frontier from a ParetoFrontierResults object. Args: frontier (ParetoFrontierResults): The results of the Pareto frontier computation. CI_level (float, optional): The confidence level, i.e. 0.95 (95%) show_parameterization_on_hover (bool, optional): If True, show the parameterization of the points on the frontier on hover. Returns: AEPlotConfig: The resulting Plotly plot definition. """ primary_means = frontier.means[frontier.primary_metric] primary_sems = frontier.sems[frontier.primary_metric] secondary_means = frontier.means[frontier.secondary_metric] secondary_sems = frontier.sems[frontier.secondary_metric] absolute_metrics = frontier.absolute_metrics if CI_level is not None: Z = 0.5 * norm.ppf(1 - (1 - CI_level) / 2) else: Z = None labels = [] rel_x = frontier.secondary_metric not in absolute_metrics rel_y = frontier.primary_metric not in absolute_metrics for i, param_dict in enumerate(frontier.param_dicts): heading = "<b>Parameterization {}</b><br>".format(i + 1) x_lab = _make_label( mean=secondary_means[i], sem=secondary_sems[i], name=frontier.secondary_metric, is_relative=rel_x, Z=Z, ) y_lab = _make_label( mean=primary_means[i], sem=primary_sems[i], name=frontier.primary_metric, is_relative=rel_y, Z=Z, ) parameterization = (_format_dict(param_dict, "Parameterization") if show_parameterization_on_hover else "") labels.append("{heading}<br>{xlab}{ylab}{param_blob}".format( heading=heading, xlab=x_lab, ylab=y_lab, param_blob=parameterization)) traces = [ go.Scatter( x=secondary_means, y=primary_means, error_x={ "type": "data", "array": Z * np.array(secondary_sems), "thickness": 2, "color": rgba(COLORS.STEELBLUE.value, CI_OPACITY), }, error_y={ "type": "data", "array": Z * np.array(primary_sems), "thickness": 2, "color": rgba(COLORS.STEELBLUE.value, CI_OPACITY), }, mode="markers", text=labels, hoverinfo="text", ) ] layout = go.Layout( title="Pareto Frontier", xaxis={ "title": frontier.secondary_metric, "ticksuffix": "%" if rel_x else "", "zeroline": True, }, yaxis={ "title": frontier.primary_metric, "ticksuffix": "%" if rel_y else "", "zeroline": True, }, hovermode="closest", legend={"orientation": "h"}, width=750, height=500, margin=go.layout.Margin(pad=4, l=225, b=75, t=75), # noqa E741 ) fig = go.Figure(data=traces, layout=layout) return AxPlotConfig(data=fig, plot_type=AxPlotTypes.GENERIC)