def _get_results(names, raw_results, kwargs_list): registry = get_registry(extended=True) results = {} for name, result, inputs in zip(names, raw_results, kwargs_list): if isinstance(result, OptimizeResult): history = result.history params_history = pd.DataFrame([ tree_just_flatten(p, registry=registry) for p in history["params"] ]) criterion_history = pd.Series(history["criterion"]) time_history = pd.Series(history["runtime"]) elif isinstance(result, str): _criterion = inputs["criterion"] params_history = pd.DataFrame( tree_just_flatten(inputs["params"], registry=registry)).T criterion_history = pd.Series( _criterion(inputs["params"])["value"]) time_history = pd.Series([np.inf]) else: raise ValueError( "'result' object is expected to be of type 'dict' or 'str'.") results[name] = { "params_history": params_history, "criterion_history": criterion_history, "time_history": time_history, "solution": result, } return results
def assert_almost_equal(x, y, decimal=6): if isinstance(x, np.ndarray): x_flat = x y_flat = y else: registry = get_registry(extended=True) x_flat = np.array(tree_just_flatten(x, registry=registry)) y_flat = np.array(tree_just_flatten(x, registry=registry)) aaae(x_flat, y_flat, decimal=decimal)
def test_tree_params_numerical_derivative_sos_ls(params, algorithm): flat = np.array(tree_just_flatten(params, registry=REGISTRY)) expected = np.zeros_like(flat) res = minimize( criterion=flexible_sos_ls, params=params, algorithm=algorithm, ) calculated = np.array(tree_just_flatten(res.params, registry=REGISTRY)) aaae(calculated, expected)
def test_log_reader_read_multistart_history(example_db): reader = OptimizeLogReader(example_db) history, local_history, exploration = reader.read_multistart_history( direction="minimize") assert local_history is None assert exploration is None registry = get_registry(extended=True) assert tree_equal( tree_just_flatten(history, registry=registry), tree_just_flatten(reader.read_history(), registry=registry), )
def test_tree_params_scalar_criterion(params): flat = np.array(tree_just_flatten(params, registry=REGISTRY)) expected = np.zeros_like(flat) res = minimize( criterion=flexible_sos_scalar, derivative=flexible_sos_scalar_derivative, params=params, algorithm="scipy_lbfgsb", ) calculated = np.array(tree_just_flatten(res.params, registry=REGISTRY)) aaae(calculated, expected)
def func_flatten(func_eval): # the if condition is necessary, such that we can also accept func_evals # where the primary entry has already been extracted. This is for example # necessary if the criterion_and_derivative returns only the relevant # entry of criterion, whereas criterion returns a dict. if isinstance(func_eval, dict) and key in func_eval: func_eval = func_eval[key] return aggregate(tree_just_flatten(func_eval, registry=registry))
def test_tree_params_sos_ls(params, algorithm): flat = np.array(tree_just_flatten(params, registry=REGISTRY)) expected = np.zeros_like(flat) derivatives = { "value": flexible_sos_scalar_derivative, "root_contributions": flexible_sos_ls_derivative, } res = minimize( criterion=flexible_sos_ls, derivative=derivatives, params=params, algorithm=algorithm, ) calculated = np.array(tree_just_flatten(res.params, registry=REGISTRY)) aaae(calculated, expected)
def test_optimization_with_valid_logging(algorithm, params): res = minimize( flexible_sos_ls, params=params, algorithm=algorithm, logging="logging.db", ) registry = get_registry(extended=True) flat = np.array(tree_just_flatten(res.params, registry=registry)) aaae(flat, np.zeros(3))
def _get_selection_indices(params, selector): """Get index of selected flat params and number of flat params.""" registry = get_registry(extended=True) flat_params, params_treedef = tree_flatten(params, registry=registry) n_params = len(flat_params) indices = np.arange(n_params, dtype=int) params_indices = tree_unflatten(params_treedef, indices, registry=registry) selected = selector(params_indices) selection_indices = np.array(tree_just_flatten(selected, registry=registry), dtype=int) return selection_indices, n_params
def calculate_free_estimates(estimates, internal_estimates): mask = internal_estimates.free_mask names = internal_estimates.names registry = get_registry(extended=True) external_flat = np.array(tree_just_flatten(estimates, registry=registry)) free_estimates = FreeParams( values=external_flat[mask], free_mask=mask, all_names=names, free_names=np.array(names)[mask].tolist(), ) return free_estimates
def tree_params_converter(tree_params): registry = get_registry(extended=True) _, treedef = tree_flatten(tree_params, registry=registry) converter = TreeConverter( params_flatten=lambda params: np.array( tree_just_flatten(params, registry=registry) ), params_unflatten=lambda x: tree_unflatten( treedef, x.tolist(), registry=registry ), func_flatten=None, derivative_flatten=None, ) return converter
def _msm_criterion(params, simulate_moments, flat_empirical_moments, chol_weights, registry): """Calculate msm criterion given parameters and building blocks.""" simulated = simulate_moments(params) if isinstance(simulated, dict) and "simulated_moments" in simulated: simulated = simulated["simulated_moments"] if isinstance(simulated, np.ndarray) and simulated.ndim == 1: simulated_flat = simulated else: simulated_flat = np.array( tree_just_flatten(simulated, registry=registry)) deviations = simulated_flat - flat_empirical_moments root_contribs = deviations @ chol_weights value = root_contribs @ root_contribs out = { "value": value, "root_contributions": root_contribs, } return out
def get_msm_optimization_functions( simulate_moments, empirical_moments, weights, *, simulate_moments_kwargs=None, jacobian=None, jacobian_kwargs=None, ): """Construct criterion functions and their derivatives for msm estimation. Args: simulate_moments (callable): Function that takes params and potentially other keyworrd arguments and returns simulated moments as a pandas Series. Alternatively, the function can return a dict with any number of entries as long as one of those entries is "simulated_moments". empirical_moments (pandas.Series): A pandas series with the empirical equivalents of the simulated moments. weights (pytree): The weighting matrix as block pytree. simulate_moments_kwargs (dict): Additional keyword arguments for ``simulate_moments``. jacobian (callable or pandas.DataFrame): A function that take ``params`` and potentially other keyword arguments and returns the jacobian of simulate_moments with respect to the params. Alternatively you can pass a pandas.DataFrame with the jacobian at the optimal parameters. This is only possible if you pass ``optimize_options=False``. jacobian_kwargs (dict): Additional keyword arguments for jacobian. Returns: dict: Dictionary containing at least the entry "criterion". If enough inputs are provided it also contains the entries "derivative" and "criterion_and_derivative". All values are functions that take params as only argument. """ flat_weights = block_tree_to_matrix( weights, outer_tree=empirical_moments, inner_tree=empirical_moments, ) chol_weights = np.linalg.cholesky(flat_weights) registry = get_registry(extended=True) flat_emp_mom = tree_just_flatten(empirical_moments, registry=registry) _simulate_moments = _partial_kwargs(simulate_moments, simulate_moments_kwargs) _jacobian = _partial_kwargs(jacobian, jacobian_kwargs) criterion = functools.partial( _msm_criterion, simulate_moments=_simulate_moments, flat_empirical_moments=flat_emp_mom, chol_weights=chol_weights, registry=registry, ) out = {"criterion": criterion} if _jacobian is not None: raise NotImplementedError( "Closed form jacobians are not yet supported in estimate_msm") return out
def create_convergence_histories(problems, results, stopping_criterion, x_precision, y_precision): """Create tidy DataFrame with all information needed for the benchmarking plots. Args: problems (dict): estimagic benchmarking problems dictionary. Keys are the problem names. Values contain information on the problem, including the solution value. results (dict): estimagic benchmarking results dictionary. Keys are tuples of the form (problem, algorithm), values are dictionaries of the collected information on the benchmark run, including 'criterion_history' and 'time_history'. stopping_criterion (str): one of "x_and_y", "x_or_y", "x", "y". Determines how convergence is determined from the two precisions. x_precision (float or None): how close an algorithm must have gotten to the true parameter values (as percent of the Euclidean distance between start and solution parameters) before the criterion for clipping and convergence is fulfilled. y_precision (float or None): how close an algorithm must have gotten to the true criterion values (as percent of the distance between start and solution criterion value) before the criterion for clipping and convergence is fulfilled. Returns: pandas.DataFrame: tidy DataFrame with the following columns: - problem - algorithm - n_evaluations - walltime - criterion - criterion_normalized - monotone_criterion - monotone_criterion_normalized - parameter_distance - parameter_distance_normalized - monotone_parameter_distance - monotone_parameter_distance_normalized """ # get solution values for each problem registry = get_registry(extended=True) x_opt = { name: tree_just_flatten(prob["solution"]["params"], registry=registry) for name, prob in problems.items() } f_opt = pd.Series( {name: prob["solution"]["value"] for name, prob in problems.items()}) # build df from results time_sr = _get_history_as_stacked_sr_from_results(results, "time_history") time_sr.name = "walltime" criterion_sr = _get_history_as_stacked_sr_from_results( results, "criterion_history") x_dist_sr = _get_history_of_the_parameter_distance(results, x_opt) df = pd.concat([time_sr, criterion_sr, x_dist_sr], axis=1) df.index = df.index.rename({"evaluation": "n_evaluations"}) df = df.sort_index().reset_index() first_evaluations = df.query("n_evaluations == 0").groupby("problem") f_0 = first_evaluations["criterion"].mean() x_0_dist = first_evaluations["parameter_distance"].mean() x_opt_dist = {name: 0 for name in problems} # normalizations df["criterion_normalized"] = _normalize(df=df, col="criterion", start_values=f_0, target_values=f_opt) df["parameter_distance_normalized"] = _normalize( df=df, col="parameter_distance", start_values=x_0_dist, target_values=x_opt_dist, ) # create monotone versions of columns df["monotone_criterion"] = _make_history_monotone(df, "criterion") df["monotone_parameter_distance"] = _make_history_monotone( df, "parameter_distance") df["monotone_criterion_normalized"] = _make_history_monotone( df, "criterion_normalized") df["monotone_parameter_distance_normalized"] = _make_history_monotone( df, "parameter_distance_normalized") if stopping_criterion is not None: df, converged_info = _clip_histories( df=df, stopping_criterion=stopping_criterion, x_precision=x_precision, y_precision=y_precision, ) else: converged_info = None return df, converged_info
def evaluator(params): raw = constraint["selector"](params) flat = tree_just_flatten(raw, registry=registry) return flat
def params_to_internal(self, params): registry = get_registry(extended=True) return np.array(tree_just_flatten(params, registry=registry))
def evaluator(params): raw = [sel(params) for sel in constraint["selectors"]] flat = [tree_just_flatten(r, registry=registry) for r in raw] return flat
def calculate_estimation_summary( summary_data, names, free_names, ): """Create estimation summary using pre-calculated results. Args: summary_data (dict): Dictionary with entries ['params', 'p_value', 'ci_lower', 'ci_upper', 'standard_error']. names (List[str]): List of parameter names, corresponding to result_object. free_names (List[str]): List of parameter names for free parameters. Returns: pytree: A pytree with the same structure as params. Each leaf in the params tree is replaced by a DataFrame containing columns "value", "standard_error", "pvalue", "ci_lower" and "ci_upper". Parameters that do not have a standard error (e.g. because they were fixed during estimation) contain NaNs in all but the "value" column. The value column is only reproduced for convenience. """ # ================================================================================== # Flatten summary and construct data frame for flat estimates # ================================================================================== registry = get_registry(extended=True) flat_data = { key: tree_just_flatten(val, registry=registry) for key, val in summary_data.items() } df = pd.DataFrame(flat_data, index=names) df.loc[free_names, "stars"] = pd.cut( df.loc[free_names, "p_value"], bins=[-1, 0.01, 0.05, 0.1, 2], labels=["***", "**", "*", ""], ) # ================================================================================== # Map summary data into params tree structure # ================================================================================== # create tree with values corresponding to indices of df indices = tree_unflatten(summary_data["value"], names, registry=registry) estimates_flat = tree_just_flatten(summary_data["value"]) indices_flat = tree_just_flatten(indices) # use index chunks in indices_flat to access the corresponding sub data frame of df, # and use the index information stored in estimates_flat to form the correct (multi) # index for the resulting leaf. summary_flat = [] for index_leaf, params_leaf in zip(indices_flat, estimates_flat): if np.isscalar(params_leaf): loc = [index_leaf] index = [0] elif isinstance(params_leaf, pd.DataFrame) and "value" in params_leaf: loc = index_leaf["value"].to_numpy().flatten() index = params_leaf.index elif isinstance(params_leaf, pd.DataFrame): loc = index_leaf.to_numpy().flatten() # use product of existing index and columns for regular pd.DataFrame index = pd.MultiIndex.from_tuples([ (*row, col) if isinstance(row, tuple) else (row, col) for row in params_leaf.index for col in params_leaf.columns ]) elif isinstance(params_leaf, pd.Series): loc = index_leaf.to_numpy().flatten() index = params_leaf.index else: # array case (numpy or jax) loc = index_leaf.flatten() if params_leaf.ndim == 1: index = pd.RangeIndex(stop=params_leaf.size) else: index = pd.MultiIndex.from_arrays( np.unravel_index(np.arange(params_leaf.size), params_leaf.shape)) df_chunk = df.loc[loc] df_chunk.index = index summary_flat.append(df_chunk) summary = tree_unflatten(summary_data["value"], summary_flat) return summary
def flexible_sos_scalar(params): flat = np.array(tree_just_flatten(params, registry=REGISTRY)) return flat @ flat
def params_plot( result, selector=None, max_evaluations=None, template=PLOTLY_TEMPLATE, show_exploration=False, ): """Plot the params history of an optimization. Args: result (Union[OptimizeResult, pathlib.Path, str]): An optimization results with collected history. If dict, then the key is used as the name in a legend. selector (callable): A callable that takes params and returns a subset of params. If provided, only the selected subset of params is plotted. max_evaluations (int): Clip the criterion history after that many entries. template (str): The template for the figure. Default is "plotly_white". show_exploration (bool): If True, exploration samples of a multistart optimization are visualized. Default is False. Returns: plotly.graph_objs._figure.Figure: The figure. """ # ================================================================================== # Process inputs # ================================================================================== if isinstance(result, OptimizeResult): data = _extract_plotting_data_from_results_object( result, stack_multistart=True, show_exploration=show_exploration, plot_name="params_plot", ) start_params = result.start_params elif isinstance(result, (str, Path)): data = _extract_plotting_data_from_database( result, stack_multistart=True, show_exploration=show_exploration, ) start_params = data["start_params"] else: raise ValueError("result must be an OptimizeResult or a path to a log file.") if data["stacked_local_histories"] is not None: history = data["stacked_local_histories"]["params"] else: history = data["history"]["params"] # ================================================================================== # Create figure # ================================================================================== fig = go.Figure() registry = get_registry(extended=True) hist_arr = np.array([tree_just_flatten(p, registry=registry) for p in history]).T names = leaf_names(start_params, registry=registry) if selector is not None: flat, treedef = tree_flatten(start_params, registry=registry) helper = tree_unflatten(treedef, list(range(len(flat))), registry=registry) selected = np.array(tree_just_flatten(selector(helper), registry=registry)) names = [names[i] for i in selected] hist_arr = hist_arr[selected] for name, data in zip(names, hist_arr): if max_evaluations is not None and len(data) > max_evaluations: data = data[:max_evaluations] trace = go.Scatter( x=np.arange(len(data)), y=data, mode="lines", name=name, ) fig.add_trace(trace) fig.update_layout( template=template, xaxis_title_text="No. of criterion evaluations", yaxis_title_text="Parameter value", legend={"yanchor": "top", "xanchor": "right", "y": 0.95, "x": 0.95}, ) return fig
def func(data, **kwargs): raw = calculate_moments(data, **kwargs) out = pd.Series(tree_just_flatten( raw, registry=registry)) # xxxx won't be necessary soon! return out
def dashboard_app( doc, session_data, updating_options, ): """Create plots showing the development of the criterion and parameters. Args: doc (bokeh.Document): Argument required by bokeh. session_data (dict): Infos to be passed between and within apps. Keys of this app's entry are: - last_retrieved (int): last iteration currently in the ColumnDataSource. - database_path (str or pathlib.Path) - callbacks (dict): dictionary to be populated with callbacks. updating_options (dict): Specification how to update the plotting data. It contains rollover, update_frequency, update_chunk, jump and stride. """ # style the Document template_folder = Path(__file__).resolve().parent # conversion to string from pathlib Path is necessary for FileSystemLoader env = Environment(loader=FileSystemLoader(str(template_folder))) doc.template = env.get_template("index.html") # process inputs database = load_database(path=session_data["database_path"]) start_point = _calculate_start_point(database, updating_options) session_data["last_retrieved"] = start_point # build start_params DataFrame registry = get_registry(extended=True) start_params_tree = read_start_params(path_or_database=database) internal_params = tree_just_flatten(tree=start_params_tree, registry=registry) full_names = leaf_names(start_params_tree, registry=registry) optimization_problem = read_last_rows( database=database, table_name="optimization_problem", n_rows=1, return_type="dict_of_lists", ) free_mask = optimization_problem["free_mask"][0] params_groups, short_names = get_params_groups_and_short_names( params=start_params_tree, free_mask=free_mask ) start_params = pd.DataFrame( { "full_name": full_names, "name": short_names, "group": params_groups, "value": internal_params, } ) start_params["id"] = _create_id_column(start_params) group_to_param_ids = _map_group_to_other_column(start_params, "id") group_to_param_names = _map_group_to_other_column(start_params, "name") criterion_history, params_history = _create_cds_for_dashboard(group_to_param_ids) # create elements title_text = """<h1 style="font-size:30px;">estimagic Dashboard</h1>""" title = Row( children=[ Div( text=title_text, sizing_mode="scale_width", ) ], name="title", margin=(5, 5, -20, 5), ) plots = _create_initial_plots( criterion_history=criterion_history, params_history=params_history, group_to_param_ids=group_to_param_ids, group_to_param_names=group_to_param_names, ) restart_button = _create_restart_button( doc=doc, database=database, session_data=session_data, start_params=start_params, updating_options=updating_options, ) button_row = Row( children=[restart_button], name="button_row", ) # add elements to bokeh Document grid = Column(children=[title, button_row, *plots], sizing_mode="stretch_width") doc.add_root(grid) # start the convergence plot immediately restart_button.active = True
def derivative_flatten(derivative_eval): flat = np.array( tree_just_flatten(derivative_eval, registry=registry) ).astype(float) return flat
def outcome_flat(data): return tree_just_flatten(outcome(data), registry=registry)
def params_flatten(params): return np.array(tree_just_flatten(params, registry=registry)).astype(float)
def slice_plot( func, params, lower_bounds=None, upper_bounds=None, func_kwargs=None, selector=None, n_cores=DEFAULT_N_CORES, n_gridpoints=20, plots_per_row=2, param_names=None, share_y=True, expand_yrange=0.02, share_x=False, color="#497ea7", template=PLOTLY_TEMPLATE, title=None, return_dict=False, make_subplot_kwargs=None, batch_evaluator="joblib", ): """Plot criterion along coordinates at given and random values. Generates plots for each parameter and optionally combines them into a figure with subplots. Args: criterion (callable): criterion function that takes params and returns a scalar value or dictionary with the entry "value". params (pytree): A pytree with parameters. lower_bounds (pytree): A pytree with same structure as params. Must be specified and finite for all parameters unless params is a DataFrame containing with "lower_bound" column. upper_bounds (pytree): A pytree with same structure as params. Must be specified and finite for all parameters unless params is a DataFrame containing with "lower_bound" column. selector (callable): Function that takes params and returns a subset of params for which we actually want to generate the plot. n_cores (int): Number of cores. n_gridpoins (int): Number of gridpoints on which the criterion function is evaluated. This is the number per plotted line. plots_per_row (int): Number of plots per row. param_names (dict or NoneType): Dictionary mapping old parameter names to new ones. share_y (bool): If True, the individual plots share the scale on the yaxis and plots in one row actually share the y axis. share_x (bool): If True, set the same range of x axis for all plots and share the x axis for all plots in one column. expand_y (float): The ration by which to expand the range of the (shared) y axis, such that the axis is not cropped at exactly max of Criterion Value. color: The line color. template (str): The template for the figure. Default is "plotly_white". layout_kwargs (dict or NoneType): Dictionary of key word arguments used to update layout of plotly Figure object. If None, the default kwargs defined in the function will be used. title (str): The figure title. return_dict (bool): If True, return dictionary with individual plots of each parameter, else, ombine individual plots into a figure with subplots. make_subplot_kwargs (dict or NoneType): Dictionary of keyword arguments used to instantiate plotly Figure with multiple subplots. Is used to define properties such as, for example, the spacing between subplots (governed by 'horizontal_spacing' and 'vertical_spacing'). If None, default arguments defined in the function are used. batch_evaluator (str or callable): See :ref:`batch_evaluators`. Returns: out (dict or plotly.Figure): Returns either dictionary with individual slice plots for each parameter or a plotly Figure combining the individual plots. """ layout_kwargs = None if title is not None: title_kwargs = {"text": title} else: title_kwargs = None if func_kwargs is not None: func = partial(func, **func_kwargs) func_eval = func(params) converter, internal_params = get_converter( params=params, constraints=None, lower_bounds=lower_bounds, upper_bounds=upper_bounds, func_eval=func_eval, primary_key="value", scaling=False, scaling_options=None, ) n_params = len(internal_params.values) selected = np.arange(n_params, dtype=int) if selector is not None: helper = converter.params_from_internal(selected) registry = get_registry(extended=True) selected = np.array(tree_just_flatten(selector(helper), registry=registry), dtype=int) if not np.isfinite(internal_params.lower_bounds[selected]).all(): raise ValueError( "All selected parameters must have finite lower bounds.") if not np.isfinite(internal_params.upper_bounds[selected]).all(): raise ValueError( "All selected parameters must have finite upper bounds.") evaluation_points, metadata = [], [] for pos in selected: lb = internal_params.lower_bounds[pos] ub = internal_params.upper_bounds[pos] grid = np.linspace(lb, ub, n_gridpoints) name = internal_params.names[pos] for param_value in grid: if param_value != internal_params.values[pos]: meta = { "name": name, "Parameter Value": param_value, } x = internal_params.values.copy() x[pos] = param_value point = converter.params_from_internal(x) evaluation_points.append(point) metadata.append(meta) batch_evaluator = process_batch_evaluator(batch_evaluator) func_values = batch_evaluator( func=func, arguments=evaluation_points, error_handling="continue", n_cores=n_cores, ) # add NaNs where an evaluation failed func_values = [ converter.func_to_internal(val) if not isinstance(val, str) else np.nan for val in func_values ] func_values += [converter.func_to_internal(func_eval)] * len(selected) for pos in selected: meta = { "name": internal_params.names[pos], "Parameter Value": internal_params.values[pos], } metadata.append(meta) plot_data = pd.DataFrame(metadata) plot_data["Function Value"] = func_values if param_names is not None: plot_data["name"] = plot_data["name"].replace(param_names) lb = plot_data["Function Value"].min() ub = plot_data["Function Value"].max() y_range = ub - lb yaxis_ub = ub + y_range * expand_yrange yaxis_lb = lb - y_range * expand_yrange layout_kwargs = get_layout_kwargs( layout_kwargs, None, title_kwargs, template, False, ) plots_dict = {} for pos in selected: par_name = internal_params.names[pos] if param_names is not None and par_name in param_names: par_name = param_names[par_name] df = plot_data[plot_data["name"] == par_name].sort_values( "Parameter Value") subfig = px.line( df, y="Function Value", x="Parameter Value", color_discrete_sequence=[color], ) subfig.add_trace( go.Scatter( x=[internal_params.values[pos]], y=[converter.func_to_internal(func_eval)], marker={"color": color}, )) subfig.update_layout(**layout_kwargs) subfig.update_xaxes(title={"text": par_name}) subfig.update_yaxes(title={"text": "Function Value"}) if share_y is True: subfig.update_yaxes(range=[yaxis_lb, yaxis_ub]) plots_dict[par_name] = subfig if return_dict: out = plots_dict else: plots = list(plots_dict.values()) out = combine_plots( plots=plots, plots_per_row=plots_per_row, sharex=share_x, sharey=share_y, share_yrange_all=share_y, share_xrange_all=share_x, expand_yrange=expand_yrange, make_subplot_kwargs=make_subplot_kwargs, showlegend=False, template=template, clean_legend=True, layout_kwargs=layout_kwargs, legend_kwargs={}, title_kwargs=title_kwargs, ) return out
error_handling=error_handling, batch_evaluator=batch_evaluator, ) all_outcomes = existing_outcomes + new_outcomes else: random_indices = rng.choice(n_existing, n_draws, replace=False) all_outcomes = [existing_outcomes[k] for k in random_indices] # ================================================================================== # Process results # ================================================================================== registry = get_registry(extended=True) flat_outcomes = [ tree_just_flatten(_outcome, registry=registry) for _outcome in all_outcomes ] internal_outcomes = np.array(flat_outcomes) result = BootstrapResult( _base_outcome=base_outcome, _internal_outcomes=internal_outcomes, _internal_cov=np.cov(internal_outcomes, rowvar=False), ) return result @dataclass class BootstrapResult: