def matrix_to_block_tree(matrix, outer_tree, inner_tree):
"""Convert a matrix (2-dimensional array) to block-tree.
A block tree most often arises when one applies an operation to a function that maps
between two trees. For certain functions this results in a 2-dimensional data array.
Two main examples are the Jacobian of the function f : inner_tree -> outer_tree,
which results in a block tree structure, or the covariance matrix of a tree, in
which case outer_tree = inner_tree.
Args:
matrix (numpy.ndarray): 2d representation of the block tree. Has shape (m, n).
outer_tree: A pytree. If flattened to scalars has length m.
inner_tree: A pytree. If flattened to scalars has length n.
Returns:
block_tree: A (block) pytree.
"""
_check_dimensions_matrix(matrix, outer_tree, inner_tree)
flat_outer, treedef_outer = tree_flatten(outer_tree)
flat_inner, treedef_inner = tree_flatten(inner_tree)
flat_outer_np = [
_convert_to_numpy(leaf, only_pandas=True) for leaf in flat_outer
]
flat_inner_np = [
_convert_to_numpy(leaf, only_pandas=True) for leaf in flat_inner
]
shapes_outer = [np.shape(a) for a in flat_outer_np]
shapes_inner = [np.shape(a) for a in flat_inner_np]
block_bounds_outer = np.cumsum(
[int(np.product(s)) for s in shapes_outer[:-1]])
block_bounds_inner = np.cumsum(
[int(np.product(s)) for s in shapes_inner[:-1]])
blocks = []
for leaf_outer, s1, submat in zip(
flat_outer, shapes_outer,
np.split(matrix, block_bounds_outer, axis=0)):
row = []
for leaf_inner, s2, block_values in zip(
flat_inner, shapes_inner,
np.split(submat, block_bounds_inner, axis=1)):
raw_block = block_values.reshape((*s1, *s2))
block = _convert_raw_block_to_pandas(raw_block, leaf_outer,
leaf_inner)
row.append(block)
blocks.append(row)
block_tree = tree_unflatten(
treedef_outer, [tree_unflatten(treedef_inner, row) for row in blocks])
return block_tree
def hessian_to_block_tree(hessian, f_tree, params_tree):
"""Convert a Hessian array to block-tree format.
Remark: In comparison to Jax we need this formatting function because we calculate
the second derivative using second-order finite differences. Jax computes the
second derivative by applying their jacobian function twice, which produces the
desired block-tree shape of the Hessian automatically. If we apply our first
derivative function twice we get the same block-tree shape.
Args:
hessian (np.ndarray): The Hessian, 2- or 3-dimensional array representation of
the resulting block-tree.
f_tree (pytree): The function evaluated at params_tree.
params_tree (pytree): The params_tree.
Returns:
hessian_block_tree (pytree): The pytree
"""
_check_dimensions_hessian(hessian, f_tree, params_tree)
if hessian.ndim == 2:
hessian = hessian[np.newaxis]
flat_f, treedef_f = tree_flatten(f_tree)
flat_p, treedef_p = tree_flatten(params_tree)
flat_f_np = [_convert_to_numpy(leaf, only_pandas=True) for leaf in flat_f]
flat_p_np = [_convert_to_numpy(leaf, only_pandas=True) for leaf in flat_p]
shapes_f = [np.shape(a) for a in flat_f_np]
shapes_p = [np.shape(a) for a in flat_p_np]
block_bounds_f = np.cumsum([int(np.product(s)) for s in shapes_f[:-1]])
block_bounds_p = np.cumsum([int(np.product(s)) for s in shapes_p[:-1]])
sub_block_trees = []
for s0, subarr in zip(shapes_f, np.split(hessian, block_bounds_f, axis=0)):
blocks = []
for leaf_outer, s1, submat in zip(
flat_p, shapes_p, np.split(subarr, block_bounds_p, axis=1)):
row = []
for leaf_inner, s2, block_values in zip(
flat_p, shapes_p, np.split(submat, block_bounds_p,
axis=2)):
raw_block = block_values.reshape(((*s0, *s1, *s2)))
raw_block = np.squeeze(raw_block)
block = _convert_raw_block_to_pandas(raw_block, leaf_outer,
leaf_inner)
row.append(block)
blocks.append(row)
block_tree = tree_unflatten(
treedef_p, [tree_unflatten(treedef_p, row) for row in blocks])
sub_block_trees.append(block_tree)
hessian_block_tree = tree_unflatten(treedef_f, sub_block_trees)
return hessian_block_tree
def ci(self, ci_method="percentile", ci_level=0.95):
"""Calculate confidence intervals.
Args:
ci_method (str): Method of choice for computing confidence intervals.
The default is "percentile".
ci_level (float): Confidence level for the calculation of confidence
intervals. The default is 0.95.
Returns:
Any: Pytree with the same structure as base_outcome containing lower
bounds of confidence intervals.
Any: Pytree with the same structure as base_outcome containing upper
bounds of confidence intervals.
"""
registry = get_registry(extended=True)
base_outcome_flat, treedef = tree_flatten(self._base_outcome,
registry=registry)
lower_flat, upper_flat = calculate_ci(base_outcome_flat,
self._internal_outcomes,
ci_method, ci_level)
lower = tree_unflatten(treedef, lower_flat, registry=registry)
upper = tree_unflatten(treedef, upper_flat, registry=registry)
return lower, upper
def __post_init__(self):
_database = _load_database(self.path)
_start_params = read_start_params(_database)
_registry = get_registry(extended=True)
_, _treedef = tree_flatten(_start_params, registry=_registry)
self._database = _database
self._registry = _registry
self._treedef = _treedef
self._start_params = _start_params
def _get_selection_indices(params, selector):
"""Get index of selected flat params and number of flat params."""
registry = get_registry(extended=True)
flat_params, params_treedef = tree_flatten(params, registry=registry)
n_params = len(flat_params)
indices = np.arange(n_params, dtype=int)
params_indices = tree_unflatten(params_treedef, indices, registry=registry)
selected = selector(params_indices)
selection_indices = np.array(tree_just_flatten(selected,
registry=registry),
dtype=int)
return selection_indices, n_params
def outcomes(self):
"""Returns the estimated bootstrap outcomes.
Returns:
List[Any]: The boostrap outcomes as a list of pytrees.
"""
registry = get_registry(extended=True)
_, treedef = tree_flatten(self._base_outcome, registry=registry)
outcomes = [
tree_unflatten(treedef, out, registry=registry)
for out in self._internal_outcomes
]
return outcomes
def se(self):
"""Calculate standard errors.
Returns:
Any: The standard errors of the estimated parameters as a block-pytree,
numpy.ndarray, or pandas.DataFrame.
"""
cov = self._internal_cov
se = np.sqrt(np.diagonal(cov))
registry = get_registry(extended=True)
_, treedef = tree_flatten(self._base_outcome, registry=registry)
se = tree_unflatten(treedef, se, registry=registry)
return se
def tree_params_converter(tree_params):
registry = get_registry(extended=True)
_, treedef = tree_flatten(tree_params, registry=registry)
converter = TreeConverter(
params_flatten=lambda params: np.array(
tree_just_flatten(params, registry=registry)
),
params_unflatten=lambda x: tree_unflatten(
treedef, x.tolist(), registry=registry
),
func_flatten=None,
derivative_flatten=None,
)
return converter
def get_tree_converter(
params,
lower_bounds,
upper_bounds,
func_eval,
primary_key,
derivative_eval=None,
soft_lower_bounds=None,
soft_upper_bounds=None,
add_soft_bounds=False,
):
"""Get flatten and unflatten functions for criterion and its derivative.
The function creates a converter with methods to convert parameters, derivatives
and the output of the criterion function between the user provided pytree structure
and flat representations.
The main motivation for bundling all of this together (as opposed to handling
parameters, derivatives and function outputs separately) is that the derivative
conversion needs to know about the structure of params and the criterion output.
Args:
params (pytree): The user provided parameters.
lower_bounds (pytree): The user provided lower_bounds
upper_bounds (pytree): The user provided upper bounds
func_eval (float, dict or pytree): An evaluation of ``func`` at ``params``.
Used to deterimine how the function output has to be transformed for the
optimizer.
primary_key (str): One of "value", "contributions" and "root_contributions".
Used to determine how the function and derivative output has to be
transformed for the optimzer.
derivative_eval (dict, pytree or None): Evaluation of the derivative of
func at params. Used for consistency checks.
soft_lower_bounds (pytree): As lower_bounds
soft_upper_bounds (pytree): As upper_bounds
add_soft_bounds (bool): Whether soft bounds should be added to the flat_params
Returns:
TreeConverter: NamedTuple with flatten and unflatten methods.
FlatParams: NamedTuple of 1d arrays with flattened bounds and param names.
"""
_registry = get_registry(extended=True)
_params_vec, _params_treedef = tree_flatten(params, registry=_registry)
_params_vec = np.array(_params_vec).astype(float)
_lower, _upper = get_bounds(
params=params,
lower_bounds=lower_bounds,
upper_bounds=upper_bounds,
registry=_registry,
)
if add_soft_bounds:
_soft_lower, _soft_upper = get_bounds(
params=params,
lower_bounds=lower_bounds,
upper_bounds=upper_bounds,
registry=_registry,
soft_lower_bounds=soft_lower_bounds,
soft_upper_bounds=soft_upper_bounds,
add_soft_bounds=add_soft_bounds,
)
else:
_soft_lower, _soft_upper = None, None
_param_names = leaf_names(params, registry=_registry)
flat_params = FlatParams(
values=_params_vec,
lower_bounds=_lower,
upper_bounds=_upper,
names=_param_names,
soft_lower_bounds=_soft_lower,
soft_upper_bounds=_soft_upper,
)
_params_flatten = _get_params_flatten(registry=_registry)
_params_unflatten = _get_params_unflatten(
registry=_registry, treedef=_params_treedef
)
_func_flatten = _get_func_flatten(
registry=_registry,
func_eval=func_eval,
primary_key=primary_key,
)
_derivative_flatten = _get_derivative_flatten(
registry=_registry,
primary_key=primary_key,
params=params,
func_eval=func_eval,
derivative_eval=derivative_eval,
)
converter = TreeConverter(
params_flatten=_params_flatten,
params_unflatten=_params_unflatten,
func_flatten=_func_flatten,
derivative_flatten=_derivative_flatten,
)
return converter, flat_params
def second_derivative(
func,
params,
*,
func_kwargs=None,
method="central_cross",
n_steps=1,
base_steps=None,
scaling_factor=1,
lower_bounds=None,
upper_bounds=None,
step_ratio=2,
min_steps=None,
f0=None,
n_cores=DEFAULT_N_CORES,
error_handling="continue",
batch_evaluator="joblib",
return_func_value=False,
return_info=False,
key=None,
):
"""Evaluate second derivative of func at params according to method and step options
Internally, the function is converted such that it maps from a 1d array to a 1d
array. Then the Hessians of that function are calculated. The resulting derivative
estimate is always a :class:`numpy.ndarray`.
The parameters and the function output can be pandas objects (Series or DataFrames
with value column). In that case the output of second_derivative is also a pandas
object and with appropriate index and columns.
Detailed description of all options that influence the step size as well as an
explanation of how steps are adjusted to bounds in case of a conflict,
see :func:`~estimagic.differentiation.generate_steps.generate_steps`.
Args:
func (callable): Function of which the derivative is calculated.
params (numpy.ndarray, pandas.Series or pandas.DataFrame): 1d numpy array or
:class:`pandas.DataFrame` with parameters at which the derivative is
calculated. If it is a DataFrame, it can contain the columns "lower_bound"
and "upper_bound" for bounds. See :ref:`params`.
func_kwargs (dict): Additional keyword arguments for func, optional.
method (str): One of {"forward", "backward", "central_average", "central_cross"}
These correspond to the finite difference approximations defined in
equations [7, x, 8, 9] in Rideout [2009], where ("backward", x) is not found
in Rideout [2009] but is the natural extension of equation 7 to the backward
case. Default "central_cross".
n_steps (int): Number of steps needed. For central methods, this is
the number of steps per direction. It is 1 if no Richardson extrapolation
is used.
base_steps (numpy.ndarray, optional): 1d array of the same length as params.
base_steps * scaling_factor is the absolute value of the first (and possibly
only) step used in the finite differences approximation of the derivative.
If base_steps * scaling_factor conflicts with bounds, the actual steps will
be adjusted. If base_steps is not provided, it will be determined according
to a rule of thumb as long as this does not conflict with min_steps.
scaling_factor (numpy.ndarray or float): Scaling factor which is applied to
base_steps. If it is an numpy.ndarray, it needs to be as long as params.
scaling_factor is useful if you want to increase or decrease the base_step
relative to the rule-of-thumb or user provided base_step, for example to
benchmark the effect of the step size. Default 1.
lower_bounds (numpy.ndarray): 1d array with lower bounds for each parameter. If
params is a DataFrame and has the columns "lower_bound", this will be taken
as lower_bounds if now lower_bounds have been provided explicitly.
upper_bounds (numpy.ndarray): 1d array with upper bounds for each parameter. If
params is a DataFrame and has the columns "upper_bound", this will be taken
as upper_bounds if no upper_bounds have been provided explicitly.
step_ratio (float, numpy.array): Ratio between two consecutive Richardson
extrapolation steps in the same direction. default 2.0. Has to be larger
than one. The step ratio is only used if n_steps > 1.
min_steps (numpy.ndarray): Minimal possible step sizes that can be chosen to
accommodate bounds. Must have same length as params. By default min_steps is
equal to base_steps, i.e step size is not decreased beyond what is optimal
according to the rule of thumb.
f0 (numpy.ndarray): 1d numpy array with func(x), optional.
n_cores (int): Number of processes used to parallelize the function
evaluations. Default 1.
error_handling (str): One of "continue" (catch errors and continue to calculate
derivative estimates. In this case, some derivative estimates can be
missing but no errors are raised), "raise" (catch errors and continue
to calculate derivative estimates at fist but raise an error if all
evaluations for one parameter failed) and "raise_strict" (raise an error
as soon as a function evaluation fails).
batch_evaluator (str or callable): Name of a pre-implemented batch evaluator
(currently 'joblib' and 'pathos_mp') or Callable with the same interface
as the estimagic batch_evaluators.
return_func_value (bool): If True, return function value at params, stored in
output dict under "func_value". Default False. This is useful when using
first_derivative during optimization.
return_info (bool): If True, return additional information on function
evaluations and internal derivative candidates, stored in output dict under
"func_evals" and "derivative_candidates". Derivative candidates are only
returned if n_steps > 1. Default False.
key (str): If func returns a dictionary, take the derivative of
func(params)[key].
Returns:
result (dict): Result dictionary with keys:
- "derivative" (numpy.ndarray, pandas.Series or pandas.DataFrame): The
estimated second derivative of func at params. The shape of the output
depends on the dimension of params and func(params):
- f: R -> R leads to shape (1,), usually called second derivative
- f: R^m -> R leads to shape (m, m), usually called Hessian
- f: R -> R^n leads to shape (n,), usually called Hessian
- f: R^m -> R^n leads to shape (n, m, m), usually called Hessian tensor
- "func_value" (numpy.ndarray, pandas.Series or pandas.DataFrame): Function
value at params, returned if return_func_value is True.
- "func_evals_one_step" (pandas.DataFrame): Function evaluations produced by
internal derivative method when altering the params vector at one
dimension, returned if return_info is True.
- "func_evals_two_step" (pandas.DataFrame): This features is not implemented
yet and therefore set to None. Once implemented it will contain
function evaluations produced by internal derivative method when
altering the params vector at two dimensions, returned if return_info is
True.
- "func_evals_cross_step" (pandas.DataFrame): This features is not
implemented yet and therefore set to None. Once implemented it will
contain function evaluations produced by internal derivative method when
altering the params vector at two dimensions in different directions,
returned if return_info is True.
"""
lower_bounds, upper_bounds = get_bounds(params, lower_bounds, upper_bounds)
# handle keyword arguments
func_kwargs = {} if func_kwargs is None else func_kwargs
partialed_func = functools.partial(func, **func_kwargs)
# convert params to numpy
registry = get_registry(extended=True)
x, params_treedef = tree_flatten(params, registry=registry)
x = np.atleast_1d(x).astype(np.float64)
if np.isnan(x).any():
raise ValueError("The parameter vector must not contain NaNs.")
implemented_methods = {
"forward", "backward", "central_average", "central_cross"
}
if method not in implemented_methods:
raise ValueError(f"Method has to be in {implemented_methods}.")
# generate the step array
steps = generate_steps(
x=x,
method=("central" if "central" in method else method),
n_steps=n_steps,
target="second_derivative",
base_steps=base_steps,
scaling_factor=scaling_factor,
lower_bounds=lower_bounds,
upper_bounds=upper_bounds,
step_ratio=step_ratio,
min_steps=min_steps,
)
# generate parameter vectors at which func has to be evaluated as numpy arrays
evaluation_points = {"one_step": [], "two_step": [], "cross_step": []}
for step_arr in steps:
# single direction steps
for i, j in product(range(n_steps), range(len(x))):
if np.isnan(step_arr[i, j]):
evaluation_points["one_step"].append(np.nan)
else:
point = x.copy()
point[j] += step_arr[i, j]
evaluation_points["one_step"].append(point)
# two and cross direction steps
for i, j, k in product(range(n_steps), range(len(x)), range(len(x))):
if j > k or np.isnan(step_arr[i, j]) or np.isnan(step_arr[i, k]):
evaluation_points["two_step"].append(np.nan)
evaluation_points["cross_step"].append(np.nan)
else:
point = x.copy()
point[j] += step_arr[i, j]
point[k] += step_arr[i, k]
evaluation_points["two_step"].append(point)
if j == k:
evaluation_points["cross_step"].append(np.nan)
else:
point = x.copy()
point[j] += step_arr[i, j]
point[k] -= step_arr[i, k]
evaluation_points["cross_step"].append(point)
# convert the numpy arrays to whatever is needed by func
evaluation_points = {
# entries are either a numpy.ndarray or np.nan, we unflatten only
step_type:
[_unflatten_if_not_nan(p, params_treedef, registry) for p in points]
for step_type, points in evaluation_points.items()
}
# we always evaluate f0, so we can fall back to one-sided derivatives if
# two-sided derivatives fail. The extra cost is negligible in most cases.
if f0 is None:
evaluation_points["one_step"].append(params)
# do the function evaluations for one and two step, including error handling
batch_error_handling = "raise" if error_handling == "raise_strict" else "continue"
raw_evals = _nan_skipping_batch_evaluator(
func=partialed_func,
arguments=list(
itertools.chain.from_iterable(evaluation_points.values())),
n_cores=n_cores,
error_handling=batch_error_handling,
batch_evaluator=batch_evaluator,
)
# extract information on exceptions that occurred during function evaluations
exc_info = "\n\n".join([val for val in raw_evals if isinstance(val, str)])
raw_evals = [
val if not isinstance(val, str) else np.nan for val in raw_evals
]
n_one_step, n_two_step, n_cross_step = map(len, evaluation_points.values())
raw_evals = {
"one_step": raw_evals[:n_one_step],
"two_step": raw_evals[n_one_step:n_two_step + n_one_step],
"cross_step": raw_evals[n_two_step + n_one_step:],
}
# store full function value at params as func_value and a processed version of it
# that we need to calculate derivatives as f0
if f0 is None:
f0 = raw_evals["one_step"][-1]
raw_evals["one_step"] = raw_evals["one_step"][:-1]
func_value = f0
f0_tree = f0[key] if key is not None and isinstance(f0, dict) else f0
f0 = tree_leaves(f0_tree, registry=registry)
f0 = np.array(f0, dtype=np.float64)
# convert the raw evaluations to numpy arrays
raw_evals = {
step_type: _convert_evals_to_numpy(evals, key, registry)
for step_type, evals in raw_evals.items()
}
# reshape arrays into dimension (n_steps, dim_f, dim_x) or (n_steps, dim_f, dim_x,
# dim_x) for finite differences
evals = {}
evals["one_step"] = _reshape_one_step_evals(raw_evals["one_step"], n_steps,
len(x))
evals["two_step"] = _reshape_two_step_evals(raw_evals["two_step"], n_steps,
len(x))
evals["cross_step"] = _reshape_cross_step_evals(raw_evals["cross_step"],
n_steps, len(x), f0)
# apply finite difference formulae
hess_candidates = {}
for m in ["forward", "backward", "central_average", "central_cross"]:
hess_candidates[m] = finite_differences.hessian(evals, steps, f0, m)
# get the best derivative estimate out of all derivative estimates that could be
# calculated, given the function evaluations.
orders = {
"central_cross":
["central_cross", "central_average", "forward", "backward"],
"central_average":
["central_average", "central_cross", "forward", "backward"],
"forward": ["forward", "backward", "central_average", "central_cross"],
"backward":
["backward", "forward", "central_average", "central_cross"],
}
if n_steps == 1:
hess = _consolidate_one_step_derivatives(hess_candidates,
orders[method])
updated_candidates = None
else:
raise ValueError(
"Richardson extrapolation is not implemented for the second derivative yet."
)
# raise error if necessary
if error_handling in ("raise", "raise_strict") and np.isnan(hess).any():
raise Exception(exc_info)
# results processing
derivative = hessian_to_block_tree(hess, f0_tree, params)
result = {"derivative": derivative}
if return_func_value:
result["func_value"] = func_value
if return_info:
info = _collect_additional_info(steps,
evals,
updated_candidates,
target="second_derivative")
result = {**result, **info}
return result
def first_derivative(
func,
params,
*,
func_kwargs=None,
method="central",
n_steps=1,
base_steps=None,
scaling_factor=1,
lower_bounds=None,
upper_bounds=None,
step_ratio=2,
min_steps=None,
f0=None,
n_cores=DEFAULT_N_CORES,
error_handling="continue",
batch_evaluator="joblib",
return_func_value=False,
return_info=False,
key=None,
):
"""Evaluate first derivative of func at params according to method and step options.
Internally, the function is converted such that it maps from a 1d array to a 1d
array. Then the Jacobian of that function is calculated.
The parameters and the function output can be estimagic-pytrees; for more details on
estimagi-pytrees see :ref:`eeppytrees`. By default the resulting Jacobian will be
returned as a block-pytree.
For a detailed description of all options that influence the step size as well as an
explanation of how steps are adjusted to bounds in case of a conflict, see
:func:`~estimagic.differentiation.generate_steps.generate_steps`.
Args:
func (callable): Function of which the derivative is calculated.
params (pytree): A pytree. See :ref:`params`.
func_kwargs (dict): Additional keyword arguments for func, optional.
method (str): One of ["central", "forward", "backward"], default "central".
n_steps (int): Number of steps needed. For central methods, this is
the number of steps per direction. It is 1 if no Richardson extrapolation
is used.
base_steps (numpy.ndarray, optional): 1d array of the same length as params.
base_steps * scaling_factor is the absolute value of the first (and possibly
only) step used in the finite differences approximation of the derivative.
If base_steps * scaling_factor conflicts with bounds, the actual steps will
be adjusted. If base_steps is not provided, it will be determined according
to a rule of thumb as long as this does not conflict with min_steps.
scaling_factor (numpy.ndarray or float): Scaling factor which is applied to
base_steps. If it is an numpy.ndarray, it needs to be as long as params.
scaling_factor is useful if you want to increase or decrease the base_step
relative to the rule-of-thumb or user provided base_step, for example to
benchmark the effect of the step size. Default 1.
lower_bounds (pytree): To be written.
upper_bounds (pytree): To be written.
step_ratio (float, numpy.array): Ratio between two consecutive Richardson
extrapolation steps in the same direction. default 2.0. Has to be larger
than one. The step ratio is only used if n_steps > 1.
min_steps (numpy.ndarray): Minimal possible step sizes that can be chosen to
accommodate bounds. Must have same length as params. By default min_steps is
equal to base_steps, i.e step size is not decreased beyond what is optimal
according to the rule of thumb.
f0 (numpy.ndarray): 1d numpy array with func(x), optional.
n_cores (int): Number of processes used to parallelize the function
evaluations. Default 1.
error_handling (str): One of "continue" (catch errors and continue to calculate
derivative estimates. In this case, some derivative estimates can be
missing but no errors are raised), "raise" (catch errors and continue
to calculate derivative estimates at fist but raise an error if all
evaluations for one parameter failed) and "raise_strict" (raise an error
as soon as a function evaluation fails).
batch_evaluator (str or callable): Name of a pre-implemented batch evaluator
(currently 'joblib' and 'pathos_mp') or Callable with the same interface
as the estimagic batch_evaluators.
return_func_value (bool): If True, return function value at params, stored in
output dict under "func_value". Default False. This is useful when using
first_derivative during optimization.
return_info (bool): If True, return additional information on function
evaluations and internal derivative candidates, stored in output dict under
"func_evals" and "derivative_candidates". Derivative candidates are only
returned if n_steps > 1. Default False.
key (str): If func returns a dictionary, take the derivative of
func(params)[key].
Returns:
result (dict): Result dictionary with keys:
- "derivative" (numpy.ndarray, pandas.Series or pandas.DataFrame): The
estimated first derivative of func at params. The shape of the output
depends on the dimension of params and func(params):
- f: R -> R leads to shape (1,), usually called derivative
- f: R^m -> R leads to shape (m, ), usually called Gradient
- f: R -> R^n leads to shape (n, 1), usually called Jacobian
- f: R^m -> R^n leads to shape (n, m), usually called Jacobian
- "func_value" (numpy.ndarray, pandas.Series or pandas.DataFrame): Function
value at params, returned if return_func_value is True.
- "func_evals" (pandas.DataFrame): Function evaluations produced by internal
derivative method, returned if return_info is True.
- "derivative_candidates" (pandas.DataFrame): Derivative candidates from
Richardson extrapolation, returned if return_info is True and n_steps >
1.
"""
_is_fast_params = isinstance(params, np.ndarray) and params.ndim == 1
registry = get_registry(extended=True)
lower_bounds, upper_bounds = get_bounds(params, lower_bounds, upper_bounds)
# handle keyword arguments
func_kwargs = {} if func_kwargs is None else func_kwargs
partialed_func = functools.partial(func, **func_kwargs)
# convert params to numpy
if not _is_fast_params:
x, params_treedef = tree_flatten(params, registry=registry)
x = np.array(x, dtype=np.float64)
else:
x = params.astype(float)
if np.isnan(x).any():
raise ValueError("The parameter vector must not contain NaNs.")
# generate the step array
steps = generate_steps(
x=x,
method=method,
n_steps=n_steps,
target="first_derivative",
base_steps=base_steps,
scaling_factor=scaling_factor,
lower_bounds=lower_bounds,
upper_bounds=upper_bounds,
step_ratio=step_ratio,
min_steps=min_steps,
)
# generate parameter vectors at which func has to be evaluated as numpy arrays
evaluation_points = []
for step_arr in steps:
for i, j in product(range(n_steps), range(len(x))):
if np.isnan(step_arr[i, j]):
evaluation_points.append(np.nan)
else:
point = x.copy()
point[j] += step_arr[i, j]
evaluation_points.append(point)
# convert the numpy arrays to whatever is needed by func
if not _is_fast_params:
evaluation_points = [
# entries are either a numpy.ndarray or np.nan
_unflatten_if_not_nan(p, params_treedef, registry)
for p in evaluation_points
]
# we always evaluate f0, so we can fall back to one-sided derivatives if
# two-sided derivatives fail. The extra cost is negligible in most cases.
if f0 is None:
evaluation_points.append(params)
# do the function evaluations, including error handling
batch_error_handling = "raise" if error_handling == "raise_strict" else "continue"
raw_evals = _nan_skipping_batch_evaluator(
func=partialed_func,
arguments=evaluation_points,
n_cores=n_cores,
error_handling=batch_error_handling,
batch_evaluator=batch_evaluator,
)
# extract information on exceptions that occurred during function evaluations
exc_info = "\n\n".join([val for val in raw_evals if isinstance(val, str)])
raw_evals = [
val if not isinstance(val, str) else np.nan for val in raw_evals
]
# store full function value at params as func_value and a processed version of it
# that we need to calculate derivatives as f0
if f0 is None:
f0 = raw_evals[-1]
raw_evals = raw_evals[:-1]
func_value = f0
use_key = key is not None and isinstance(f0, dict)
f0_tree = f0[key] if use_key else f0
scalar_out = np.isscalar(f0_tree)
vector_out = isinstance(f0_tree, np.ndarray) and f0_tree.ndim == 1
if scalar_out:
f0 = np.array([f0_tree], dtype=float)
elif vector_out:
f0 = f0_tree.astype(float)
else:
f0 = tree_leaves(f0_tree, registry=registry)
f0 = np.array(f0, dtype=np.float64)
# convert the raw evaluations to numpy arrays
raw_evals = _convert_evals_to_numpy(
raw_evals=raw_evals,
key=key,
registry=registry,
is_scalar_out=scalar_out,
is_vector_out=vector_out,
)
# apply finite difference formulae
evals = np.array(raw_evals).reshape(2, n_steps, len(x), -1)
evals = np.transpose(evals, axes=(0, 1, 3, 2))
evals = Evals(pos=evals[0], neg=evals[1])
jac_candidates = {}
for m in ["forward", "backward", "central"]:
jac_candidates[m] = finite_differences.jacobian(evals, steps, f0, m)
# get the best derivative estimate out of all derivative estimates that could be
# calculated, given the function evaluations.
orders = {
"central": ["central", "forward", "backward"],
"forward": ["forward", "backward"],
"backward": ["backward", "forward"],
}
if n_steps == 1:
jac = _consolidate_one_step_derivatives(jac_candidates, orders[method])
updated_candidates = None
else:
richardson_candidates = _compute_richardson_candidates(
jac_candidates, steps, n_steps)
jac, updated_candidates = _consolidate_extrapolated(
richardson_candidates)
# raise error if necessary
if error_handling in ("raise", "raise_strict") and np.isnan(jac).any():
raise Exception(exc_info)
# results processing
if _is_fast_params and vector_out:
derivative = jac
elif _is_fast_params and scalar_out:
derivative = jac.flatten()
else:
derivative = matrix_to_block_tree(jac, f0_tree, params)
result = {"derivative": derivative}
if return_func_value:
result["func_value"] = func_value
if return_info:
info = _collect_additional_info(steps,
evals,
updated_candidates,
target="first_derivative")
result = {**result, **info}
return result
def test_unflatten_partially_numeric_df(other_df):
registry = get_registry(extended=True)
_, treedef = tree_flatten(other_df, registry=registry)
unflat = tree_unflatten(treedef, [1, 2, 3, 4, 5, 6], registry=registry)
other_df = other_df.assign(b=[1, 3, 5], c=[2, 4, 6])
assert_frame_equal(unflat, other_df, check_dtype=False)
def test_flatten_partially_numeric_df(other_df):
registry = get_registry(extended=True)
flat, _ = tree_flatten(other_df, registry=registry)
assert flat == [0, 3.14, 1, 3.14, 2, 3.14]
def test_unflatten_df_with_value_column(value_df):
registry = get_registry(extended=True)
_, treedef = tree_flatten(value_df, registry=registry)
unflat = tree_unflatten(treedef, [10, 11, 12], registry=registry)
assert unflat.equals(value_df.assign(value=[10, 11, 12]))
def test_flatten_df_with_value_column(value_df):
registry = get_registry(extended=True)
flat, _ = tree_flatten(value_df, registry=registry)
assert flat == [1, 3, 5]
def params_plot(
result,
selector=None,
max_evaluations=None,
template=PLOTLY_TEMPLATE,
show_exploration=False,
):
"""Plot the params history of an optimization.
Args:
result (Union[OptimizeResult, pathlib.Path, str]): An optimization results with
collected history. If dict, then the key is used as the name in a legend.
selector (callable): A callable that takes params and returns a subset
of params. If provided, only the selected subset of params is plotted.
max_evaluations (int): Clip the criterion history after that many entries.
template (str): The template for the figure. Default is "plotly_white".
show_exploration (bool): If True, exploration samples of a multistart
optimization are visualized. Default is False.
Returns:
plotly.graph_objs._figure.Figure: The figure.
"""
# ==================================================================================
# Process inputs
# ==================================================================================
if isinstance(result, OptimizeResult):
data = _extract_plotting_data_from_results_object(
result,
stack_multistart=True,
show_exploration=show_exploration,
plot_name="params_plot",
)
start_params = result.start_params
elif isinstance(result, (str, Path)):
data = _extract_plotting_data_from_database(
result,
stack_multistart=True,
show_exploration=show_exploration,
)
start_params = data["start_params"]
else:
raise ValueError("result must be an OptimizeResult or a path to a log file.")
if data["stacked_local_histories"] is not None:
history = data["stacked_local_histories"]["params"]
else:
history = data["history"]["params"]
# ==================================================================================
# Create figure
# ==================================================================================
fig = go.Figure()
registry = get_registry(extended=True)
hist_arr = np.array([tree_just_flatten(p, registry=registry) for p in history]).T
names = leaf_names(start_params, registry=registry)
if selector is not None:
flat, treedef = tree_flatten(start_params, registry=registry)
helper = tree_unflatten(treedef, list(range(len(flat))), registry=registry)
selected = np.array(tree_just_flatten(selector(helper), registry=registry))
names = [names[i] for i in selected]
hist_arr = hist_arr[selected]
for name, data in zip(names, hist_arr):
if max_evaluations is not None and len(data) > max_evaluations:
data = data[:max_evaluations]
trace = go.Scatter(
x=np.arange(len(data)),
y=data,
mode="lines",
name=name,
)
fig.add_trace(trace)
fig.update_layout(
template=template,
xaxis_title_text="No. of criterion evaluations",
yaxis_title_text="Parameter value",
legend={"yanchor": "top", "xanchor": "right", "y": 0.95, "x": 0.95},
)
return fig
