def wrapper_numpy_interface(x, *args, **kwargs):
# Handle usage in :func:`internal_function` for gradients.
if constraints is None:
p = params.copy()
p["value"] = x
# Handle usage in :func:`internal_criterion`.
else:
p = reparametrize_from_internal(
internal=x,
fixed_values=params["_internal_fixed_value"].to_numpy(),
pre_replacements=params["_pre_replacements"].to_numpy().
astype(int),
processed_constraints=constraints,
post_replacements=(
params["_post_replacements"].to_numpy().astype(int)),
processed_params=params,
)
criterion_value = func(p, *args, **kwargs)
if isinstance(criterion_value, (pd.DataFrame, pd.Series)):
criterion_value = criterion_value.to_numpy()
return criterion_value
def _process_optimization_results(results, results_arguments):
"""Expand the solutions back to the original problems.
Args:
results (list):
list of dictionaries with the harmonized results objects.
results_arguments (list):
each element is a dictionary supplying the start params DataFrame
and the constraints to the original problem.
The keys are "params", "constraints" and "keep_dashboard_alive".
Returns:
results (tuple): Tuple of the harmonized result info dictionary and the params
DataFrame with the minimizing parameter values of the untransformed problem
as specified of the user.
"""
new_results = []
for res, args in zip(results, results_arguments):
res["x"] = list(res["x"])
start_params = args["params"]
params = reparametrize_from_internal(
internal=res["x"],
fixed_values=start_params["_internal_fixed_value"].to_numpy(),
pre_replacements=start_params["_pre_replacements"].to_numpy(dtype="int"),
processed_constraints=args["constraints"],
post_replacements=start_params["_post_replacements"].to_numpy(dtype="int"),
processed_params=start_params,
)
new_results.append((res, params))
if len(new_results) == 1:
new_results = new_results[0]
return new_results
def test_reparametrize_from_internal(internal, expected_external, category):
constr = constraints(expected_external)
calculated = reparametrize_from_internal(internal, constr,
expected_external, None)["value"]
assert_series_equal(calculated[category], expected_external.loc[category,
"value"])
def _process_results(res, params, internal_params, constraints, origin):
"""Convert optimization results into json serializable dictionary.
Args:
res: Result from numerical optimizer.
params (DataFrame): See :ref:`params`.
internal_params (DataFrame): See :ref:`params`.
constraints (list): constraints for the optimization
origin (str): takes the values "pygmo", "nlopt", "scipy"
"""
if origin == "scipy":
res_dict = {**res}
for key, value in res_dict.items():
if isinstance(value, np.ndarray):
res_dict[key] = value.tolist()
x = res.x
elif origin in ["pygmo", "nlopt"]:
x = res.champion_x
res_dict = {"fun": res.champion_f[0]}
elif origin in ["tao"]:
x = res["x"]
res_dict = {**res}
params = reparametrize_from_internal(
internal=x,
fixed_values=params["_internal_fixed_value"].to_numpy(),
pre_replacements=params["_pre_replacements"].to_numpy().astype(int),
processed_constraints=constraints,
post_replacements=params["_post_replacements"].to_numpy().astype(int),
processed_params=params,
)
return res_dict, params
def internal_criterion(x, counter=c):
p = reparametrize_from_internal(
internal=x,
fixed_values=params["_internal_fixed_value"].to_numpy(),
pre_replacements=params["_pre_replacements"].to_numpy().astype(
int),
processed_constraints=constraints,
post_replacements=params["_post_replacements"].to_numpy().astype(
int),
processed_params=params,
)
fitness_eval = criterion(p, **criterion_kwargs)
# For Pounders, return the sum of squared errors.
_fitness_eval = (fitness_eval.sum() if isinstance(
fitness_eval, np.ndarray) else fitness_eval)
if queue is not None:
queue.put(
QueueEntry(
iteration=counter[0],
params=p,
fitness=fitness_factor * _fitness_eval,
))
counter += 1
return fitness_eval
def test_reparametrize_from_internal(internal, expected_external, category):
constr = constraints(expected_external)
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore", message="indexing past lexsort depth may impact performance."
)
calculated = reparametrize_from_internal(internal, constr, expected_external)
assert_series_equal(
calculated[category], expected_external.loc[category, "value"]
)
def _params_from_x(x, internal_params, constraints, params, scaling_factor):
internal_params = internal_params.copy(deep=True)
# :func:`internal_criterion` always assumes that `x` is a NumPy array, but if we
# pass the internal criterion function to :func:`gradient`, x is a DataFrame.
# Setting a series to a DataFrame will convert the column "value" to object type
# which causes trouble in following NumPy routines assuming a numeric type.
internal_params["value"] = x["value"] if isinstance(x, pd.DataFrame) else x
updated_params = reparametrize_from_internal(internal_params, constraints,
params, scaling_factor)
return updated_params
def _back_and_forth_transformation_and_assert(params, constraints):
pc, pp = process_constraints(constraints, params)
internal = reparametrize_to_internal(pp, pc)
external = reparametrize_from_internal(
internal=internal,
fixed_values=pp["_internal_fixed_value"].to_numpy(),
pre_replacements=pp["_pre_replacements"].to_numpy(),
processed_constraints=pc,
post_replacements=pp["_post_replacements"].to_numpy(),
processed_params=pp,
)
assert_series_equal(external["value"], params["value"])
return internal, external
def wrapper_handle_exceptions(x, *args, **kwargs):
try:
out = func(x, *args, **kwargs)
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
# Adjust the criterion value at the start.
start_criterion_value = general_options[
"start_criterion_value"]
constant, slope = general_options.get(
"criterion_exception_penalty", (None, None))
constant = 2 * start_criterion_value if constant is None else constant
slope = 0.1 * start_criterion_value if slope is None else slope
raise_exc = general_options.get("criterion_exception_raise",
False)
if raise_exc:
raise e
else:
if database:
exception_info = traceback.format_exc()
p = reparametrize_from_internal(
internal=x,
fixed_values=params["_internal_fixed_value"].
to_numpy(),
pre_replacements=params["_pre_replacements"].
to_numpy().astype(int),
processed_constraints=constraints,
post_replacements=(params["_post_replacements"].
to_numpy().astype(int)),
processed_params=params,
)
msg = (exception_info + "\n\n" +
"The parameters are\n\n" +
p["value"].to_csv(sep="\t", header=True))
append_rows(database, "exceptions", {"value": msg})
out = min(
MAX_CRITERION_PENALTY,
constant + slope * np.linalg.norm(x - start_params),
)
return out
def wrapper_numpy_interface(x, *args, **kwargs):
if isinstance(x, pd.DataFrame):
p = x
elif isinstance(x, np.ndarray):
p = params.copy()
p["value"] = reparametrize_from_internal(
internal=x,
fixed_values=fixed_values,
pre_replacements=pre_replacements,
processed_constraints=pc,
post_replacements=post_replacements,
)
else:
raise ValueError(
"x must be a numpy array or DataFrame with 'value' column."
)
criterion_value = func(p, *args, **kwargs)
if isinstance(criterion_value,
(pd.DataFrame, pd.Series)) and numpy_output:
criterion_value = criterion_value.to_numpy()
return criterion_value
def _internal_minimize(
criterion,
criterion_kwargs,
params,
internal_params,
constraints,
algorithm,
algo_options,
gradient,
gradient_options,
general_options,
database,
queue,
fitness_factor,
):
"""Create the internal criterion function and minimize it.
Args:
criterion (function):
Python function that takes a pandas DataFrame with parameters as the first
argument and returns a scalar floating point value.
criterion_kwargs (dict):
additional keyword arguments for criterion
params (pd.DataFrame):
See :ref:`params`.
internal_params (DataFrame):
See :ref:`params`.
constraints (list):
list with constraint dictionaries. See for details.
algorithm (str):
specifies the optimization algorithm. See :ref:`list_of_algorithms`.
algo_options (dict):
algorithm specific configurations for the optimization
gradient (callable or None):
Gradient function.
gradient_options (dict):
Options for the gradient function.
general_options (dict):
additional configurations for the optimization
database (sqlalchemy.sql.schema.MetaData). The engine that connects to the
database can be accessed via ``database.bind``.
queue (Queue):
queue to which the fitness evaluations and params DataFrames are supplied.
fitness_factor (float):
multiplicative factor for the fitness displayed in the dashboard.
Set to -1 for maximizations to plot the fitness that is being maximized.
"""
logging_decorator = functools.partial(
log_evaluation,
database=database,
tables=["params_history", "criterion_history", "comparison_plot"],
)
exception_decorator = functools.partial(
handle_exceptions,
database=database,
params=params,
constraints=constraints,
start_params=internal_params,
general_options=general_options,
)
internal_criterion = create_internal_criterion(
criterion=criterion,
params=params,
constraints=constraints,
criterion_kwargs=criterion_kwargs,
logging_decorator=logging_decorator,
exception_decorator=exception_decorator,
queue=queue,
fitness_factor=fitness_factor,
)
internal_gradient = create_internal_gradient(
gradient=gradient,
gradient_options=gradient_options,
criterion=criterion,
params=params,
internal_params=internal_params,
constraints=constraints,
criterion_kwargs=criterion_kwargs,
database=database,
exception_decorator=exception_decorator,
fitness_factor=fitness_factor,
algorithm=algorithm,
general_options=general_options,
)
current_dir_path = Path(__file__).resolve().parent
with open(current_dir_path / "algo_dict.json") as j:
algos = json.load(j)
origin, algo_name = algorithm.split("_", 1)
try:
assert algo_name in algos[
origin], "Invalid algorithm requested: {}".format(algorithm)
except (AssertionError, KeyError):
proposals = propose_algorithms(algorithm, algos)
raise NotImplementedError(
f"{algorithm} is not a valid choice. Did you mean one of {proposals}?"
)
bounds = tuple(
params.query("_internal_free")[["lower", "upper"]].to_numpy().T)
if database:
update_scalar_field(database, "optimization_status", "running")
if origin in ["nlopt", "pygmo"]:
results = minimize_pygmo_np(
internal_criterion,
internal_params,
bounds,
origin,
algo_name,
algo_options,
internal_gradient,
)
elif origin == "scipy":
results = minimize_scipy_np(
internal_criterion,
internal_params,
bounds=bounds,
algo_name=algo_name,
algo_options=algo_options,
gradient=internal_gradient,
)
elif origin == "tao":
crit_val = general_options["start_criterion_value"]
len_criterion_value = 1 if np.isscalar(crit_val) else len(crit_val)
results = minimize_pounders_np(
internal_criterion,
internal_params,
bounds,
n_errors=len_criterion_value,
**algo_options,
)
else:
raise NotImplementedError("Invalid algorithm requested.")
if database:
update_scalar_field(database, "optimization_status", results["status"])
params = reparametrize_from_internal(
internal=results["x"],
fixed_values=params["_internal_fixed_value"].to_numpy(),
pre_replacements=params["_pre_replacements"].to_numpy().astype(int),
processed_constraints=constraints,
post_replacements=params["_post_replacements"].to_numpy().astype(int),
processed_params=params,
)
return results, params
def _params_from_x(x, internal_params, constraints, params):
internal_params = internal_params.copy(deep=True)
internal_params["value"] = x
updated_params = reparametrize_from_internal(internal_params, constraints,
params)
return updated_params
@pytest.mark.parametrize("case, number", to_test)
def test_reparametrize_from_internal(example_params, all_constraints, case,
number):
constraints = all_constraints[case]
params = reduce_params(example_params, constraints)
params["value"] = params[f"value{number}"]
keep = params[f"internal_value{number}"].notnull()
pc, pp = process_constraints(constraints, params)
external = reparametrize_from_internal(
internal=params[f"internal_value{number}"][keep].to_numpy(),
fixed_values=pp["_internal_fixed_value"].to_numpy(),
pre_replacements=pp["_pre_replacements"].to_numpy(),
processed_constraints=pc,
post_replacements=pp["_post_replacements"].to_numpy(),
processed_params=pp,
)
calculated_external_value = external["value"]
expected_external_value = params["value"]
assert_series_equal(calculated_external_value, expected_external_value)
invalid_cases = [
"basic_probability",
"uncorrelated_covariance",
"basic_covariance",
"basic_increasing",
params = reduce_params(example_params, constraints)
params["value"] = params[f"value{number}"]
keep = params[f"internal_value{number}"].notnull()
pc, pp = process_constraints(constraints, params)
internal_p = params[f"internal_value{number}"][keep].to_numpy()
fixed_val = pp["_internal_fixed_value"].to_numpy()
pre_repl = pp["_pre_replacements"].to_numpy()
post_repl = pp["_post_replacements"].to_numpy()
external = reparametrize_from_internal(
internal=internal_p,
fixed_values=fixed_val,
pre_replacements=pre_repl,
processed_constraints=pc,
post_replacements=post_repl,
processed_params=pp,
)
calculated_external_value = external["value"]
expected_external_value = params["value"]
assert_series_equal(calculated_external_value, expected_external_value)
invalid_cases = [
"basic_probability",
"uncorrelated_covariance",
"basic_covariance",
"basic_increasing",
def test_reparametrize_from_internal(internal, expected_external):
calculated = reparametrize_from_internal(internal,
constraints(expected_external),
expected_external)
assert_series_equal(calculated, expected_external["value"])
def transform_covariance(
params,
internal_cov,
constraints,
n_samples,
bounds_handling,
):
"""Transform the internal covariance matrix to an external one, given constraints.
Args:
params (pd.DataFrame): DataFrame where the "value" column contains estimated
parameters of a likelihood model. See :ref:`params` for details.
internal_cov (np.ndarray) with a covariance matrix of the internal parameter
vector. For background information about internal and external params
see :ref:`implementation_of_constraints`.
constraints (list): List with constraint dictionaries.
See .. _link: ../../docs/source/how_to_guides/how_to_use_constraints.ipynb
n_samples (int): Number of samples used to transform the covariance matrix of
the internal parameter vector into the covariance matrix of the external
parameters.
bounds_handling (str): One of "clip", "raise", "ignore". Determines how bounds
are handled. If "clip", confidence intervals are clipped at the bounds.
Standard errors are only adjusted if a sampling step is necessary due to
additional constraints. If "raise" and any lower or upper bound is binding,
we raise an error. If "ignore", boundary problems are simply ignored.
Returns:
pd.DataFrame: Quadratic DataFrame containing the covariance matrix of the free
parameters. If parameters were fixed (explicitly or by other constraints),
the index is a subset of params.index. The columns are the same as the
index.
"""
processed_constraints, processed_params = process_constraints(
constraints, params)
free_index = processed_params.query("_internal_free").index
if processed_constraints:
free = processed_params.loc[free_index]
is_free = processed_params["_internal_free"].to_numpy()
pre_replacements = processed_params["_pre_replacements"].to_numpy()
post_replacements = processed_params["_post_replacements"].to_numpy()
fixed_values = processed_params["_internal_fixed_value"].to_numpy()
lower_bounds = free["_internal_lower"]
upper_bounds = free["_internal_upper"]
internal_mean = reparametrize_to_internal(
external=params["value"].to_numpy(),
internal_free=is_free,
processed_constraints=processed_constraints,
)
sample = np.random.multivariate_normal(
mean=internal_mean,
cov=internal_cov,
size=n_samples,
)
transformed_free = []
for params_vec in sample:
if bounds_handling == "clip":
params_vec = np.clip(params_vec,
a_min=lower_bounds,
a_max=upper_bounds)
elif bounds_handling == "raise":
if (params_vec < lower_bounds).any() or (params_vec >
upper_bounds).any():
raise ValueError()
transformed = reparametrize_from_internal(
internal=params_vec,
fixed_values=fixed_values,
pre_replacements=pre_replacements,
processed_constraints=processed_constraints,
post_replacements=post_replacements,
)
transformed_free.append(transformed[is_free])
free_cov = np.cov(
np.array(transformed_free),
rowvar=False,
)
else:
free_cov = internal_cov
res = pd.DataFrame(data=free_cov, columns=free_index, index=free_index)
return res
