def wrapper_numpy_interface(x, *args, **kwargs): # Handle usage in :func:`internal_function` for gradients. if constraints is None: p = params.copy() p["value"] = x # Handle usage in :func:`internal_criterion`. else: p = reparametrize_from_internal( internal=x, fixed_values=params["_internal_fixed_value"].to_numpy(), pre_replacements=params["_pre_replacements"].to_numpy(). astype(int), processed_constraints=constraints, post_replacements=( params["_post_replacements"].to_numpy().astype(int)), processed_params=params, ) criterion_value = func(p, *args, **kwargs) if isinstance(criterion_value, (pd.DataFrame, pd.Series)): criterion_value = criterion_value.to_numpy() return criterion_value
def _process_optimization_results(results, results_arguments): """Expand the solutions back to the original problems. Args: results (list): list of dictionaries with the harmonized results objects. results_arguments (list): each element is a dictionary supplying the start params DataFrame and the constraints to the original problem. The keys are "params", "constraints" and "keep_dashboard_alive". Returns: results (tuple): Tuple of the harmonized result info dictionary and the params DataFrame with the minimizing parameter values of the untransformed problem as specified of the user. """ new_results = [] for res, args in zip(results, results_arguments): res["x"] = list(res["x"]) start_params = args["params"] params = reparametrize_from_internal( internal=res["x"], fixed_values=start_params["_internal_fixed_value"].to_numpy(), pre_replacements=start_params["_pre_replacements"].to_numpy(dtype="int"), processed_constraints=args["constraints"], post_replacements=start_params["_post_replacements"].to_numpy(dtype="int"), processed_params=start_params, ) new_results.append((res, params)) if len(new_results) == 1: new_results = new_results[0] return new_results
def test_reparametrize_from_internal(internal, expected_external, category): constr = constraints(expected_external) calculated = reparametrize_from_internal(internal, constr, expected_external, None)["value"] assert_series_equal(calculated[category], expected_external.loc[category, "value"])
def _process_results(res, params, internal_params, constraints, origin): """Convert optimization results into json serializable dictionary. Args: res: Result from numerical optimizer. params (DataFrame): See :ref:`params`. internal_params (DataFrame): See :ref:`params`. constraints (list): constraints for the optimization origin (str): takes the values "pygmo", "nlopt", "scipy" """ if origin == "scipy": res_dict = {**res} for key, value in res_dict.items(): if isinstance(value, np.ndarray): res_dict[key] = value.tolist() x = res.x elif origin in ["pygmo", "nlopt"]: x = res.champion_x res_dict = {"fun": res.champion_f[0]} elif origin in ["tao"]: x = res["x"] res_dict = {**res} params = reparametrize_from_internal( internal=x, fixed_values=params["_internal_fixed_value"].to_numpy(), pre_replacements=params["_pre_replacements"].to_numpy().astype(int), processed_constraints=constraints, post_replacements=params["_post_replacements"].to_numpy().astype(int), processed_params=params, ) return res_dict, params
def internal_criterion(x, counter=c): p = reparametrize_from_internal( internal=x, fixed_values=params["_internal_fixed_value"].to_numpy(), pre_replacements=params["_pre_replacements"].to_numpy().astype( int), processed_constraints=constraints, post_replacements=params["_post_replacements"].to_numpy().astype( int), processed_params=params, ) fitness_eval = criterion(p, **criterion_kwargs) # For Pounders, return the sum of squared errors. _fitness_eval = (fitness_eval.sum() if isinstance( fitness_eval, np.ndarray) else fitness_eval) if queue is not None: queue.put( QueueEntry( iteration=counter[0], params=p, fitness=fitness_factor * _fitness_eval, )) counter += 1 return fitness_eval
def test_reparametrize_from_internal(internal, expected_external, category): constr = constraints(expected_external) with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message="indexing past lexsort depth may impact performance." ) calculated = reparametrize_from_internal(internal, constr, expected_external) assert_series_equal( calculated[category], expected_external.loc[category, "value"] )
def _params_from_x(x, internal_params, constraints, params, scaling_factor): internal_params = internal_params.copy(deep=True) # :func:`internal_criterion` always assumes that `x` is a NumPy array, but if we # pass the internal criterion function to :func:`gradient`, x is a DataFrame. # Setting a series to a DataFrame will convert the column "value" to object type # which causes trouble in following NumPy routines assuming a numeric type. internal_params["value"] = x["value"] if isinstance(x, pd.DataFrame) else x updated_params = reparametrize_from_internal(internal_params, constraints, params, scaling_factor) return updated_params
def _back_and_forth_transformation_and_assert(params, constraints): pc, pp = process_constraints(constraints, params) internal = reparametrize_to_internal(pp, pc) external = reparametrize_from_internal( internal=internal, fixed_values=pp["_internal_fixed_value"].to_numpy(), pre_replacements=pp["_pre_replacements"].to_numpy(), processed_constraints=pc, post_replacements=pp["_post_replacements"].to_numpy(), processed_params=pp, ) assert_series_equal(external["value"], params["value"]) return internal, external
def wrapper_handle_exceptions(x, *args, **kwargs): try: out = func(x, *args, **kwargs) except (KeyboardInterrupt, SystemExit): raise except Exception as e: # Adjust the criterion value at the start. start_criterion_value = general_options[ "start_criterion_value"] constant, slope = general_options.get( "criterion_exception_penalty", (None, None)) constant = 2 * start_criterion_value if constant is None else constant slope = 0.1 * start_criterion_value if slope is None else slope raise_exc = general_options.get("criterion_exception_raise", False) if raise_exc: raise e else: if database: exception_info = traceback.format_exc() p = reparametrize_from_internal( internal=x, fixed_values=params["_internal_fixed_value"]. to_numpy(), pre_replacements=params["_pre_replacements"]. to_numpy().astype(int), processed_constraints=constraints, post_replacements=(params["_post_replacements"]. to_numpy().astype(int)), processed_params=params, ) msg = (exception_info + "\n\n" + "The parameters are\n\n" + p["value"].to_csv(sep="\t", header=True)) append_rows(database, "exceptions", {"value": msg}) out = min( MAX_CRITERION_PENALTY, constant + slope * np.linalg.norm(x - start_params), ) return out
def wrapper_numpy_interface(x, *args, **kwargs): if isinstance(x, pd.DataFrame): p = x elif isinstance(x, np.ndarray): p = params.copy() p["value"] = reparametrize_from_internal( internal=x, fixed_values=fixed_values, pre_replacements=pre_replacements, processed_constraints=pc, post_replacements=post_replacements, ) else: raise ValueError( "x must be a numpy array or DataFrame with 'value' column." ) criterion_value = func(p, *args, **kwargs) if isinstance(criterion_value, (pd.DataFrame, pd.Series)) and numpy_output: criterion_value = criterion_value.to_numpy() return criterion_value
def _internal_minimize( criterion, criterion_kwargs, params, internal_params, constraints, algorithm, algo_options, gradient, gradient_options, general_options, database, queue, fitness_factor, ): """Create the internal criterion function and minimize it. Args: criterion (function): Python function that takes a pandas DataFrame with parameters as the first argument and returns a scalar floating point value. criterion_kwargs (dict): additional keyword arguments for criterion params (pd.DataFrame): See :ref:`params`. internal_params (DataFrame): See :ref:`params`. constraints (list): list with constraint dictionaries. See for details. algorithm (str): specifies the optimization algorithm. See :ref:`list_of_algorithms`. algo_options (dict): algorithm specific configurations for the optimization gradient (callable or None): Gradient function. gradient_options (dict): Options for the gradient function. general_options (dict): additional configurations for the optimization database (sqlalchemy.sql.schema.MetaData). The engine that connects to the database can be accessed via ``database.bind``. queue (Queue): queue to which the fitness evaluations and params DataFrames are supplied. fitness_factor (float): multiplicative factor for the fitness displayed in the dashboard. Set to -1 for maximizations to plot the fitness that is being maximized. """ logging_decorator = functools.partial( log_evaluation, database=database, tables=["params_history", "criterion_history", "comparison_plot"], ) exception_decorator = functools.partial( handle_exceptions, database=database, params=params, constraints=constraints, start_params=internal_params, general_options=general_options, ) internal_criterion = create_internal_criterion( criterion=criterion, params=params, constraints=constraints, criterion_kwargs=criterion_kwargs, logging_decorator=logging_decorator, exception_decorator=exception_decorator, queue=queue, fitness_factor=fitness_factor, ) internal_gradient = create_internal_gradient( gradient=gradient, gradient_options=gradient_options, criterion=criterion, params=params, internal_params=internal_params, constraints=constraints, criterion_kwargs=criterion_kwargs, database=database, exception_decorator=exception_decorator, fitness_factor=fitness_factor, algorithm=algorithm, general_options=general_options, ) current_dir_path = Path(__file__).resolve().parent with open(current_dir_path / "algo_dict.json") as j: algos = json.load(j) origin, algo_name = algorithm.split("_", 1) try: assert algo_name in algos[ origin], "Invalid algorithm requested: {}".format(algorithm) except (AssertionError, KeyError): proposals = propose_algorithms(algorithm, algos) raise NotImplementedError( f"{algorithm} is not a valid choice. Did you mean one of {proposals}?" ) bounds = tuple( params.query("_internal_free")[["lower", "upper"]].to_numpy().T) if database: update_scalar_field(database, "optimization_status", "running") if origin in ["nlopt", "pygmo"]: results = minimize_pygmo_np( internal_criterion, internal_params, bounds, origin, algo_name, algo_options, internal_gradient, ) elif origin == "scipy": results = minimize_scipy_np( internal_criterion, internal_params, bounds=bounds, algo_name=algo_name, algo_options=algo_options, gradient=internal_gradient, ) elif origin == "tao": crit_val = general_options["start_criterion_value"] len_criterion_value = 1 if np.isscalar(crit_val) else len(crit_val) results = minimize_pounders_np( internal_criterion, internal_params, bounds, n_errors=len_criterion_value, **algo_options, ) else: raise NotImplementedError("Invalid algorithm requested.") if database: update_scalar_field(database, "optimization_status", results["status"]) params = reparametrize_from_internal( internal=results["x"], fixed_values=params["_internal_fixed_value"].to_numpy(), pre_replacements=params["_pre_replacements"].to_numpy().astype(int), processed_constraints=constraints, post_replacements=params["_post_replacements"].to_numpy().astype(int), processed_params=params, ) return results, params
def _params_from_x(x, internal_params, constraints, params): internal_params = internal_params.copy(deep=True) internal_params["value"] = x updated_params = reparametrize_from_internal(internal_params, constraints, params) return updated_params
@pytest.mark.parametrize("case, number", to_test) def test_reparametrize_from_internal(example_params, all_constraints, case, number): constraints = all_constraints[case] params = reduce_params(example_params, constraints) params["value"] = params[f"value{number}"] keep = params[f"internal_value{number}"].notnull() pc, pp = process_constraints(constraints, params) external = reparametrize_from_internal( internal=params[f"internal_value{number}"][keep].to_numpy(), fixed_values=pp["_internal_fixed_value"].to_numpy(), pre_replacements=pp["_pre_replacements"].to_numpy(), processed_constraints=pc, post_replacements=pp["_post_replacements"].to_numpy(), processed_params=pp, ) calculated_external_value = external["value"] expected_external_value = params["value"] assert_series_equal(calculated_external_value, expected_external_value) invalid_cases = [ "basic_probability", "uncorrelated_covariance", "basic_covariance", "basic_increasing",
params = reduce_params(example_params, constraints) params["value"] = params[f"value{number}"] keep = params[f"internal_value{number}"].notnull() pc, pp = process_constraints(constraints, params) internal_p = params[f"internal_value{number}"][keep].to_numpy() fixed_val = pp["_internal_fixed_value"].to_numpy() pre_repl = pp["_pre_replacements"].to_numpy() post_repl = pp["_post_replacements"].to_numpy() external = reparametrize_from_internal( internal=internal_p, fixed_values=fixed_val, pre_replacements=pre_repl, processed_constraints=pc, post_replacements=post_repl, processed_params=pp, ) calculated_external_value = external["value"] expected_external_value = params["value"] assert_series_equal(calculated_external_value, expected_external_value) invalid_cases = [ "basic_probability", "uncorrelated_covariance", "basic_covariance", "basic_increasing",
def test_reparametrize_from_internal(internal, expected_external): calculated = reparametrize_from_internal(internal, constraints(expected_external), expected_external) assert_series_equal(calculated, expected_external["value"])
def transform_covariance( params, internal_cov, constraints, n_samples, bounds_handling, ): """Transform the internal covariance matrix to an external one, given constraints. Args: params (pd.DataFrame): DataFrame where the "value" column contains estimated parameters of a likelihood model. See :ref:`params` for details. internal_cov (np.ndarray) with a covariance matrix of the internal parameter vector. For background information about internal and external params see :ref:`implementation_of_constraints`. constraints (list): List with constraint dictionaries. See .. _link: ../../docs/source/how_to_guides/how_to_use_constraints.ipynb n_samples (int): Number of samples used to transform the covariance matrix of the internal parameter vector into the covariance matrix of the external parameters. bounds_handling (str): One of "clip", "raise", "ignore". Determines how bounds are handled. If "clip", confidence intervals are clipped at the bounds. Standard errors are only adjusted if a sampling step is necessary due to additional constraints. If "raise" and any lower or upper bound is binding, we raise an error. If "ignore", boundary problems are simply ignored. Returns: pd.DataFrame: Quadratic DataFrame containing the covariance matrix of the free parameters. If parameters were fixed (explicitly or by other constraints), the index is a subset of params.index. The columns are the same as the index. """ processed_constraints, processed_params = process_constraints( constraints, params) free_index = processed_params.query("_internal_free").index if processed_constraints: free = processed_params.loc[free_index] is_free = processed_params["_internal_free"].to_numpy() pre_replacements = processed_params["_pre_replacements"].to_numpy() post_replacements = processed_params["_post_replacements"].to_numpy() fixed_values = processed_params["_internal_fixed_value"].to_numpy() lower_bounds = free["_internal_lower"] upper_bounds = free["_internal_upper"] internal_mean = reparametrize_to_internal( external=params["value"].to_numpy(), internal_free=is_free, processed_constraints=processed_constraints, ) sample = np.random.multivariate_normal( mean=internal_mean, cov=internal_cov, size=n_samples, ) transformed_free = [] for params_vec in sample: if bounds_handling == "clip": params_vec = np.clip(params_vec, a_min=lower_bounds, a_max=upper_bounds) elif bounds_handling == "raise": if (params_vec < lower_bounds).any() or (params_vec > upper_bounds).any(): raise ValueError() transformed = reparametrize_from_internal( internal=params_vec, fixed_values=fixed_values, pre_replacements=pre_replacements, processed_constraints=processed_constraints, post_replacements=post_replacements, ) transformed_free.append(transformed[is_free]) free_cov = np.cov( np.array(transformed_free), rowvar=False, ) else: free_cov = internal_cov res = pd.DataFrame(data=free_cov, columns=free_index, index=free_index) return res