def __init__(self, model):
super(ParameterEstimator, self).__init__(model)
self.__var = VariableNames()
self.hessian = None
self.cov = None
self._estimability = False
self._idx_to_variable = dict()
self.model_variance = True
self.termination_condition = None
# This should be a subclass or a mixin
self.G_contribution = None
self.unwanted_G = False
self.time_variant_G = False
self.time_invariant_G = False
self.time_invariant_G_decompose = False
self.time_invariant_G_no_decompose = False
# Parameter name list
self.param_names_full = define_free_parameters(self.model, kind='full')
self.param_names = define_free_parameters(self.model, kind='simple')
# for new huplc structure (CS):
if self._huplc_given:
self._list_huplcabs = self._huplc_absorbing
# self._n_huplc = len(self._list_huplcabs)
self.n_val = self._n_components
self.cov = None
def model_info(model):
"""This method provides a dict of model attributes to be used in various
classes.
.. note ::
This may be moved to the TemplateBuilder since this may be redundant
:param ConcreteModel model: A Pyomo model
:return: Useful model attributes
:rtype: dict
"""
__var = VariableNames()
model_attrs = {}
model_attrs['mixture_components'] = list(model.mixture_components)
model_attrs['complementary_states'] = list(model.complementary_states)
model_attrs['algebraics'] = list(model.algebraics)
model_attrs['n_components'] = len(model_attrs['mixture_components'])
model_attrs['n_algebraics'] = len(model_attrs['algebraics'])
model_attrs['n_complementary_states'] = len(
model_attrs['complementary_states'])
model_attrs['known_absorbance_data'] = None
model_attrs['non_absorbing'] = None
if hasattr(model, 'non_absorbing'):
model_attrs['non_absorbing'] = list(model.non_absorbing)
model_attrs['known_absorbance'] = None
if hasattr(model, 'known_absorbance'):
model_attrs['known_absorbance'] = list(model.known_absorbance)
if hasattr(model, 'abs_components'):
model_attrs['abs_components'] = list(model.abs_components.keys())
model_attrs['nabs_components'] = len(model_attrs['abs_components'])
model_attrs['huplc_absorbing'] = None
if hasattr(model, 'huplc_absorbing'):
model_attrs['huplc_absorbing'] = list(model.huplc_absorbing)
if hasattr(model, 'huplcabs_components'):
model_attrs['huplcabs_components'] = list(model.huplcabs_components)
model_attrs['nhuplcabs_components'] = len(
model_attrs['huplcabs_components'])
model_attrs['ipopt_scaled'] = False
model_attrs['spectra_given'] = hasattr(model, __var.spectra_data)
model_attrs['concentration_given'] = hasattr(
model, __var.concentration_measured) or hasattr(
model, __var.user_defined) or hasattr(model, __var.state)
model_attrs['conplementary_states_given'] = hasattr(model, __var.state)
model_attrs['absorption_given'] = hasattr(model, __var.spectra_species)
model_attrs['huplc_given'] = hasattr(model, 'Chat')
model_attrs['smoothparam_given'] = hasattr(model, __var.smooth_parameter)
return model_attrs
def __init__(self, reaction_models):
self.reaction_models = reaction_models
self.experiments = list(self.reaction_models.keys())
self.variances = {
name: model.variances
for name, model in self.reaction_models.items()
}
self.global_params = None
self.parameter_means = False
self.rm_cov = None
self.__var = VariableNames()
class AlgebraicBlock(ModelElementBlock):
"""Class to hold ModelElements of type algebraics and provide specific
methods.
:Methods:
- :func:`fixed`
- :func:`steps`
"""
__var = VariableNames()
def __init__(self, *args, **kwargs):
super().__init__(class_name='model_algebraic')
@property
def fixed(self):
"""Return a list of those algebraics that are fixed_states
This formats the list in the correct manner that fix_from_trajectory
can use to access the data
:return fix_from_traj: The list of modal var, component, and data for
fixed trajectories/states
:rtype: list
"""
fix_from_traj = []
for alg in self.algebraics.values():
if alg.data is not None:
fix_from_traj.append(
[self.__var.algebraic, alg.name, alg.data])
return fix_from_traj
@property
def steps(self):
"""Gathers a list of which algebraics are step variables
:return set steps: A set of step variables
:rtype: set
"""
steps = {}
for alg in self.algebraics.values():
if alg.step is not None:
steps[alg.name] = alg
return steps
def __init__(self, model):
"""Simulator constructor.
:param ConcreteModel model: The Pyomo model of the ReactionModel
"""
self.__var = VariableNames()
self.model = model
# Most of the attributes are model attributes
self.attrs = model_info(self.model)
for key, value in self.attrs.items():
setattr(self, f'_{key}', value)
# Creates a scaling factor suffix
if not hasattr(self.model, 'scaling_factor'):
self.model.scaling_factor = Suffix(direction=Suffix.EXPORT)
self.comps = {}
if hasattr(self.model, 'abs_components'):
self.comps['absorbing'] = [k for k in self.model.abs_components
] # all that absorb
else:
self.comps['absorbing'] = []
if hasattr(self.model, 'known_absorbance'):
self.comps['known_absorbance'] = [
k for k in self.model.known_absorbance
] # all that are known
else:
self.comps['known_absorbance'] = []
self.comps['all'] = [k for k in self.model.mixture_components
] # all species
self.comps['unknown_absorbance'] = [
k for k in self.comps['absorbing']
if k not in self.comps['known_absorbance']
]
def __init__(self, model):
"""It takes in a standard Kipet/Pyomo model, rewrites it and calls
fe_factory. More information on fe_factory is included in that class
description.
:param ConcreteModel model: Pyomo model passed from KIPET
"""
super(FESimulator, self).__init__(model)
self.__var = VariableNames()
self.p_sim = PyomoSimulator(model)
self.model = self.p_sim.model
self.param_dict = {}
self.param_name = self.__var.model_parameter
self.c_sim = self.model.clone()
# Check all parameters are fixed before simulating
if hasattr(self.model, self.__var.model_parameter):
for param_obj in getattr(self.model,
self.__var.model_parameter).values():
if not param_obj.fixed:
param_obj.fixed = True
# Build the parameter dictionary in the format that fe_factory uses
model_var = self.__var.model_parameter
for k, v in getattr(self.model, model_var).items():
self.param_dict[model_var, k] = v.value
# Build the initial condition dictionary in the format that fe_factory uses
vars_to_init = [
self.__var.concentration_model,
self.__var.state_model,
]
self.ics_ = {}
for var in vars_to_init:
self._var_init(var)
# Third library import
import numpy as np
import pandas as pd
from pyomo.environ import Suffix
from pyomo.core.base import TransformationFactory
from pyomo.opt import SolverFactory
# KIPET library imports
from kipet.calculation_tools.interpolation import interpolate_trajectory
from kipet.model_tools.visitor_classes import ScalingVisitor
from kipet.estimator_tools.results_object import ResultsObject
from kipet.model_tools.pyomo_model_tools import (get_index_sets,
index_set_info, model_info)
from kipet.general_settings.variable_names import VariableNames
__var = VariableNames()
"""Simulator based on pyomo.dae discretization strategies.
:Methods:
- :func:`apply_discretization`
- :func:`scale_model`
- :func:`scale_parameters`
- :func:`scale_expression`
- :func:`fix_from_trajectory`
- :func:`unfix_time_dependent_variable`
- :func:`initialize_parameters`
- :func:`build_set_new`
- :func:`initialize_from_trajectory`
- :fuch:`scale_variables_from_trajectory`
- :func:`run_sim`
class Comp:
"""This class is needed to handle pyomo variables and return a dict of these variables.
This will not be used by the user directly. This is one of the key classes that allows the user to use
dummy pyomo variables in building the KIPET models.
:param ConcreteModel model: The model we are going to use (dummy model)
:param
"""
var = VariableNames()
def __init__(self, model):
self._model = model
self._model_vars = self.var.model_vars
self._rate_vars = self.var.rate_vars
self.var_dict = {}
self.var_unit_dict = {}
self.assign_vars()
self.assign_rate_vars()
def assign_vars(self):
"""Digs through and assigns the variables as top-level attributes
:return: None
"""
list_of_vars = self._model_vars
for mv in list_of_vars:
if hasattr(self._model, mv):
mv_obj = getattr(self._model, mv)
index_sets = get_index_sets(mv_obj)
dim = len(index_sets)
if dim > 1:
indx = 1
else:
indx = 0
comp_set = list(index_sets[indx].keys())
for comp in comp_set:
comp = str(comp)
if isinstance(comp, str):
if isinstance(comp[0], int):
comp.insert(0, 'y')
comp_name = comp
for k, v in string_replace_dict.items():
comp_name.replace(k, v)
self.var_dict[comp_name] = ModalVar(
comp_name, comp_name, mv, self._model)
if dim > 1:
setattr(self, comp_name, mv_obj[0, comp])
else:
setattr(self, comp_name, mv_obj[comp])
return None
def assign_rate_vars(self):
"""Assigns the rate variables as top-level attributes
:return: None
"""
for mv in self._rate_vars:
if hasattr(self._model, mv):
mv_obj = getattr(self._model, mv)
index_sets = get_index_sets(mv_obj)
comp_set = list(index_sets[-1].keys())
dim = len(index_sets)
for comp in comp_set:
if isinstance(comp, str):
comp_name = comp
comp_name.replace(' ', '_')
elif isinstance(comp, int):
comp_name = f'y{comp}'
comp_name = f'd{comp_name}dt'
self.var_dict[comp_name] = ModalVar(
comp_name, str(comp), mv, self._model)
if dim > 1:
setattr(self, comp_name, mv_obj[0, comp])
else:
setattr(self, comp_name, mv_obj[comp])
return None
@property
def get_var_list(self):
"""Returns a list of variables in the Comp dict
:return: A list of variables
:rtype: list
"""
return list(self.var_dict.keys())
class StateBlock(ModelElementBlock):
"""Class to hold ModelElements of type constant and provide specific
methods.
:Methods:
- :func:`var_variances`
- :func:`component_set`
- :func:`variances`
- :func:`init_values`
- :func:`known_values`
- :func:`has_all_variances`
"""
__var = VariableNames()
def __init__(self, *args, **kwargs):
super().__init__(class_name='model_state')
def var_variances(self):
"""Returns a dict of state variances
:return: dict of variances
:rtype: dict
"""
sigma_dict = {}
for component in self.states.values():
if component.state == 'trajectory':
continue
sigma_dict[component.name] = component.variance
return sigma_dict
def component_set(self, category):
"""Returns a list of state categories
:param str category: A state category
:return: list of states with the given category
:rtype: list
"""
component_set = []
for component in self.states.values():
if component.state == category:
component_set.append(component.name)
return component_set
@property
def variances(self):
"""Returns a dict of state variances
:return: dict of known states
:rtype: dict
"""
return {comp.name: comp.variance for comp in self.states.values()}
@property
def init_values(self):
"""Returns a dict of initial state values
:return: dict of initial values
:rtype: dict
"""
return {comp.name: comp.value for comp in self.states.values()}
@property
def known_values(self):
"""Returns a dict of states that have known values (fixed)
:return: dict of known states
:rtype: dict
"""
return {comp.name: comp.known for comp in self.states.values()}
@property
def has_all_variances(self):
"""Returns True if all states have variances
:return all_component_variances: Boolean indicating if all state variances are provided
:rtype: bool
"""
all_component_variances = True
for comp in self.states.values():
if comp.variance is None:
all_component_variances = False
if not all_component_variances:
break
return all_component_variances
@property
def fixed(self):
fix_from_traj = []
for state in self.states.values():
if state.data is not None:
fix_from_traj.append(
[self.__var.state_model, state.name, state.data])
return fix_from_traj
def __init__(self,
model_orig,
src_mod,
init_con=None,
param_name=None,
param_values=None,
inputs=None,
inputs_sub=None,
jump_times=None,
jump_states=None,
):
"""
The `the paran name` might be a list of strings or a single string
corresponding to the parameters of the model.
The `param_values` dictionary needs to be declared with the following
syntax: `param_dict["P", "k0"] = 49.7796`
Where the first key corresponds to one of the parameter names, and the
second to the corresponding index (if any).
A similar structure is expected for the initial conditions and inputs.
The `inputs` and `input_sub` parameters are in place depending of
whether there is a single index input or a multiple index input.
Note that if the user does not provide correct information to
fe_factory; an exception will be thrown because of the n_var and m_eqn
check for simulation.
Once the constructor is called, one can initialize the model with the
following sintax: `self.load_initial_conditions(init_cond=ics_dict)`
Finally, to run the initialization and automatic data patching to tgt
model use: `self.run()`
If a given finite element problem fails, we do will try once again with
relaxed options. It is recommended to go back and check the model for
better understanding of the issue.
Finally, an explicit function of time on the right hand side is
prohibited. Please put this information into an input (fixed variable)
instead.
:param ConcreteModel tgt_mod: The original fully discretized model that we want to patch the information to.
:param ConcreteModel src_mod: The undiscretized reference model.
:param str init_con: The initial constraint name (corresponds to a Constraint object).
:param list param_name: The param name list. (Each element must correspond to a pyomo Var)
:param dict param_values: The corresponding values: `param_dict["param_name", "param_index"] = 49.7796`
:param dict inputs: The input dictionary. Use this dictonary for single index (time) inputs
:param dict inputs_sub: The multi-index dictionary. Use this dictionary for multi-index inputs.
"""
self.__var = VariableNames()
self.ip = SolverFactory('ipopt')
self.ip.options['halt_on_ampl_error'] = 'yes'
self.ip.options['print_user_options'] = 'yes'
self.model_orig = model_orig
self.model_ref = src_mod.clone()
self.volume_name = self.__var.volume_name
if self.volume_name is None:
raise ValueError('A volume name must exist')
time_index = None
for i in self.model_ref.component_objects(ContinuousSet):
time_index = i
break
if time_index is None:
raise Exception('no continuous_set')
self.time_set = time_index.name
model_original_time_set = getattr(self.model_orig, self.time_set)
self.ncp = model_original_time_set.get_discretization_info()['ncp']
fe_l = model_original_time_set.get_finite_elements()
self.fe_list = [fe_l[i + 1] - fe_l[i] for i in range(0, len(fe_l) - 1)]
self.nfe = len(self.fe_list)
#: Re-construct the model with [0,1] time domain
times = getattr(self.model_ref, self.time_set)
change_continuous_set(times, [0, 1])
for var in self.model_ref.component_objects(Var):
var.clear()
var.reconstruct()
for con in self.model_ref.component_objects(Constraint):
con.clear()
con.construct()
# self.model_ref.display(filename="selfmoddisc0.txt")
#: Discretize
d = TransformationFactory('dae.collocation')
d.apply_to(self.model_ref, nfe=1, ncp=self.ncp, scheme='LAGRANGE-RADAU')
#: Find out the differential variables
self.dvs_names = []
self.dvar_names = []
for con in self.model_ref.component_objects(Constraint):
name = con.name
namel = name.split('_', 1)
if len(namel) > 1:
if namel[1] == "disc_eq":
realname = getattr(self.model_ref, namel[0])
self.dvar_names.append(namel[0])
self.dvs_names.append(realname.get_state_var().name)
self.model_ref.h_i = Param(times, mutable=True, default=1.0) #: Length of finite element
#: Modify the collocation equations to introduce h_i (the length of finite element)
for i in self.dvar_names:
con = getattr(self.model_ref, i + '_disc_eq')
dv = getattr(self.model_ref, i)
e_dict = {}
fun_tup = True
for k in con.keys():
if isinstance(k, tuple):
pass
else:
k = (k,)
fun_tup = False
e = con[k].expr.args[0]
e_dict[k] = e * self.model_ref.h_i[k[0]] + dv[k] * (
1 - self.model_ref.h_i[k[0]]) == 0.0 #: As long as you don't clone
if fun_tup:
self.model_ref.add_component(i + "_deq_aug",
Constraint(con.index_set(),
rule=lambda m, *j: e_dict[j] if j[
0] > 0.0 else Constraint.Skip))
else:
self.model_ref.add_component(i + "_deq_aug",
Constraint(con.index_set(),
rule=lambda m, j: e_dict[j] if j > 0.0 else Constraint.Skip))
self.model_ref.del_component(con)
#: Sets for iteration
#: Differential variables
self.remaining_set = {}
for i in self.dvs_names:
dv = getattr(self.model_ref, i)
if dv.index_set().name == times.name: #: Just time set
# print(i, 'here')
self.remaining_set[i] = None
continue
# set_i = dv._implicit_subsets #: More than just time set
# set_i = dv._index._implicit_subsets #Update for pyomo 5.6.8 KH.L
set_i = identify_member_sets(dv)
# set_i = identify_member_sets(dv.index_set())
# print(f'set_i = {set_i}')
remaining_set = set_i[1]
for s in set_i[2:]:
remaining_set *= s
if isinstance(remaining_set, list):
self.remaining_set[i] = remaining_set
else:
self.remaining_set[i] = []
self.remaining_set[i].append(remaining_set)
#: Algebraic variables
self.weird_vars = [] #:Not indexed by time
self.remaining_set_alg = {}
# with open('model_check.txt', 'w') as f5:
# self.model_ref.pprint(ostream = f5)
for av in self.model_ref.component_objects(Var):
# print(av.name)
if av.name in self.dvs_names:
continue
if av.index_set().name == times.name: #: Just time set
self.remaining_set_alg[av.name] = None
continue
# set_i = av._implicit_subsets
# set_i = av._index._implicit_subsets #Update for pyomo 5.6.8 KH.L
set_i = identify_member_sets(av)
if set_i is None or not times in set_i:
self.weird_vars.append(av.name) #: Not indexed by time!
continue #: if this happens we might be in trouble
remaining_set = set_i[1] #: Index by time and others
for s in set_i[2:]:
if s.name == times.name:
self.remaining_set_alg[av.name] = None
continue
else:
remaining_set *= s
if isinstance(remaining_set, list):
self.remaining_set_alg[av.name] = remaining_set
else:
self.remaining_set_alg[av.name] = []
self.remaining_set_alg[av.name].append(remaining_set)
if init_con is not None: #: Delete the initial conditions (we use .fix() instead)
ic = getattr(self.model_ref, init_con)
self.model_ref.del_component(ic)
if isinstance(param_name, list): #: Time independent parameters
if param_values:
if isinstance(param_values, dict):
for pname in param_name:
p = getattr(self.model_ref, pname)
for key in p.keys():
try:
val = param_values[pname, key]
p[key].set_value(val)
except KeyError:
raise Exception("Missing a key of the param_values\n"
"Please provide all the required keys.\n"
"missing: {}".format(key))
p[key].fix()
else:
Exception("Arg param_values should be provided in a dictionary")
else:
Exception("Arg param_values should be provided in a dictionary")
elif isinstance(param_name, str):
if param_values:
if isinstance(param_values, dict):
p = getattr(self.model_ref, param_name)
for key in p.keys():
try:
val = param_values[param_name, key]
p[key].set_value(val)
except KeyError:
raise Exception("Missing a key of the param_values\n"
"Please provide all the required keys.\n"
"missing: {}".format(key))
p[key].fix()
elif not param_name:
pass
else:
raise Exception("wrong type for param_name")
#: Fix initial conditions
for i in self.dvs_names:
dv = getattr(self.model_ref, i)
if self.remaining_set[i] is None:
dv[0].fix()
for rs in self.remaining_set[i]:
for k in rs:
k = k if isinstance(k, tuple) else (k,)
dv[(0,) + k].fix()
self.inputs = None
self.input_remaining_set = {}
#: Check if inputs are declared
# if self.inputs is not None:
# if not isinstance(inputs, dict) or isinstance(inputs, str):
# raise Exception("Must be a dict or str")
# if isinstance(inputs, str):
# self.inputs = [self.inputs]
# for i in self.inputs:
# p = getattr(self.model_ref, i)
# p.fix()
# if p.index_set().name == times.name: #: Only time-set
# self.input_remaining_set[i] = None
# continue
# #set_i = p._implicit_subsets
# #set_i = p._index._implicit_subsets #Update for pyomo 5.6.8 KH.L
# set_i = identify_member_sets(p)
# if not times in set_i:
# raise RuntimeError("{} is not by index by time, this can't be an input".format(i))
# remaining_set = set_i[1]
# for s in set_i[2:]:
# if s.name == times.name: #: would this ever happen?
# continue
# else:
# remaining_set *= s
# if isinstance(remaining_set, list):
# self.input_remaining_set[i] = remaining_set
# else:
# self.input_remaining_set[i] = []
# self.input_remaining_set[i].append(remaining_set)
# self.inputs_sub = None
# inputs_sub['some_var'] = ['index0', 'index1', ('index2a', 'index2b')]
self.inputs_sub = inputs_sub
# print([i.name for i in get_index_sets(getattr(self.model_ref, 'Dose'))])
if self.inputs_sub is not None:
for key in self.inputs_sub.keys():
model_var_obj = getattr(self.model_ref, key)
# This finds the index of the set
# if identify_member_sets(model_var_obj) is None:
# raise RuntimeError("This variable is does not have multiple indices"
# "Pass {} as part of the inputs keyarg instead.".format(key))
# elif model_var_obj.index_set().name == times.name:
# raise RuntimeError("This variable is indexed over time"
# "Pass {} as part of the inputs keyarg instead.".format(key))
# else:
# if times not in identify_member_sets(model_var_obj):
# raise RuntimeError("{} is not indexed over time; it can not be an input".format(key))
for k in self.inputs_sub[key]:
if isinstance(k, str) or isinstance(k, int) or isinstance(k, tuple):
k = (k,) if not isinstance(k, tuple) else k
else:
raise RuntimeError("{} is not a valid index".format(k))
for t in times:
model_var_obj[(t,) + k].fix()
if hasattr(self.model_ref, self.__var.time_step_change):
for time_step in getattr(self.model_ref, self.__var.time_step_change):
getattr(self.model_ref, self.__var.time_step_change)[time_step].fix()
if hasattr(self.model_ref, self.__var.model_constant):
for param, obj in getattr(self.model_ref, self.__var.model_constant).items():
obj.fix()
# if hasattr(self.model_ref, self.__var.algebraic):
# model_var_obj = getattr(self.model_ref, self.__var.algebraic)
# for k in ['f']:
# if isinstance(k, str) or isinstance(k, int) or isinstance(k, tuple):
# k = (k,) if not isinstance(k, tuple) else k
# else:
# raise RuntimeError("{} is not a valid index".format(k))
# for t in times:
# print(model_var_obj.display())
# model_var_obj[(t,) + k].fix()
# for param, obj in getattr(self.model_ref, self.__var.algebraic).items():
# print(param)
# print(obj)
# if param[1] == 'f' or param[1] == 'Csat':
# for t in times:
# print(t, param[1])
# obj[(t,) + (param[1],)].fix()
# : Check n vars and m equations
(n, m) = reconcile_nvars_mequations(self.model_ref)
if n != m:
raise Exception("Inconsistent problem; n={}, m={}".format(n, m))
self.jump = False
self.con_num = 0