def initialize(self, cell_line):
"""Prepares experiment for requested cell line.
Prepares simulator for experiment. It does though by giving the initial treatment zero
treatment to the cell after which it runs the simulation until it reaches a steady state.
Args:
cell_line: String specifying cell line to simulate.
Returns:
relative_proliferation: The proliferation rate for the zero treatment which is always 1.
Raises:
ValueError: If provided cell line is unknown.
"""
assert not self.initialized, "Simulator has already been initialized."
if self.zero_term is not None:
assert self.line == cell_line, "Need to modify initialization scheme in order to get new zero term."
self.model.setTimepoints([np.infty])
self.load_conditions(cell_line)
self.load_drug_concentrations(ZERO_TREATMENT)
# compute growth term for zero treatment only once
if self.zero_term is None:
edata_ref = amici.ExpData(self.model.get())
edatas = [edata_ref]
rdatas = amici.runAmiciSimulations(self.model, self.solver, edatas)
self.zero_term = rdatas[0]["y"][0, 0]
self.line = cell_line
self.R = 1
self.initialized = True
return self.R
def apply_treatment(self, concentrations, verbose=False):
'''Loads specified drug simulation into model and runs simulation for until next steady state.
Assumes simulator has been initialzed for specific cell line first
Args:
concentrations: Specifies the drug cocktail.
verbose: If set to true, prints drug concentrations.
Returns:
overall_proliferation_rate: Describes the overall proliferation of the cell line at the
at the time at which the stable state has been reached.
Raises:
AssertionError: If simulator has not been initialized.
'''
assert self.initialized, "Simulator has not been initialized before first use."
self.load_drug_concentrations(concentrations, verbose=verbose)
edata_cond = amici.ExpData(self.model.get())
edatas = [edata_cond]
rdatas = amici.runAmiciSimulations(self.model, self.solver, edatas)
cond_term = rdatas[0]["y"][0, 0]
self.R = self.R * (cond_term / self.zero_term)
if verbose:
print(f'time to steadystate {rdatas[0]["t_steadystate"]}')
print(f'relative proliferation' f'{self.R}')
return self.R
def run_simulation(
drug_concs,
cell_line='A2058',
):
condition = conditions.loc[conditions.conditionId ==
f'TUMOR-{cell_line}-cellline-01-01', :]
if len(condition) == 0:
raise ValueError(
f'Requested cell-line "{cell_line}" has no condition data.')
# set the fixed parameters
for col in condition.columns:
if col in model.getFixedParameterIds():
model.setFixedParameterById(col, condition[col].values[0])
edata_ref = amici.ExpData(model.get())
for drug, conc in drug_concs.items():
parameter_id = model.getFixedParameterIds()[
model.getFixedParameterNames().index(drug)]
model.setFixedParameterById(parameter_id, conc)
print(f'{drug}: {conc}')
# extract simulation conditions that were specified in the model
edata_cond = amici.ExpData(model.get())
edatas = [edata_ref, edata_cond]
rdatas = amici.runAmiciSimulations(model, solver, edatas)
print(f'time to steadystate {rdatas[0]["t_steadystate"]}')
print(f'relative proliferation '
f'{rdatas[1]["y"][0, 0]/rdatas[0]["y"][0, 0]}')
return amici.getSimulationStatesAsDataFrame(model, edatas, rdatas)
def test_likelihoods(model_test_likelihoods):
"""Test the custom noise distributions used to define cost functions."""
model = model_test_likelihoods.getModel()
model.setTimepoints(np.linspace(0, 60, 60))
solver = model.getSolver()
solver.setSensitivityOrder(amici.SensitivityOrder.first)
# run model once to create an edata
rdata = amici.runAmiciSimulation(model, solver)
edata = [amici.ExpData(rdata, 1, 0)]
# just make all observables positive since some are logarithmic
for ed in edata:
y = ed.getObservedData()
y = tuple([max(val, 1e-4) for val in y])
ed.setObservedData(y)
# and now run for real and also compute likelihood values
rdata = amici.runAmiciSimulations(model, solver, edata)[0]
# output for easy debugging
for key in ['llh', 'sllh']:
print(key, rdata[key])
# it would be good to compute the expected llh+sllh by hand,
# here, we only check if they make overall sense
assert np.isfinite(rdata['llh'])
assert np.all(np.isfinite(rdata['sllh']))
assert np.any(rdata['sllh'])
def test_likelihoods(model_test_likelihoods):
"""Test the custom noise distributions used to define cost functions."""
model = model_test_likelihoods.getModel()
model.setTimepoints(np.linspace(0, 60, 60))
solver = model.getSolver()
solver.setSensitivityOrder(amici.SensitivityOrder.first)
# run model once to create an edata
rdata = amici.runAmiciSimulation(model, solver)
sigmas = rdata['y'].max(axis=0) * 0.05
edata = amici.ExpData(rdata, sigmas, [])
# just make all observables positive since some are logarithmic
while min(edata.getObservedData()) < 0:
edata = amici.ExpData(rdata, sigmas, [])
# and now run for real and also compute likelihood values
rdata = amici.runAmiciSimulations(model, solver, [edata])[0]
# check if the values make overall sense
assert np.isfinite(rdata['llh'])
assert np.all(np.isfinite(rdata['sllh']))
assert np.any(rdata['sllh'])
rdata_df = amici.getSimulationObservablesAsDataFrame(model,
edata,
rdata,
by_id=True)
edata_df = amici.getDataObservablesAsDataFrame(model, edata, by_id=True)
# check correct likelihood value
llh_exp = -sum([
normal_nllh(edata_df['o1'], rdata_df['o1'], sigmas[0]),
log_normal_nllh(edata_df['o2'], rdata_df['o2'], sigmas[1]),
log10_normal_nllh(edata_df['o3'], rdata_df['o3'], sigmas[2]),
laplace_nllh(edata_df['o4'], rdata_df['o4'], sigmas[3]),
log_laplace_nllh(edata_df['o5'], rdata_df['o5'], sigmas[4]),
log10_laplace_nllh(edata_df['o6'], rdata_df['o6'], sigmas[5]),
custom_nllh(edata_df['o7'], rdata_df['o7'], sigmas[6]),
])
assert np.isclose(rdata['llh'], llh_exp)
# check gradient
for sensi_method in [
amici.SensitivityMethod.forward, amici.SensitivityMethod.adjoint
]:
solver = model.getSolver()
solver.setSensitivityMethod(sensi_method)
solver.setSensitivityOrder(amici.SensitivityOrder.first)
solver.setRelativeTolerance(1e-12)
solver.setAbsoluteTolerance(1e-12)
check_derivatives(model,
solver,
edata,
assert_fun,
atol=1e-2,
rtol=1e-2,
epsilon=1e-5,
check_least_squares=False)
def test_steadystate_simulation(model_steadystate_module):
model = model_steadystate_module.getModel()
model.setTimepoints(np.linspace(0, 60, 60))
solver = model.getSolver()
solver.setSensitivityOrder(amici.SensitivityOrder.first)
rdata = amici.runAmiciSimulation(model, solver)
edata = [amici.ExpData(rdata, 1, 0)]
rdata = amici.runAmiciSimulations(model, solver, edata)
# check roundtripping of DataFrame conversion
df_edata = amici.getDataObservablesAsDataFrame(model, edata)
edata_reconstructed = amici.getEdataFromDataFrame(model, df_edata)
assert np.isclose(
amici.ExpDataView(edata[0])['observedData'],
amici.ExpDataView(edata_reconstructed[0])['observedData']).all()
assert np.isclose(
amici.ExpDataView(edata[0])['observedDataStdDev'],
amici.ExpDataView(edata_reconstructed[0])['observedDataStdDev']).all()
if len(edata[0].fixedParameters):
assert list(edata[0].fixedParameters) \
== list(edata_reconstructed[0].fixedParameters)
else:
assert list(model.getFixedParameters()) \
== list(edata_reconstructed[0].fixedParameters)
assert list(edata[0].fixedParametersPreequilibration) == \
list(edata_reconstructed[0].fixedParametersPreequilibration)
df_state = amici.getSimulationStatesAsDataFrame(model, edata, rdata)
assert np.isclose(rdata[0]['x'],
df_state[list(model.getStateIds())].values).all()
df_obs = amici.getSimulationObservablesAsDataFrame(model, edata, rdata)
assert np.isclose(rdata[0]['y'],
df_obs[list(model.getObservableIds())].values).all()
amici.getResidualsAsDataFrame(model, edata, rdata)
solver.setRelativeTolerance(1e-12)
solver.setAbsoluteTolerance(1e-12)
check_derivatives(model,
solver,
edata[0],
assert_fun,
atol=1e-3,
rtol=1e-3,
epsilon=1e-4)
# Run some additional tests which need a working Model,
# but don't need precomputed expectations.
_test_set_parameters_by_dict(model_steadystate_module)
def __call__(self, x_dct: Dict, sensi_order: int, mode: str,
amici_model: AmiciModel, amici_solver: AmiciSolver,
edatas: List['amici.ExpData'], n_threads: int,
x_ids: Sequence[str], parameter_mapping: 'ParameterMapping'):
"""Perform the actual AMICI call.
Called within the :func:`AmiciObjective.__call__` method.
Parameters
----------
x_dct:
Parameters for which to compute function value and derivatives.
sensi_order:
Maximum sensitivity order.
mode:
Call mode (function value or residual based).
amici_model:
The AMICI model.
amici_solver:
The AMICI solver.
edatas:
The experimental data.
n_threads:
Number of threads for AMICI call.
x_ids:
Ids of optimization parameters.
parameter_mapping:
Mapping of optimization to simulation parameters.
"""
# set order in solver
amici_solver.setSensitivityOrder(sensi_order)
# fill in parameters
# TODO (#226) use plist to compute only required derivatives
amici.parameter_mapping.fill_in_parameters(
edatas=edatas,
problem_parameters=x_dct,
scaled_parameters=True,
parameter_mapping=parameter_mapping,
amici_model=amici_model)
# run amici simulation
rdatas = amici.runAmiciSimulations(
amici_model,
amici_solver,
edatas,
num_threads=min(n_threads, len(edatas)),
)
return calculate_function_values(rdatas, sensi_order, mode,
amici_model, amici_solver, edatas,
x_ids, parameter_mapping)
def test_steadystate_scaled(self):
"""
Test SBML import and simulation from AMICI python interface
"""
def assert_fun(x):
return self.assertTrue(x)
sbmlFile = os.path.join(os.path.dirname(__file__), '..', 'python',
'examples', 'example_steadystate',
'model_steadystate_scaled.xml')
sbmlImporter = amici.SbmlImporter(sbmlFile)
observables = amici.assignmentRules2observables(
sbmlImporter.sbml,
filter_function=lambda variable:
variable.getId().startswith('observable_') and
not variable.getId().endswith('_sigma')
)
outdir = 'test_model_steadystate_scaled'
sbmlImporter.sbml2amici('test_model_steadystate_scaled',
outdir,
observables=observables,
constantParameters=['k0'],
sigmas={'observable_x1withsigma':
'observable_x1withsigma_sigma'})
sys.path.insert(0, outdir)
import test_model_steadystate_scaled as modelModule
model = modelModule.getModel()
model.setTimepoints(np.linspace(0, 60, 60))
solver = model.getSolver()
solver.setSensitivityOrder(amici.SensitivityOrder_first)
rdata = amici.runAmiciSimulation(model, solver)
edata = [amici.ExpData(rdata, 1, 0)]
rdata = amici.runAmiciSimulations(model, solver, edata)
# check roundtripping of DataFrame conversion
df_edata = amici.getDataObservablesAsDataFrame(model, edata)
edata_reconstructed = amici.getEdataFromDataFrame(model, df_edata)
self.assertTrue(
np.isclose(
amici.ExpDataView(edata[0])
['observedData'],
amici.ExpDataView(edata_reconstructed[0])
['observedData'],
).all()
)
self.assertTrue(
np.isclose(
amici.ExpDataView(edata[0])
['observedDataStdDev'],
amici.ExpDataView(edata_reconstructed[0])
['observedDataStdDev'],
).all()
)
if len(edata[0].fixedParameters):
self.assertListEqual(
list(edata[0].fixedParameters),
list(edata_reconstructed[0].fixedParameters),
)
else:
self.assertListEqual(
list(model.getFixedParameters()),
list(edata_reconstructed[0].fixedParameters),
)
self.assertListEqual(
list(edata[0].fixedParametersPreequilibration),
list(edata_reconstructed[0].fixedParametersPreequilibration),
)
df_state = amici.getSimulationStatesAsDataFrame(model, edata, rdata)
self.assertTrue(
np.isclose(
rdata[0]['x'],
df_state[list(model.getStateIds())].values
).all()
)
df_obs = amici.getSimulationObservablesAsDataFrame(model, edata, rdata)
self.assertTrue(
np.isclose(
rdata[0]['y'],
df_obs[list(model.getObservableIds())].values
).all()
)
amici.getResidualsAsDataFrame(model, edata, rdata)
solver.setRelativeTolerance(1e-12)
solver.setAbsoluteTolerance(1e-12)
check_derivatives(model, solver, edata[0], assert_fun, atol=1e-3,
rtol=1e-3, epsilon=1e-4)
def test_likelihoods(self):
"""
Test the custom noise distributions used to define cost functions.
"""
def assert_fun(x):
return self.assertTrue(x)
sbmlFile = os.path.join(os.path.dirname(__file__), '..', 'python',
'examples', 'example_steadystate',
'model_steadystate_scaled.xml')
sbmlImporter = amici.SbmlImporter(sbmlFile)
observables = amici.assignmentRules2observables(
sbmlImporter.sbml,
filter_function=lambda variable:
variable.getId().startswith('observable_') and
not variable.getId().endswith('_sigma')
)
# assign different noise models
obs_keys = list(observables.keys())
# exponentiate observable formulas
obs1 = observables[obs_keys[1]]
obs3 = observables[obs_keys[3]]
obs1['formula'] = '10^(' + obs1['formula'] + ')'
obs3['formula'] = 'exp(' + obs3['formula'] + ')'
# customize noise distributions
noise_distributions = {
obs_keys[0]: 'normal',
obs_keys[1]: 'log-normal',
obs_keys[2]: 'laplace',
obs_keys[3]: 'log10-laplace',
}
outdir = 'test_likelihoods'
sbmlImporter.sbml2amici('test_likelihoods',
outdir,
observables=observables,
constantParameters=['k0'],
sigmas={'observable_x1withsigma':
'observable_x1withsigma_sigma'},
noise_distributions=noise_distributions
)
sys.path.insert(0, outdir)
import test_likelihoods as modelModule
model = modelModule.getModel()
model.setTimepoints(np.linspace(0, 60, 60))
solver = model.getSolver()
solver.setSensitivityOrder(amici.SensitivityOrder_first)
# run model once to create an edata
rdata = amici.runAmiciSimulation(model, solver)
edata = [amici.ExpData(rdata, 1, 0)]
# just make all observables positive since some are logarithmic
for ed in edata:
y = ed.getObservedData()
y = tuple([max(val, 1e-4) for val in y])
ed.setObservedData(y)
# and now run for real and also compute likelihood values
rdata = amici.runAmiciSimulations(model, solver, edata)[0]
# output for easy debugging
for key in ['llh', 'sllh']:
print(key, rdata[key])
# it would be good to compute the expected llh+sllh by hand,
# here, we only check if they make overall sense
self.assertTrue(np.isfinite(rdata['llh']))
self.assertTrue(np.all(np.isfinite(rdata['sllh'])))
self.assertTrue(np.any(rdata['sllh']))
def runTest(self):
"""
test runner routine that loads data expectedResults.h5 hdf file and
runs individual models/settings as subTests
"""
expected_results = h5py.File(self.expectedResultsFile, 'r')
for subTest in expected_results.keys():
for case in list(expected_results[subTest].keys()):
if case.startswith('sensi2'):
model_name = subTest + '_o2'
else:
model_name = subTest
with self.subTest(modelName=model_name, caseName=case):
print(f'running {model_name}::{case}')
def assert_fun(x):
return self.assertTrue(x)
model_swig_folder = \
os.path.join(os.path.dirname(__file__), '..',
'build', 'tests', 'cpputest',
f'external_{model_name}-prefix',
'src', f'external_{model_name}-build',
'swig')
sys.path.insert(0, model_swig_folder)
test_model_module = importlib.import_module(model_name)
self.model = test_model_module.getModel()
self.solver = self.model.getSolver()
amici.readModelDataFromHDF5(
self.expectedResultsFile, self.model.get(),
f'/{subTest}/{case}/options'
)
amici.readSolverSettingsFromHDF5(
self.expectedResultsFile, self.solver.get(),
f'/{subTest}/{case}/options'
)
edata = None
if 'data' in expected_results[subTest][case].keys():
edata = amici.readSimulationExpData(
self.expectedResultsFile,
f'/{subTest}/{case}/data', self.model.get()
)
rdata = amici.runAmiciSimulation(self.model, self.solver,
edata)
check_derivative_opts = dict()
if model_name == 'model_nested_events':
check_derivative_opts['rtol'] = 1e-2
elif model_name == 'model_events':
check_derivative_opts['atol'] = 1e-3
if edata \
and self.solver.getSensitivityMethod() \
and self.solver.getSensitivityOrder() \
and len(self.model.getParameterList()) \
and not model_name.startswith('model_neuron') \
and not case.endswith('byhandpreeq'):
check_derivatives(self.model, self.solver, edata,
assert_fun, **check_derivative_opts)
verify_simulation_opts = dict()
if model_name.startswith('model_neuron'):
verify_simulation_opts['atol'] = 1e-5
verify_simulation_opts['rtol'] = 1e-2
if model_name.startswith('model_robertson') and \
case == 'sensiforwardSPBCG':
verify_simulation_opts['atol'] = 1e-3
verify_simulation_opts['rtol'] = 1e-3
verify_simulation_results(
rdata, expected_results[subTest][case]['results'],
assert_fun, **verify_simulation_opts
)
if model_name == 'model_steadystate' and \
case == 'sensiforwarderrorint':
edata = amici.amici.ExpData(self.model.get())
if edata and model_name != 'model_neuron_o2' and not (
model_name == 'model_robertson' and
case == 'sensiforwardSPBCG'
):
# Test runAmiciSimulations: ensure running twice
# with same ExpData yields same results
if isinstance(edata, amici.amici.ExpData):
edatas = [edata, edata]
else:
edatas = [edata.get(), edata.get()]
rdatas = amici.runAmiciSimulations(
self.model, self.solver, edatas, num_threads=2
)
verify_simulation_results(
rdatas[0],
expected_results[subTest][case]['results'],
assert_fun, **verify_simulation_opts
)
verify_simulation_results(
rdatas[1],
expected_results[subTest][case]['results'],
assert_fun, **verify_simulation_opts
)
self.assertRaises(
RuntimeError,
self.model.getParameterByName,
'thisParameterDoesNotExist'
)
def _call_amici(self, x: np.ndarray, sensi_orders: Tuple[int, ...],
mode: str) -> Dict:
# amici is built such that only the maximum sensitivity is required,
# the lower orders are then automatically computed
sensi_order = min(max(sensi_orders), 1)
# order 2 currently not implemented, we are using the FIM
# check if the requested sensitivities can be computed
if sensi_order > self.max_sensi_order:
raise Exception("Sensitivity order not allowed.")
sensi_method = self.amici_solver.getSensitivityMethod()
# prepare outputs
nllh = 0.0
snllh = np.zeros(self.dim)
s2nllh = np.zeros([self.dim, self.dim])
res = np.zeros([0])
sres = np.zeros([0, self.dim])
# set order in solver
self.amici_solver.setSensitivityOrder(sensi_order)
x_dct = self.par_arr_to_dct(x)
# fill in parameters
# TODO (#226) use plist to compute only required derivatives
amici.parameter_mapping.fill_in_parameters(
edatas=self.edatas,
problem_parameters=x_dct,
scaled_parameters=True,
parameter_mapping=self.parameter_mapping,
amici_model=self.amici_model)
# update steady state
for data_ix, edata in enumerate(self.edatas):
if self.guess_steadystate and \
self.steadystate_guesses['fval'] < np.inf:
self.apply_steadystate_guess(data_ix, x_dct)
# run amici simulation
rdatas = amici.runAmiciSimulations(
self.amici_model,
self.amici_solver,
self.edatas,
num_threads=min(self.n_threads, len(self.edatas)),
)
par_sim_ids = list(self.amici_model.getParameterIds())
for data_ix, rdata in enumerate(rdatas):
log_simulation(data_ix, rdata)
# check if the computation failed
if rdata['status'] < 0.0:
return self.get_error_output(rdatas)
condition_map_sim_var = \
self.parameter_mapping[data_ix].map_sim_var
nllh -= rdata['llh']
# compute objective
if mode == MODE_FUN:
if sensi_order > 0:
add_sim_grad_to_opt_grad(self.x_ids,
par_sim_ids,
condition_map_sim_var,
rdata['sllh'],
snllh,
coefficient=-1.0)
if sensi_method == 1:
# TODO Compute the full Hessian, and check here
add_sim_hess_to_opt_hess(self.x_ids,
par_sim_ids,
condition_map_sim_var,
rdata['FIM'],
s2nllh,
coefficient=+1.0)
elif mode == MODE_RES:
res = np.hstack([res, rdata['res']]) \
if res.size else rdata['res']
if sensi_order > 0:
opt_sres = sim_sres_to_opt_sres(self.x_ids,
par_sim_ids,
condition_map_sim_var,
rdata['sres'],
coefficient=1.0)
sres = np.vstack([sres, opt_sres]) \
if sres.size else opt_sres
# check whether we should update data for preequilibration guesses
if self.guess_steadystate and \
nllh <= self.steadystate_guesses['fval']:
self.steadystate_guesses['fval'] = nllh
for data_ix, rdata in enumerate(rdatas):
self.store_steadystate_guess(data_ix, x_dct, rdata)
return {
FVAL: nllh,
GRAD: snllh,
HESS: s2nllh,
RES: res,
SRES: sres,
RDATAS: rdatas
}
def simulate_petab(
petab_problem: petab.Problem,
amici_model: AmiciModel,
solver: Optional[amici.Solver] = None,
problem_parameters: Optional[Dict[str, float]] = None,
simulation_conditions: Union[pd.DataFrame, Dict] = None,
edatas: List[AmiciExpData] = None,
parameter_mapping: ParameterMapping = None,
scaled_parameters: Optional[bool] = False,
log_level: int = logging.WARNING
) -> Dict[str, Any]:
"""Simulate PEtab model.
:param petab_problem:
PEtab problem to work on.
:param amici_model:
AMICI Model assumed to be compatible with ``petab_problem``.
:param solver:
An AMICI solver. Will use default options if None.
:param problem_parameters:
Run simulation with these parameters. If None, PEtab `nominalValues`
will be used). To be provided as dict, mapping PEtab problem
parameters to SBML IDs.
:param simulation_conditions:
Result of `petab.get_simulation_conditions`. Can be provided to save
time if this has be obtained before.
Not required if `edatas` and `parameter_mapping` are provided.
:param edatas:
Experimental data. Parameters are inserted in-place for simulation.
:param parameter_mapping:
Optional precomputed PEtab parameter mapping for efficiency, as
generated by `create_parameter_mapping`.
:param scaled_parameters:
If True, problem_parameters are assumed to be on the scale provided
in the PEtab parameter table and will be unscaled. If False, they
are assumed to be in linear scale.
:param log_level:
Log level, see :mod:`amici.logging` module.
:return:
Dictionary of
* cost function value (LLH),
* const function sensitivity w.r.t. parameters (SLLH),
(**NOTE**: Sensitivities are computed for the scaled parameters)
* list of `ReturnData` (RDATAS),
corresponding to the different simulation conditions.
For ordering of simulation conditions, see
:meth:`petab.Problem.get_simulation_conditions_from_measurement_df`.
"""
logger.setLevel(log_level)
if solver is None:
solver = amici_model.getSolver()
# Get parameters
if problem_parameters is None:
# Use PEtab nominal values as default
problem_parameters = {t.Index: getattr(t, NOMINAL_VALUE) for t in
petab_problem.parameter_df.itertuples()}
scaled_parameters = False
# number of amici simulations will be number of unique
# (preequilibrationConditionId, simulationConditionId) pairs.
# Can be optimized by checking for identical condition vectors.
if simulation_conditions is None and parameter_mapping is None \
and edatas is None:
simulation_conditions = \
petab_problem.get_simulation_conditions_from_measurement_df()
# Get parameter mapping
if parameter_mapping is None:
parameter_mapping = create_parameter_mapping(
petab_problem=petab_problem,
simulation_conditions=simulation_conditions,
scaled_parameters=scaled_parameters,
amici_model=amici_model)
# Get edatas
if edatas is None:
# Generate ExpData with all condition-specific information
edatas = create_edatas(
amici_model=amici_model,
petab_problem=petab_problem,
simulation_conditions=simulation_conditions)
# Fill parameters in ExpDatas (in-place)
fill_in_parameters(
edatas=edatas,
problem_parameters=problem_parameters,
scaled_parameters=scaled_parameters,
parameter_mapping=parameter_mapping,
amici_model=amici_model)
# Simulate
rdatas = amici.runAmiciSimulations(amici_model, solver, edata_list=edatas)
# Compute total llh
llh = sum(rdata['llh'] for rdata in rdatas)
# Compute total sllh
sllh = aggregate_sllh(amici_model=amici_model, rdatas=rdatas,
parameter_mapping=parameter_mapping)
# Log results
sim_cond = petab_problem.get_simulation_conditions_from_measurement_df()
for i, rdata in enumerate(rdatas):
logger.debug(f"Condition: {sim_cond.iloc[i, :].values}, status: "
f"{rdata['status']}, llh: {rdata['llh']}")
return {
LLH: llh,
SLLH: sllh,
RDATAS: rdatas
}
if model_name == 'model_steadystate' and \
case == 'sensiforwarderrorint':
edata = amici.amici.ExpData(model.get())
# Test runAmiciSimulations: ensure running twice
# with same ExpData yields same results
if edata and model_name != 'model_neuron_o2' and not (
model_name == 'model_robertson' and case == 'sensiforwardSPBCG'):
if isinstance(edata, amici.amici.ExpData):
edatas = [edata, edata]
else:
edatas = [edata.get(), edata.get()]
rdatas = amici.runAmiciSimulations(model,
solver,
edatas,
num_threads=2,
failfast=False)
verify_simulation_results(rdatas[0],
expected_results[sub_test][case]['results'],
**verify_simulation_opts)
verify_simulation_results(rdatas[1],
expected_results[sub_test][case]['results'],
**verify_simulation_opts)
# test residuals mode
if solver.getSensitivityMethod() == amici.SensitivityMethod.adjoint:
with pytest.raises(RuntimeError):
solver.setReturnDataReportingMode(amici.RDataReporting.residuals)
else:
solver.setReturnDataReportingMode(amici.RDataReporting.residuals)
def test_special_likelihoods(model_special_likelihoods):
"""Test special likelihood functions."""
model = model_special_likelihoods.getModel()
model.setTimepoints(np.linspace(0, 60, 10))
solver = model.getSolver()
solver.setSensitivityOrder(amici.SensitivityOrder.first)
# Test in region with positive density
# run model once to create an edata
rdata = amici.runAmiciSimulation(model, solver)
edata = amici.ExpData(rdata, 0.001, 0)
# make sure measurements are smaller for non-degenerate probability
y = edata.getObservedData()
y = tuple([val * np.random.uniform(0, 1) for val in y])
edata.setObservedData(y)
# set sigmas
sigma = 0.2
sigmas = sigma * np.ones(len(y))
edata.setObservedDataStdDev(sigmas)
# and now run for real and also compute likelihood values
rdata = amici.runAmiciSimulations(model, solver, [edata])[0]
# check if the values make overall sense
assert np.isfinite(rdata['llh'])
assert np.all(np.isfinite(rdata['sllh']))
assert np.any(rdata['sllh'])
rdata_df = amici.getSimulationObservablesAsDataFrame(model,
edata,
rdata,
by_id=True)
edata_df = amici.getDataObservablesAsDataFrame(model, edata, by_id=True)
# check correct likelihood value
llh_exp = -sum([
binomial_nllh(edata_df['o1'], rdata_df['o1'], sigma),
negative_binomial_nllh(edata_df['o2'], rdata_df['o2'], sigma),
])
assert np.isclose(rdata['llh'], llh_exp)
# check gradient
for sensi_method in [
amici.SensitivityMethod.forward, amici.SensitivityMethod.adjoint
]:
solver = model.getSolver()
solver.setSensitivityMethod(sensi_method)
solver.setSensitivityOrder(amici.SensitivityOrder.first)
check_derivatives(model,
solver,
edata,
assert_fun,
atol=1e-1,
rtol=1e-1,
check_least_squares=False)
# Test for m > y, i.e. in region with 0 density
rdata = amici.runAmiciSimulation(model, solver)
edata = amici.ExpData(rdata, 0.001, 0)
# make sure measurements are smaller for non-degenerate probability
y = edata.getObservedData()
y = tuple([val * np.random.uniform(0.5, 3) for val in y])
edata.setObservedData(y)
edata.setObservedDataStdDev(sigmas)
# and now run for real and also compute likelihood values
rdata = amici.runAmiciSimulations(model, solver, [edata])[0]
# m > y -> outside binomial domain -> 0 density
assert rdata['llh'] == -np.inf
# check for non-informative gradient
assert all(np.isnan(rdata['sllh']))
def test_steadystate_scaled(self):
'''
Test SBML import and simulation from AMICI python interface
'''
sbmlFile = os.path.join(os.path.dirname(__file__), '..', 'python',
'examples', 'example_steadystate',
'model_steadystate_scaled.xml')
sbmlImporter = amici.SbmlImporter(sbmlFile)
observables = amici.assignmentRules2observables(
sbmlImporter.sbml,
filter_function=lambda variable: variable.getId().startswith(
'observable_') and not variable.getId().endswith('_sigma'))
outdir = 'test_model_steadystate_scaled'
sbmlImporter.sbml2amici(
'test_model_steadystate_scaled',
outdir,
observables=observables,
constantParameters=['k0'],
sigmas={'observable_x1withsigma': 'observable_x1withsigma_sigma'})
sys.path.insert(0, outdir)
import test_model_steadystate_scaled as modelModule
model = modelModule.getModel()
model.setTimepoints(amici.DoubleVector(np.linspace(0, 60, 60)))
solver = model.getSolver()
rdata = amici.runAmiciSimulation(model, solver)
edata = [amici.ExpData(rdata, 0.01, 0)]
rdata = amici.runAmiciSimulations(model, solver, edata)
# check roundtripping of DataFrame conversion
df_edata = amici.getDataObservablesAsDataFrame(model, edata)
edata_reconstructed = amici.getEdataFromDataFrame(model, df_edata)
self.assertTrue(
np.isclose(
amici.edataToNumPyArrays(edata[0])['observedData'],
amici.edataToNumPyArrays(
edata_reconstructed[0])['observedData'],
).all())
self.assertTrue(
np.isclose(
amici.edataToNumPyArrays(edata[0])['observedDataStdDev'],
amici.edataToNumPyArrays(
edata_reconstructed[0])['observedDataStdDev'],
).all())
if edata[0].fixedParameters.size():
self.assertListEqual(
list(edata[0].fixedParameters),
list(edata_reconstructed[0].fixedParameters),
)
else:
self.assertListEqual(
list(model.getFixedParameters()),
list(edata_reconstructed[0].fixedParameters),
)
self.assertListEqual(
list(edata[0].fixedParametersPreequilibration),
list(edata_reconstructed[0].fixedParametersPreequilibration),
)
df_state = amici.getSimulationStatesAsDataFrame(model, edata, rdata)
self.assertTrue(
np.isclose(rdata[0]['x'],
df_state[list(model.getStateIds())].values).all())
df_obs = amici.getSimulationObservablesAsDataFrame(model, edata, rdata)
self.assertTrue(
np.isclose(rdata[0]['y'],
df_obs[list(model.getObservableIds())].values).all())
amici.getResidualsAsDataFrame(model, edata, rdata)
def run(self,
tspan=None,
initials=None,
param_values=None,
num_processors=1):
"""
Run a simulation and returns the result (trajectories)
.. note::
In early versions of the Simulator class, ``tspan``, ``initials``
and ``param_values`` supplied to this method persisted to future
:func:`run` calls. This is no longer the case.
Parameters
----------
tspan
initials
param_values
See parameter definitions in :class:`ScipyOdeSimulator`.
num_processors : int
Number of processes to use (default: 1). Set to a larger number
(e.g. the number of CPU cores available) for parallel execution of
simulations. This is only useful when simulating with more than one
set of initial conditions and/or parameters.
Returns
-------
A :class:`SimulationResult` object
"""
super(AmiciSimulator, self).run(tspan=tspan,
initials=initials,
param_values=param_values,
_run_kwargs=[])
n_sims = len(self.param_values)
num_processors = min(n_sims, num_processors)
if num_processors == 1:
self._logger.debug('Single processor (serial) mode')
else:
if not amici.compiledWithOpenMP():
raise EnvironmentError(
'AMICI/model was not compiled with openMP support, which '
'is required for parallel simulation. Please see '
'https://github.com/ICB-DCM/AMICI/blob/master'
'/documentation/PYTHON.md for details on how to compile '
'AMICI with openMP support.')
self._logger.debug('Multi-processor (parallel) mode using {} '
'processes'.format(num_processors))
edatas = self._simulationspecs_to_edatas()
rdatas = amici.runAmiciSimulations(model=self.amici_model,
solver=self.amici_solver,
edata_list=edatas,
failfast=False,
num_threads=num_processors)
self._logger.info('All simulation(s) complete')
return SimulationResult(self, np.array([self.tspan] * n_sims),
self._rdatas_to_trajectories(rdatas))
def _call_amici(self, x, sensi_orders, mode):
# amici is built such that only the maximum sensitivity is required,
# the lower orders are then automatically computed
sensi_order = min(max(sensi_orders), 1)
# order 2 currently not implemented, we are using the FIM
# check if the requested sensitivities can be computed
if sensi_order > self.max_sensi_order:
raise Exception("Sensitivity order not allowed.")
# prepare outputs
nllh = 0.0
snllh = np.zeros(self.dim)
s2nllh = np.zeros([self.dim, self.dim])
res = np.zeros([0])
sres = np.zeros([0, self.dim])
# set order in solver
self.amici_solver.setSensitivityOrder(sensi_order)
# loop over experimental data
for data_ix, edata in enumerate(self.edatas):
# set model parameter scale for condition index
self.set_parameter_scale(data_ix)
# set parameters in model, according to mapping
self.set_par_sim_for_condition(data_ix, x)
# set parameter list according to mapping
self.set_plist_for_condition(data_ix)
if self.guess_steadystate and \
self.steadystate_guesses['fval'] < np.inf:
self.apply_steadystate_guess(data_ix, x)
# run amici simulation
rdatas = amici.runAmiciSimulations(
self.amici_model,
self.amici_solver,
self.edatas,
num_threads=min(self.n_threads, len(self.edatas)),
)
for data_ix, rdata in enumerate(rdatas):
log_simulation(data_ix, rdata)
# check if the computation failed
if rdata['status'] < 0.0:
return self.get_error_output(rdatas)
# compute objective
if mode == MODE_FUN:
nllh -= rdata['llh']
if sensi_order > 0:
add_sim_grad_to_opt_grad(
self.x_ids,
self.mapping_par_opt_to_par_sim[data_ix],
rdata['sllh'],
snllh,
coefficient=-1.0)
# TODO: Compute the full Hessian, and check here
add_sim_hess_to_opt_hess(
self.x_ids,
self.mapping_par_opt_to_par_sim[data_ix],
rdata['FIM'],
s2nllh,
coefficient=-1.0)
elif mode == MODE_RES:
res = np.hstack([res, rdata['res']]) \
if res.size else rdata['res']
if sensi_order > 0:
opt_sres = sim_sres_to_opt_sres(
self.x_ids,
self.mapping_par_opt_to_par_sim[data_ix],
rdata['sres'],
coefficient=1.0)
sres = np.vstack([sres, opt_sres]) \
if sres.size else opt_sres
# check whether we should update data for preequilibration guesses
if self.guess_steadystate and \
nllh <= self.steadystate_guesses['fval']:
self.steadystate_guesses['fval'] = nllh
for data_ix, rdata in enumerate(rdatas):
self.store_steadystate_guess(data_ix, x, rdata)
return {
FVAL: nllh,
GRAD: snllh,
HESS: s2nllh,
RES: res,
SRES: sres,
RDATAS: rdatas
}
def runTest(self):
'''
test runner routine that loads data expectedResults.h5 hdf file and runs individual models/settings
as subTests
'''
expectedResults = h5py.File(self.expectedResultsFile, 'r')
for subTest in expectedResults.keys():
for case in list(expectedResults[subTest].keys()):
if re.search('^sensi2',case) != None:
modelName = subTest + '_o2'
else:
modelName = subTest
with self.subTest(modelName=modelName, caseName=case):
print('running subTest modelName = ' + modelName + ', caseName = ' + case)
modelSwigFolder = os.path.join(os.path.dirname(__file__), '..', 'build', 'tests',
'cpputest', 'external_' + modelName + '-prefix',
'src', 'external_' + modelName + '-build', 'swig')
sys.path.insert(0, modelSwigFolder)
testModelModule = importlib.import_module(modelName)
self.model = testModelModule.getModel()
self.solver = self.model.getSolver()
amici.readModelDataFromHDF5(self.expectedResultsFile,
self.model.get(),
"/" + subTest + "/" + case + "/options")
amici.readSolverSettingsFromHDF5(self.expectedResultsFile,
self.solver.get(),
"/" + subTest + "/" + case + "/options")
edata = None
if 'data' in expectedResults[subTest][case].keys():
edata = amici.readSimulationExpData(self.expectedResultsFile,
"/" + subTest + "/" + case + "/data",
self.model.get())
rdata = amici.runAmiciSimulation(self.model, self.solver, edata)
# todo: set higher tolerances in testcase options and
# regenerate results
if modelName == 'model_jakstat_adjoint_o2':
self.solver.setRelativeTolerance(1e-10)
self.solver.setAbsoluteTolerance(1e-10)
if modelName in [
'model_jakstat_adjoint', 'model_nested_events',
'model_steadystate'
]:
self.solver.setRelativeTolerance(1e-12)
self.solver.setAbsoluteTolerance(1e-12)
if edata \
and self.solver.getSensitivityMethod() \
and self.solver.getSensitivityOrder() \
and len(self.model.getParameterList()) \
and not modelName.startswith('model_neuron') \
and not case.endswith('byhandpreeq'):
checkDerivatives(self.model, self.solver, edata)
if modelName == 'model_neuron_o2':
self.solver.setRelativeTolerance(1e-12)
verifySimulationResults(rdata, expectedResults[subTest][case]['results'],atol=1e-6,rtol=1e-2)
else:
verifySimulationResults(rdata, expectedResults[subTest][case]['results'])
if edata and modelName != 'model_neuron_o2':
# Test runAmiciSimulations: ensure running twice with same ExpData yields same results
edatas = [edata.get(), edata.get()]
rdatas = amici.runAmiciSimulations(self.model, self.solver, edatas, num_threads=2)
verifySimulationResults(rdatas[0], expectedResults[subTest][case]['results'])
verifySimulationResults(rdatas[1], expectedResults[subTest][case]['results'])
self.assertRaises(
RuntimeError,
self.model.getParameterByName,
'thisParameterDoesNotExist'
)
def __call__(self,
x_dct: Dict,
sensi_order: int,
mode: str,
amici_model: AmiciModel,
amici_solver: AmiciSolver,
edatas: List['amici.ExpData'],
n_threads: int,
x_ids: Sequence[str],
parameter_mapping: 'ParameterMapping',
fim_for_hess: bool):
"""Perform the actual AMICI call.
Called within the :func:`AmiciObjective.__call__` method.
Parameters
----------
x_dct:
Parameters for which to compute function value and derivatives.
sensi_order:
Maximum sensitivity order.
mode:
Call mode (function value or residual based).
amici_model:
The AMICI model.
amici_solver:
The AMICI solver.
edatas:
The experimental data.
n_threads:
Number of threads for AMICI call.
x_ids:
Ids of optimization parameters.
parameter_mapping:
Mapping of optimization to simulation parameters.
fim_for_hess:
Whether to use the FIM (if available) instead of the Hessian (if
requested).
"""
# set order in solver
if sensi_order == 2 and fim_for_hess:
# we use the FIM
amici_solver.setSensitivityOrder(sensi_order-1)
else:
amici_solver.setSensitivityOrder(sensi_order)
# fill in parameters
# TODO (#226) use plist to compute only required derivatives
amici.parameter_mapping.fill_in_parameters(
edatas=edatas,
problem_parameters=x_dct,
scaled_parameters=True,
parameter_mapping=parameter_mapping,
amici_model=amici_model
)
# run amici simulation
rdatas = amici.runAmiciSimulations(
amici_model,
amici_solver,
edatas,
num_threads=min(n_threads, len(edatas)),
)
if not self._known_least_squares_safe and mode == MODE_RES and \
sensi_order > 0:
if any(
((r['ssigmay'] is not None and np.any(r['ssigmay']))
or
(r['ssigmaz'] is not None and np.any(r['ssigmaz'])))
for r in rdatas
):
raise RuntimeError('Cannot use least squares solver with'
'parameter dependent sigma!')
self._known_least_squares_safe = True # don't check this again
return calculate_function_values(
rdatas=rdatas, sensi_order=sensi_order, mode=mode,
amici_model=amici_model, amici_solver=amici_solver, edatas=edatas,
x_ids=x_ids, parameter_mapping=parameter_mapping,
fim_for_hess=fim_for_hess)
