def test_choose_param_init(pheno_path, testdata):
model = Model(pheno_path)
params = (model.parameters['OMEGA(1,1)'], model.parameters['OMEGA(2,2)'])
rvs = RandomVariables(model.random_variables.etas)
init = _choose_param_init(model, rvs, params)
assert init == 0.0118179
model = Model(pheno_path)
model.source.path = testdata # Path where there is no .ext-file
init = _choose_param_init(model, rvs, params)
assert init == 0.0031045
model = Model(pheno_path)
omega1 = S('OMEGA(3,3)')
x = stats.Normal('ETA(3)', 0, sympy.sqrt(omega1))
x.variability_level = VariabilityLevel.IIV
rvs.add(x)
ie = model.modelfit_results.individual_estimates
ie['ETA(3)'] = ie['ETA(1)']
model.modelfit_results = ModelfitResults(individual_estimates=ie)
init = _choose_param_init(model, rvs, params)
assert init == 0.0118179
def create_rv_block(model, list_of_rvs=None):
"""
Creates a full or partial block structure of etas. The etas must be IIVs and cannot
be fixed. Initial estimates for covariance between the etas is dependent on whether
the model has results from a previous results. In that case, the correlation will
be calculated from individual estimates, otherwise correlation will be set to 10%.
Parameters
----------
model : Model
Pharmpy model to create block effect on.
list_of_rvs : list
List of etas to create a block structure from. If None, all etas that are IIVs and
non-fixed will be used (full block). None is default.
"""
rvs = model.random_variables
if list_of_rvs is None:
iiv_rvs = model.random_variables.iiv
nonfix = RandomVariables()
for rv in iiv_rvs:
for name in iiv_rvs.parameter_names:
if model.parameters[name].fix:
break
else:
nonfix.append(rv)
list_of_rvs = nonfix.names
else:
for name in list_of_rvs:
if name in rvs and rvs[name].level == 'IOV':
raise ValueError(
f'{name} describes IOV: Joining IOV random variables is currently not supported'
)
if len(list_of_rvs) == 1:
raise ValueError('At least two random variables are needed')
sset = model.statements
paramnames = []
for rv in list_of_rvs:
statements = sset.find_assignment(rv, is_symbol=False, last=False)
parameter_names = '_'.join([s.symbol.name for s in statements])
paramnames.append(parameter_names)
cov_to_params = rvs.join(list_of_rvs,
name_template='IIV_{}_IIV_{}',
param_names=paramnames)
pset = model.parameters
for cov_name, param_names in cov_to_params.items():
parent_params = (pset[param_names[0]], pset[param_names[1]])
covariance_init = _choose_param_init(model, rvs, parent_params)
param_new = Parameter(cov_name, covariance_init)
pset.append(param_new)
return model
def _get_rvs(model, list_of_rvs):
full_block = False
if list_of_rvs is None:
list_of_rvs = [rv for rv in model.random_variables.etas]
full_block = True
elif len(list_of_rvs) == 1:
raise RVInputException('At least two random variables are needed')
rvs = []
for rv_str in list_of_rvs:
try:
rv = model.random_variables[rv_str]
except KeyError:
raise RVInputException(f'Random variable does not exist: {rv_str}')
if _has_fixed_params(model, rv):
if not full_block:
raise RVInputException(f'Random variable cannot be set to fixed: {rv}')
continue
if rv.variability_level == VariabilityLevel.IOV:
if not full_block:
raise RVInputException(f'Random variable cannot be IOV: {rv}')
continue
rvs.append(rv)
return RandomVariables(rvs)
def test_all_parameters():
omega1 = symbol('OMEGA(1,1)')
eta1 = stats.Normal('ETA(1)', 0, sympy.sqrt(omega1))
omega2 = symbol('OMEGA(2,2)')
eta2 = stats.Normal('ETA(2)', 0, sympy.sqrt(omega2))
sigma = symbol('SIGMA(1,1)')
eps = stats.Normal('EPS(1)', 0, sympy.sqrt(sigma))
rvs = RandomVariables([eta1, eta2, eps])
assert len(rvs) == 3
params = rvs.all_parameters()
assert len(params) == 3
assert params == ['OMEGA(1,1)', 'OMEGA(2,2)', 'SIGMA(1,1)']
def test_rv():
omega1 = sympy.Symbol('OMEGA(1,1)')
x = stats.Normal('ETA(1)', 0, sympy.sqrt(omega1))
rvs = RandomVariables([x])
assert len(rvs) == 1
retrieved = rvs['ETA(1)']
assert retrieved.name == 'ETA(1)'
assert retrieved.pspace.distribution.mean == 0
def random_variables(self):
try:
return self._random_variables
except AttributeError:
pass
rvs = RandomVariables()
next_omega = 1
prev_cov = None
for omega_record in self.control_stream.get_records('OMEGA'):
etas, next_omega, prev_cov, _ = omega_record.random_variables(
next_omega, prev_cov)
rvs.extend(etas)
self.adjust_iovs(rvs)
next_sigma = 1
prev_cov = None
for sigma_record in self.control_stream.get_records('SIGMA'):
epsilons, next_sigma, prev_cov, _ = sigma_record.random_variables(
next_sigma, prev_cov)
rvs.extend(epsilons)
self._random_variables = rvs
self._old_random_variables = rvs.copy()
if ('comment' in pharmpy.plugins.nonmem.conf.parameter_names
or 'abbr' in pharmpy.plugins.nonmem.conf.parameter_names):
self.statements
return self._random_variables
def test_validate_parameters():
a, b, c, d = (symbol('a'), symbol('b'), symbol('c'), symbol('d'))
rvs = JointNormalSeparate(['ETA(1)', 'ETA(2)'], [0, 0], [[a, b], [b, c]])
rvs = RandomVariables(rvs)
rvs.add(stats.Normal('ETA(3)', 0.5, d))
params = {'a': 2, 'b': 0.1, 'c': 1, 'd': 23}
assert rvs.validate_parameters(params)
params2 = {'a': 2, 'b': 2, 'c': 1, 'd': 23}
assert not rvs.validate_parameters(params2)
def test_distributions():
rvs = JointNormalSeparate(['ETA(1)', 'ETA(2)'], [0, 0],
[[3, 0.25], [0.25, 1]])
rvs = RandomVariables(rvs)
rvs.add(stats.Normal('ETA(3)', 0.5, 2))
dists = rvs.distributions()
symbols, dist = dists[0]
assert symbols[0].name == 'ETA(1)'
assert symbols[1].name == 'ETA(2)'
assert len(symbols) == 2
assert dist == rvs[0].pspace.distribution
symbols, dist = dists[1]
assert symbols[0].name == 'ETA(3)'
assert len(symbols) == 1
assert dist == rvs[2].pspace.distribution
def test_extract_from_block():
etas = JointNormalSeparate(['ETA(1)', 'ETA(2)'], [0, 0],
[[3, 0.25], [0.25, 1]])
for rv in etas:
rv.variability_level = VariabilityLevel.IIV
rvs = RandomVariables(etas)
eta3 = stats.Normal('ETA(3)', 0.5, 2)
eta3.variability_level = VariabilityLevel.IIV
rvs.add(eta3)
dists = rvs.distributions()
assert len(dists) == 2
rvs.extract_from_block(etas[0])
dists = rvs.distributions()
assert len(dists) == 3
assert rvs[0].name == 'ETA(1)'
assert rvs[2].name == 'ETA(3)'
def test_distributions():
rvs = JointNormalSeparate(['ETA(1)', 'ETA(2)'], [0, 0],
[[3, 0.25], [0.25, 1]])
rvs = RandomVariables(rvs)
rvs.add(stats.Normal('ETA(3)', 0.5, 2))
gen = rvs.distributions()
symbols, dist = next(gen)
assert symbols[0].name == 'ETA(1)'
assert symbols[1].name == 'ETA(2)'
assert len(symbols) == 2
assert dist == rvs[0].pspace.distribution
symbols, dist = next(gen)
assert symbols[0].name == 'ETA(3)'
assert len(symbols) == 1
assert dist == rvs[2].pspace.distribution
with pytest.raises(StopIteration):
symbols, dist = next(gen)
def create_record_maps(etas, rvs, name_map, eta_map):
cov_rvs = RandomVariables(rvs).covariance_matrix()
cov_rec = RandomVariables(etas).covariance_matrix()
for row in range(cov_rvs.shape[0]):
for col in range(cov_rvs.shape[1]):
if row >= col:
elem_new = cov_rvs.row(row).col(col)[0]
elem_rec = cov_rec.row(row).col(col)[0]
name_map[str(elem_new)] = name_map.pop(str(elem_rec))
for rv, eta in zip(rvs, etas):
eta_map[str(rv)] = eta_map.pop(str(eta))
def test_has_same_order():
omega1 = symbol('OMEGA(1,1)')
eta1 = stats.Normal('ETA(1)', 0, sympy.sqrt(omega1))
omega2 = symbol('OMEGA(2,2)')
eta2 = stats.Normal('ETA(2)', 0, sympy.sqrt(omega2))
omega3 = symbol('OMEGA(1,1)')
eta3 = stats.Normal('ETA(3)', 0, sympy.sqrt(omega3))
rvs_full = RandomVariables([eta1, eta2, eta3])
assert rvs_full.are_consecutive(rvs_full)
rvs_sub = RandomVariables([eta1, eta2])
assert rvs_full.are_consecutive(rvs_sub)
rvs_rev = RandomVariables([eta3, eta2, eta1])
assert not rvs_full.are_consecutive(rvs_rev)
def test_merge_normal_distributions():
rvs = JointNormalSeparate(['ETA(1)', 'ETA(2)'], [0, 0],
[[3, 0.25], [0.25, 1]])
rvs = RandomVariables(rvs)
rvs.add(stats.Normal('ETA(3)', 0.5, 2))
rvs.merge_normal_distributions()
assert len(rvs) == 3
assert rvs['ETA(1)'].name == 'ETA(1)'
assert rvs[1].name == 'ETA(2)'
assert rvs[2].name == 'ETA(3)'
assert rvs[0].pspace is rvs[1].pspace
assert rvs[0].pspace is rvs[2].pspace
dist = rvs[0].pspace.distribution
assert dist.mu == sympy.Matrix([0, 0, 0.5])
assert dist.sigma == sympy.Matrix([[3, 0.25, 0], [0.25, 1, 0], [0, 0, 4]])
rvs.merge_normal_distributions(fill=1)
dist = rvs[0].pspace.distribution
assert dist.sigma == sympy.Matrix([[3, 0.25, 1], [0.25, 1, 1], [1, 1, 4]])
def test_merge_normal_distributions():
rvs = JointNormalSeparate(['ETA(1)', 'ETA(2)'], [0, 0],
[[3, 0.25], [0.25, 1]])
for rv in rvs:
rv.variability_level = VariabilityLevel.IIV
rvs = RandomVariables(rvs)
eta3 = stats.Normal('ETA(3)', 0.5, 2)
eta3.variability_level = VariabilityLevel.IIV
rvs.add(eta3)
rvs.merge_normal_distributions()
assert len(rvs) == 3
assert rvs['ETA(1)'].name == 'ETA(1)'
assert rvs[1].name == 'ETA(2)'
assert rvs[2].name == 'ETA(3)'
assert rvs[0].pspace is rvs[1].pspace
assert rvs[0].pspace is rvs[2].pspace
dist = rvs[0].pspace.distribution
assert dist.mu == sympy.Matrix([0, 0, 0.5])
assert dist.sigma == sympy.Matrix([[3, 0.25, 0], [0.25, 1, 0], [0, 0, 4]])
def create_rv_block(model, list_of_rvs=None):
"""
Creates a full or partial block structure of etas. The etas must be IIVs and cannot
be fixed. Initial estimates for covariance between the etas is dependent on whether
the model has results from a previous results. In that case, the correlation will
be calculated from individual estimates, otherwise correlation will be set to 10%.
Parameters
----------
model : Model
Pharmpy model to create block effect on.
list_of_rvs : list
List of etas to create a block structure from. If None, all etas that are IIVs and
non-fixed will be used (full block). None is default.
"""
rvs_full = model.random_variables
rvs_block = _get_rvs(model, list_of_rvs)
if list_of_rvs is not None:
for rv in rvs_block:
if isinstance(rv.pspace.distribution, MultivariateNormalDistribution):
if rvs_full.get_rvs_from_same_dist(rv):
rv_extracted = rvs_full.extract_from_block(rv)
rvs_block.discard(rv)
rvs_block.add(rv_extracted)
pset = _merge_rvs(model, rvs_block)
rvs_new = RandomVariables()
are_consecutive = rvs_full.are_consecutive(rvs_block)
for rv in rvs_full:
if rv.name not in [rv.name for rv in rvs_block.etas]:
rvs_new.add(rv)
elif are_consecutive:
rvs_new.add(rvs_block[rv.name])
if not are_consecutive:
{rvs_new.add(rv) for rv in rvs_block} # Add new block last
model.random_variables = rvs_new
model.parameters = pset
return model
def random_variables(self):
rvs = RandomVariables()
next_omega = 1
prev_cov = None
for omega_record in self.control_stream.get_records('OMEGA'):
etas, next_omega, prev_cov, _ = omega_record.random_variables(
next_omega, prev_cov)
rvs.update(etas)
next_sigma = 1
prev_cov = None
for sigma_record in self.control_stream.get_records('SIGMA'):
epsilons, next_sigma, prev_cov, _ = sigma_record.random_variables(
next_sigma, prev_cov)
rvs.update(epsilons)
return rvs
def _get_etas(model, list_to_remove):
rvs = model.random_variables
sset = model.statements
symbols_all = [
s.symbol.name for s in model.statements if isinstance(s, Assignment)
]
if list_to_remove is None:
list_to_remove = [rv for rv in rvs.etas]
etas = []
symbols = []
for variable in list_to_remove:
try:
eta = rvs[variable]
except KeyError:
if variable in symbols_all:
symbols.append(variable)
continue
else:
raise RVInputException(
f'Random variable does not exist: {variable}')
if eta.variability_level == VariabilityLevel.IOV:
if list_to_remove:
raise RVInputException(
f'Random variable cannot be IOV: {variable}')
continue
etas.append(eta)
for symbol in symbols:
terms = sset.find_assignment(symbol).free_symbols
eta_set = terms.intersection(rvs.free_symbols)
etas += [rvs[eta.name] for eta in eta_set]
return RandomVariables(etas)
def random_variables(self):
try:
return self._random_variables
except AttributeError:
pass
rvs = RandomVariables()
next_omega = 1
prev_cov = None
for omega_record in self.control_stream.get_records('OMEGA'):
etas, next_omega, prev_cov, _ = omega_record.random_variables(
next_omega, prev_cov)
rvs.update(etas)
self.adjust_iovs(rvs)
next_sigma = 1
prev_cov = None
for sigma_record in self.control_stream.get_records('SIGMA'):
epsilons, next_sigma, prev_cov, _ = sigma_record.random_variables(
next_sigma, prev_cov)
rvs.update(epsilons)
self._random_variables = rvs
self._old_random_variables = rvs.copy()
return rvs
def update_random_variable_records(model, rvs_diff, rec_dict, comment_dict):
removed = []
eta_number = 1
number_of_records = 0
for i, (op, (rvs, _)) in enumerate(rvs_diff):
if op == '+':
if len(rvs) == 1:
create_omega_single(model, rvs[0], eta_number,
number_of_records, comment_dict)
eta_number += 1
else:
create_omega_block(model, RandomVariables(rvs), eta_number,
number_of_records, comment_dict)
eta_number += len(rvs)
number_of_records += 1
elif op == '-':
if len(rvs) == 1:
recs_to_remove = [rec_dict[rvs[0].name]]
else:
recs_to_remove = list({rec_dict[rv.name] for rv in rvs})
recs_to_remove = [
rec for rec in recs_to_remove if rec not in removed
]
if recs_to_remove:
model.control_stream.remove_records(recs_to_remove)
removed += [rec for rec in recs_to_remove]
else:
if len(rvs) == 1 and rec_dict[
rvs[0].name] in removed: # Account for etas in diagonal
create_omega_single(model, rvs[0], eta_number,
number_of_records, comment_dict)
eta_number += 1
else:
eta_number += len(rvs)
number_of_records += 1
def random_variables(self, start_omega, previous_cov=None):
"""Get a RandomVariableSet for this omega record
start_omega - the first omega in this record
previous_sigma - the matrix of the previous omega block
"""
next_cov = None # The cov matrix if a block
block = self.root.find('block')
bare_block = self.root.find('bare_block')
zero_fix = []
if not (block or bare_block):
rvs = RandomVariables()
i = start_omega
numetas = len(self.root.all('diag_item'))
for node in self.root.all('diag_item'):
init = node.init.NUMERIC
fixed = bool(node.find('FIX'))
name = self._rv_name(i)
if not (init == 0 and fixed): # 0 FIX are not RVs
eta = sympy.stats.Normal(name, 0, sympy.sqrt(
sympy.Symbol(f'{self.name}({i},{i})')))
rvs.add(eta)
else:
zero_fix.append(name)
i += 1
else:
if bare_block:
numetas = previous_cov.rows
else:
numetas = self.root.block.size.INT
same = bool(self.root.find('same'))
params, _, _ = self.parameters(start_omega, previous_cov.rows if
hasattr(previous_cov, 'rows') else None)
all_zero_fix = True
for param in params:
if not (param.init == 0 and param.fix):
all_zero_fix = False
if all_zero_fix and len(params) > 0 or (previous_cov == 'ZERO' and same):
names = [self._rv_name(i) for i in range(start_omega, start_omega + numetas)]
return RandomVariables(), start_omega + numetas, 'ZERO', names
if numetas > 1:
names = [self._rv_name(i) for i in range(start_omega, start_omega + numetas)]
means = [0] * numetas
if same:
rvs = JointNormalSeparate(names, means, previous_cov)
next_cov = previous_cov
else:
cov = sympy.zeros(numetas)
for row in range(numetas):
for col in range(row + 1):
cov[row, col] = sympy.Symbol(
f'{self.name}({start_omega + row},{start_omega + col})')
if row != col:
cov[col, row] = cov[row, col]
next_cov = cov
rvs = JointNormalSeparate(names, means, cov)
else:
rvs = RandomVariables()
name = self._rv_name(start_omega)
if same:
symbol = previous_cov
else:
symbol = sympy.Symbol(f'{self.name}({start_omega},{start_omega})')
eta = sympy.stats.Normal(name, 0, sympy.sqrt(symbol))
next_cov = symbol
rvs.add(eta)
if self.name == 'OMEGA':
level = VariabilityLevel.IIV
else:
level = VariabilityLevel.RUV
for rv in rvs:
rv.variability_level = level
return rvs, start_omega + numetas, next_cov, zero_fix
def _get_etas(rvs):
etas = [eta for eta in rvs if eta.variability_level == VariabilityLevel.IOV]
return RandomVariables(etas)
def random_variables(self, start_omega, previous_cov=None):
"""Get a RandomVariableSet for this omega record
start_omega - the first omega in this record
previous_cov - the matrix of the previous omega block
"""
same = bool(self.root.find('same'))
if not hasattr(self, 'name_map') and not same:
if isinstance(previous_cov, sympy.Symbol):
prev_size = 1
elif previous_cov is not None:
prev_size = len(previous_cov)
else:
prev_size = None
self.parameters(start_omega, prev_size)
if hasattr(self, 'name_map'):
rev_map = {value: key for key, value in self.name_map.items()}
next_cov = None # The cov matrix if a block
block = self.root.find('block')
bare_block = self.root.find('bare_block')
zero_fix = []
etas = []
if not (block or bare_block):
rvs = RandomVariables()
i = start_omega
numetas = len(self.root.all('diag_item'))
for node in self.root.all('diag_item'):
init = node.init.NUMERIC
fixed = bool(node.find('FIX'))
name = self._rv_name(i)
if not (init == 0 and fixed): # 0 FIX are not RVs
eta = RandomVariable.normal(name, 'iiv', 0, symbol(rev_map[(i, i)]))
rvs.append(eta)
etas.append(eta.name)
else:
zero_fix.append(name)
etas.append(name)
i += 1
else:
if bare_block:
numetas = previous_cov.rows
else:
numetas = self.root.block.size.INT
params, _, _ = self.parameters(
start_omega, previous_cov.rows if hasattr(previous_cov, 'rows') else None
)
all_zero_fix = True
for param in params:
if not (param.init == 0 and param.fix):
all_zero_fix = False
if all_zero_fix and len(params) > 0 or (previous_cov == 'ZERO' and same):
names = [self._rv_name(i) for i in range(start_omega, start_omega + numetas)]
self.eta_map = {eta: start_omega + i for i, eta in enumerate(names)}
return RandomVariables(), start_omega + numetas, 'ZERO', names
if numetas > 1:
names = [self._rv_name(i) for i in range(start_omega, start_omega + numetas)]
means = [0] * numetas
if same:
rvs = RandomVariable.joint_normal(names, 'iiv', means, previous_cov)
etas = [rv.name for rv in rvs]
next_cov = previous_cov
else:
cov = sympy.zeros(numetas)
for row in range(numetas):
for col in range(row + 1):
cov[row, col] = symbol(rev_map[(start_omega + row, start_omega + col)])
if row != col:
cov[col, row] = cov[row, col]
next_cov = cov
rvs = RandomVariable.joint_normal(names, 'iiv', means, cov)
etas = [rv.name for rv in rvs]
else:
rvs = RandomVariables()
name = self._rv_name(start_omega)
if same:
sym = previous_cov
else:
sym = symbol(rev_map[(start_omega, start_omega)])
eta = RandomVariable.normal(name, 'iiv', 0, sym)
next_cov = sym
rvs.append(eta)
etas.append(eta.name)
if self.name == 'OMEGA':
if same:
level = 'IOV'
else:
level = 'IIV'
else:
level = 'RUV'
for rv in rvs:
rv.level = level
self.eta_map = {eta: start_omega + i for i, eta in enumerate(etas)}
return rvs, start_omega + numetas, next_cov, zero_fix
def _get_etas(rvs):
etas = [eta for eta in rvs if eta.level == 'IOV']
return RandomVariables(etas)