def _read_phi_table(self):
if not self._read_phi:
self._load()
for result_obj in self:
result_obj._individual_ofv = None
result_obj._individual_estimates = None
result_obj._individual_estimates_covariance = None
rv_names = [
rv.name for rv in self.model.random_variables
if rv.name.startswith('ETA')
]
try:
phi_tables = NONMEMTableFile(self._path.with_suffix('.phi'))
except FileNotFoundError:
return
for result_obj in self:
table = phi_tables.table_no(result_obj.table_number)
if table is not None:
result_obj._individual_ofv = table.iofv
result_obj._individual_estimates = table.etas[rv_names]
covs = table.etcs
covs = covs.transform(
lambda cov: cov[rv_names].loc[rv_names])
result_obj._individual_estimates_covariance = covs
self._read_phi = True
def _read_coi_table(self):
if not self._read_coi:
self._load()
coi_table = NONMEMTableFile(self._path.with_suffix('.coi'))
for result_obj in self:
df = coi_table.table_no(result_obj.table_number).data_frame
if df is not None:
if self.model:
df = df.rename(
index=self.model.parameter_translation())
df.columns = df.index
result_obj._information_matrix = df
self._read_coi = True
def _read_cor_table(self):
if not self._read_cor:
self._load()
cor_table = NONMEMTableFile(self._path.with_suffix('.cor'))
for result_obj in self:
cor = cor_table.table_no(result_obj.table_number).data_frame
if cor is not None:
if self.model:
cor = cor.rename(
index=self.model.parameter_translation())
cor.columns = cor.index
np.fill_diagonal(cor.values, 1)
result_obj._correlation_matrix = cor
self._read_cor = True
def initial_individual_estimates(self):
"""Initial individual estimates
These are taken from the $ETAS FILE. 0 FIX ETAs are removed.
If no $ETAS is present None will be returned.
Setter assumes that all IDs are present
"""
try:
return self._initial_individual_estimates
except AttributeError:
pass
etas = self.control_stream.get_records('ETAS')
if etas:
path = Path(etas[0].path)
if not path.is_absolute():
source_dir = self.source.path.parent
path = source_dir / path
path = path.resolve()
phi_tables = NONMEMTableFile(path)
rv_names = [
rv.name for rv in self.random_variables
if rv.name.startswith('ETA')
]
etas = next(phi_tables).etas[rv_names]
self._initial_individual_estimates = etas
else:
self._initial_individual_estimates = None
return self._initial_individual_estimates
def _read_ext_table(self):
if not self._read_ext:
ext_tables = NONMEMTableFile(self._path.with_suffix('.ext'))
for table, result_obj in zip(ext_tables, self):
ests = table.final_parameter_estimates
fix = table.fixed
ests = ests[~fix]
result_obj._parameter_estimates = ests
sdcorr = table.omega_sigma_stdcorr[~fix]
sdcorr_ests = ests.copy()
sdcorr_ests.update(sdcorr)
result_obj._parameter_estimates_sdcorr = sdcorr_ests
try:
ses = table.standard_errors
except Exception:
result_obj._standard_errors = None
else:
ses = ses[~fix]
result_obj._standard_errors = ses
sdcorr = table.omega_sigma_se_stdcorr[~fix]
sdcorr_ses = ses.copy()
sdcorr_ses.update(sdcorr)
result_obj._standard_errors_sdcorr = sdcorr_ses
result_obj._ofv = table.final_ofv
self._read_ext = True
def _update_initial_individual_estimates(self, path, nofiles=False):
"""Update $ETAS
Could have 0 FIX in model. Need to read these
"""
if path is None: # What to do here?
phi_path = Path('.')
else:
phi_path = path.parent
phi_path /= f'{self.name}_input.phi'
if self._initial_individual_estimates_updated:
etas = self.initial_individual_estimates
zero_fix = self._zero_fix_rvs(eta=True)
if zero_fix:
for eta in zero_fix:
etas[eta] = 0
etas = self._sort_eta_columns(etas)
if not nofiles:
phi = PhiTable(df=etas)
table_file = NONMEMTableFile(tables=[phi])
table_file.write(phi_path)
# FIXME: This is a common operation
eta_records = self.control_stream.get_records('ETAS')
if eta_records:
record = eta_records[0]
else:
record = self.control_stream.append_record('$ETAS ')
record.path = phi_path
elif self._updated_etas_file:
eta_records = self.control_stream.get_records('ETAS')
if eta_records:
record = eta_records[0]
else:
record = self.control_stream.append_record('$ETAS')
shutil.copy(self._updated_etas_file, phi_path)
record.path = phi_path
if self._initial_individual_estimates_updated or self._updated_etas_file:
first_est_record = self.control_stream.get_records('ESTIMATION')[0]
try:
first_est_record.option_pairs['MCETA']
except KeyError:
first_est_record.set_option('MCETA', 1)
self._updated_etas_file = None
self._initial_individual_estimates_updated = False
def _read_cor_table(self):
if not self._read_cor:
cor_table = NONMEMTableFile(self._path.with_suffix('.cor'))
cor = next(cor_table).data_frame
if not hasattr(
self[-1],
'_standard_errors'): # In case read from the ext-file
self[-1]._standard_errors = pd.Series(np.diag(cor),
index=cor.index)
np.fill_diagonal(cor.values, 1)
self[-1]._correlation_matrix = cor
self._read_cor = True
def _read_phi_table(self):
if not self._read_phi:
rv_names = [
rv.name for rv in self.model.random_variables
if rv.name.startswith('ETA')
]
phi_tables = NONMEMTableFile(self._path.with_suffix('.phi'))
for table, result_obj in zip(phi_tables, self):
result_obj._individual_ofv = table.iofv
result_obj._individual_estimates = table.etas[rv_names]
covs = table.etcs
for index, cov in covs.iteritems(
): # Remove ETCs for 0 FIX OMEGAs
covs[index] = cov[rv_names].loc[rv_names]
result_obj._individual_estimates_covariance = covs
self._read_phi = True
def psn_bootstrap_results(path):
"""Create bootstrapresults from a PsN bootstrap run
:param path: Path to PsN boostrap run directory
:return: A :class:`BootstrapResults` object
"""
path = Path(path)
models = [Model(p) for p in model_paths(path, 'bs_pr1_*.mod')]
# Read the results already now to give an appropriate error if no results exists
results = [
m.modelfit_results for m in models if m.modelfit_results is not None
]
if not results:
raise FileNotFoundError("No model results available in m1")
try:
base_model = Model(cmd_line_model_path(path))
except FileNotFoundError:
base_model = None
# Read dOFV results in NONMEM specific way. Models have multiple $PROBLEM
# Create proper result objects to pass to calculate_results
dofv_results = None
if (path / 'm1' / 'dofv_1.mod').is_file():
from pharmpy.plugins.nonmem.table import NONMEMTableFile
dofv_results = []
for table_path in (path / 'm1').glob('dofv_*.ext'):
table_file = NONMEMTableFile(table_path)
next_table = 1
for table in table_file:
while next_table != table.number:
dofv_results.append(None)
next_table += 1
res = ModelfitResults(ofv=table.final_ofv)
dofv_results.append(res)
next_table += 1
incinds = pd.read_csv(path / 'included_individuals1.csv',
header=None).values.tolist()
res = calculate_results(models,
original_model=base_model,
included_individuals=incinds,
dofv_results=dofv_results)
return res
def _read_from_tables(self, columns, result_obj):
self._load()
table_recs = self.model.control_stream.get_records('TABLE')
found = []
df = pd.DataFrame()
for table_rec in table_recs:
columns_in_table = []
for key, value in table_rec.all_options:
if key in columns and key not in found and value is None:
# FIXME: Cannot handle synonyms here
colname = key
elif value in columns and value not in found:
colname = value
else:
continue
found.append(colname)
columns_in_table.append(colname)
if columns_in_table:
table_file = NONMEMTableFile(self.model.source.path.parent /
table_rec.path)
table = table_file.tables[0]
df[columns_in_table] = table.data_frame[columns_in_table]
return df
def _read_coi_table(self):
if not self._read_coi:
coi_table = NONMEMTableFile(self._path.with_suffix('.coi'))
self[-1]._information_matrix = next(coi_table).data_frame
self._read_coi = True
def _read_cov_table(self):
if not self._read_cov:
cov_table = NONMEMTableFile(self._path.with_suffix('.cov'))
self[-1]._covariance_matrix = next(cov_table).data_frame
self._read_cov = True
def _read_ext_table(self):
if not self._read_ext:
self._read_ext = True
try:
ext_tables = NONMEMTableFile(self._path.with_suffix('.ext'))
except ValueError:
# The ext-file is illegal
warnings.warn(
f"Broken ext-file {self._path.with_suffix('.ext')}")
self._read_ext = True
res = NONMEMModelfitResults(self)
res.model_name = self._path.stem
res.model = self.model
# Parameter esitimates NaN for all parameters that should be estimated
pe = pd.Series(
np.nan,
index=list(self.model.parameters.nonfixed_inits.keys()))
res._parameter_estimates = pe
res._ofv = np.nan
self.append(res)
return
for table in ext_tables:
if self._subproblem and table.subproblem != self._subproblem:
continue
result_obj = NONMEMModelfitResults(self)
result_obj.model_name = self._path.stem
result_obj.model = self.model
result_obj.table_number = table.number
result_obj._ofv = table.final_ofv
result_obj._evaluation_ofv = table.initial_ofv
if table.is_evaluation:
result_obj._set_estimation_status(results_file=None,
requested=False)
ests = table.final_parameter_estimates
try:
fix = table.fixed
except KeyError:
# NM 7.2 does not have row -1000000006 indicating FIXED status
if self.model:
fixed = pd.Series(self.model.parameters.fix)
fix = pd.concat([
fixed,
pd.Series(True,
index=ests.index.difference(fixed.index))
])
ests = ests[~fix]
if self.model:
ests = ests.rename(
index=self.model.parameter_translation())
result_obj._parameter_estimates = ests
try:
sdcorr = table.omega_sigma_stdcorr[~fix]
except KeyError:
pass
else:
if self.model:
sdcorr = sdcorr.rename(
index=self.model.parameter_translation())
sdcorr_ests = ests.copy()
sdcorr_ests.update(sdcorr)
result_obj._parameter_estimates_sdcorr = sdcorr_ests
try:
ses = table.standard_errors
except Exception:
# If there are no standard errors in ext-file it means
# there can be no cov, cor or coi either
result_obj._standard_errors = None
result_obj._covariance_matrix = None
result_obj._correlation_matrix = None
result_obj._information_matrix = None
result_obj._condition_number = None
result_obj._set_covariance_status(None)
else:
ses = ses[~fix]
sdcorr = table.omega_sigma_se_stdcorr[~fix]
if self.model:
ses = ses.rename(
index=self.model.parameter_translation())
sdcorr = sdcorr.rename(
index=self.model.parameter_translation())
result_obj._standard_errors = ses
sdcorr_ses = ses.copy()
sdcorr_ses.update(sdcorr)
if self.model:
sdcorr_ses = sdcorr_ses.rename(
index=self.model.parameter_translation())
result_obj._standard_errors_sdcorr = sdcorr_ses
try:
condition_number = table.condition_number
except Exception:
pass # PRINT=E not set in $COV, but could compute from correlation matrix
else:
result_obj._condition_number = condition_number
self.append(result_obj)
def _read_ext_table(self):
if not self._read_ext:
ext_tables = NONMEMTableFile(self._path.with_suffix('.ext'))
for table in ext_tables:
result_obj = NONMEMModelfitResults(self)
result_obj.model_name = self._path.stem
result_obj.model = self.model
result_obj.table_number = table.number
result_obj._ofv = table.final_ofv
result_obj._evaluation_ofv = table.initial_ofv
if table.is_evaluation:
result_obj._set_estimation_status(results_file=None,
requested=False)
ests = table.final_parameter_estimates
try:
fix = table.fixed
except KeyError:
# NM 7.2 does not have row -1000000006 indicating FIXED status
if self.model:
fixed = pd.Series(self.model.parameters.fix)
fix = pd.concat([
fixed,
pd.Series(True,
index=ests.index.difference(fixed.index))
])
ests = ests[~fix]
if self.model:
ests = ests.rename(
index=self.model.parameter_translation())
result_obj._parameter_estimates = ests
sdcorr = table.omega_sigma_stdcorr[~fix]
sdcorr_ests = ests.copy()
sdcorr_ests.update(sdcorr)
result_obj._parameter_estimates_sdcorr = sdcorr_ests
try:
ses = table.standard_errors
except Exception:
# If there are no standard errors in ext-file it means
# there can be no cov, cor or coi either
result_obj._standard_errors = None
result_obj._covariance_matrix = None
result_obj._correlation_matrix = None
result_obj._information_matrix = None
result_obj._condition_number = None
result_obj._set_covariance_status(None)
else:
ses = ses[~fix]
if self.model:
ses = ses.rename(
index=self.model.parameter_translation())
result_obj._standard_errors = ses
sdcorr = table.omega_sigma_se_stdcorr[~fix]
sdcorr_ses = ses.copy()
sdcorr_ses.update(sdcorr)
if self.model:
sdcorr_ses = sdcorr_ses.rename(
index=self.model.parameter_translation())
result_obj._standard_errors_sdcorr = sdcorr_ses
try:
condition_number = table.condition_number
except Exception:
pass # PRINT=E not set in $COV, but could compute from correlation matrix
else:
result_obj._condition_number = condition_number
self.append(result_obj)
self._read_ext = True