def simulate(path: Union[str, Path], n_sim: int):
"""
Simulate from a database, within a database.
Parameters
----------
path
A path to the database object to create simulations in.
n_sim
Number of simulations to create.
"""
d = DismodIO(path=path)
try:
if d.fit_var.empty:
raise SampleError(
"Cannot run sample simulate on a database without fit_var!")
except ValueError:
raise SampleError(
"Cannot run sample simulate on a database without fit_var!"
"Does not have the fit_var table yet.")
# Create n_sim simulation datasets based on the fitted parameters
run_dismod_commands(dm_file=path,
commands=[
'set start_var fit_var', 'set truth_var fit_var',
'set scale_var fit_var', f'simulate {n_sim}'
])
def test_predict_sample(mi, settings, dismod):
alchemy = Alchemy(settings)
fill_avgint_with_priors_grid(inputs=mi,
alchemy=alchemy,
settings=settings,
source_db_path=NAME,
child_locations=[72],
child_sexes=[2])
run_dismod_commands(dm_file=NAME, commands=['predict sample'])
di = DismodIO(NAME)
assert len(di.predict) == 2 * len(di.avgint)
def __call__(self, index=None):
index_db = self.context.db_file(location_id=self.location_id,
sex_id=self.sex_id,
index=index)
if index is not None:
copy2(src=str(self.main_db), dst=str(index_db))
run_dismod_commands(dm_file=index_db,
commands=[f'fit {self.fit_type} {index}'])
db = DismodIO(path=index_db)
fit = db.fit_var
fit['sample_index'] = index
return fit
def _process(self, db: str):
run_dismod_commands(dm_file=db,
commands=[f'fit {self.fit_type} {self.index}'])
db = DismodIO(path=db)
fit = db.fit_var
fit['sample_index'] = self.index
fit.rename(columns={
'fit_var_id': 'var_id',
'fit_var_value': 'var_value'
},
inplace=True)
return fit
def main(args=None):
"""
Creates a dismod database using the saved inputs and the file
structure specified in the context.
Then runs an optional set of commands on the database passed
in the --commands argument.
Also passes an optional argument --options as a dictionary to
the dismod database to fill/modify the options table.
"""
args = get_args(args=args)
logging.basicConfig(level=LEVELS[args.loglevel])
if args.test_dir:
context = Context(model_version_id=args.model_version_id,
configure_application=False,
root_directory=args.test_dir)
else:
context = Context(model_version_id=args.model_version_id)
inputs, alchemy, settings = context.read_inputs()
# If we want to override the rate priors with posteriors from a previous
# database, pass them in here.
if args.prior_parent or args.prior_sex:
if not (args.prior_parent and args.prior_sex):
raise RuntimeError("Need to pass both prior parent and sex or neither.")
child_prior = DismodExtractor(path=context.db_file(
location_id=args.prior_parent,
sex_id=args.prior_sex
)).gather_draws_for_prior_grid(
location_id=args.parent_location_id,
sex_id=args.sex_id,
rates=[r.rate for r in settings.rate]
)
else:
child_prior = None
df = DismodFiller(
path=context.db_file(location_id=args.parent_location_id, sex_id=args.sex_id),
settings_configuration=settings,
measurement_inputs=inputs,
grid_alchemy=alchemy,
parent_location_id=args.parent_location_id,
sex_id=args.sex_id,
child_prior=child_prior
)
df.fill_for_parent_child(**args.options)
run_dismod_commands(dm_file=df.path.absolute(), commands=args.commands)
def main():
"""
Takes dismod databases that have already had a fit run on them and simulates new datasets, refitting
on all of them, then combining the results back into one database.
Returns:
"""
args = get_args()
logging.basicConfig(level=LEVELS[args.loglevel])
context = Context(model_version_id=args.model_version_id)
main_db = context.db_file(location_id=args.parent_location_id,
sex_id=args.sex_id)
d = DismodIO(path=main_db)
if d.fit_var.empty:
raise RuntimeError(
"Cannot run sample / simulate on a database without fit_var!")
# Create n_sim simulation datasets based on the fitted parameters
run_dismod_commands(dm_file=main_db,
commands=[
'set start_var fit_var'
'set truth_var fit_var', 'set scale_var fit_var',
f'simulate {args.n_sim}'
])
if args.n_pool > 1:
# Make a pool and fit to each of the simulations (uses the __call__ method)
fit_sample = FitSample(context=context,
location_id=args.location_id,
sex_id=args.sex_id,
fit_type=args.fit_type)
p = Pool(args.n_pool)
fits = list(p.map(fit_sample, range(args.n_sim)))
p.close()
# Reconstruct the sample table with all n_sim fits
sample = pd.DataFrame().append(fits).reset_index(drop=True)
sample.rename(columns={
'fit_var_id': 'var_id',
'fit_var_value': 'var_value'
},
inplace=True)
d.sample = sample
else:
# If we only have one pool that means we aren't going to run in parallel
run_dismod_commands(dm_file=main_db,
commands=[f'sample simulate {args.n_sim}'])
def sample_simulate_sequence(path: Union[str, Path], n_sim: int,
fit_type: str):
"""
Fit the samples in a database in sequence.
Parameters
----------
path
A path to the database object to create simulations in.
n_sim
Number of simulations to create.
fit_type
Type of fit -- fixed or both
"""
run_dismod_commands(dm_file=path,
commands=[f'sample simulate {fit_type} {n_sim}'])
def _process(self, db: str):
dbio = DismodIO(path=db)
n_var = len(dbio.var)
this_sample = dbio.sample.loc[dbio.sample.sample_index ==
self.index].copy()
this_sample['sample_index'] = 0
this_sample['sample_id'] = this_sample['var_id']
dbio.sample = this_sample
del dbio
run_dismod_commands(dm_file=db, commands=[f'predict sample'])
dbio = DismodIO(path=db)
predict = dbio.predict
predict['sample_index'] = self.index
return predict
def main():
args = get_args()
logging.basicConfig(level=LEVELS[args.loglevel])
context = Context(model_version_id=args.model_version_id)
inputs, alchemy, settings = context.read_inputs()
sourceDB = DismodIO(path=context.db_file(
location_id=args.source_location, sex_id=args.source_sex, make=False))
rates = [r.rate for r in settings.rate]
posterior_grid = get_prior_avgint_grid(settings=settings,
integrands=rates,
sexes=args.target_sexes,
locations=args.target_locations,
midpoint=False)
posterior_grid = inputs.add_covariates_to_data(df=posterior_grid)
posterior_grid = prep_data_avgint(df=posterior_grid,
node_df=sourceDB.node,
covariate_df=sourceDB.covariate)
posterior_grid.rename(columns={'sex_id': 'c_sex_id'}, inplace=True)
sourceDB.avgint = posterior_grid
run_dismod_commands(dm_file=sourceDB, commands=['predict sample'])
def sample_asymptotic(path: Union[str, Path], n_sim: int, fit_type: str):
"""
Fit the samples in a database in sequence.
Parameters
----------
path
A path to the database object to create simulations in.
n_sim
Number of simulations to create.
fit_type
Type of fit -- fixed or both
"""
return run_dismod_commands(dm_file=path,
commands=[
'set start_var fit_var',
'set truth_var fit_var',
'set scale_var fit_var',
f'sample asymptotic {fit_type} {n_sim}'
])
def dismod_db(model_version_id: int,
parent_location_id: int,
sex_id: int,
dm_commands: List[str],
dm_options: Dict[str, Union[int, str, float]],
prior_samples: bool = False,
prior_parent: Optional[int] = None,
prior_sex: Optional[int] = None,
prior_mulcov_model_version_id: Optional[int] = None,
test_dir: Optional[str] = None,
fill: bool = False,
save_fit: bool = True,
save_prior: bool = True) -> None:
"""
Creates a dismod database using the saved inputs and the file
structure specified in the context. Alternatively it will
skip the filling stage and move straight to the command
stage if you don't pass --fill.
Then runs an optional set of commands on the database passed
in the --commands argument.
Also passes an optional argument --options as a dictionary to
the dismod database to fill/modify the options table.
Parameters
----------
model_version_id
The model version ID
parent_location_id
The parent location for the database
sex_id
The parent sex for the database
dm_commands
A list of commands to pass to the run_dismod_commands function, executed
directly on the dismod database
dm_options
A dictionary of options to pass to the the dismod option table
prior_parent
An optional parent location ID that specifies where to pull the prior
information from.
prior_sex
An optional parent sex ID that specifies where to pull the prior information from.
test_dir
A test directory to create the database in rather than the database
specified by the IHME file system context.
fill
Whether or not to fill the database with new inputs based on the model_version_id,
parent_location_id, and sex_id. If not filling, this script can be used
to just execute commands on the database instead.
save_fit
Whether or not to save the fit from this database as the parent fit.
save_prior
Whether or not to save the prior for the children as the prior fit.
"""
if test_dir is not None:
context = Context(model_version_id=model_version_id,
configure_application=False,
root_directory=test_dir)
else:
context = Context(model_version_id=model_version_id)
db_path = context.db_file(location_id=parent_location_id, sex_id=sex_id)
inputs, alchemy, settings = context.read_inputs()
# If we want to override the rate priors with posteriors from a previous
# database, pass them in here.
if prior_parent or prior_sex:
if not (prior_parent and prior_sex):
raise DismodDBError(
"Need to pass both prior parent and sex or neither.")
prior_db = context.db_file(location_id=prior_parent, sex_id=prior_sex)
child_prior = get_prior(path=prior_db,
location_id=parent_location_id,
sex_id=sex_id,
rates=[r.rate for r in settings.rate],
samples=prior_samples)
if save_prior:
save_predictions(db_file=prior_db,
locations=[parent_location_id],
sexes=[sex_id],
model_version_id=model_version_id,
gbd_round_id=settings.gbd_round_id,
out_dir=context.prior_dir)
else:
child_prior = None
if save_prior:
raise DismodDBError(
"Cannot save the prior because there was no argument"
"passed in for the prior_parent or prior_sex.")
if prior_mulcov_model_version_id is not None:
LOG.info(
f'Passing mulcov prior from model version id = {prior_mulcov_model_version_id}'
)
mulcov_priors = get_mulcov_priors(prior_mulcov_model_version_id)
else:
mulcov_priors = None
if fill:
fill_database(
path=db_path,
inputs=inputs,
alchemy=alchemy,
settings=settings,
parent_location_id=parent_location_id,
sex_id=sex_id,
child_prior=child_prior,
options=dm_options,
mulcov_prior=mulcov_priors,
)
if dm_commands:
run_dismod_commands(dm_file=str(db_path), commands=dm_commands)
if save_fit:
save_predictions(db_file=context.db_file(
location_id=parent_location_id, sex_id=sex_id),
model_version_id=model_version_id,
gbd_round_id=settings.gbd_round_id,
out_dir=context.fit_dir)
def predict_sample_sequence(path: Union[str, Path], table: str):
"""
Runs predict for either fit_var or sample, based on the table.
"""
run_dismod_commands(dm_file=path, commands=[f'predict {table}'])
def test_sample_simulate(filler, dismod):
run_dismod_commands(NAME, commands=['init', 'fit fixed'])
simulate(NAME, n_sim=2)
