def simulate(path: Union[str, Path], n_sim: int): """ Simulate from a database, within a database. Parameters ---------- path A path to the database object to create simulations in. n_sim Number of simulations to create. """ d = DismodIO(path=path) try: if d.fit_var.empty: raise SampleError( "Cannot run sample simulate on a database without fit_var!") except ValueError: raise SampleError( "Cannot run sample simulate on a database without fit_var!" "Does not have the fit_var table yet.") # Create n_sim simulation datasets based on the fitted parameters run_dismod_commands(dm_file=path, commands=[ 'set start_var fit_var', 'set truth_var fit_var', 'set scale_var fit_var', f'simulate {n_sim}' ])
def test_predict_sample(mi, settings, dismod): alchemy = Alchemy(settings) fill_avgint_with_priors_grid(inputs=mi, alchemy=alchemy, settings=settings, source_db_path=NAME, child_locations=[72], child_sexes=[2]) run_dismod_commands(dm_file=NAME, commands=['predict sample']) di = DismodIO(NAME) assert len(di.predict) == 2 * len(di.avgint)
def __call__(self, index=None): index_db = self.context.db_file(location_id=self.location_id, sex_id=self.sex_id, index=index) if index is not None: copy2(src=str(self.main_db), dst=str(index_db)) run_dismod_commands(dm_file=index_db, commands=[f'fit {self.fit_type} {index}']) db = DismodIO(path=index_db) fit = db.fit_var fit['sample_index'] = index return fit
def _process(self, db: str): run_dismod_commands(dm_file=db, commands=[f'fit {self.fit_type} {self.index}']) db = DismodIO(path=db) fit = db.fit_var fit['sample_index'] = self.index fit.rename(columns={ 'fit_var_id': 'var_id', 'fit_var_value': 'var_value' }, inplace=True) return fit
def main(args=None): """ Creates a dismod database using the saved inputs and the file structure specified in the context. Then runs an optional set of commands on the database passed in the --commands argument. Also passes an optional argument --options as a dictionary to the dismod database to fill/modify the options table. """ args = get_args(args=args) logging.basicConfig(level=LEVELS[args.loglevel]) if args.test_dir: context = Context(model_version_id=args.model_version_id, configure_application=False, root_directory=args.test_dir) else: context = Context(model_version_id=args.model_version_id) inputs, alchemy, settings = context.read_inputs() # If we want to override the rate priors with posteriors from a previous # database, pass them in here. if args.prior_parent or args.prior_sex: if not (args.prior_parent and args.prior_sex): raise RuntimeError("Need to pass both prior parent and sex or neither.") child_prior = DismodExtractor(path=context.db_file( location_id=args.prior_parent, sex_id=args.prior_sex )).gather_draws_for_prior_grid( location_id=args.parent_location_id, sex_id=args.sex_id, rates=[r.rate for r in settings.rate] ) else: child_prior = None df = DismodFiller( path=context.db_file(location_id=args.parent_location_id, sex_id=args.sex_id), settings_configuration=settings, measurement_inputs=inputs, grid_alchemy=alchemy, parent_location_id=args.parent_location_id, sex_id=args.sex_id, child_prior=child_prior ) df.fill_for_parent_child(**args.options) run_dismod_commands(dm_file=df.path.absolute(), commands=args.commands)
def main(): """ Takes dismod databases that have already had a fit run on them and simulates new datasets, refitting on all of them, then combining the results back into one database. Returns: """ args = get_args() logging.basicConfig(level=LEVELS[args.loglevel]) context = Context(model_version_id=args.model_version_id) main_db = context.db_file(location_id=args.parent_location_id, sex_id=args.sex_id) d = DismodIO(path=main_db) if d.fit_var.empty: raise RuntimeError( "Cannot run sample / simulate on a database without fit_var!") # Create n_sim simulation datasets based on the fitted parameters run_dismod_commands(dm_file=main_db, commands=[ 'set start_var fit_var' 'set truth_var fit_var', 'set scale_var fit_var', f'simulate {args.n_sim}' ]) if args.n_pool > 1: # Make a pool and fit to each of the simulations (uses the __call__ method) fit_sample = FitSample(context=context, location_id=args.location_id, sex_id=args.sex_id, fit_type=args.fit_type) p = Pool(args.n_pool) fits = list(p.map(fit_sample, range(args.n_sim))) p.close() # Reconstruct the sample table with all n_sim fits sample = pd.DataFrame().append(fits).reset_index(drop=True) sample.rename(columns={ 'fit_var_id': 'var_id', 'fit_var_value': 'var_value' }, inplace=True) d.sample = sample else: # If we only have one pool that means we aren't going to run in parallel run_dismod_commands(dm_file=main_db, commands=[f'sample simulate {args.n_sim}'])
def sample_simulate_sequence(path: Union[str, Path], n_sim: int, fit_type: str): """ Fit the samples in a database in sequence. Parameters ---------- path A path to the database object to create simulations in. n_sim Number of simulations to create. fit_type Type of fit -- fixed or both """ run_dismod_commands(dm_file=path, commands=[f'sample simulate {fit_type} {n_sim}'])
def _process(self, db: str): dbio = DismodIO(path=db) n_var = len(dbio.var) this_sample = dbio.sample.loc[dbio.sample.sample_index == self.index].copy() this_sample['sample_index'] = 0 this_sample['sample_id'] = this_sample['var_id'] dbio.sample = this_sample del dbio run_dismod_commands(dm_file=db, commands=[f'predict sample']) dbio = DismodIO(path=db) predict = dbio.predict predict['sample_index'] = self.index return predict
def main(): args = get_args() logging.basicConfig(level=LEVELS[args.loglevel]) context = Context(model_version_id=args.model_version_id) inputs, alchemy, settings = context.read_inputs() sourceDB = DismodIO(path=context.db_file( location_id=args.source_location, sex_id=args.source_sex, make=False)) rates = [r.rate for r in settings.rate] posterior_grid = get_prior_avgint_grid(settings=settings, integrands=rates, sexes=args.target_sexes, locations=args.target_locations, midpoint=False) posterior_grid = inputs.add_covariates_to_data(df=posterior_grid) posterior_grid = prep_data_avgint(df=posterior_grid, node_df=sourceDB.node, covariate_df=sourceDB.covariate) posterior_grid.rename(columns={'sex_id': 'c_sex_id'}, inplace=True) sourceDB.avgint = posterior_grid run_dismod_commands(dm_file=sourceDB, commands=['predict sample'])
def sample_asymptotic(path: Union[str, Path], n_sim: int, fit_type: str): """ Fit the samples in a database in sequence. Parameters ---------- path A path to the database object to create simulations in. n_sim Number of simulations to create. fit_type Type of fit -- fixed or both """ return run_dismod_commands(dm_file=path, commands=[ 'set start_var fit_var', 'set truth_var fit_var', 'set scale_var fit_var', f'sample asymptotic {fit_type} {n_sim}' ])
def dismod_db(model_version_id: int, parent_location_id: int, sex_id: int, dm_commands: List[str], dm_options: Dict[str, Union[int, str, float]], prior_samples: bool = False, prior_parent: Optional[int] = None, prior_sex: Optional[int] = None, prior_mulcov_model_version_id: Optional[int] = None, test_dir: Optional[str] = None, fill: bool = False, save_fit: bool = True, save_prior: bool = True) -> None: """ Creates a dismod database using the saved inputs and the file structure specified in the context. Alternatively it will skip the filling stage and move straight to the command stage if you don't pass --fill. Then runs an optional set of commands on the database passed in the --commands argument. Also passes an optional argument --options as a dictionary to the dismod database to fill/modify the options table. Parameters ---------- model_version_id The model version ID parent_location_id The parent location for the database sex_id The parent sex for the database dm_commands A list of commands to pass to the run_dismod_commands function, executed directly on the dismod database dm_options A dictionary of options to pass to the the dismod option table prior_parent An optional parent location ID that specifies where to pull the prior information from. prior_sex An optional parent sex ID that specifies where to pull the prior information from. test_dir A test directory to create the database in rather than the database specified by the IHME file system context. fill Whether or not to fill the database with new inputs based on the model_version_id, parent_location_id, and sex_id. If not filling, this script can be used to just execute commands on the database instead. save_fit Whether or not to save the fit from this database as the parent fit. save_prior Whether or not to save the prior for the children as the prior fit. """ if test_dir is not None: context = Context(model_version_id=model_version_id, configure_application=False, root_directory=test_dir) else: context = Context(model_version_id=model_version_id) db_path = context.db_file(location_id=parent_location_id, sex_id=sex_id) inputs, alchemy, settings = context.read_inputs() # If we want to override the rate priors with posteriors from a previous # database, pass them in here. if prior_parent or prior_sex: if not (prior_parent and prior_sex): raise DismodDBError( "Need to pass both prior parent and sex or neither.") prior_db = context.db_file(location_id=prior_parent, sex_id=prior_sex) child_prior = get_prior(path=prior_db, location_id=parent_location_id, sex_id=sex_id, rates=[r.rate for r in settings.rate], samples=prior_samples) if save_prior: save_predictions(db_file=prior_db, locations=[parent_location_id], sexes=[sex_id], model_version_id=model_version_id, gbd_round_id=settings.gbd_round_id, out_dir=context.prior_dir) else: child_prior = None if save_prior: raise DismodDBError( "Cannot save the prior because there was no argument" "passed in for the prior_parent or prior_sex.") if prior_mulcov_model_version_id is not None: LOG.info( f'Passing mulcov prior from model version id = {prior_mulcov_model_version_id}' ) mulcov_priors = get_mulcov_priors(prior_mulcov_model_version_id) else: mulcov_priors = None if fill: fill_database( path=db_path, inputs=inputs, alchemy=alchemy, settings=settings, parent_location_id=parent_location_id, sex_id=sex_id, child_prior=child_prior, options=dm_options, mulcov_prior=mulcov_priors, ) if dm_commands: run_dismod_commands(dm_file=str(db_path), commands=dm_commands) if save_fit: save_predictions(db_file=context.db_file( location_id=parent_location_id, sex_id=sex_id), model_version_id=model_version_id, gbd_round_id=settings.gbd_round_id, out_dir=context.fit_dir)
def predict_sample_sequence(path: Union[str, Path], table: str): """ Runs predict for either fit_var or sample, based on the table. """ run_dismod_commands(dm_file=path, commands=[f'predict {table}'])
def test_sample_simulate(filler, dismod): run_dismod_commands(NAME, commands=['init', 'fit fixed']) simulate(NAME, n_sim=2)