def test_vector_drug_param_sweep(self): b = ModBuilder.from_combos([ ModFn(set_species_param, 'gambiae', 'Required_Habitat_Factor', value=v) for v in [(100, 50), (200, 100)] ], [ ModFn(set_drug_param, 'Artemether', 'Max_Drug_IRBC_Kill', value=v) for v in [4, 2] ]) md = [(4, (100, 50)), (2, (100, 50)), (4, (200, 100)), (2, (200, 100))] ngenerated = 0 for i, ml in enumerate(b.mod_generator): for m in ml: m(self.cb) self.assertListEqual([v for v in b.metadata.values()], list(md[i])) self.assertEqual( get_species_param(self.cb, 'gambiae', 'Required_Habitat_Factor'), md[i][1]) self.assertEqual( get_drug_param(self.cb, 'Artemether', 'Max_Drug_IRBC_Kill'), md[i][0]) ngenerated += 1 self.assertEqual(ngenerated, 4)
def test_generic_sweep(self): def verify(b): md = [(0.05, 100, 'Namawala'), (0.05, 100, 'Matsari'), (0.1, 100, 'Namawala'), (0.1, 100, 'Matsari')] ngenerated = 0 for i, ml in enumerate(b.mod_generator): for m in ml: m(self.cb) mdd = dict( zip(('x_Temporary_Larval_Habitat', 'Simulation_Duration', '_site_'), md[i])) mdd.update({'population_scale': 1}) self.assertEqual(b.metadata, mdd) self.assertEqual( self.cb.get_param('x_Temporary_Larval_Habitat'), md[i][0]) ngenerated += 1 self.assertEqual(ngenerated, 4) b = ModBuilder.from_combos([ ModFn(DTKConfigBuilder.set_param, 'x_Temporary_Larval_Habitat', v) for v in [0.05, 0.1] ], [ModFn(DTKConfigBuilder.set_param, 'Simulation_Duration', 100) ], [ModFn(configure_site, s) for s in ['Namawala', 'Matsari']]) verify(b) b = GenericSweepBuilder.from_dict({ 'x_Temporary_Larval_Habitat': [0.05, 0.1], 'Simulation_Duration': [100], '_site_': ['Namawala', 'Matsari'] }) verify(b)
def exp_builder_func(self, next_params, n_replicates=None): if not n_replicates: n_replicates = self.sim_runs_per_param_set return ModBuilder.from_combos( [ModFn(self.config_builder.__class__.set_param, 'Run_Number', i+1) for i in range(n_replicates)], [ModFn(site.setup_fn) for site in self.sites], [ModFn(self.map_sample_to_model_input_fn, index, samples) for index, samples in enumerate(next_params)] )
def __init__(self, nsims): self.tags = {} self.mod_generator = ( self.set_mods( [ModFn(DTKConfigBuilder.set_param, 'Run_Number', i)] ) for i in range(nsims) )
def set_immune_forcing_builder(transition_matrix=None, scale_factor_array=[2, 5, 10, 100]): builder = ModBuilder.from_combos([ ModFn(set_transition_matrix, transition_matrix, scale_factor) for scale_factor in scale_factor_array ]) return builder
def set_immune_forcing_builder(transition_matrix=None, scale_factor_array=[2, 5, 10, 100]): """Creates an experiment builder setting up immunity based on scale factor""" builder = ModBuilder.from_combos([ ModFn(set_transition_matrix, transition_matrix, scale_factor) for scale_factor in scale_factor_array ]) return builder
def get_modifier_functions(self): """ Returns a ModBuilder ModFn that sets file contents and values in config builder according to the dynamic parameters. """ return [ ModFn(self.mod_dynamic_parameters, dict(zip(self.header, row))) for row in self.table ]
def test_param_fn(self): k, v = ('Simulation_Duration', 100) fn = ModFn(DTKConfigBuilder.set_param, k, v) fn(self.cb) logging.debug('Parameter %s has value %s set from %s', k, self.cb.get_param(k), v) logging.debug('ModBuilder metadata: %s', ModBuilder.metadata) self.assertEqual(self.cb.get_param(k), v) self.assertEqual(ModBuilder.metadata, {k: v})
def modfn_sweep_over_burnins(archetype): burnin_df = pd.read_csv("burnins.csv") burnin_df = burnin_df[burnin_df["archetype"] == archetype] modlist = [] for i, row in burnin_df.iterrows(): modlist.append(ModFn(draw_from_serialized_file, dict(row))) return modlist
def test_calibsite_fn(self): from calibtool.study_sites.NamawalaCalibSite import NamawalaCalibSite s = NamawalaCalibSite() fn = ModFn(set_calibration_site, s) fn(self.cb) self.assertEqual( self.cb.campaign['Events'][0]['Event_Coordinator_Config'] ['Intervention_Config']['class'], 'InputEIR') self.assertEqual(self.cb.custom_reports[0].type, 'MalariaSummaryReport') self.assertEqual(ModBuilder.metadata, {'_site_': 'Namawala'})
def test_custom_fn(self): v = [100, 50] fn = ModFn(set_species_param, 'gambiae', 'Required_Habitat_Factor', value=v) fn(self.cb) self.assertListEqual( get_species_param(self.cb, 'gambiae', 'Required_Habitat_Factor'), v) self.assertEqual(ModBuilder.metadata, {'gambiae.Required_Habitat_Factor': v})
def test_custom_fn(self): v = [100, 50] self.cb.set_param('nested', {'foo': {'bar': [0, 0]}}) def custom_fn(cb, foo, bar, value): cb.config['nested'][foo][bar] = value return {'.'.join([foo, bar]): value} fn = ModFn(custom_fn, 'foo', 'bar', value=v) fn(self.cb) self.assertListEqual(self.cb.get_param('nested')['foo']['bar'], v) self.assertEqual(ModBuilder.metadata, {'foo.bar': v})
def test_site_fn(self): s = 'Namawala' fn = ModFn(configure_site, s) fn(self.cb) logging.debug('Demographics_Filenames: %s', self.cb.get_param('Demographics_Filenames')) logging.debug('ModBuilder metadata: %s', ModBuilder.metadata) self.assertTrue( 'Namawala' in self.cb.get_param('Demographics_Filenames')[0]) self.assertEqual(ModBuilder.metadata, { '_site_': 'Namawala', 'population_scale': 1 })
def modfn_sweep_over_scenarios(archetype, specific_scenarios_to_run=None): scenario_df = pd.read_csv("scenario_master_list.csv") scenario_df = scenario_df[scenario_df["archetype"] == archetype].reset_index(drop=True) if specific_scenarios_to_run == None: scenario_numbers = list(scenario_df["scenario_number"]) else: scenario_numbers = specific_scenarios_to_run # scenario_numbers = scenario_numbers[:2] #TESTING ONLY modlist = [ ModFn(add_scenario_specific_interventions, ns, archetype) for ns in scenario_numbers ] return modlist
def modfn_sweep_over_timings(archetype, num_years=1): if archetype == "Southern": timings_df = pd.read_csv("southern_term_sweep_scenarios.csv") elif archetype == "Sahel": timings_df = pd.read_csv("sahel_term_sweep_scenarios.csv") elif archetype == "Central": timings_df = pd.read_csv("central_term_sweep_scenarios.csv") else: raise NotImplementedError scenario_numbers = list(timings_df["scenario_number"]) # scenario_numbers = range(5) #TEST ONLY modlist = [ ModFn(add_ipt_for_timing_sweep, ns, archetype, num_years) for ns in scenario_numbers ] return modlist
def test_multiple_site_exception(self): def verify(b): with self.assertRaises(ValueError): for ml in b.mod_generator: pass b = ModBuilder.from_combos([ModFn(set_calibration_site, 'Matsari')], [ModFn(configure_site, 'Namawala')]) verify(b) b = ModBuilder.from_combos([ModFn(configure_site, 'Matsari')], [ModFn(configure_site, 'Namawala')]) verify(b) b = ModBuilder.from_combos([ModFn(set_calibration_site, 'Matsari')], [ModFn(set_calibration_site, 'Namawala')]) verify(b)
# +-------------------+----------------+ # | larvicide_start | Run_Number | # +-------------------+----------------+ # | 0 | 0 | # | 5 | 0 | # | 10 | 0 | # | ... | ... | # | 5 | 1 | # | 10 | 1 | # | 365 | 1 | # | ... | ... | # | 365 | 4 | # | 730 | 4 | # +-------------------+----------------+ builder = ModBuilder.from_combos( [ModFn(set_larvicides, start_time) for start_time in (0, 5, 10, 365, 730)], [ ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed) for seed in range(5) ], ) # The run_sim_args is a dictionary informing the command line of: # - What config builder to use # - What is the name we want for the experiment # - Which experiment builder to use run_sim_args = { 'config_builder': cb, 'exp_name': 'Sample larvicides epxeriment', 'exp_builder': builder }
sim_duration = 365 * 1 num_seeds = 2 SetupParser.init('LOCAL') cb = DTKConfigBuilder.from_defaults('MALARIA_SIM') intervention_days = [180] coverages = [x for x in np.arange(0.0, 1.01, 0.1)] target_groups_IVM = ['Excludingwomen'] expanded_SMC_groups_IVM = ['ExpandedSMCexcludingwomenonly'] # DP + Ivermectin (expanded and regular) SMC_IVM = [[ ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed), ModFn(DTKConfigBuilder.set_param, 'Simulation_Duration', smc_start_day + 365), ModFn( add_smc_group, start_days=[smc_start_day], coverage=smc_coverage, drug=drug, agemax=agemax, ), ModFn(add_ivermectin_group, coverage=smc_coverage, start_days=[smc_start_day], drug_code=drug_code, smc_code=drug + str(agemax), target_group=target_group),
# arab_burnin = 9.388 # funest_burnin = 10.2778 arab_burnin = 10 funest_burnin = 9.4 set_ento(cb, arab_burnin, funest_burnin, arab_times, arab_spline, funest_times, funest_spline) SetupParser.init() SetupParser.set("HPC", "priority", priority) SetupParser.set("HPC", "node_group", coreset) modlists = [] new_modlist = [ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed) for seed in range(num_seeds)] modlists.append(new_modlist) new_modlist = [ModFn(load_burnin_x_temp, x_temp) for x_temp in [2,3,4,6,7,8,9,10]] modlists.append(new_modlist) new_modlist = [ModFn(iver_sweep, intervention_df_dict, "ITN only", "none", -1, -1), ModFn(iver_sweep, intervention_df_dict, "ITN and IRS", "none", -1, -1), ModFn(iver_sweep, intervention_df_dict, "ITN and IRS", "MDA without IVM", 0.2, -1), ModFn(iver_sweep, intervention_df_dict, "ITN and IRS", "MDA with IVM", 0.8, 90)] modlists.append(new_modlist) builder = ModBuilder.from_combos(*modlists) exp_manager = ExperimentManagerFactory.init()
event.Event_Coordinator_Config.Intervention_Config, 'Delay_Period_Mean'): event.Event_Coordinator_Config.Intervention_Config.Delay_Period_Mean = value else: raise Exception('Unknown campaign parameter for setting: %s' % param) return update_values SWEEPS = [ # 143 sims per base config { 'name': 'delay_and_supression', # TODO: [min, max), step 'mods': [ ModFn(update_all_configs, update_values=values) for values in [{ 'Delay_Period_Mean': x, 'ART_Viral_Suppression_Multiplier': y } for x, y in it.product(range(0, 360 + 1, 60), np.arange(0, 0.2 + 0.001, 0.02))] ] }, # 156 sims per base config # cannot exceed 5 for Acute_Duration_In_Months, model limitation { 'name': 'acuteness', 'mods': [ ModFn(update_all_configs,
random.uniform(0.4, 0.99), SIACov[ind], random.randint(1, 1e6), Migration[ind], RuralMultiplier[ind], UrbanMultiplier[ind], MCV1Days[ind], mAbProfile[ind], Dropout[ind], MCV2Days[ind], BirthRateScale[ind], basePop, 3.0 ] else: values = [ 0.4 + 0.59 * math.sqrt(random.uniform(0, 1)), 0.4 + 0.59 * math.sqrt(random.uniform(0, 1)), SIACov[ind], random.randint(1, 1e6), Migration[ind], RuralMultiplier[ind], UrbanMultiplier[ind], MCV1Days[ind], mAbProfile[ind], Dropout[ind], MCV2Days[ind], BirthRateScale[ind], basePop, 3.0 ] mod_fns.append(ModFn(sample_point_fn, names, values)) builder = ModBuilder.from_combos(mod_fns) # Name the experiment exp_name = 'Measles RI targets' exp_manager = ExperimentManagerFactory.from_cb(cb) # suite_id = exp_manager.create_suite("My experiment - Iteration 0") run_sim_args = { 'config_builder': cb, 'exp_name': exp_name, 'exp_builder': builder } exp_manager.experiment_tags = {}
# Let's use a standard DTKConfigBuilder. config_builder = DTKConfigBuilder() # ignore_missing is required when using templates config_builder.ignore_missing = True # Use the default COMPS 2.10 and the SamplesInput folder for input files # See examples\AssetManagement\use_custom_executable.py for more information config_builder.set_exe_collection('EMOD 2.10') # For the experiment builder in the example, we use a ModBuilder from_combos to run # each of the configurations for two separate run numbers. # The first line needs to be the modifier functions coming from the templates experiment_builder = ModBuilder.from_combos( templates.get_modifier_functions(), # <-- Do this first! [ ModFn(DTKConfigBuilder.set_param, 'Run_Number', rn) for rn in range(2, 4) ]) # This dictionary is used by the `dtk run` command line and allow to pass it some parameters # If you are running through `python example_templates.py`, this dictionary is unused run_sim_args = { 'exp_builder': experiment_builder, 'exp_name': 'TemplateDemo', 'config_builder': config_builder } # This part is needed if you are running the script through `python example_templates.py` # It is not used by the command line interface if __name__ == "__main__": # The first thing to do is always to initialize the environment
adult_life_expectacy) set_species_param(cb, 'gambiae', 'Anthropophily', 0.85) set_species_param(cb, 'gambiae', 'Vector_Sugar_Feeding_Frequency', "VECTOR_SUGAR_FEEDING_NONE") set_species_param(cb, 'funestus', "Indoor_Feeding_Fraction", 0.5) set_species_param(cb, 'funestus', 'Adult_Life_Expectancy', adult_life_expectacy) set_species_param(cb, 'funestus', 'Anthropophily', 0.65) set_species_param(cb, 'funestus', 'Vector_Sugar_Feeding_Frequency', "VECTOR_SUGAR_FEEDING_NONE") return {"adult_life_expectancy": adult_life_expectacy} modlist = [ModFn(change_adult_lifespan, ale) for ale in range(20, 50)] modlists.append(modlist) #################### # Reports and logs # #################### add_burnin_reports(cb, include_inset=True) ############################### # Submission/COMPs parameters # ############################### comps_experiment_name = "central_lifespan_sweep" # comps_priority = "Normal" comps_priority = "Normal" comps_coreset = "emod_abcd"
"Decay_Time_Constant": 30 } }] add_ATSB(cb, coverage=coverage, start=100, duration=365, kill_cfg=killing_cfg[1]) return {'coverage': coverage, 'initial_killing': killing} # add_ATSB(cb, start=100, coverage=1, duration=365) builder = ModBuilder.from_list([[ ModFn(atsb_fn, cov, kill), ModFn(DTKConfigBuilder.set_param, 'Run_Number', x) ] for cov in [0, 1] for kill in [0.15] for x in range(1)]) run_sim_args = { 'config_builder': cb, 'exp_name': exp_name, 'exp_builder': builder } if __name__ == "__main__": SetupParser.init('HPC') exp_manager = ExperimentManagerFactory.init() exp_manager.run_simulations(**run_sim_args) # Wait for the simulations to be done exp_manager.wait_for_finished(verbose=True)
cb = DTKConfigBuilder.from_defaults('MALARIA_SIM', Simulation_Duration=int(365 * num_years)) update_config_params(cb, direc=dir, geography=geography) add_insecticides(cb, pyrethroid_killing=0.125, carbamate_killing=0.25) serialized_file_list = ['', ''] cb.update_params({ 'Serialized_Population_Filenames': serialized_file_list }) ########################## VECTOR GENETICS ############################################ three_year_gap = [ [ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed), ModFn(change_vector_params, species='gambiae', mutation_rate1=mutation_rate1, mutation_rate2=mutation_rate2, serialization_seed=0), ModFn(DTKConfigBuilder.set_param, 'Serialized_Population_Path', ''), ModFn(add_ITNs, start=start, insecticide='pyrethroid', label='3year'), ModFn(add_ITNs, start=start+3*365, insecticide='carbamate', label='3year') ] for seed in range(num_seeds) for mutation_rate1 in [pow(10, x) for x in range(-4, -1)] for mutation_rate2 in [pow(10, x) for x in range(-4, -1)] for start in [180 + extra*365] ] no_rotation_three_year = [ [ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed), ModFn(change_vector_params, species='gambiae', mutation_rate1=mutation_rate1,
tuples_list.append([True, True, True, True, False, False]) tuples_list.append([False, True, True, True, True, False]) tuples_list.append([True, False, True, True, True, False]) tuples_list.append([True, True, False, False, True, False]) return tuples_list if __name__ == "__main__": SetupParser.init() SetupParser.set("HPC", "priority", priority) SetupParser.set("HPC", "node_group", coreset) modlists = [] new_modlist = [ ModFn(pickup_from_sample, i) for i in range(len(arab_spline_list) * len(funest_spline_list)) ] modlists.append(new_modlist) # # new_modlist = [ModFn(counterfactuals, itn, irs, mda, msat, chw_hs, chw_rcd) # for itn in [True] # for irs in [True] # for mda in [True] # for msat in [True] # for chw_hs in [False, True] # for chw_rcd in [False, True]] tuples_list = generate_input_variations() new_modlist = [ ModFn(counterfactuals, x[0], x[1], x[2], x[3], x[4], x[5])
def test_param_fn(self): k, v = ('foo', 'bar') fn = ModFn(SimConfigBuilder.set_param, k, v) fn(self.cb) self.assertEqual(self.cb.get_param('foo'), 'bar') self.assertDictEqual(ModBuilder.metadata, dict(foo='bar'))
"a1": 0.0 }, "Mutations": { "a0:a1": 0.0, "a1:a0": 0.0 } }] } set_species_genes(cb, genes) copy_to_likelihoods = np.arange(0.5, 0.59, 0.1).tolist() # Add gene trait modifiers gene_drive = [[ ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed), ModFn(add_release, label='Gene_drive'), ModFn(add_driver, copy_to_likelihood=likelihood), ] for seed in range(num_seeds) for likelihood in copy_to_likelihoods] builder = ModBuilder.from_list(gene_drive) # Call output reporters cb.add_reports( BaseVectorGeneticsReport( type='ReportVectorGenetics', species='arabiensis', gender='VECTOR_FEMALE', include_vector_state_columns=1, stratify_by='SPECIFIC_GENOME', combine_similar_genomes=1,
set_species_param(cb, 'arabiensis', 'Adult_Life_Expectancy', 10) set_species_param(cb, 'arabiensis', 'Transmission_Rate', 0.5) set_species_param(cb, 'arabiensis', 'Indoor_Feeding_Fraction', 1.0) set_species_param(cb, 'funestus', 'Larval_Habitat_Types', {"CONSTANT": 11250000000}) set_species_param(cb, 'funestus', 'Male_Life_Expectancy', 5) set_species_param(cb, 'funestus', 'Adult_Life_Expectancy', 10) set_species_param(cb, 'funestus', 'Transmission_Rate', 0.5) set_species_param(cb, 'funestus', 'Indoor_Feeding_Fraction', 1.0) ########################## VECTOR GENETICS #################################################### add_genes(cb) mendelian = [[ ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed), ModFn(add_release, label=label, number=10000), ] for seed in range(num_seeds) for label in ['No_inrogression']] builder = ModBuilder.from_list(mendelian) # Call output reporters cb.add_reports( BaseVectorGeneticsReport( type='ReportVectorGenetics', species='arabiensis', gender='VECTOR_FEMALE', include_vector_state_columns=1, stratify_by='SPECIFIC_GENOME', combine_similar_genomes=1, specific_genome_combinations_for_stratification=[{
# Add genes genes = {'gambiae': [{ "Alleles": {"a0": 1.0, "a1": 0.0}, "Mutations": {"a1:a0": 0.05} }] } set_species_genes(cb, genes) release_numbers = [1000] net_coverages = [0.8] copy_to_likelihoods = np.arange(1.0, 1.01, 0.1).tolist() release_nodes = [6] transmission_probs = [0.3] start_days = [180] VC_and_GM = [ [ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed), ModFn(DTKConfigBuilder.set_param, 'Serialized_Population_Path', ''), ModFn(add_nets, coverage=net_coverage, number=number, num_nodes=numnodes, start_day=start_day), ModFn(add_release, number=number, num_nodes=numnodes, start_day=start_day), ModFn(add_drivers, copy_to_likelihood=likelihood), ModFn(add_trait_modifiers, transmission_to_human=transmission_prob) ] for seed in range(num_seeds) for start_day in start_days for number in release_numbers for numnodes in release_nodes for net_coverage in net_coverages for likelihood in copy_to_likelihoods for transmission_prob in transmission_probs ]