def test_generic_sweep(self):
def verify(b):
md = [(0.05, 100, 'Namawala'), (0.05, 100, 'Matsari'),
(0.1, 100, 'Namawala'), (0.1, 100, 'Matsari')]
ngenerated = 0
for i, ml in enumerate(b.mod_generator):
for m in ml:
m(self.cb)
mdd = dict(
zip(('x_Temporary_Larval_Habitat', 'Simulation_Duration',
'_site_'), md[i]))
mdd.update({'population_scale': 1})
self.assertEqual(b.metadata, mdd)
self.assertEqual(
self.cb.get_param('x_Temporary_Larval_Habitat'), md[i][0])
ngenerated += 1
self.assertEqual(ngenerated, 4)
b = ModBuilder.from_combos([
ModFn(DTKConfigBuilder.set_param, 'x_Temporary_Larval_Habitat', v)
for v in [0.05, 0.1]
], [ModFn(DTKConfigBuilder.set_param, 'Simulation_Duration', 100)
], [ModFn(configure_site, s) for s in ['Namawala', 'Matsari']])
verify(b)
b = GenericSweepBuilder.from_dict({
'x_Temporary_Larval_Habitat': [0.05, 0.1],
'Simulation_Duration': [100],
'_site_': ['Namawala', 'Matsari']
})
verify(b)
def test_vector_drug_param_sweep(self):
b = ModBuilder.from_combos([
ModFn(set_species_param,
'gambiae',
'Required_Habitat_Factor',
value=v) for v in [(100, 50), (200, 100)]
], [
ModFn(set_drug_param, 'Artemether', 'Max_Drug_IRBC_Kill', value=v)
for v in [4, 2]
])
md = [(4, (100, 50)), (2, (100, 50)), (4, (200, 100)), (2, (200, 100))]
ngenerated = 0
for i, ml in enumerate(b.mod_generator):
for m in ml:
m(self.cb)
self.assertListEqual([v for v in b.metadata.values()], list(md[i]))
self.assertEqual(
get_species_param(self.cb, 'gambiae',
'Required_Habitat_Factor'), md[i][1])
self.assertEqual(
get_drug_param(self.cb, 'Artemether', 'Max_Drug_IRBC_Kill'),
md[i][0])
ngenerated += 1
self.assertEqual(ngenerated, 4)
def set_immune_forcing_builder(transition_matrix=None,
scale_factor_array=[2, 5, 10, 100]):
builder = ModBuilder.from_combos([
ModFn(set_transition_matrix, transition_matrix, scale_factor)
for scale_factor in scale_factor_array
])
return builder
def test_multiple_site_exception(self):
def verify(b):
with self.assertRaises(ValueError):
for ml in b.mod_generator:
pass
b = ModBuilder.from_combos([ModFn(set_calibration_site, 'Matsari')],
[ModFn(configure_site, 'Namawala')])
verify(b)
b = ModBuilder.from_combos([ModFn(configure_site, 'Matsari')],
[ModFn(configure_site, 'Namawala')])
verify(b)
b = ModBuilder.from_combos([ModFn(set_calibration_site, 'Matsari')],
[ModFn(set_calibration_site, 'Namawala')])
verify(b)
def set_immune_forcing_builder(transition_matrix=None,
scale_factor_array=[2, 5, 10, 100]):
"""Creates an experiment builder setting up immunity based on scale factor"""
builder = ModBuilder.from_combos([
ModFn(set_transition_matrix, transition_matrix, scale_factor)
for scale_factor in scale_factor_array
])
return builder
def exp_builder_func(self, next_params, n_replicates=None):
if not n_replicates:
n_replicates = self.sim_runs_per_param_set
return ModBuilder.from_combos(
[ModFn(self.config_builder.__class__.set_param, 'Run_Number', i+1) for i in range(n_replicates)],
[ModFn(site.setup_fn) for site in self.sites],
[ModFn(self.map_sample_to_model_input_fn, index, samples) for index, samples in enumerate(next_params)]
)
# create a config builder for the current experiment, derived from the base simulation input files
config_filepath = os.path.join(simulation_directory, 'config.json')
campaign_filepath = os.path.join(
simulation_directory,
'campaign_Swaziland_v05Sep2018_FINAL_reftracktopup_2011_v2.json')
cb = DTKConfigBuilder.from_files(config_name=config_filepath,
campaign_name=campaign_filepath)
cb.set_input_files_root(
path=simulation_directory
) # discoverability of other input files, e.g. PFA_Overlay.json
exp_name = EXP_NAME_PATTERN % (sweep['name'], i)
# Create a generator of the per-simulation differences from the base simulation
builder = ModBuilder.from_combos(sweep['mods'])
# keep track of each run_sim_args dictionary, one per experiment
run_sets.append({
'exp_name': exp_name,
'exp_builder': builder,
'config_builder': cb,
'base_simulation_id': sim_id,
'sweep_name': sweep['name']
})
# # ck4, for catalyst this block
# run_sim_args = run_sets[1]
# run_sim_args.pop('base_simulation_id')
# run_sim_args.pop('sweep_name')
temp_h = [12.424280532176581]
habitat_scales = it.product(temp_h, const_h)
site = 'Hautkatanga'
demographics_filename = "Hautkatanga_node13.json"
air_temp_filename = "DemocraticRepublicOfTheCongo_30arcsec_air_temperature_daily.bin"
land_temp_filename = "DemocraticRepublicOfTheCongo_30arcsec_air_temperature_daily.bin"
rainfall_filename = "DemocraticRepublicOfTheCongo_30arcsec_rainfall_daily.bin"
relative_humidity_filename = "DemocraticRepublicOfTheCongo_30arcsec_relative_humidity_daily.bin"
exp_name = site + '_baseline_single'
nyears = 50
builder = ModBuilder.from_combos(
[ModFn(set_habs_scale, habs)
for habs in habitat_scales], # set habitat scales
[ModFn(DTKConfigBuilder.set_param, 'Run_Number', r) for r in range(1)])
cb = DTKConfigBuilder.from_defaults('MALARIA_SIM')
cb.update_params({
"Num_Cores": 1,
"Age_Initialization_Distribution_Type": 'DISTRIBUTION_COMPLEX',
"Simulation_Duration": nyears * 365,
"Enable_Demographics_Reporting": 1,
"Enable_Demographics_Builtin": 0,
"Valid_Intervention_States": [],
"New_Diagnostic_Sensitivity": 0.05 # 20/uL
})
# match demographics file for constant population size (with exponential age distribution)
# Let's use a standard DTKConfigBuilder.
config_builder = DTKConfigBuilder()
# ignore_missing is required when using templates
config_builder.ignore_missing = True
# Use the default COMPS 2.10 and the SamplesInput folder for input files
# See examples\AssetManagement\use_custom_executable.py for more information
config_builder.set_exe_collection('EMOD 2.10')
# For the experiment builder in the example, we use a ModBuilder from_combos to run
# each of the configurations for two separate run numbers.
# The first line needs to be the modifier functions coming from the templates
experiment_builder = ModBuilder.from_combos(
templates.get_modifier_functions(), # <-- Do this first!
[
ModFn(DTKConfigBuilder.set_param, 'Run_Number', rn)
for rn in range(2, 4)
])
# This dictionary is used by the `dtk run` command line and allow to pass it some parameters
# If you are running through `python example_templates.py`, this dictionary is unused
run_sim_args = {
'exp_builder': experiment_builder,
'exp_name': 'TemplateDemo',
'config_builder': config_builder
}
# This part is needed if you are running the script through `python example_templates.py`
# It is not used by the command line interface
if __name__ == "__main__":
# The first thing to do is always to initialize the environment
RuralMultiplier[ind], UrbanMultiplier[ind], MCV1Days[ind],
mAbProfile[ind], Dropout[ind], MCV2Days[ind],
BirthRateScale[ind], basePop, 3.0
]
else:
values = [
0.4 + 0.59 * math.sqrt(random.uniform(0, 1)),
0.4 + 0.59 * math.sqrt(random.uniform(0, 1)), SIACov[ind],
random.randint(1, 1e6), Migration[ind],
RuralMultiplier[ind], UrbanMultiplier[ind], MCV1Days[ind],
mAbProfile[ind], Dropout[ind], MCV2Days[ind],
BirthRateScale[ind], basePop, 3.0
]
mod_fns.append(ModFn(sample_point_fn, names, values))
builder = ModBuilder.from_combos(mod_fns)
# Name the experiment
exp_name = 'Measles RI targets'
exp_manager = ExperimentManagerFactory.from_cb(cb)
# suite_id = exp_manager.create_suite("My experiment - Iteration 0")
run_sim_args = {
'config_builder': cb,
'exp_name': exp_name,
'exp_builder': builder
}
exp_manager.experiment_tags = {}
for name, value in zip(names, values):
if name not in [
# Construct the table
table = [list(c.values()) for c in combined]
# Change some things in the config.json
config.set_param('Config_Name', scenario_name)
# Initialize the template
tpl = TemplateHelper()
tpl.set_dynamic_header_table(headers, table)
tpl.active_templates = [
config, campaign_tpl, demog, demog_pfa, demog_asrt, demog_acc
]
# Create an experiment builder
experiment_builder = ModBuilder.from_combos(
tpl.get_modifier_functions())
experiment_manager = ExperimentManagerFactory.from_cb(
config_builder)
COMPS_experiment_name = scenario_name
# COMPS_experiment_name = suite_name # I want hover-over in COMPS to be the suite name
experiment_manager.run_simulations(exp_name=COMPS_experiment_name,
exp_builder=experiment_builder,
suite_id=suite_id)
experiments.append(experiment_manager)
experiments_ids.append(experiment_manager.experiment.exp_id)
# Dump the experiment ids for resume
with open('ids.json', 'w') as out:
json.dump(experiments_ids, out)
def submit_experiment(self,
cb,
num_seeds=1,
intervention_sweep=False,
migration_sweep=False,
vector_migration_sweep=False,
simple_intervention_sweep=False,
custom_name=None):
# Implement the actual (not dummy) baseline healthseeking
self.implement_baseline_healthseeking(cb)
modlists = []
if num_seeds > 1:
new_modlist = [
ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed)
for seed in range(num_seeds)
]
modlists.append(new_modlist)
if migration_sweep:
new_modlist = [
ModFn(DTKConfigBuilder.set_param, 'x_Local_Migration', x)
for x in [0.5, 1, 5, 10]
]
modlists.append(new_modlist)
if vector_migration_sweep:
new_modlist = [
ModFn(self.vector_migration_sweeper, vector_migration_on)
for vector_migration_on in [True, False]
]
modlists.append(new_modlist)
if simple_intervention_sweep:
new_modlist = [
ModFn(self.implement_interventions, True, False, False, False,
False),
ModFn(self.implement_interventions, False, True, False, False,
False),
ModFn(self.implement_interventions, False, False, True, False,
False),
ModFn(self.implement_interventions, False, False, False, True,
False),
ModFn(self.implement_interventions, False, False, False, False,
True),
ModFn(self.implement_interventions, True, True, True, True,
True)
]
modlists.append(new_modlist)
else:
# new_modlist = [ModFn(self.implement_interventions, True, True, True, True, True)]
new_modlist = [
ModFn(self.implement_interventions, True, True, False, False,
False)
]
modlists.append(new_modlist)
builder = ModBuilder.from_combos(*modlists)
run_name = self.exp_name
if custom_name:
run_name = custom_name
# SetupParser.init()
# SetupParser.set("HPC","priority","Normal")
# exp_manager = ExperimentManagerFactory.init()
# exp_manager.run_simulations(config_builder=self.cb, exp_name=run_name, exp_builder=builder)
# return self.cb
exp_manager = ExperimentManagerFactory.init()
exp_manager.run_simulations(config_builder=cb,
exp_name=run_name,
exp_builder=builder)
return cb
prefix = "GarkiNE"
exp_name = prefix + "_pop_rep_3_locations_con_sw_w_seasonality"
num_cores = 24
num_years = 1
builder = ModBuilder.from_combos(
[ModFn(configure_site, site)],
[
ModFn(DTKConfigBuilder.set_param, "x_Vector_Migration_Local", vm)
for vm in [1]
], #, 5*1e-1, 1]],
[ModFn(DTKConfigBuilder.set_param, 'Run_Number', r) for r in range(1, 2)],
# [ModFn(DTKConfigBuilder.set_param, 'HEG_Infection_Modification', im) for im in [0.05]],
# [ModFn(DTKConfigBuilder.set_param, 'HEG_Fecundity_Limiting', fl) for fl in [0.05]],
# [ModFn(DTKConfigBuilder.set_param, 'HEG_Homing_Rate', hr) for hr in [0.95]],
# [ModFn(heg_release, num_released, num_repetitions = 10, released_species = "arabiensis", start_day = d, node_list = range(200, 300, 45)) for (num_released, d) in [(500, 480)]],
#[ModFn(heg_release, num_released, num_repetitions = 52, released_species = "arabiensis", start_day = d, node_list = [1]) for (num_released, d) in [(500, 365)]],
#[ModFn(heg_release, num_released, num_repetitions = 52, released_species = "arabiensis", start_day = d, node_list = range(1, 400, 20)) for (num_released, d) in [(500, 365)]],
)
cb = DTKConfigBuilder.from_defaults('VECTOR_SIM',
Num_Cores=num_cores,
Simulation_Duration=int(365 * num_years))
# set demographics file name
cb.update_params({
'Demographics_Filenames':
'Demographics_Filenames' : ['Malariatherapy_demographics.json']
})
set_climate_constant(cb)
analyzers = [DurationsAnalyzer()
]
# Specify immune parameters --------------------------------------------------------------------------------------
if immunity_forcing_on == 1:
#Pull the naive transition matrix values from the config
transition_matrix = cb.config['parameters']['Parasite_Peak_Density_Probabilities']
#Specify the scaling of the transitio matrix to be applied (representing increasing immune pressure)
scale_factor_array = [2,5,10,100]
builder = ModBuilder.from_combos(
[ModFn(set_transition_matrix, transition_matrix, scale_factor) for scale_factor in scale_factor_array]
)
else:
builder = ''
# Add source of infection (challenge bite or forced EIR) --------------------------------------------------------
add_challenge_trial(cb,start_day=0)
# ---- CUSTOM REPORTS ----
add_patient_report(cb)
# Run args
run_sim_args = {'config_builder': cb,
'exp_name': exp_name,
'exp_builder': builder
}
# +-------------------+----------------+
# | 0 | 0 |
# | 5 | 0 |
# | 10 | 0 |
# | ... | ... |
# | 5 | 1 |
# | 10 | 1 |
# | 365 | 1 |
# | ... | ... |
# | 365 | 4 |
# | 730 | 4 |
# +-------------------+----------------+
builder = ModBuilder.from_combos(
[ModFn(set_larvicides, start_time) for start_time in (0, 5, 10, 365, 730)],
[
ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed)
for seed in range(5)
],
)
# The run_sim_args is a dictionary informing the command line of:
# - What config builder to use
# - What is the name we want for the experiment
# - Which experiment builder to use
run_sim_args = {
'config_builder': cb,
'exp_name': 'Sample larvicides epxeriment',
'exp_builder': builder
}
# In this file we also decided to include the analyzers.
builder = ModBuilder.from_combos(
[
ModFn(DTKConfigBuilder.set_param, 'HEG_Infection_Modification', im)
for im in [0.05]
],
#[ModFn(DTKConfigBuilder.set_param, 'HEG_Fecundity_Limiting', fl) for fl in [0.9,0.3]],
[
ModFn(DTKConfigBuilder.set_param, 'HEG_Fecundity_Limiting', fl)
for fl in [0.1]
],
#[ModFn(DTKConfigBuilder.set_param, 'HEG_Homing_Rate', hr) for hr in [0.95,0.4]],
[
ModFn(DTKConfigBuilder.set_param, 'HEG_Homing_Rate', hr)
for hr in [0.95]
],
[ModFn(set_habs_scale, habs)
for habs in habitat_scales], # set habitat scales
#[ModFn(heg_release, num_released, num_repetitions = 1, node_list = [1535977513], start_day = d) for (num_released, d) in release_combs],
#[ModFn(configure_site, site)],
[
ModFn(heg_release,
num_released,
num_repetitions=1,
node_list=[1535977513],
start_day=d) for (num_released, d) in release_combs
],
[ModFn(configure_site, site)],
#[ModFn(DTKConfigBuilder.set_param, 'Run_Number', r) for r in range(1, 21)]
[ModFn(DTKConfigBuilder.set_param, 'Run_Number', r) for r in range(1, 2)])
from simtools.ModBuilder import ModFn, ModBuilder
from dtk.utils.core.DTKConfigBuilder import DTKConfigBuilder
from malaria.reports.MalariaReport import add_patient_report
from dtk.vector.study_sites import configure_site
from dtk.vector.species import set_species_param
from simtools.SetupParser import SetupParser
SetupParser.default_block = 'HPC'
exp_name = 'DrugCampaignVectorParamSweep'
builder = ModBuilder.from_combos([
ModFn(DTKConfigBuilder.set_param, 'x_Temporary_Larval_Habitat', h)
for h in [0.05]
], [ModFn(configure_site, 'Namawala')], [
ModFn(set_species_param, 'gambiae', 'Required_Habitat_Factor', value=v)
for v in [(100, 50), (200, 100)]
], [
ModFn(set_drug_param, 'Artemether', 'Max_Drug_IRBC_Kill', value=v)
for v in [4, 2]
])
cb = DTKConfigBuilder.from_defaults('MALARIA_SIM')
cb.update_params({
'Num_Cores': 1,
'Base_Population_Scale_Factor': 0.01,
'Simulation_Duration': 30
})
add_drug_campaign(cb, 'DHA', start_days=[10])
add_patient_report(cb)
SetupParser.set("HPC", "priority", priority)
SetupParser.set("HPC", "node_group", coreset)
modlists = []
new_modlist = [
ModFn(pickup_from_sample, i)
for i in range(len(arab_spline_list) * len(funest_spline_list))
]
modlists.append(new_modlist)
#
# new_modlist = [ModFn(counterfactuals, itn, irs, mda, msat, chw_hs, chw_rcd)
# for itn in [True]
# for irs in [True]
# for mda in [True]
# for msat in [True]
# for chw_hs in [False, True]
# for chw_rcd in [False, True]]
tuples_list = generate_input_variations()
new_modlist = [
ModFn(counterfactuals, x[0], x[1], x[2], x[3], x[4], x[5])
for x in tuples_list
]
modlists.append(new_modlist)
builder = ModBuilder.from_combos(*modlists)
exp_manager = ExperimentManagerFactory.init()
exp_manager.run_simulations(config_builder=cb,
exp_name=experiment_name,
exp_builder=builder)
exp_name = site + '_95%ITN-test'
nyears = 15
baseline_year = 0
'''
start_days = [(365 * baseline_year) + 1]
num_released = [200] # released mosquitoes
release_combs = it.product(num_released, start_days) # NOTE: this generates 656 pairs; coupled with 20 random seeds below results in 13120 sims; beware if running on a local machine...
'''
builder = ModBuilder.from_combos(
#[ModFn(DTKConfigBuilder.set_param, 'HEG_Infection_Modification', im) for im in [1.0]],
#[ModFn(DTKConfigBuilder.set_param, 'HEG_Fecundity_Limiting', fl) for fl in [0.05, 0.1, 0.15]],
#[ModFn(DTKConfigBuilder.set_param, 'HEG_Homing_Rate', hr) for hr in [0.9, 0.95, 1.0]],
#[ModFn(heg_release, num_released, d) for (num_released, d) in release_combs],
[ModFn(set_habs_scale, habs)
for habs in habitat_scales], # set habitat scales
[ModFn(DTKConfigBuilder.set_param, 'Run_Number', r) for r in range(1)])
cb = DTKConfigBuilder.from_defaults('MALARIA_SIM')
for year in range(0, nyears, 3):
add_ITN_age_season(cb,
start=(365 * (baseline_year + year)) + 1,
coverage_all=0.95,
seasonal_dep={
'times': [0, 365],
'values': [0.65, 0.65]
})
'''