def test_vector_drug_param_sweep(self):
b = ModBuilder.from_combos([
ModFn(set_species_param,
'gambiae',
'Required_Habitat_Factor',
value=v) for v in [(100, 50), (200, 100)]
], [
ModFn(set_drug_param, 'Artemether', 'Max_Drug_IRBC_Kill', value=v)
for v in [4, 2]
])
md = [(4, (100, 50)), (2, (100, 50)), (4, (200, 100)), (2, (200, 100))]
ngenerated = 0
for i, ml in enumerate(b.mod_generator):
for m in ml:
m(self.cb)
self.assertListEqual([v for v in b.metadata.values()], list(md[i]))
self.assertEqual(
get_species_param(self.cb, 'gambiae',
'Required_Habitat_Factor'), md[i][1])
self.assertEqual(
get_drug_param(self.cb, 'Artemether', 'Max_Drug_IRBC_Kill'),
md[i][0])
ngenerated += 1
self.assertEqual(ngenerated, 4)
def test_generic_sweep(self):
def verify(b):
md = [(0.05, 100, 'Namawala'), (0.05, 100, 'Matsari'),
(0.1, 100, 'Namawala'), (0.1, 100, 'Matsari')]
ngenerated = 0
for i, ml in enumerate(b.mod_generator):
for m in ml:
m(self.cb)
mdd = dict(
zip(('x_Temporary_Larval_Habitat', 'Simulation_Duration',
'_site_'), md[i]))
mdd.update({'population_scale': 1})
self.assertEqual(b.metadata, mdd)
self.assertEqual(
self.cb.get_param('x_Temporary_Larval_Habitat'), md[i][0])
ngenerated += 1
self.assertEqual(ngenerated, 4)
b = ModBuilder.from_combos([
ModFn(DTKConfigBuilder.set_param, 'x_Temporary_Larval_Habitat', v)
for v in [0.05, 0.1]
], [ModFn(DTKConfigBuilder.set_param, 'Simulation_Duration', 100)
], [ModFn(configure_site, s) for s in ['Namawala', 'Matsari']])
verify(b)
b = GenericSweepBuilder.from_dict({
'x_Temporary_Larval_Habitat': [0.05, 0.1],
'Simulation_Duration': [100],
'_site_': ['Namawala', 'Matsari']
})
verify(b)
def exp_builder_func(self, next_params, n_replicates=None):
if not n_replicates:
n_replicates = self.sim_runs_per_param_set
return ModBuilder.from_combos(
[ModFn(self.config_builder.__class__.set_param, 'Run_Number', i+1) for i in range(n_replicates)],
[ModFn(site.setup_fn) for site in self.sites],
[ModFn(self.map_sample_to_model_input_fn, index, samples) for index, samples in enumerate(next_params)]
)
def __init__(self, nsims):
self.tags = {}
self.mod_generator = (
self.set_mods(
[ModFn(DTKConfigBuilder.set_param, 'Run_Number', i)]
) for i in range(nsims)
)
def set_immune_forcing_builder(transition_matrix=None,
scale_factor_array=[2, 5, 10, 100]):
builder = ModBuilder.from_combos([
ModFn(set_transition_matrix, transition_matrix, scale_factor)
for scale_factor in scale_factor_array
])
return builder
def set_immune_forcing_builder(transition_matrix=None,
scale_factor_array=[2, 5, 10, 100]):
"""Creates an experiment builder setting up immunity based on scale factor"""
builder = ModBuilder.from_combos([
ModFn(set_transition_matrix, transition_matrix, scale_factor)
for scale_factor in scale_factor_array
])
return builder
def get_modifier_functions(self):
"""
Returns a ModBuilder ModFn that sets file contents and values in config builder according to the dynamic parameters.
"""
return [
ModFn(self.mod_dynamic_parameters, dict(zip(self.header, row)))
for row in self.table
]
def test_param_fn(self):
k, v = ('Simulation_Duration', 100)
fn = ModFn(DTKConfigBuilder.set_param, k, v)
fn(self.cb)
logging.debug('Parameter %s has value %s set from %s', k,
self.cb.get_param(k), v)
logging.debug('ModBuilder metadata: %s', ModBuilder.metadata)
self.assertEqual(self.cb.get_param(k), v)
self.assertEqual(ModBuilder.metadata, {k: v})
def modfn_sweep_over_burnins(archetype):
burnin_df = pd.read_csv("burnins.csv")
burnin_df = burnin_df[burnin_df["archetype"] == archetype]
modlist = []
for i, row in burnin_df.iterrows():
modlist.append(ModFn(draw_from_serialized_file, dict(row)))
return modlist
def test_calibsite_fn(self):
from calibtool.study_sites.NamawalaCalibSite import NamawalaCalibSite
s = NamawalaCalibSite()
fn = ModFn(set_calibration_site, s)
fn(self.cb)
self.assertEqual(
self.cb.campaign['Events'][0]['Event_Coordinator_Config']
['Intervention_Config']['class'], 'InputEIR')
self.assertEqual(self.cb.custom_reports[0].type,
'MalariaSummaryReport')
self.assertEqual(ModBuilder.metadata, {'_site_': 'Namawala'})
def test_custom_fn(self):
v = [100, 50]
fn = ModFn(set_species_param,
'gambiae',
'Required_Habitat_Factor',
value=v)
fn(self.cb)
self.assertListEqual(
get_species_param(self.cb, 'gambiae', 'Required_Habitat_Factor'),
v)
self.assertEqual(ModBuilder.metadata,
{'gambiae.Required_Habitat_Factor': v})
def test_custom_fn(self):
v = [100, 50]
self.cb.set_param('nested', {'foo': {'bar': [0, 0]}})
def custom_fn(cb, foo, bar, value):
cb.config['nested'][foo][bar] = value
return {'.'.join([foo, bar]): value}
fn = ModFn(custom_fn, 'foo', 'bar', value=v)
fn(self.cb)
self.assertListEqual(self.cb.get_param('nested')['foo']['bar'], v)
self.assertEqual(ModBuilder.metadata, {'foo.bar': v})
def test_site_fn(self):
s = 'Namawala'
fn = ModFn(configure_site, s)
fn(self.cb)
logging.debug('Demographics_Filenames: %s',
self.cb.get_param('Demographics_Filenames'))
logging.debug('ModBuilder metadata: %s', ModBuilder.metadata)
self.assertTrue(
'Namawala' in self.cb.get_param('Demographics_Filenames')[0])
self.assertEqual(ModBuilder.metadata, {
'_site_': 'Namawala',
'population_scale': 1
})
def modfn_sweep_over_scenarios(archetype, specific_scenarios_to_run=None):
scenario_df = pd.read_csv("scenario_master_list.csv")
scenario_df = scenario_df[scenario_df["archetype"] ==
archetype].reset_index(drop=True)
if specific_scenarios_to_run == None:
scenario_numbers = list(scenario_df["scenario_number"])
else:
scenario_numbers = specific_scenarios_to_run
# scenario_numbers = scenario_numbers[:2] #TESTING ONLY
modlist = [
ModFn(add_scenario_specific_interventions, ns, archetype)
for ns in scenario_numbers
]
return modlist
def modfn_sweep_over_timings(archetype, num_years=1):
if archetype == "Southern":
timings_df = pd.read_csv("southern_term_sweep_scenarios.csv")
elif archetype == "Sahel":
timings_df = pd.read_csv("sahel_term_sweep_scenarios.csv")
elif archetype == "Central":
timings_df = pd.read_csv("central_term_sweep_scenarios.csv")
else:
raise NotImplementedError
scenario_numbers = list(timings_df["scenario_number"])
# scenario_numbers = range(5) #TEST ONLY
modlist = [
ModFn(add_ipt_for_timing_sweep, ns, archetype, num_years)
for ns in scenario_numbers
]
return modlist
def test_multiple_site_exception(self):
def verify(b):
with self.assertRaises(ValueError):
for ml in b.mod_generator:
pass
b = ModBuilder.from_combos([ModFn(set_calibration_site, 'Matsari')],
[ModFn(configure_site, 'Namawala')])
verify(b)
b = ModBuilder.from_combos([ModFn(configure_site, 'Matsari')],
[ModFn(configure_site, 'Namawala')])
verify(b)
b = ModBuilder.from_combos([ModFn(set_calibration_site, 'Matsari')],
[ModFn(set_calibration_site, 'Namawala')])
verify(b)
# +-------------------+----------------+
# | larvicide_start | Run_Number |
# +-------------------+----------------+
# | 0 | 0 |
# | 5 | 0 |
# | 10 | 0 |
# | ... | ... |
# | 5 | 1 |
# | 10 | 1 |
# | 365 | 1 |
# | ... | ... |
# | 365 | 4 |
# | 730 | 4 |
# +-------------------+----------------+
builder = ModBuilder.from_combos(
[ModFn(set_larvicides, start_time) for start_time in (0, 5, 10, 365, 730)],
[
ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed)
for seed in range(5)
],
)
# The run_sim_args is a dictionary informing the command line of:
# - What config builder to use
# - What is the name we want for the experiment
# - Which experiment builder to use
run_sim_args = {
'config_builder': cb,
'exp_name': 'Sample larvicides epxeriment',
'exp_builder': builder
}
sim_duration = 365 * 1
num_seeds = 2
SetupParser.init('LOCAL')
cb = DTKConfigBuilder.from_defaults('MALARIA_SIM')
intervention_days = [180]
coverages = [x for x in np.arange(0.0, 1.01, 0.1)]
target_groups_IVM = ['Excludingwomen']
expanded_SMC_groups_IVM = ['ExpandedSMCexcludingwomenonly']
# DP + Ivermectin (expanded and regular)
SMC_IVM = [[
ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed),
ModFn(DTKConfigBuilder.set_param, 'Simulation_Duration',
smc_start_day + 365),
ModFn(
add_smc_group,
start_days=[smc_start_day],
coverage=smc_coverage,
drug=drug,
agemax=agemax,
),
ModFn(add_ivermectin_group,
coverage=smc_coverage,
start_days=[smc_start_day],
drug_code=drug_code,
smc_code=drug + str(agemax),
target_group=target_group),
# arab_burnin = 9.388
# funest_burnin = 10.2778
arab_burnin = 10
funest_burnin = 9.4
set_ento(cb, arab_burnin, funest_burnin, arab_times, arab_spline, funest_times, funest_spline)
SetupParser.init()
SetupParser.set("HPC", "priority", priority)
SetupParser.set("HPC", "node_group", coreset)
modlists = []
new_modlist = [ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed) for seed in range(num_seeds)]
modlists.append(new_modlist)
new_modlist = [ModFn(load_burnin_x_temp, x_temp) for x_temp in [2,3,4,6,7,8,9,10]]
modlists.append(new_modlist)
new_modlist = [ModFn(iver_sweep, intervention_df_dict, "ITN only", "none", -1, -1),
ModFn(iver_sweep, intervention_df_dict, "ITN and IRS", "none", -1, -1),
ModFn(iver_sweep, intervention_df_dict, "ITN and IRS", "MDA without IVM", 0.2, -1),
ModFn(iver_sweep, intervention_df_dict, "ITN and IRS", "MDA with IVM", 0.8, 90)]
modlists.append(new_modlist)
builder = ModBuilder.from_combos(*modlists)
exp_manager = ExperimentManagerFactory.init()
event.Event_Coordinator_Config.Intervention_Config,
'Delay_Period_Mean'):
event.Event_Coordinator_Config.Intervention_Config.Delay_Period_Mean = value
else:
raise Exception('Unknown campaign parameter for setting: %s' %
param)
return update_values
SWEEPS = [
# 143 sims per base config
{
'name':
'delay_and_supression', # TODO: [min, max), step
'mods': [
ModFn(update_all_configs, update_values=values)
for values in [{
'Delay_Period_Mean': x,
'ART_Viral_Suppression_Multiplier': y
} for x, y in it.product(range(0, 360 + 1, 60),
np.arange(0, 0.2 + 0.001, 0.02))]
]
},
# 156 sims per base config
# cannot exceed 5 for Acute_Duration_In_Months, model limitation
{
'name':
'acuteness',
'mods': [
ModFn(update_all_configs,
random.uniform(0.4, 0.99), SIACov[ind],
random.randint(1, 1e6), Migration[ind],
RuralMultiplier[ind], UrbanMultiplier[ind], MCV1Days[ind],
mAbProfile[ind], Dropout[ind], MCV2Days[ind],
BirthRateScale[ind], basePop, 3.0
]
else:
values = [
0.4 + 0.59 * math.sqrt(random.uniform(0, 1)),
0.4 + 0.59 * math.sqrt(random.uniform(0, 1)), SIACov[ind],
random.randint(1, 1e6), Migration[ind],
RuralMultiplier[ind], UrbanMultiplier[ind], MCV1Days[ind],
mAbProfile[ind], Dropout[ind], MCV2Days[ind],
BirthRateScale[ind], basePop, 3.0
]
mod_fns.append(ModFn(sample_point_fn, names, values))
builder = ModBuilder.from_combos(mod_fns)
# Name the experiment
exp_name = 'Measles RI targets'
exp_manager = ExperimentManagerFactory.from_cb(cb)
# suite_id = exp_manager.create_suite("My experiment - Iteration 0")
run_sim_args = {
'config_builder': cb,
'exp_name': exp_name,
'exp_builder': builder
}
exp_manager.experiment_tags = {}
# Let's use a standard DTKConfigBuilder.
config_builder = DTKConfigBuilder()
# ignore_missing is required when using templates
config_builder.ignore_missing = True
# Use the default COMPS 2.10 and the SamplesInput folder for input files
# See examples\AssetManagement\use_custom_executable.py for more information
config_builder.set_exe_collection('EMOD 2.10')
# For the experiment builder in the example, we use a ModBuilder from_combos to run
# each of the configurations for two separate run numbers.
# The first line needs to be the modifier functions coming from the templates
experiment_builder = ModBuilder.from_combos(
templates.get_modifier_functions(), # <-- Do this first!
[
ModFn(DTKConfigBuilder.set_param, 'Run_Number', rn)
for rn in range(2, 4)
])
# This dictionary is used by the `dtk run` command line and allow to pass it some parameters
# If you are running through `python example_templates.py`, this dictionary is unused
run_sim_args = {
'exp_builder': experiment_builder,
'exp_name': 'TemplateDemo',
'config_builder': config_builder
}
# This part is needed if you are running the script through `python example_templates.py`
# It is not used by the command line interface
if __name__ == "__main__":
# The first thing to do is always to initialize the environment
adult_life_expectacy)
set_species_param(cb, 'gambiae', 'Anthropophily', 0.85)
set_species_param(cb, 'gambiae', 'Vector_Sugar_Feeding_Frequency',
"VECTOR_SUGAR_FEEDING_NONE")
set_species_param(cb, 'funestus', "Indoor_Feeding_Fraction", 0.5)
set_species_param(cb, 'funestus', 'Adult_Life_Expectancy',
adult_life_expectacy)
set_species_param(cb, 'funestus', 'Anthropophily', 0.65)
set_species_param(cb, 'funestus', 'Vector_Sugar_Feeding_Frequency',
"VECTOR_SUGAR_FEEDING_NONE")
return {"adult_life_expectancy": adult_life_expectacy}
modlist = [ModFn(change_adult_lifespan, ale) for ale in range(20, 50)]
modlists.append(modlist)
####################
# Reports and logs #
####################
add_burnin_reports(cb, include_inset=True)
###############################
# Submission/COMPs parameters #
###############################
comps_experiment_name = "central_lifespan_sweep"
# comps_priority = "Normal"
comps_priority = "Normal"
comps_coreset = "emod_abcd"
"Decay_Time_Constant": 30
}
}]
add_ATSB(cb,
coverage=coverage,
start=100,
duration=365,
kill_cfg=killing_cfg[1])
return {'coverage': coverage, 'initial_killing': killing}
# add_ATSB(cb, start=100, coverage=1, duration=365)
builder = ModBuilder.from_list([[
ModFn(atsb_fn, cov, kill),
ModFn(DTKConfigBuilder.set_param, 'Run_Number', x)
] for cov in [0, 1] for kill in [0.15] for x in range(1)])
run_sim_args = {
'config_builder': cb,
'exp_name': exp_name,
'exp_builder': builder
}
if __name__ == "__main__":
SetupParser.init('HPC')
exp_manager = ExperimentManagerFactory.init()
exp_manager.run_simulations(**run_sim_args)
# Wait for the simulations to be done
exp_manager.wait_for_finished(verbose=True)
cb = DTKConfigBuilder.from_defaults('MALARIA_SIM',
Simulation_Duration=int(365 * num_years))
update_config_params(cb, direc=dir, geography=geography)
add_insecticides(cb, pyrethroid_killing=0.125, carbamate_killing=0.25)
serialized_file_list = ['', '']
cb.update_params({
'Serialized_Population_Filenames': serialized_file_list
})
########################## VECTOR GENETICS ############################################
three_year_gap = [
[ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed),
ModFn(change_vector_params, species='gambiae', mutation_rate1=mutation_rate1,
mutation_rate2=mutation_rate2, serialization_seed=0),
ModFn(DTKConfigBuilder.set_param, 'Serialized_Population_Path', ''),
ModFn(add_ITNs, start=start, insecticide='pyrethroid', label='3year'),
ModFn(add_ITNs, start=start+3*365, insecticide='carbamate', label='3year')
]
for seed in range(num_seeds)
for mutation_rate1 in [pow(10, x) for x in range(-4, -1)]
for mutation_rate2 in [pow(10, x) for x in range(-4, -1)]
for start in [180 + extra*365]
]
no_rotation_three_year = [
[ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed),
ModFn(change_vector_params, species='gambiae', mutation_rate1=mutation_rate1,
tuples_list.append([True, True, True, True, False, False])
tuples_list.append([False, True, True, True, True, False])
tuples_list.append([True, False, True, True, True, False])
tuples_list.append([True, True, False, False, True, False])
return tuples_list
if __name__ == "__main__":
SetupParser.init()
SetupParser.set("HPC", "priority", priority)
SetupParser.set("HPC", "node_group", coreset)
modlists = []
new_modlist = [
ModFn(pickup_from_sample, i)
for i in range(len(arab_spline_list) * len(funest_spline_list))
]
modlists.append(new_modlist)
#
# new_modlist = [ModFn(counterfactuals, itn, irs, mda, msat, chw_hs, chw_rcd)
# for itn in [True]
# for irs in [True]
# for mda in [True]
# for msat in [True]
# for chw_hs in [False, True]
# for chw_rcd in [False, True]]
tuples_list = generate_input_variations()
new_modlist = [
ModFn(counterfactuals, x[0], x[1], x[2], x[3], x[4], x[5])
def test_param_fn(self):
k, v = ('foo', 'bar')
fn = ModFn(SimConfigBuilder.set_param, k, v)
fn(self.cb)
self.assertEqual(self.cb.get_param('foo'), 'bar')
self.assertDictEqual(ModBuilder.metadata, dict(foo='bar'))
"a1": 0.0
},
"Mutations": {
"a0:a1": 0.0,
"a1:a0": 0.0
}
}]
}
set_species_genes(cb, genes)
copy_to_likelihoods = np.arange(0.5, 0.59, 0.1).tolist()
# Add gene trait modifiers
gene_drive = [[
ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed),
ModFn(add_release, label='Gene_drive'),
ModFn(add_driver, copy_to_likelihood=likelihood),
] for seed in range(num_seeds) for likelihood in copy_to_likelihoods]
builder = ModBuilder.from_list(gene_drive)
# Call output reporters
cb.add_reports(
BaseVectorGeneticsReport(
type='ReportVectorGenetics',
species='arabiensis',
gender='VECTOR_FEMALE',
include_vector_state_columns=1,
stratify_by='SPECIFIC_GENOME',
combine_similar_genomes=1,
set_species_param(cb, 'arabiensis', 'Adult_Life_Expectancy', 10)
set_species_param(cb, 'arabiensis', 'Transmission_Rate', 0.5)
set_species_param(cb, 'arabiensis', 'Indoor_Feeding_Fraction', 1.0)
set_species_param(cb, 'funestus', 'Larval_Habitat_Types',
{"CONSTANT": 11250000000})
set_species_param(cb, 'funestus', 'Male_Life_Expectancy', 5)
set_species_param(cb, 'funestus', 'Adult_Life_Expectancy', 10)
set_species_param(cb, 'funestus', 'Transmission_Rate', 0.5)
set_species_param(cb, 'funestus', 'Indoor_Feeding_Fraction', 1.0)
########################## VECTOR GENETICS ####################################################
add_genes(cb)
mendelian = [[
ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed),
ModFn(add_release, label=label, number=10000),
] for seed in range(num_seeds) for label in ['No_inrogression']]
builder = ModBuilder.from_list(mendelian)
# Call output reporters
cb.add_reports(
BaseVectorGeneticsReport(
type='ReportVectorGenetics',
species='arabiensis',
gender='VECTOR_FEMALE',
include_vector_state_columns=1,
stratify_by='SPECIFIC_GENOME',
combine_similar_genomes=1,
specific_genome_combinations_for_stratification=[{
# Add genes
genes = {'gambiae': [{
"Alleles": {"a0": 1.0, "a1": 0.0},
"Mutations": {"a1:a0": 0.05}
}]
}
set_species_genes(cb, genes)
release_numbers = [1000]
net_coverages = [0.8]
copy_to_likelihoods = np.arange(1.0, 1.01, 0.1).tolist()
release_nodes = [6]
transmission_probs = [0.3]
start_days = [180]
VC_and_GM = [
[ModFn(DTKConfigBuilder.set_param, 'Run_Number', seed),
ModFn(DTKConfigBuilder.set_param, 'Serialized_Population_Path', ''),
ModFn(add_nets, coverage=net_coverage, number=number, num_nodes=numnodes, start_day=start_day),
ModFn(add_release, number=number, num_nodes=numnodes, start_day=start_day),
ModFn(add_drivers, copy_to_likelihood=likelihood),
ModFn(add_trait_modifiers, transmission_to_human=transmission_prob)
]
for seed in range(num_seeds)
for start_day in start_days
for number in release_numbers
for numnodes in release_nodes
for net_coverage in net_coverages
for likelihood in copy_to_likelihoods
for transmission_prob in transmission_probs
]
