def test_has_option(self):
filename = os.path.join(self.input_path, SetupParser.ini_filename)
SetupParser.init(selected_block='TEST',
setup_file=filename,
is_testing=True)
self.assertTrue(SetupParser.has_option('environment'))
self.assertFalse(SetupParser.has_option('notanoption'))
def initialize_SetupParser_from_args(args, unknownArgs):
# determine the selected environment
if hasattr(args, 'block'):
selected_block = args.block # we should use this going forward (add flags in commands_args.py)
else:
# current, back-compatible route. This should go away once ini block selection is done via a known flag
# in all commands
if len(unknownArgs) == 1:
import re
arg = unknownArgs[0]
block_regex = re.compile('^--(?P<block>\w+)$')
if block_regex.search(arg):
selected_block = block_regex.search(arg).group('block')
else:
selected_block = None # we use the default block selection in SetupParser
elif len(unknownArgs) > 1:
raise Exception(
"Only zero or one unknown args are allowed for simtools.ini block selection."
)
else: # len(unknownArgs) == 0, use SetupParser default block selection
selected_block = None
# some HPC environment overrides that may have been provided on the command line
overrides = grab_HPC_overrides(args)
#if hasattr(args, 'ini'):
# provided_ini = args.ini
#else:
# provided_ini = None
# If a config module (experiment .py file) was provided, it MAY have set some SetupParser defaults, so we load
# it first.
load_config(args)
SetupParser.init(selected_block=selected_block, overrides=overrides)
def test_items(self):
filename = os.path.join(self.input_path, SetupParser.ini_filename)
SetupParser.init(selected_block='TEST',
setup_file=filename,
is_testing=True)
self.assertEqual(len(SetupParser.items()), 24)
self.assertEqual(len(SetupParser.items(block='HPC')), 20)
def test_verify_uninit(self):
self.assertFalse(SetupParser.initialized)
SetupParser.init()
self.assertTrue(SetupParser.initialized)
SetupParser._uninit()
self.assertFalse(SetupParser.initialized)
self.assertEqual(SetupParser.singleton, None)
def setUp(self):
self.cb = SimConfigBuilder.from_defaults('DUMMY')
SetupParser.init(selected_block='LOCAL')
self.input_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'input')
self.dummy_exe_folder = os.path.join(self.input_path,
'dummy_exe_folder')
self.another_dummy_exe_folder = os.path.join(
self.input_path, 'another_dummy_exe_folder')
def run(self):
SetupParser.init(singleton=self.setup_parser_singleton)
try:
self.process()
except Exception as e:
print("Exception during commission")
import traceback
traceback.print_exc()
exit()
def test_regression_968(self):
"""
Investigate a difference of behavior between UNIX systems and Windows
Test for https://github.com/InstituteforDiseaseModeling/dtk-tools/issues/968
"""
SetupParser.init()
singleton = SetupParser.singleton
SetupParser.init(singleton=singleton)
SetupParser._uninit()
def test_no_overlay_default_block(self):
SetupParser.init()
self.assertEqual(SetupParser.selected_block, 'LOCAL')
self.assertEqual(SetupParser.get('type'), 'LOCAL')
# User should be there
self.assertIsNotNone(SetupParser.get('user'))
# We dont want HPC or unknown params
self.assertRaises(ValueError, SetupParser.get, 'use_comps_asset_svc')
self.assertRaises(ValueError, SetupParser.get, 'WRONG')
def test_regression_965(self):
"""
Allow substitutions in simtools.ini
Test for https://github.com/InstituteforDiseaseModeling/dtk-tools/issues/965
"""
SetupParser.init(selected_block='LOCAL',
setup_file=os.path.join(self.input_path, '965',
'simtools.ini'),
is_testing=True)
self.assertTrue(SetupParser.get('block'), 'PATH1')
SetupParser._uninit()
def test_explicit_overrides(self):
filename = os.path.join(self.input_path, SetupParser.ini_filename)
overrides = {'in_all': 'yellow'}
SetupParser.init(selected_block='TEST',
setup_file=filename,
overrides=overrides,
is_testing=True)
self.assertEqual(SetupParser.get('in_all'), overrides['in_all'])
# verify this works even after block change
SetupParser.override_block(block='LOCAL2')
self.assertEqual(SetupParser.get('in_all'), overrides['in_all'])
def test_verify_override_block(self):
"""
1) set the SetupParser once to get the expected value, unset it, then 2) load a
different selected block & then override it to the first. Should get same answer
both routes.
"""
SetupParser.init(selected_block='LOCAL', is_testing=True)
expected_params = dict(SetupParser.items(SetupParser.selected_block))
SetupParser._uninit()
SetupParser.init(selected_block='HPC', is_testing=True)
SetupParser.override_block('LOCAL')
override_params = dict(SetupParser.items(SetupParser.selected_block))
self.assertEqual(override_params, expected_params)
def run_single_analyzer(exp_id, analyzer, savefile_prefix=""):
SetupParser.default_block = 'HPC'
SetupParser.init()
am = AnalyzeManager()
am.add_analyzer(analyzer())
exp = retrieve_experiment(exp_id)
am.add_experiment(exp)
am.analyze()
df_return = am.analyzers[0].results
df_return.to_csv("{}_{}.csv".format(savefile_prefix, exp_id), index=False)
return df_return
def test_init_with_singleton_provided(self):
"""
This is covering the case where we call (in a new process): SetupParser(singleton=some_object)
:return:
"""
attr = 'johns_favorite_color' # an attribute NOT in the ini file to make sure we keep any set values
value = 'purple'
SetupParser('LOCAL')
SetupParser.singleton.setup.set('LOCAL', attr, value)
singleton = SetupParser.singleton
SetupParser._uninit()
SetupParser.init(singleton=singleton)
self.assertEqual(SetupParser.get(attr), value)
def test_get_boolean(self):
filename = os.path.join(self.input_path, SetupParser.ini_filename)
SetupParser.init(selected_block='TEST',
setup_file=filename,
is_testing=True)
# assertIs checks if the items evaluate to the same object (not truthy comparison)
self.assertIs(SetupParser.getboolean('this_is_true'), True)
self.assertIs(SetupParser.getboolean('this_is_false'), False)
kwargs = {'parameter': 'notaparameter'}
self.assertRaises(ValueError, SetupParser.getboolean,
**kwargs) # not present in TEST
kwargs = {'parameter': 'environment'}
self.assertRaises(ValueError, SetupParser.getboolean,
**kwargs) # not a boolean, but is present in TEST.
def setUp(self):
current_dir = os.path.dirname(os.path.realpath(__file__))
input_dir = os.path.join(current_dir, 'input')
self.am_simtools = os.path.join(input_dir, 'am_simtools.ini')
# Adjust path sensitive parameters in the ini file
cp = ConfigParser()
cp.read(self.am_simtools)
cp.set('DEFAULT', 'path', input_dir)
cp.set('DEFAULT', 'example_dir',
os.path.join(current_dir, '..', '..', 'examples'))
cp.write(open(self.am_simtools, 'w'))
SetupParser.init(selected_block=self.SELECTED_BLOCK,
setup_file=self.am_simtools)
self.config_builder = DTKConfigBuilder.from_defaults('VECTOR_SIM')
configure_site(self.config_builder, 'Namawala')
self.config_builder.add_reports(BaseReport(type="VectorHabitatReport"))
def test_get(self):
filename = os.path.join(self.input_path, SetupParser.ini_filename)
SetupParser.init(selected_block='TEST',
setup_file=filename,
is_testing=True)
default_value = 'abcdefg'
# param present and default value not used
kwargs = {'parameter': 'made_up_param', 'default': default_value}
self.assertEqual(SetupParser.get(**kwargs), '1')
# param not present and no default provided
kwargs = {'parameter': 'notaparameter'}
self.assertRaises(ValueError, SetupParser.get, **kwargs)
# default value provided and used
kwargs = {'parameter': 'notaparameter', 'default': default_value}
test_value = SetupParser.get(**kwargs)
self.assertEqual(test_value, default_value)
def run_experiment(configbuilder, experiment_name, experiment_builder,
analyzers):
run_sim_args = {
'config_builder': configbuilder,
'exp_name': experiment_name,
'exp_builder': experiment_builder
}
if not SetupParser.initialized:
SetupParser.init('HPC')
exp_manager = ExperimentManagerFactory.init()
exp_manager.run_simulations(**run_sim_args)
exp_manager.wait_for_finished(verbose=True)
assert (exp_manager.succeeded())
am = AnalyzeManager(exp_manager.experiment)
for a in analyzers:
am.add_analyzer(a)
am.analyze()
def test_overlay_file_values(self):
filename = os.path.join(self.input_path, SetupParser.ini_filename)
SetupParser.init(selected_block='TEST',
setup_file=filename,
is_testing=True)
# verify the selected block is correct
self.assertEqual(SetupParser.selected_block, 'TEST')
# data overlay - param in TEST, no conflict with DEFAULT or default ini (TEST wins)
self.assertEqual(SetupParser.get('made_up_param'), '1')
# data overlay - param in TEST, conflict with DEFAULT and default ini (TEST wins)
self.assertEqual(SetupParser.get('environment'), 'TestEnv')
# data overlay - DEFAULT conflict with default ini (DEFAULT wins)
self.assertEqual(SetupParser.get('max_threads'), '0')
# data overlay - only in default ini DEFAULT, (default ini DEFAULT wins, obviously!)
self.assertIsNotNone(SetupParser.get('exe_path'))
def setUp(self):
SetupParser._uninit()
current_dir = os.path.dirname(os.path.realpath(__file__))
SetupParser.init(selected_block=self.SELECTED_BLOCK,
setup_file=os.path.join(current_dir, 'input',
'am_simtools.ini'))
COMPS_login(SetupParser.get('server_endpoint'))
self.existing_collection = AssetCollection(
base_collection=get_asset_collection(self.EXISTING_COLLECTION_ID))
self.existing_collection.prepare(location='HPC')
self.existing_COMPS_asset_files = self.existing_collection.asset_files_to_use
# a FileList object
dir = os.path.join(self.INPUT_DIR, 'files')
files = [os.path.join(dir, f) for f in os.listdir(dir)]
root = os.path.dirname(os.path.dirname(files[0]))
files = [
os.path.join(
os.path.split(os.path.dirname(f))[1], os.path.basename(f))
for f in files
]
self.local_files = FileList(root=root, files_in_root=files)
# take one away
files = [
os.path.join(f.relative_path, f.file_name)
for f in self.local_files.files
if f.file_name != 'file1_REMOTE_ONLY'
]
# add some more
dir = os.path.join(self.INPUT_DIR, 'additional_files')
additional_files = [os.path.join(dir, f) for f in os.listdir(dir)]
additional_files = [
os.path.join(
os.path.split(os.path.dirname(f))[1], os.path.basename(f))
for f in additional_files
]
files += additional_files
self.local_files_plus_minus = FileList(root=root, files_in_root=files)
def test_regression_959(self):
"""
Improper 'type' inheritance in simtools.ini
Test for https://github.com/InstituteforDiseaseModeling/dtk-tools/issues/959
"""
SetupParser.init(selected_block='AM',
setup_file=os.path.join(self.input_path, '959',
'simtools1.ini'),
is_testing=True)
self.assertEqual(SetupParser.get('base_collection_id_dll'), "in AM")
SetupParser._uninit()
SetupParser.init(selected_block='AM',
setup_file=os.path.join(self.input_path, '959',
'simtools2.ini'),
is_testing=True)
self.assertEqual(SetupParser.get('base_collection_id_dll'), "in HPC")
SetupParser._uninit()
SetupParser.init(selected_block='AM',
setup_file=os.path.join(self.input_path, '959',
'simtools3.ini'),
is_testing=True)
self.assertEqual(SetupParser.get('base_collection_id_dll'), "in HPC")
SetupParser._uninit()
def run_analyzers(exp_id, analyzers, savefile_prefix=""):
def _remove_duplicate_columns(df):
columns_to_keep = []
for c in df.columns:
if "_duplicated" not in c:
columns_to_keep.append(c)
return df[columns_to_keep]
SetupParser.default_block = 'HPC'
SetupParser.init()
am = AnalyzeManager()
for a in analyzers:
am.add_analyzer(a())
exp = retrieve_experiment(exp_id)
am.add_experiment(exp)
am.analyze()
if len(analyzers) == 1:
df_return = am.analyzers[0].results
elif len(analyzers) > 1:
df_list = [x.results for x in am.analyzers]
df_return = pd.merge(df_list[0],
df_list[1],
on="sim_id",
suffixes=["", "_duplicated"])
# Drop duplicated columns
# for c in df_result.columns:
# if "_duplicated" in c:
# df_result.drop(c, inplace=True)
df_return = _remove_duplicate_columns(df_return)
else:
raise ValueError
df_return.to_csv("{}_{}.csv".format(savefile_prefix, exp_id), index=False)
return df_return
def test_listed_events(self):
from simtools.SetupParser import SetupParser
SetupParser.init()
# Create a builder based on the files
builder = DTKConfigBuilder.from_files(
os.path.join(self.input_path, 'customevents', 'configbad.json'),
os.path.join(self.input_path, 'customevents', 'campaign.json'))
# Call the file writter
def fake_write(name, content):
return
builder.file_writer(fake_write)
# Try if the listed events worked
self.assertEqual(
set(builder.config['parameters']['Listed_Events']),
set([
"Received_ITN", "TestedPositive", "Received_Campaign_Drugs",
"Received_Treatment", "Received_Test", "Received_Survey",
"Diagnostic_Survey_5", "Give_Drugs"
]))
# listening_duration of -1 indicates that this intervention will listen forever and perform the tasks whenever Action1 is sent out
add_node_IRS(cb, 30, trigger_condition_list=["Party"], listening_duration=-1)
# this intervention is distributed and starts listening on day 60, but distributes the IRS 1000 days after it is triggered
# please note if the listening duration was less than triggered day + campaign delay, the intervention would not run.
add_node_IRS(cb,
60,
triggered_campaign_delay=1000,
trigger_condition_list=["PartyHarder"])
# listens for 15 days and, as the result, hears nothing and does nothing.
add_node_IRS(cb,
60,
trigger_condition_list=["PartyHarder"],
listening_duration=15)
# run_sim_args is what the `dtk run` command will look for
run_sim_args = {
'exp_name': 'Example Event Counter and Triggered and Delayed IRS',
'config_builder': cb
}
# If you prefer running with `python example_example_event_count_and_triggered_events.py`, you will need the
# following block
if __name__ == "__main__":
SetupParser.init()
exp_manager = ExperimentManagerFactory.init()
exp_manager.run_simulations(**run_sim_args)
# Wait for the simulations to be done
exp_manager.wait_for_finished(verbose=True)
assert (exp_manager.succeeded())
def setUp(self):
SetupParser.init(selected_block='LOCAL')
def setUp(self):
SetupParser.init(selected_block='HPC')
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib as mpl
from simtools.Analysis.AnalyzeManager import AnalyzeManager
from simtools.Analysis.BaseAnalyzers import BaseAnalyzer
from simtools.SetupParser import SetupParser
from scipy import interpolate
import os
mpl.rcParams['pdf.fonttype'] = 42
if not SetupParser.initialized:
SetupParser.init('HPC')
wdir = os.path.join(os.getcwd(), 'output')
expids = [
'4f157763-d898-ea11-a2c5-c4346bcb1550',
'3f4bf407-fa96-ea11-a2c5-c4346bcb1550'
]
n_exps = len(expids)
descriptors = ['no seasonality', 'with seasonality']
expnames = ['EIR_sweep'] * n_exps
counters = np.arange(n_exps)
cmap = ['#ff0000', '#0000ff']
df_all = pd.DataFrame()
fig, ax = plt.subplots()
center_repeats=1,
sigma_r=sigma_r)
calib_manager = CalibManager(
name=COMPS_calib_exp_name,
config_builder=mozamb_exp.cb,
map_sample_to_model_input_fn=map_sample_to_model_input,
sites=sites,
next_point=optimtool,
sim_runs_per_param_set=sim_runs_per_param_set,
max_iterations=max_iterations,
plotters=plotters)
run_calib_args = {"calib_manager": calib_manager}
if __name__ == "__main__":
if parser_location == "LOCAL":
SetupParser.init("LOCAL")
else:
SetupParser.init()
SetupParser.set("HPC", "priority", priority)
SetupParser.set("HPC", "node_group", coreset)
if calib_stage >= 1:
cm = run_calib_args["calib_manager"]
cm.run_calibration()
elif calib_stage == 0:
print("File generation complete.")
def test_verify_user_in_default_block(self):
SetupParser.init(selected_block='LOCAL')
self.assertTrue(
'user' in SetupParser.singleton.setup.defaults().keys())
self.assertNotEqual(SetupParser.singleton.setup.defaults()['user'],
None)
def test_reinitialization_not_allowed(self):
SetupParser.init(selected_block="LOCAL")
kwargs = {"selected_block": "LOCAL"}
self.assertRaises(SetupParser.AlreadyInitialized, SetupParser.init,
**kwargs)
def test_verify_block_selection(self):
SetupParser.init(selected_block='LOCAL')
self.assertEqual(SetupParser.selected_block, 'LOCAL')
