def test_has_option(self): filename = os.path.join(self.input_path, SetupParser.ini_filename) SetupParser.init(selected_block='TEST', setup_file=filename, is_testing=True) self.assertTrue(SetupParser.has_option('environment')) self.assertFalse(SetupParser.has_option('notanoption'))
def initialize_SetupParser_from_args(args, unknownArgs): # determine the selected environment if hasattr(args, 'block'): selected_block = args.block # we should use this going forward (add flags in commands_args.py) else: # current, back-compatible route. This should go away once ini block selection is done via a known flag # in all commands if len(unknownArgs) == 1: import re arg = unknownArgs[0] block_regex = re.compile('^--(?P<block>\w+)$') if block_regex.search(arg): selected_block = block_regex.search(arg).group('block') else: selected_block = None # we use the default block selection in SetupParser elif len(unknownArgs) > 1: raise Exception( "Only zero or one unknown args are allowed for simtools.ini block selection." ) else: # len(unknownArgs) == 0, use SetupParser default block selection selected_block = None # some HPC environment overrides that may have been provided on the command line overrides = grab_HPC_overrides(args) #if hasattr(args, 'ini'): # provided_ini = args.ini #else: # provided_ini = None # If a config module (experiment .py file) was provided, it MAY have set some SetupParser defaults, so we load # it first. load_config(args) SetupParser.init(selected_block=selected_block, overrides=overrides)
def test_items(self): filename = os.path.join(self.input_path, SetupParser.ini_filename) SetupParser.init(selected_block='TEST', setup_file=filename, is_testing=True) self.assertEqual(len(SetupParser.items()), 24) self.assertEqual(len(SetupParser.items(block='HPC')), 20)
def test_verify_uninit(self): self.assertFalse(SetupParser.initialized) SetupParser.init() self.assertTrue(SetupParser.initialized) SetupParser._uninit() self.assertFalse(SetupParser.initialized) self.assertEqual(SetupParser.singleton, None)
def setUp(self): self.cb = SimConfigBuilder.from_defaults('DUMMY') SetupParser.init(selected_block='LOCAL') self.input_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'input') self.dummy_exe_folder = os.path.join(self.input_path, 'dummy_exe_folder') self.another_dummy_exe_folder = os.path.join( self.input_path, 'another_dummy_exe_folder')
def run(self): SetupParser.init(singleton=self.setup_parser_singleton) try: self.process() except Exception as e: print("Exception during commission") import traceback traceback.print_exc() exit()
def test_regression_968(self): """ Investigate a difference of behavior between UNIX systems and Windows Test for https://github.com/InstituteforDiseaseModeling/dtk-tools/issues/968 """ SetupParser.init() singleton = SetupParser.singleton SetupParser.init(singleton=singleton) SetupParser._uninit()
def test_no_overlay_default_block(self): SetupParser.init() self.assertEqual(SetupParser.selected_block, 'LOCAL') self.assertEqual(SetupParser.get('type'), 'LOCAL') # User should be there self.assertIsNotNone(SetupParser.get('user')) # We dont want HPC or unknown params self.assertRaises(ValueError, SetupParser.get, 'use_comps_asset_svc') self.assertRaises(ValueError, SetupParser.get, 'WRONG')
def test_regression_965(self): """ Allow substitutions in simtools.ini Test for https://github.com/InstituteforDiseaseModeling/dtk-tools/issues/965 """ SetupParser.init(selected_block='LOCAL', setup_file=os.path.join(self.input_path, '965', 'simtools.ini'), is_testing=True) self.assertTrue(SetupParser.get('block'), 'PATH1') SetupParser._uninit()
def test_explicit_overrides(self): filename = os.path.join(self.input_path, SetupParser.ini_filename) overrides = {'in_all': 'yellow'} SetupParser.init(selected_block='TEST', setup_file=filename, overrides=overrides, is_testing=True) self.assertEqual(SetupParser.get('in_all'), overrides['in_all']) # verify this works even after block change SetupParser.override_block(block='LOCAL2') self.assertEqual(SetupParser.get('in_all'), overrides['in_all'])
def test_verify_override_block(self): """ 1) set the SetupParser once to get the expected value, unset it, then 2) load a different selected block & then override it to the first. Should get same answer both routes. """ SetupParser.init(selected_block='LOCAL', is_testing=True) expected_params = dict(SetupParser.items(SetupParser.selected_block)) SetupParser._uninit() SetupParser.init(selected_block='HPC', is_testing=True) SetupParser.override_block('LOCAL') override_params = dict(SetupParser.items(SetupParser.selected_block)) self.assertEqual(override_params, expected_params)
def run_single_analyzer(exp_id, analyzer, savefile_prefix=""): SetupParser.default_block = 'HPC' SetupParser.init() am = AnalyzeManager() am.add_analyzer(analyzer()) exp = retrieve_experiment(exp_id) am.add_experiment(exp) am.analyze() df_return = am.analyzers[0].results df_return.to_csv("{}_{}.csv".format(savefile_prefix, exp_id), index=False) return df_return
def test_init_with_singleton_provided(self): """ This is covering the case where we call (in a new process): SetupParser(singleton=some_object) :return: """ attr = 'johns_favorite_color' # an attribute NOT in the ini file to make sure we keep any set values value = 'purple' SetupParser('LOCAL') SetupParser.singleton.setup.set('LOCAL', attr, value) singleton = SetupParser.singleton SetupParser._uninit() SetupParser.init(singleton=singleton) self.assertEqual(SetupParser.get(attr), value)
def test_get_boolean(self): filename = os.path.join(self.input_path, SetupParser.ini_filename) SetupParser.init(selected_block='TEST', setup_file=filename, is_testing=True) # assertIs checks if the items evaluate to the same object (not truthy comparison) self.assertIs(SetupParser.getboolean('this_is_true'), True) self.assertIs(SetupParser.getboolean('this_is_false'), False) kwargs = {'parameter': 'notaparameter'} self.assertRaises(ValueError, SetupParser.getboolean, **kwargs) # not present in TEST kwargs = {'parameter': 'environment'} self.assertRaises(ValueError, SetupParser.getboolean, **kwargs) # not a boolean, but is present in TEST.
def setUp(self): current_dir = os.path.dirname(os.path.realpath(__file__)) input_dir = os.path.join(current_dir, 'input') self.am_simtools = os.path.join(input_dir, 'am_simtools.ini') # Adjust path sensitive parameters in the ini file cp = ConfigParser() cp.read(self.am_simtools) cp.set('DEFAULT', 'path', input_dir) cp.set('DEFAULT', 'example_dir', os.path.join(current_dir, '..', '..', 'examples')) cp.write(open(self.am_simtools, 'w')) SetupParser.init(selected_block=self.SELECTED_BLOCK, setup_file=self.am_simtools) self.config_builder = DTKConfigBuilder.from_defaults('VECTOR_SIM') configure_site(self.config_builder, 'Namawala') self.config_builder.add_reports(BaseReport(type="VectorHabitatReport"))
def test_get(self): filename = os.path.join(self.input_path, SetupParser.ini_filename) SetupParser.init(selected_block='TEST', setup_file=filename, is_testing=True) default_value = 'abcdefg' # param present and default value not used kwargs = {'parameter': 'made_up_param', 'default': default_value} self.assertEqual(SetupParser.get(**kwargs), '1') # param not present and no default provided kwargs = {'parameter': 'notaparameter'} self.assertRaises(ValueError, SetupParser.get, **kwargs) # default value provided and used kwargs = {'parameter': 'notaparameter', 'default': default_value} test_value = SetupParser.get(**kwargs) self.assertEqual(test_value, default_value)
def run_experiment(configbuilder, experiment_name, experiment_builder, analyzers): run_sim_args = { 'config_builder': configbuilder, 'exp_name': experiment_name, 'exp_builder': experiment_builder } if not SetupParser.initialized: SetupParser.init('HPC') exp_manager = ExperimentManagerFactory.init() exp_manager.run_simulations(**run_sim_args) exp_manager.wait_for_finished(verbose=True) assert (exp_manager.succeeded()) am = AnalyzeManager(exp_manager.experiment) for a in analyzers: am.add_analyzer(a) am.analyze()
def test_overlay_file_values(self): filename = os.path.join(self.input_path, SetupParser.ini_filename) SetupParser.init(selected_block='TEST', setup_file=filename, is_testing=True) # verify the selected block is correct self.assertEqual(SetupParser.selected_block, 'TEST') # data overlay - param in TEST, no conflict with DEFAULT or default ini (TEST wins) self.assertEqual(SetupParser.get('made_up_param'), '1') # data overlay - param in TEST, conflict with DEFAULT and default ini (TEST wins) self.assertEqual(SetupParser.get('environment'), 'TestEnv') # data overlay - DEFAULT conflict with default ini (DEFAULT wins) self.assertEqual(SetupParser.get('max_threads'), '0') # data overlay - only in default ini DEFAULT, (default ini DEFAULT wins, obviously!) self.assertIsNotNone(SetupParser.get('exe_path'))
def setUp(self): SetupParser._uninit() current_dir = os.path.dirname(os.path.realpath(__file__)) SetupParser.init(selected_block=self.SELECTED_BLOCK, setup_file=os.path.join(current_dir, 'input', 'am_simtools.ini')) COMPS_login(SetupParser.get('server_endpoint')) self.existing_collection = AssetCollection( base_collection=get_asset_collection(self.EXISTING_COLLECTION_ID)) self.existing_collection.prepare(location='HPC') self.existing_COMPS_asset_files = self.existing_collection.asset_files_to_use # a FileList object dir = os.path.join(self.INPUT_DIR, 'files') files = [os.path.join(dir, f) for f in os.listdir(dir)] root = os.path.dirname(os.path.dirname(files[0])) files = [ os.path.join( os.path.split(os.path.dirname(f))[1], os.path.basename(f)) for f in files ] self.local_files = FileList(root=root, files_in_root=files) # take one away files = [ os.path.join(f.relative_path, f.file_name) for f in self.local_files.files if f.file_name != 'file1_REMOTE_ONLY' ] # add some more dir = os.path.join(self.INPUT_DIR, 'additional_files') additional_files = [os.path.join(dir, f) for f in os.listdir(dir)] additional_files = [ os.path.join( os.path.split(os.path.dirname(f))[1], os.path.basename(f)) for f in additional_files ] files += additional_files self.local_files_plus_minus = FileList(root=root, files_in_root=files)
def test_regression_959(self): """ Improper 'type' inheritance in simtools.ini Test for https://github.com/InstituteforDiseaseModeling/dtk-tools/issues/959 """ SetupParser.init(selected_block='AM', setup_file=os.path.join(self.input_path, '959', 'simtools1.ini'), is_testing=True) self.assertEqual(SetupParser.get('base_collection_id_dll'), "in AM") SetupParser._uninit() SetupParser.init(selected_block='AM', setup_file=os.path.join(self.input_path, '959', 'simtools2.ini'), is_testing=True) self.assertEqual(SetupParser.get('base_collection_id_dll'), "in HPC") SetupParser._uninit() SetupParser.init(selected_block='AM', setup_file=os.path.join(self.input_path, '959', 'simtools3.ini'), is_testing=True) self.assertEqual(SetupParser.get('base_collection_id_dll'), "in HPC") SetupParser._uninit()
def run_analyzers(exp_id, analyzers, savefile_prefix=""): def _remove_duplicate_columns(df): columns_to_keep = [] for c in df.columns: if "_duplicated" not in c: columns_to_keep.append(c) return df[columns_to_keep] SetupParser.default_block = 'HPC' SetupParser.init() am = AnalyzeManager() for a in analyzers: am.add_analyzer(a()) exp = retrieve_experiment(exp_id) am.add_experiment(exp) am.analyze() if len(analyzers) == 1: df_return = am.analyzers[0].results elif len(analyzers) > 1: df_list = [x.results for x in am.analyzers] df_return = pd.merge(df_list[0], df_list[1], on="sim_id", suffixes=["", "_duplicated"]) # Drop duplicated columns # for c in df_result.columns: # if "_duplicated" in c: # df_result.drop(c, inplace=True) df_return = _remove_duplicate_columns(df_return) else: raise ValueError df_return.to_csv("{}_{}.csv".format(savefile_prefix, exp_id), index=False) return df_return
def test_listed_events(self): from simtools.SetupParser import SetupParser SetupParser.init() # Create a builder based on the files builder = DTKConfigBuilder.from_files( os.path.join(self.input_path, 'customevents', 'configbad.json'), os.path.join(self.input_path, 'customevents', 'campaign.json')) # Call the file writter def fake_write(name, content): return builder.file_writer(fake_write) # Try if the listed events worked self.assertEqual( set(builder.config['parameters']['Listed_Events']), set([ "Received_ITN", "TestedPositive", "Received_Campaign_Drugs", "Received_Treatment", "Received_Test", "Received_Survey", "Diagnostic_Survey_5", "Give_Drugs" ]))
# listening_duration of -1 indicates that this intervention will listen forever and perform the tasks whenever Action1 is sent out add_node_IRS(cb, 30, trigger_condition_list=["Party"], listening_duration=-1) # this intervention is distributed and starts listening on day 60, but distributes the IRS 1000 days after it is triggered # please note if the listening duration was less than triggered day + campaign delay, the intervention would not run. add_node_IRS(cb, 60, triggered_campaign_delay=1000, trigger_condition_list=["PartyHarder"]) # listens for 15 days and, as the result, hears nothing and does nothing. add_node_IRS(cb, 60, trigger_condition_list=["PartyHarder"], listening_duration=15) # run_sim_args is what the `dtk run` command will look for run_sim_args = { 'exp_name': 'Example Event Counter and Triggered and Delayed IRS', 'config_builder': cb } # If you prefer running with `python example_example_event_count_and_triggered_events.py`, you will need the # following block if __name__ == "__main__": SetupParser.init() exp_manager = ExperimentManagerFactory.init() exp_manager.run_simulations(**run_sim_args) # Wait for the simulations to be done exp_manager.wait_for_finished(verbose=True) assert (exp_manager.succeeded())
def setUp(self): SetupParser.init(selected_block='LOCAL')
def setUp(self): SetupParser.init(selected_block='HPC')
import pandas as pd import numpy as np import matplotlib.pyplot as plt import seaborn as sns import matplotlib as mpl from simtools.Analysis.AnalyzeManager import AnalyzeManager from simtools.Analysis.BaseAnalyzers import BaseAnalyzer from simtools.SetupParser import SetupParser from scipy import interpolate import os mpl.rcParams['pdf.fonttype'] = 42 if not SetupParser.initialized: SetupParser.init('HPC') wdir = os.path.join(os.getcwd(), 'output') expids = [ '4f157763-d898-ea11-a2c5-c4346bcb1550', '3f4bf407-fa96-ea11-a2c5-c4346bcb1550' ] n_exps = len(expids) descriptors = ['no seasonality', 'with seasonality'] expnames = ['EIR_sweep'] * n_exps counters = np.arange(n_exps) cmap = ['#ff0000', '#0000ff'] df_all = pd.DataFrame() fig, ax = plt.subplots()
center_repeats=1, sigma_r=sigma_r) calib_manager = CalibManager( name=COMPS_calib_exp_name, config_builder=mozamb_exp.cb, map_sample_to_model_input_fn=map_sample_to_model_input, sites=sites, next_point=optimtool, sim_runs_per_param_set=sim_runs_per_param_set, max_iterations=max_iterations, plotters=plotters) run_calib_args = {"calib_manager": calib_manager} if __name__ == "__main__": if parser_location == "LOCAL": SetupParser.init("LOCAL") else: SetupParser.init() SetupParser.set("HPC", "priority", priority) SetupParser.set("HPC", "node_group", coreset) if calib_stage >= 1: cm = run_calib_args["calib_manager"] cm.run_calibration() elif calib_stage == 0: print("File generation complete.")
def test_verify_user_in_default_block(self): SetupParser.init(selected_block='LOCAL') self.assertTrue( 'user' in SetupParser.singleton.setup.defaults().keys()) self.assertNotEqual(SetupParser.singleton.setup.defaults()['user'], None)
def test_reinitialization_not_allowed(self): SetupParser.init(selected_block="LOCAL") kwargs = {"selected_block": "LOCAL"} self.assertRaises(SetupParser.AlreadyInitialized, SetupParser.init, **kwargs)
def test_verify_block_selection(self): SetupParser.init(selected_block='LOCAL') self.assertEqual(SetupParser.selected_block, 'LOCAL')