'agemax': 200,
'seek': 1,
'rate': 0.0
}])
#add_drug_campaign(cb, "SMC", "SPA", start_days=[230 + 365*x for x in range(5)], repetitions=1, interval=30, coverage=1., target_group={'agemin' : 0, 'agemax' : 5})
add_summary_report(cb,
interval=365 / 12.,
description='Monthly',
age_bins=[5, 15, 200])
#add_summary_report(cb, interval=365/12., description='MonthlyUnderFive', age_bins=[5])
add_summary_report(cb,
interval=1.,
description='DailyUnderFive',
age_bins=[5, 15, 200])
#add_summary_report(cb, interval=365/12., description='AnnualUnderFive', age_bins=[5])
builder = GenericSweepBuilder.from_dict({'Run_Number': range(1)})
#builder = GenericSweepBuilder.from_dict({'Run_Number':range(100)})
run_sim_args = {
'config_builder': cb,
'exp_name': expname,
'exp_builder': builder
}
if __name__ == "__main__":
sm = ExperimentManagerFactory.from_setup(SetupParser())
sm.run_simulations(**run_sim_args)
def link(args, unknownArgs):
"""
Open browser to the COMPS Experiment/Simulation with ID or name provided
:param args:
:param unknownArgs:
:return:
"""
# get input from commands line
input_id = args.Id
# default: consider the latest experiment
if input_id is None:
latest = DataStore.get_most_recent_experiment()
input_id = latest.exp_id
try:
comps_item = retrieve_item(input_id)
except:
print('Nothing was found for {}'.format(input_id))
exit()
# check item type
id_type = ''
location = 'LOCAL'
if isinstance(comps_item, Experiment):
item_id = comps_item.exp_id
id_type = 'exp'
location = comps_item.location
elif isinstance(comps_item, Simulation):
item_id = comps_item.id
exp_id = comps_item.experiment_id
id_type = 'sim'
# retrieve location
exp = DataStore.get_experiment(exp_id)
location = exp.location
else:
print('No Experiment or Simulation was found on COMPS for {}'.format(
input_id))
exit()
# make sure it exists on COMPS
if location == 'LOCAL':
print('Item is on LOCAL not on COMPS.')
exit()
# open browser to COMPS Experiment/Simulation
import webbrowser
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
url = ''
if id_type == 'exp':
url = '%s/#explore/Experiments?filters=Id=%s&offset=0&selectedId=%s' % (
endpoint, item_id, item_id)
elif id_type == 'sim':
url = '%s/#explore/Simulations?filters=Id=%s&mode=list&orderby=DateCreated+desc&count=50&offset=0&layout=512C56&selectedId=%s' % (
endpoint, item_id, item_id)
# Open URL in new browser window
webbrowser.open_new(url) # opens in default browser
map_sample_to_model_input_fn=map_sample_to_model_input,
sites=sites,
next_point=optimtool,
sim_runs_per_param_set=sim_runs_per_param_set,
max_iterations=max_iterations,
plotters=plotters)
run_calib_args = {
"calib_manager": calib_manager
}
if __name__ == "__main__":
if parser_location == "LOCAL":
SetupParser.init("LOCAL")
else:
SetupParser.init()
SetupParser.set("HPC", "priority", priority)
SetupParser.set("HPC", "node_group", coreset)
cm = run_calib_args["calib_manager"]
if scratch_or_resume == "scratch":
cm.run_calibration()
elif scratch_or_resume == "resume":
cm.resume_calibration(iteration=0, iter_step='analyze')
simdata[sweep_var] = 0
return simdata
def finalize(self, all_data):
selected = [data for sim, data in all_data.items()]
if len(selected) == 0:
print("No data have been returned... Exiting...")
return
df = pd.concat(selected).reset_index(drop=True)
df.to_csv(os.path.join(self.working_dir, '%s.csv' % self.dir_name), index=False)
if __name__ == "__main__":
SetupParser.init("HPC")
out_dir = os.path.join('E:/', 'Dropbox (IDM)', 'Malaria Team Folder', 'projects',
'map_intervention_impact', 'lookup_tables')
sites = pd.read_csv("site_details.csv")
experiments = {
"corr_itn_irs" :"e7fd04a2-a8d8-e811-a2bd-c4346bcb1555"
}
for dirname, exp_id in experiments.items():
am = AnalyzeManager(exp_list=exp_id, analyzers=[BurdenAnalyzer(working_dir=out_dir,
dir_name=dirname,
report_names = sites["name"].tolist(),
sweep_variables=["Run_Number",
# This is REQUIRED by the templates
config_builder.ignore_missing = True
# Get the dicts
points_dict = header_table_to_dict(point_header, points, index_name='TPI')
for point in points_dict:
tpi = point.pop('TPI')
if 'TAGS' not in point:
point['TAGS'] = {}
point['TAGS']['TPI'] = tpi
scenarios_dict = header_table_to_dict(scenario_header, scenarios)
if __name__ == "__main__":
SetupParser.init()
experiments = [] # All the experiment managers for all experiments
experiments_ids = [
] # Ids of the created experiments for resuming capabilities
# Check if we want to resume
if os.path.exists('ids.json'):
print("Previous run detected... Run [N]ew, [R]esume, [A]bort?")
resp = ""
while resp not in ('N', 'R', 'A'):
resp = input()
if resp == "A":
exit()
elif resp == "R":
# In the resume case, retrieve the ids and create the managers
from dtk.tools.demographics.DemographicsFile import DemographicsFile
from simtools.Analysis.AnalyzeManager import AnalyzeManager
from simtools.Analysis.BaseAnalyzers import DownloadAnalyzer
from simtools.Utilities.COMPSUtilities import get_experiment_by_id
from COMPS.Data import QueryCriteria
import time
import random
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from PythonHelperFunctions.DemographicsHelpers import *
from PythonHelperFunctions.utils import load_dropbox_path, load_output_path, load_input_path
from pyDOE import lhs
import numpy as np
#Run locally or on HPC
SetupParser.default_block = "HPC"
SetupParser.init(selected_block=SetupParser.default_block)
ScenarioName = "HomeOnlyForever"
InterventionDuration = 2000
# Set the path for DTK input files
BaseInputPath = load_input_path()
if not os.path.exists(BaseInputPath):
os.mkdir(BaseInputPath)
OutputPath = os.path.join(load_output_path(), ScenarioName)
if not os.path.exists(OutputPath):
os.mkdir(OutputPath)
if not os.path.exists(os.path.join(OutputPath, 'input_files')):
os.mkdir(os.path.join(OutputPath, 'input_files'))
#Name the input files
DemoFile = os.path.join(BaseInputPath, 'demographics.json')
CampaignFile = os.path.join(BaseInputPath, 'campaign.json')
# Get the last time a cleanup happened
last_cleanup = DataStore.get_setting('last_log_cleanup')
if not last_cleanup or (datetime.today() - datetime.strptime(last_cleanup.value.split(' ')[0],'%Y-%m-%d')).days > 1:
# Do the cleanup
from simtools.DataAccess.LoggingDataStore import LoggingDataStore
LoggingDataStore.log_cleanup()
DataStore.save_setting(DataStore.create_setting(key='last_log_cleanup', value=datetime.today()))
if __name__ == "__main__":
logger.debug('Start Overseer pid: %d' % os.getpid())
# we technically don't care about full consistency of SetupParser with the original dtk command, as experiments
# have all been created. We can grab 'generic' max_local_sims / max_threads
SetupParser.init() # default block
max_local_sims = int(SetupParser.get('max_local_sims'))
# Create the queue
local_queue = multiprocessing.Queue(max_local_sims)
managers = OrderedDict()
# Queue to be shared among all runners in order to update the individual simulation states in the DB
manager = Manager()
# Take this opportunity to cleanup the logs
lc = threading.Thread(target=LogCleaner)
lc.start()
count = 0
def file_writer(self, write_fn):
"""
Dump all the files needed for the simulation in the simulation directory.
This includes:
* The campaign file
* The custom reporters file
* The different demographic overlays
* The other input files (``input_files`` dictionary)
* The config file
* The emodules_map file
Args:
write_fn: The function that will write the files. This function needs to take a file name and a content.
Examples:
For example, in the :py:class:`SimConfigBuilder` class, the :py:func:`dump_files` is defining the `write_fn`
like::
def write_file(name, content):
filename = os.path.join(working_directory, '%s.json' % name)
with open(filename, 'w') as f:
f.write(content)
"""
from simtools.SetupParser import SetupParser
if self.human_readability:
dump = lambda content: json.dumps(
content, sort_keys=True, indent=3, cls=NumpyEncoder).strip('"')
else:
dump = lambda content: json.dumps(
content, sort_keys=True, cls=NumpyEncoder).strip('"')
write_fn(self.config['parameters']['Campaign_Filename'],
dump(self.campaign))
if self.custom_reports:
self.set_param('Custom_Reports_Filename', 'custom_reports.json')
write_fn('custom_reports.json',
dump(format_reports(self.custom_reports)))
for name, content in self.demog_overlays.items():
self.append_overlay('%s' % name)
write_fn('%s' % name, dump(content))
for name, content in self.input_files.items():
write_fn(name, dump(content))
# Add missing item from campaign individual events into Listed_Events
self.config['parameters']['Listed_Events'] = self.check_custom_events()
write_fn('config.json', dump(self.config))
# complete the path to each dll before writing emodules_map.json
location = SetupParser.get('type')
if location == 'LOCAL':
root = self.assets.dll_root
elif location == 'HPC':
root = 'Assets'
else:
raise Exception('Unknown location: %s' % location)
for module_type in self.emodules_map.keys():
self.emodules_map[module_type] = list(
set([
os.path.join(root, dll)
for dll in self.emodules_map[module_type]
]))
write_fn('emodules_map.json', dump(self.emodules_map))
0,
"Base_Air_Temperature":
25.0,
"Enable_Demographics_Birth":
0,
"Enable_Climate_Stochasticity":
1,
"Age_Dependent_Biting_Risk_Type":
"OFF",
'.Vector_Sampling_Type':
"TRACK_ALL_VECTORS",
'Vector_Sugar_Feeding_Frequency':
'VECTOR_SUGAR_FEEDING_NONE',
"Vector_Sampling_Type":
"VECTOR_COMPARTMENTS_NUMBER",
"Vector_Species_Names": ['arabiensis'],
"Enable_Vector_Aging":
1
})
SetupParser('LOCAL')
run_sim_args = {
'config_builder': cb,
'exp_name': expname,
'exp_builder': builder
}
exp_manager = ExperimentManagerFactory.from_setup()
exp_manager.run_simulations(**run_sim_args)
exp_manager.wait_for_finished(verbose=True)
def location(self):
return SetupParser.get(
'type') if self._location is None else self._location
can be encoded in a similar fashion using custom functions rather than the generic "set_param".
'''
sample_point = prior.to_dict(sample_dimension_values)
params_to_update = dict()
params_to_update['Simulation_Duration'] = 365
for sample_dimension_name, sample_dimension_value in sample_point.items():
param_name = sample_dimension_name.replace('_LOG', '')
params_to_update[param_name] = pow(10, sample_dimension_value)
return cb.update_params(params_to_update)
next_point_kwargs = dict(initial_samples=3,
samples_per_iteration=3,
n_resamples=100)
calib_manager = CalibManager(name='test_dummy_calibration',
setup=SetupParser(),
config_builder=cb,
map_sample_to_model_input_fn=sample_point_fn,
sites=sites,
next_point=IMIS(prior, **next_point_kwargs),
sim_runs_per_param_set=1,
max_iterations=2,
plotters=plotters)
run_calib_args = {}
if __name__ == "__main__":
calib_manager.run_calibration(**run_calib_args)
[
ModFn(sweep_larval_habitat, scale_factor = habitat_multiplier),
ModFn(DTKConfigBuilder.set_param,'Run_Number',seed)
]
for habitat_multiplier in np.logspace(-2, 1, 50)
for seed in range(num_seeds)
]
)
# builder = ModBuilder.from_list( [[ ModFn(sweep_scale_factor, x)] for x in [0.01,10,100]])
# report_years = 1
# report_start = 40
# Run args
run_sim_args = {'config_builder': cb,
'exp_name': exp_name,
'exp_builder': builder
}
if __name__ == "__main__":
if not SetupParser.initialized:
SetupParser.init('LOCAL')
exp_manager = ExperimentManagerFactory.init()
exp_manager.run_simulations(**run_sim_args)
exp_manager.wait_for_finished(verbose=True)
assert (exp_manager.succeeded())
def tearDown(self):
SetupParser._uninit()
from dtk.utils.core.DTKConfigBuilder import DTKConfigBuilder
from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory
from simtools.ModBuilder import SingleSimulationBuilder
from simtools.SetupParser import SetupParser
experiment_name = "Generic Default simluation test for tools"
generic_defaults = [
'GENERIC_SIM_SIR', 'GENERIC_SIM_SEIR', 'GENERIC_SIM_SIRS',
'GENERIC_SIM_SEIRS', 'GENERIC_SIM_SI', 'GENERIC_SIM_SIS',
'GENERIC_SIM_SIR_Vaccinations_A', 'GENERIC_SIM_SIR_Vaccinations_B',
'GENERIC_SIM_SIR_Vaccinations_C'
]
if __name__ == '__main__':
# Initialize the SetupParser to use HPC
SetupParser.init(selected_block='HPC')
# Create an experiment manager
exp_mgr = ExperimentManagerFactory.init()
# For each generic simulations, create a config builder from the default and run it
for sim_example in generic_defaults:
cb = DTKConfigBuilder.from_defaults(sim_example)
cb.set_param("Config_Name", sim_example)
# Create a builder in order to run a single simulation
# This step is optional usually but it allows us to set tags on the simulation
builder = SingleSimulationBuilder()
builder.tags.update({"Simulation_Type": sim_example})
exp_mgr.run_simulations(config_builder=cb,
def test_block_inheritance(self):
"""
Issue 1246
Verify that multi-level block inheritance works properly and that 'type' percolates from the deepest-level
(root) of an inheritance chain.
"""
# # ck4, template for following tests/asserts
# SetupParser.init(selected_block='LOCAL',
# setup_file=os.path.join(self.input_path, '1246', somedir, 'simtools.ini'), is_testing=True)
# something = None
# self.assertTrue(something)
# SetupParser._uninit()
#
# Using a 3 level-inheritance scheme in all of these cases
#
# descendant block values override parent block values
# EXCEPT: 'type is inherited from the inheritance chain root
# verify that block order in the ini file does not matter for arbitrary key/values OR 'type'
values = {}
types = {}
for i in range(1, 4):
testdir = 'ordering%d' % i
SetupParser.init(selected_block='LOCALB',
setup_file=os.path.join(self.input_path, '1246',
testdir, 'simtools.ini'),
is_testing=True)
values[testdir] = SetupParser.get('a')
types[testdir] = SetupParser.get('type')
SetupParser._uninit()
unique_values = sorted(set(values.values()))
self.assertEqual(unique_values, ['3'])
unique_types = sorted(set(types.values()))
self.assertEqual(unique_types, ['LOCAL'])
# verify that the proper values are inherited, regardless of the level in the inheritance chain values are
# located.
SetupParser.init(selected_block='LOCALB',
setup_file=os.path.join(self.input_path, '1246',
'mixedLevelValues',
'simtools.ini'),
is_testing=True)
self.assertEqual(SetupParser.get('a'), '1')
self.assertEqual(SetupParser.get('b'), '3')
self.assertEqual(SetupParser.get('c'), '5')
self.assertEqual(SetupParser.get('d'), '7')
self.assertEqual(SetupParser.get('e'), '10')
SetupParser._uninit()
# Blocks used as the 'type' should fail if missing
kwargs = {
'selected_block':
'LOCALB',
'setup_file':
os.path.join(self.input_path, '1246', 'missingReferencedBlock',
'simtools.ini'),
'is_testing':
True
}
self.assertRaises(SetupParser.InvalidBlock, SetupParser.init, **kwargs)
SetupParser._uninit()
# Blocks missing 'type' should fail
kwargs = {
'selected_block':
'LOCALB',
'setup_file':
os.path.join(self.input_path, '1246', 'missingType',
'simtools.ini'),
'is_testing':
True
}
self.assertRaises(SetupParser.InvalidBlock, SetupParser.init, **kwargs)
SetupParser._uninit()
def create_simulations(self,
exp_name='test',
exp_builder=None,
suite_id=None,
verbose=True):
"""
Create an experiment with simulations modified according to the specified experiment builder.
"""
global simulations_expected
simulations_expected = 0
self.exp_builder = exp_builder or SingleSimulationBuilder()
self.cache.clear()
# Create the experiment if not present already
if not self.experiment or self.experiment.exp_name != exp_name:
self.create_experiment(experiment_name=exp_name, suite_id=suite_id)
else:
# Refresh the experiment
self.refresh_experiment()
# Save the experiment in the DB
DataStore.save_experiment(self.experiment, verbose=verbose)
# Separate the experiment builder generator into batches
sim_per_batch = int(SetupParser.get('sims_per_thread', default=50))
mods = self.exp_builder.mod_generator
max_creator_processes = min(
multiprocessing.cpu_count() - 1,
int(
SetupParser.get('max_threads',
default=multiprocessing.cpu_count() - 1)))
creator_processes = []
work_queue = Queue(max_creator_processes * 5)
simulations_created = 0
def fill_queue(mods, sim_per_batch, max_creator_processes, work_queue):
global simulations_expected
# Add the work to be done
for wbatch in batch(mods, sim_per_batch):
work_queue.put(wbatch)
simulations_expected += len(wbatch)
# Poison
for _ in range(max_creator_processes):
work_queue.put(None)
t = Thread(target=fill_queue,
args=(mods, sim_per_batch, max_creator_processes,
work_queue))
t.start()
for _ in range(max_creator_processes):
creator_process = self.get_simulation_creator(
work_queue=work_queue)
creator_process.daemon = True
creator_process.start()
creator_processes.append(creator_process)
# Display some info
if verbose:
logger.info("Creating the simulations")
logger.info(
" | Creator processes: {} ".format(max_creator_processes))
logger.info(" | Simulations per batch: {}".format(sim_per_batch))
# Status display
while simulations_created == 0 or simulations_created != simulations_expected or t.isAlive(
):
sys.stdout.write("\r {} Created simulations: {}/{}".format(
next(animation), simulations_created, simulations_expected))
sys.stdout.flush()
# Refresh the number of sims created
simulations_created = len(self.cache)
time.sleep(0.3)
for p in creator_processes:
p.join()
sys.stdout.write("\r | Created simulations: {}/{}\n".format(
simulations_created, simulations_expected))
sys.stdout.flush()
# Insert simulations in the cache
DataStore.bulk_insert_simulations(self.cache)
self.cache.clear()
# Refresh the experiment
self.refresh_experiment()
# Display sims
if verbose:
sims_to_display = 2
display = -sims_to_display if simulations_created > sims_to_display else -simulations_created
logger.info(" ")
logger.info("Simulation{} created:\n"
"----------------------".format(
"s" if simulations_created > 1 else ""))
for sim in self.experiment.simulations[display:]:
logger.info("- Simulation {}".format(sim.id))
logger.info(
json.dumps(sim.tags,
indent=2,
cls=GeneralEncoder,
sort_keys=True))
logger.info(" ")
if simulations_created > sims_to_display:
logger.info("... and %s more" %
(simulations_created + display))
from scipy.special import gammaln
#sys.path.append(os.path.abspath('Single_Node_Sites'))
from outbreakindividualdengue import add_OutbreakIndivisualDengue
from dtk.utils.core.DTKConfigBuilder import DTKConfigBuilder
from dtk.vector.study_sites import configure_site
from scipy.stats import uniform
from simtools.SetupParser import SetupParser
from ColombiaChikvTSSite_Department_admin2_nodes import ColombiaTSSite
import numpy as np
import scipy as sp
site_name = sys.argv[1]
SetupParser.default_block = "HPC"
with SetupParser.TemporarySetup("LOCAL") as setp:
input_dir = setp.get('input_root')
# @@ should be imported from elsewhere. PENDING until decided on structure for this.
def foi_time(times, widths, heights, days_sim, node_cnt):
foi_out = [
np.repeat(0, repeats=len(days_sim)) for ii in range(0, node_cnt)
]
for ss in range(0, node_cnt):
for yy in range(0, len(times[ss])):
times[ss][yy] = times[ss][yy] * len(days_sim)
foi_out[ss] = \
foi_out[ss] + \
def __init__(self, names):
self.regression_path = os.path.join(SetupParser().get('dll_root'),
'..', '..', 'Regression')
self.mod_generator = (self.add_test(n) for n in names)
from simtools.SetupParser import SetupParser
from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory
from examples.magude_multinode.build_cb import build_project_cb
from examples.magude_multinode.interventions import add_all_interventions
from examples.magude_multinode.reports import add_all_reports
# Run parameters:
run_priority = "Highest"
run_coreset = "emod_32cores"
experiment_name = "CoreMagude_Test"
if __name__ == '__main__':
cb = build_project_cb()
add_all_interventions(cb, sim_start_date="2010-01-01")
add_all_reports(cb)
# Start the simulation
SetupParser.default_block = "HPC"
SetupParser.init()
SetupParser.set("HPC", "priority", run_priority)
SetupParser.set("HPC", "node_group", run_coreset)
exp_manager = ExperimentManagerFactory.init()
exp_manager.run_simulations(config_builder=cb, exp_name=experiment_name)
def test_reinitialization_not_allowed(self):
SetupParser.init(selected_block="LOCAL")
kwargs = {"selected_block": "LOCAL"}
self.assertRaises(SetupParser.AlreadyInitialized, SetupParser.init,
**kwargs)
def _get_path(self, path_type):
return self.paths.get(path_type, None) or SetupParser.get(
self.SETUP_MAPPING[path_type])
def test_verify_user_in_default_block(self):
SetupParser.init(selected_block='LOCAL')
self.assertTrue(
'user' in SetupParser.singleton.setup.defaults().keys())
self.assertNotEqual(SetupParser.singleton.setup.defaults()['user'],
None)
from dtk.utils.core.DTKConfigBuilder import DTKConfigBuilder
from dtk.utils.builders.sweep import GenericSweepBuilder
from dtk.vector.species import set_species_param
from dtk.generic.serialization import add_SerializationTimesteps
from dtk.generic.climate import set_climate_constant
from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory
from simtools.SetupParser import SetupParser
if __name__ == "__main__":
expname = 'Matsari_closed_loop__burnin'
sim_duration = 365 * 70
num_seeds = 1
SetupParser('HPC')
cb = DTKConfigBuilder.from_defaults('MALARIA_SIM')
builder = GenericSweepBuilder.from_dict({'Run_Number': range(num_seeds)})
set_climate_constant(cb)
set_species_param(
cb,
'gambiae',
'Larval_Habitat_Types',
{
"LINEAR_SPLINE": {
"Capacity_Distribution_Per_Year": {
"Times": [
0.0, 30.417, 60.833, 91.25, 121.667, 152.083, 182.5,
def tearDown(self):
os.chdir(self.cwd)
SetupParser._uninit()
num_params / 2. * math.log(math.pi)))
optimtool = OptimTool(
params,
constrain_sample, # <-- WILL NOT BE SAVED IN ITERATION STATE
mu_r=
r, # <-- radius for numerical derivatve. CAREFUL not to go too small with integer parameters
sigma_r=r / 10., # <-- stdev of radius
center_repeats=
2, # <-- Number of times to replicate the center (current guess). Nice to compare intrinsic to extrinsic noise
samples_per_iteration=
4 # 32 # <-- Samples per iteration, includes center repeats. Actual number of sims run is this number times number of sites.
)
calib_manager = CalibManager(
name='MalariaPackageTest', # <-- Please customize this name
config_builder=cb,
map_sample_to_model_input_fn=map_sample_to_model_input,
sites=sites,
next_point=optimtool,
sim_runs_per_param_set=3, # <-- Replicates
max_iterations=1, # <-- Iterations
plotters=plotters)
run_calib_args = {"calib_manager": calib_manager}
if __name__ == "__main__":
SetupParser.init()
cm = run_calib_args["calib_manager"]
cm.run_calibration()
def test_verify_block_selection(self):
SetupParser.init(selected_block='LOCAL')
self.assertEqual(SetupParser.selected_block, 'LOCAL')
def sync(args, unknownArgs):
"""
Sync COMPS db with local db
"""
# Create a default HPC setup parser
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
user = sp.get('user')
COMPS_login(endpoint)
exp_to_save = list()
exp_deleted = 0
# Retrieve all the experiment id from COMPS for the current user
exp_ids = get_experiment_ids_for_user(user)
# Test the experiments present in the local DB to make sure they still exist in COMPS
for exp in DataStore.get_experiments():
if exp.location == "HPC":
if exp.exp_id not in exp_ids:
# The experiment doesnt exist on COMPS anymore -> delete from local
DataStore.delete_experiment(exp)
exp_deleted += 1
# Consider experiment id option
exp_id = args.exp_id if args.exp_id else None
exp_name = args.exp_name if args.exp_name else None
user = args.user if args.user else user
if exp_name:
experiments = get_experiments_by_name(exp_name, user)
for experiment_data in experiments:
experiment = COMPS_experiment_to_local_db(
exp_id=str(experiment_data.id),
endpoint=endpoint,
verbose=True,
save_new_experiment=False)
if experiment:
exp_to_save.append(experiment)
elif exp_id:
# Create a new experiment
experiment = COMPS_experiment_to_local_db(exp_id=exp_id,
endpoint=endpoint,
verbose=True,
save_new_experiment=False)
# The experiment needs to be saved
if experiment:
exp_to_save.append(experiment)
else:
# By default only get experiments created in the last month
# day_limit = args.days if args.days else day_limit_default
day_limit = 30
today = datetime.date.today()
limit_date = today - datetime.timedelta(days=int(day_limit))
# For each of them, check if they are in the db
for exp in get_experiments_per_user_and_date(user, limit_date):
# Create a new experiment
experiment = COMPS_experiment_to_local_db(
exp_id=str(exp.id),
endpoint=endpoint,
save_new_experiment=False)
# The experiment needs to be saved
if experiment:
exp_to_save.append(experiment)
# Save the experiments if any
if len(exp_to_save) > 0 and exp_deleted == 0:
DataStore.batch_save_experiments(exp_to_save)
logger.info(
"The following experiments have been added to the database:")
logger.info("\n".join(["- " + str(exp) for exp in exp_to_save]))
logger.info("%s experiments have been updated in the DB." %
len(exp_to_save))
logger.info("%s experiments have been deleted from the DB." %
exp_deleted)
else:
print("The database was already up to date.")
# Start overseer
BaseExperimentManager.check_overseer()
def test_regression_959(self):
"""
Improper 'type' inheritance in simtools.ini
Test for https://github.com/InstituteforDiseaseModeling/dtk-tools/issues/959
"""
SetupParser.init(selected_block='AM',
setup_file=os.path.join(self.input_path, '959',
'simtools1.ini'),
is_testing=True)
self.assertEqual(SetupParser.get('base_collection_id_dll'), "in AM")
SetupParser._uninit()
SetupParser.init(selected_block='AM',
setup_file=os.path.join(self.input_path, '959',
'simtools2.ini'),
is_testing=True)
self.assertEqual(SetupParser.get('base_collection_id_dll'), "in HPC")
SetupParser._uninit()
SetupParser.init(selected_block='AM',
setup_file=os.path.join(self.input_path, '959',
'simtools3.ini'),
is_testing=True)
self.assertEqual(SetupParser.get('base_collection_id_dll'), "in HPC")
SetupParser._uninit()
from dtk.tools.climate.ClimateGenerator import ClimateGenerator
from dtk.tools.climate.WeatherNode import WeatherNode
from dtk.tools.demographics.DemographicsFile import DemographicsFile
from simtools.SetupParser import SetupParser
# Set up the paths
current_dir = os.path.dirname(os.path.realpath(__file__))
output_path = os.path.join(current_dir, 'output')
intermediate_dir = os.path.join(current_dir, 'intermediate', 'climate')
# Make sure we have directory created
if not os.path.exists(intermediate_dir): os.makedirs(intermediate_dir)
if not os.path.exists(output_path): os.makedirs(output_path)
# Get a setup
SetupParser.init('HPC')
# Create the 2 nodes we need to pull weather for
node_1001 = WeatherNode(lon=27.6, lat=-17.05, name='Node 1001', pop=1000)
node_others = WeatherNode(lon=28, lat=-16.5, name='Others', pop=1000)
nodes = [node_1001, node_others]
# Create the file
dg = DemographicsFile(nodes)
climate_demog = os.path.join(intermediate_dir, 'climate_demog.json')
dg.generate_file(climate_demog)
cg = ClimateGenerator(demographics_file_path=climate_demog,
work_order_path=os.path.join(intermediate_dir,
'wo.json'),
climate_files_output_path=intermediate_dir,
def generate_climate_files(self):
# see InputDataWorker for other work options
self.wo = InputDataWorker(
demographics_file_path=self.demographics_file_path,
wo_output_path=self.work_order_path,
project_info=self.climate_project,
start_year=self.start_year,
num_years=self.num_years,
resolution=self.resolution,
idRef=self.idRef)
# login to COMPS (if not already logged in) to submit climate files generation work order
self.wo.wo_2_json()
from COMPS.Data.WorkItem import WorkerOrPluginKey, WorkItemState
from COMPS.Data import QueryCriteria
from COMPS.Data import WorkItem, WorkItemFile
from COMPS.Data import AssetCollection
workerkey = WorkerOrPluginKey(name='InputDataWorker',
version='1.0.0.0_RELEASE')
wi = WorkItem('dtk-tools InputDataWorker WorkItem', workerkey,
SetupParser.get('environment'))
wi.set_tags({
'dtk-tools': None,
'WorkItem type': 'InputDataWorker dtk-tools'
})
with open(self.work_order_path, 'rb') as workorder_file:
# wi.AddWorkOrder(workorder_file.read())
wi.add_work_order(data=workorder_file.read())
with open(self.demographics_file_path, 'rb') as demog_file:
wi.add_file(WorkItemFile(
os.path.basename(self.demographics_file_path), 'Demographics',
''),
data=demog_file.read())
wi.save()
print("Created request for climate files generation.")
print("Commissioning...")
wi.commission()
while wi.state not in (WorkItemState.Succeeded, WorkItemState.Failed,
WorkItemState.Canceled):
om('Waiting for climate generation to complete (current state: ' +
str(wi.state) + ')',
style='flushed')
time.sleep(5)
wi.refresh()
print("Climate files SUCCESSFULLY generated")
# Get the collection with our files
collections = wi.get_related_asset_collections()
collection_id = collections[0].id
comps_collection = AssetCollection.get(
collection_id,
query_criteria=QueryCriteria().select_children('assets'))
# Get the files
if len(comps_collection.assets) > 0:
print("Found output files:")
for asset in comps_collection.assets:
print("- %s (%s)" % (asset.file_name, file_size(asset.length)))
print("\nDownloading to %s..." % self.climate_files_output_path)
# Download the collection
download_asset_collection(comps_collection,
self.climate_files_output_path)
# return filenames; this use of re in conjunction w/ glob is not great; consider refactor
rain_bin_re = os.path.abspath(self.climate_files_output_path +
'/*rain*.bin')
humidity_bin_re = os.path.abspath(self.climate_files_output_path +
'/*humidity*.bin')
temperature_bin_re = os.path.abspath(
self.climate_files_output_path + '/*temperature*.bin')
rain_file_name = os.path.basename(glob.glob(rain_bin_re)[0])
humidity_file_name = os.path.basename(
glob.glob(humidity_bin_re)[0])
temperature_file_name = os.path.basename(
glob.glob(temperature_bin_re)[0])
print('Climate files SUCCESSFULLY stored.')
return rain_file_name, temperature_file_name, humidity_file_name
else:
print('No output files found')
