'agemax': 200, 'seek': 1, 'rate': 0.0 }]) #add_drug_campaign(cb, "SMC", "SPA", start_days=[230 + 365*x for x in range(5)], repetitions=1, interval=30, coverage=1., target_group={'agemin' : 0, 'agemax' : 5}) add_summary_report(cb, interval=365 / 12., description='Monthly', age_bins=[5, 15, 200]) #add_summary_report(cb, interval=365/12., description='MonthlyUnderFive', age_bins=[5]) add_summary_report(cb, interval=1., description='DailyUnderFive', age_bins=[5, 15, 200]) #add_summary_report(cb, interval=365/12., description='AnnualUnderFive', age_bins=[5]) builder = GenericSweepBuilder.from_dict({'Run_Number': range(1)}) #builder = GenericSweepBuilder.from_dict({'Run_Number':range(100)}) run_sim_args = { 'config_builder': cb, 'exp_name': expname, 'exp_builder': builder } if __name__ == "__main__": sm = ExperimentManagerFactory.from_setup(SetupParser()) sm.run_simulations(**run_sim_args)
def link(args, unknownArgs): """ Open browser to the COMPS Experiment/Simulation with ID or name provided :param args: :param unknownArgs: :return: """ # get input from commands line input_id = args.Id # default: consider the latest experiment if input_id is None: latest = DataStore.get_most_recent_experiment() input_id = latest.exp_id try: comps_item = retrieve_item(input_id) except: print('Nothing was found for {}'.format(input_id)) exit() # check item type id_type = '' location = 'LOCAL' if isinstance(comps_item, Experiment): item_id = comps_item.exp_id id_type = 'exp' location = comps_item.location elif isinstance(comps_item, Simulation): item_id = comps_item.id exp_id = comps_item.experiment_id id_type = 'sim' # retrieve location exp = DataStore.get_experiment(exp_id) location = exp.location else: print('No Experiment or Simulation was found on COMPS for {}'.format( input_id)) exit() # make sure it exists on COMPS if location == 'LOCAL': print('Item is on LOCAL not on COMPS.') exit() # open browser to COMPS Experiment/Simulation import webbrowser with SetupParser.TemporarySetup(temporary_block='HPC') as sp: endpoint = sp.get('server_endpoint') url = '' if id_type == 'exp': url = '%s/#explore/Experiments?filters=Id=%s&offset=0&selectedId=%s' % ( endpoint, item_id, item_id) elif id_type == 'sim': url = '%s/#explore/Simulations?filters=Id=%s&mode=list&orderby=DateCreated+desc&count=50&offset=0&layout=512C56&selectedId=%s' % ( endpoint, item_id, item_id) # Open URL in new browser window webbrowser.open_new(url) # opens in default browser
map_sample_to_model_input_fn=map_sample_to_model_input, sites=sites, next_point=optimtool, sim_runs_per_param_set=sim_runs_per_param_set, max_iterations=max_iterations, plotters=plotters) run_calib_args = { "calib_manager": calib_manager } if __name__ == "__main__": if parser_location == "LOCAL": SetupParser.init("LOCAL") else: SetupParser.init() SetupParser.set("HPC", "priority", priority) SetupParser.set("HPC", "node_group", coreset) cm = run_calib_args["calib_manager"] if scratch_or_resume == "scratch": cm.run_calibration() elif scratch_or_resume == "resume": cm.resume_calibration(iteration=0, iter_step='analyze')
simdata[sweep_var] = 0 return simdata def finalize(self, all_data): selected = [data for sim, data in all_data.items()] if len(selected) == 0: print("No data have been returned... Exiting...") return df = pd.concat(selected).reset_index(drop=True) df.to_csv(os.path.join(self.working_dir, '%s.csv' % self.dir_name), index=False) if __name__ == "__main__": SetupParser.init("HPC") out_dir = os.path.join('E:/', 'Dropbox (IDM)', 'Malaria Team Folder', 'projects', 'map_intervention_impact', 'lookup_tables') sites = pd.read_csv("site_details.csv") experiments = { "corr_itn_irs" :"e7fd04a2-a8d8-e811-a2bd-c4346bcb1555" } for dirname, exp_id in experiments.items(): am = AnalyzeManager(exp_list=exp_id, analyzers=[BurdenAnalyzer(working_dir=out_dir, dir_name=dirname, report_names = sites["name"].tolist(), sweep_variables=["Run_Number",
# This is REQUIRED by the templates config_builder.ignore_missing = True # Get the dicts points_dict = header_table_to_dict(point_header, points, index_name='TPI') for point in points_dict: tpi = point.pop('TPI') if 'TAGS' not in point: point['TAGS'] = {} point['TAGS']['TPI'] = tpi scenarios_dict = header_table_to_dict(scenario_header, scenarios) if __name__ == "__main__": SetupParser.init() experiments = [] # All the experiment managers for all experiments experiments_ids = [ ] # Ids of the created experiments for resuming capabilities # Check if we want to resume if os.path.exists('ids.json'): print("Previous run detected... Run [N]ew, [R]esume, [A]bort?") resp = "" while resp not in ('N', 'R', 'A'): resp = input() if resp == "A": exit() elif resp == "R": # In the resume case, retrieve the ids and create the managers
from dtk.tools.demographics.DemographicsFile import DemographicsFile from simtools.Analysis.AnalyzeManager import AnalyzeManager from simtools.Analysis.BaseAnalyzers import DownloadAnalyzer from simtools.Utilities.COMPSUtilities import get_experiment_by_id from COMPS.Data import QueryCriteria import time import random sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from PythonHelperFunctions.DemographicsHelpers import * from PythonHelperFunctions.utils import load_dropbox_path, load_output_path, load_input_path from pyDOE import lhs import numpy as np #Run locally or on HPC SetupParser.default_block = "HPC" SetupParser.init(selected_block=SetupParser.default_block) ScenarioName = "HomeOnlyForever" InterventionDuration = 2000 # Set the path for DTK input files BaseInputPath = load_input_path() if not os.path.exists(BaseInputPath): os.mkdir(BaseInputPath) OutputPath = os.path.join(load_output_path(), ScenarioName) if not os.path.exists(OutputPath): os.mkdir(OutputPath) if not os.path.exists(os.path.join(OutputPath, 'input_files')): os.mkdir(os.path.join(OutputPath, 'input_files')) #Name the input files DemoFile = os.path.join(BaseInputPath, 'demographics.json') CampaignFile = os.path.join(BaseInputPath, 'campaign.json')
# Get the last time a cleanup happened last_cleanup = DataStore.get_setting('last_log_cleanup') if not last_cleanup or (datetime.today() - datetime.strptime(last_cleanup.value.split(' ')[0],'%Y-%m-%d')).days > 1: # Do the cleanup from simtools.DataAccess.LoggingDataStore import LoggingDataStore LoggingDataStore.log_cleanup() DataStore.save_setting(DataStore.create_setting(key='last_log_cleanup', value=datetime.today())) if __name__ == "__main__": logger.debug('Start Overseer pid: %d' % os.getpid()) # we technically don't care about full consistency of SetupParser with the original dtk command, as experiments # have all been created. We can grab 'generic' max_local_sims / max_threads SetupParser.init() # default block max_local_sims = int(SetupParser.get('max_local_sims')) # Create the queue local_queue = multiprocessing.Queue(max_local_sims) managers = OrderedDict() # Queue to be shared among all runners in order to update the individual simulation states in the DB manager = Manager() # Take this opportunity to cleanup the logs lc = threading.Thread(target=LogCleaner) lc.start() count = 0
def file_writer(self, write_fn): """ Dump all the files needed for the simulation in the simulation directory. This includes: * The campaign file * The custom reporters file * The different demographic overlays * The other input files (``input_files`` dictionary) * The config file * The emodules_map file Args: write_fn: The function that will write the files. This function needs to take a file name and a content. Examples: For example, in the :py:class:`SimConfigBuilder` class, the :py:func:`dump_files` is defining the `write_fn` like:: def write_file(name, content): filename = os.path.join(working_directory, '%s.json' % name) with open(filename, 'w') as f: f.write(content) """ from simtools.SetupParser import SetupParser if self.human_readability: dump = lambda content: json.dumps( content, sort_keys=True, indent=3, cls=NumpyEncoder).strip('"') else: dump = lambda content: json.dumps( content, sort_keys=True, cls=NumpyEncoder).strip('"') write_fn(self.config['parameters']['Campaign_Filename'], dump(self.campaign)) if self.custom_reports: self.set_param('Custom_Reports_Filename', 'custom_reports.json') write_fn('custom_reports.json', dump(format_reports(self.custom_reports))) for name, content in self.demog_overlays.items(): self.append_overlay('%s' % name) write_fn('%s' % name, dump(content)) for name, content in self.input_files.items(): write_fn(name, dump(content)) # Add missing item from campaign individual events into Listed_Events self.config['parameters']['Listed_Events'] = self.check_custom_events() write_fn('config.json', dump(self.config)) # complete the path to each dll before writing emodules_map.json location = SetupParser.get('type') if location == 'LOCAL': root = self.assets.dll_root elif location == 'HPC': root = 'Assets' else: raise Exception('Unknown location: %s' % location) for module_type in self.emodules_map.keys(): self.emodules_map[module_type] = list( set([ os.path.join(root, dll) for dll in self.emodules_map[module_type] ])) write_fn('emodules_map.json', dump(self.emodules_map))
0, "Base_Air_Temperature": 25.0, "Enable_Demographics_Birth": 0, "Enable_Climate_Stochasticity": 1, "Age_Dependent_Biting_Risk_Type": "OFF", '.Vector_Sampling_Type': "TRACK_ALL_VECTORS", 'Vector_Sugar_Feeding_Frequency': 'VECTOR_SUGAR_FEEDING_NONE', "Vector_Sampling_Type": "VECTOR_COMPARTMENTS_NUMBER", "Vector_Species_Names": ['arabiensis'], "Enable_Vector_Aging": 1 }) SetupParser('LOCAL') run_sim_args = { 'config_builder': cb, 'exp_name': expname, 'exp_builder': builder } exp_manager = ExperimentManagerFactory.from_setup() exp_manager.run_simulations(**run_sim_args) exp_manager.wait_for_finished(verbose=True)
def location(self): return SetupParser.get( 'type') if self._location is None else self._location
can be encoded in a similar fashion using custom functions rather than the generic "set_param". ''' sample_point = prior.to_dict(sample_dimension_values) params_to_update = dict() params_to_update['Simulation_Duration'] = 365 for sample_dimension_name, sample_dimension_value in sample_point.items(): param_name = sample_dimension_name.replace('_LOG', '') params_to_update[param_name] = pow(10, sample_dimension_value) return cb.update_params(params_to_update) next_point_kwargs = dict(initial_samples=3, samples_per_iteration=3, n_resamples=100) calib_manager = CalibManager(name='test_dummy_calibration', setup=SetupParser(), config_builder=cb, map_sample_to_model_input_fn=sample_point_fn, sites=sites, next_point=IMIS(prior, **next_point_kwargs), sim_runs_per_param_set=1, max_iterations=2, plotters=plotters) run_calib_args = {} if __name__ == "__main__": calib_manager.run_calibration(**run_calib_args)
[ ModFn(sweep_larval_habitat, scale_factor = habitat_multiplier), ModFn(DTKConfigBuilder.set_param,'Run_Number',seed) ] for habitat_multiplier in np.logspace(-2, 1, 50) for seed in range(num_seeds) ] ) # builder = ModBuilder.from_list( [[ ModFn(sweep_scale_factor, x)] for x in [0.01,10,100]]) # report_years = 1 # report_start = 40 # Run args run_sim_args = {'config_builder': cb, 'exp_name': exp_name, 'exp_builder': builder } if __name__ == "__main__": if not SetupParser.initialized: SetupParser.init('LOCAL') exp_manager = ExperimentManagerFactory.init() exp_manager.run_simulations(**run_sim_args) exp_manager.wait_for_finished(verbose=True) assert (exp_manager.succeeded())
def tearDown(self): SetupParser._uninit()
from dtk.utils.core.DTKConfigBuilder import DTKConfigBuilder from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory from simtools.ModBuilder import SingleSimulationBuilder from simtools.SetupParser import SetupParser experiment_name = "Generic Default simluation test for tools" generic_defaults = [ 'GENERIC_SIM_SIR', 'GENERIC_SIM_SEIR', 'GENERIC_SIM_SIRS', 'GENERIC_SIM_SEIRS', 'GENERIC_SIM_SI', 'GENERIC_SIM_SIS', 'GENERIC_SIM_SIR_Vaccinations_A', 'GENERIC_SIM_SIR_Vaccinations_B', 'GENERIC_SIM_SIR_Vaccinations_C' ] if __name__ == '__main__': # Initialize the SetupParser to use HPC SetupParser.init(selected_block='HPC') # Create an experiment manager exp_mgr = ExperimentManagerFactory.init() # For each generic simulations, create a config builder from the default and run it for sim_example in generic_defaults: cb = DTKConfigBuilder.from_defaults(sim_example) cb.set_param("Config_Name", sim_example) # Create a builder in order to run a single simulation # This step is optional usually but it allows us to set tags on the simulation builder = SingleSimulationBuilder() builder.tags.update({"Simulation_Type": sim_example}) exp_mgr.run_simulations(config_builder=cb,
def test_block_inheritance(self): """ Issue 1246 Verify that multi-level block inheritance works properly and that 'type' percolates from the deepest-level (root) of an inheritance chain. """ # # ck4, template for following tests/asserts # SetupParser.init(selected_block='LOCAL', # setup_file=os.path.join(self.input_path, '1246', somedir, 'simtools.ini'), is_testing=True) # something = None # self.assertTrue(something) # SetupParser._uninit() # # Using a 3 level-inheritance scheme in all of these cases # # descendant block values override parent block values # EXCEPT: 'type is inherited from the inheritance chain root # verify that block order in the ini file does not matter for arbitrary key/values OR 'type' values = {} types = {} for i in range(1, 4): testdir = 'ordering%d' % i SetupParser.init(selected_block='LOCALB', setup_file=os.path.join(self.input_path, '1246', testdir, 'simtools.ini'), is_testing=True) values[testdir] = SetupParser.get('a') types[testdir] = SetupParser.get('type') SetupParser._uninit() unique_values = sorted(set(values.values())) self.assertEqual(unique_values, ['3']) unique_types = sorted(set(types.values())) self.assertEqual(unique_types, ['LOCAL']) # verify that the proper values are inherited, regardless of the level in the inheritance chain values are # located. SetupParser.init(selected_block='LOCALB', setup_file=os.path.join(self.input_path, '1246', 'mixedLevelValues', 'simtools.ini'), is_testing=True) self.assertEqual(SetupParser.get('a'), '1') self.assertEqual(SetupParser.get('b'), '3') self.assertEqual(SetupParser.get('c'), '5') self.assertEqual(SetupParser.get('d'), '7') self.assertEqual(SetupParser.get('e'), '10') SetupParser._uninit() # Blocks used as the 'type' should fail if missing kwargs = { 'selected_block': 'LOCALB', 'setup_file': os.path.join(self.input_path, '1246', 'missingReferencedBlock', 'simtools.ini'), 'is_testing': True } self.assertRaises(SetupParser.InvalidBlock, SetupParser.init, **kwargs) SetupParser._uninit() # Blocks missing 'type' should fail kwargs = { 'selected_block': 'LOCALB', 'setup_file': os.path.join(self.input_path, '1246', 'missingType', 'simtools.ini'), 'is_testing': True } self.assertRaises(SetupParser.InvalidBlock, SetupParser.init, **kwargs) SetupParser._uninit()
def create_simulations(self, exp_name='test', exp_builder=None, suite_id=None, verbose=True): """ Create an experiment with simulations modified according to the specified experiment builder. """ global simulations_expected simulations_expected = 0 self.exp_builder = exp_builder or SingleSimulationBuilder() self.cache.clear() # Create the experiment if not present already if not self.experiment or self.experiment.exp_name != exp_name: self.create_experiment(experiment_name=exp_name, suite_id=suite_id) else: # Refresh the experiment self.refresh_experiment() # Save the experiment in the DB DataStore.save_experiment(self.experiment, verbose=verbose) # Separate the experiment builder generator into batches sim_per_batch = int(SetupParser.get('sims_per_thread', default=50)) mods = self.exp_builder.mod_generator max_creator_processes = min( multiprocessing.cpu_count() - 1, int( SetupParser.get('max_threads', default=multiprocessing.cpu_count() - 1))) creator_processes = [] work_queue = Queue(max_creator_processes * 5) simulations_created = 0 def fill_queue(mods, sim_per_batch, max_creator_processes, work_queue): global simulations_expected # Add the work to be done for wbatch in batch(mods, sim_per_batch): work_queue.put(wbatch) simulations_expected += len(wbatch) # Poison for _ in range(max_creator_processes): work_queue.put(None) t = Thread(target=fill_queue, args=(mods, sim_per_batch, max_creator_processes, work_queue)) t.start() for _ in range(max_creator_processes): creator_process = self.get_simulation_creator( work_queue=work_queue) creator_process.daemon = True creator_process.start() creator_processes.append(creator_process) # Display some info if verbose: logger.info("Creating the simulations") logger.info( " | Creator processes: {} ".format(max_creator_processes)) logger.info(" | Simulations per batch: {}".format(sim_per_batch)) # Status display while simulations_created == 0 or simulations_created != simulations_expected or t.isAlive( ): sys.stdout.write("\r {} Created simulations: {}/{}".format( next(animation), simulations_created, simulations_expected)) sys.stdout.flush() # Refresh the number of sims created simulations_created = len(self.cache) time.sleep(0.3) for p in creator_processes: p.join() sys.stdout.write("\r | Created simulations: {}/{}\n".format( simulations_created, simulations_expected)) sys.stdout.flush() # Insert simulations in the cache DataStore.bulk_insert_simulations(self.cache) self.cache.clear() # Refresh the experiment self.refresh_experiment() # Display sims if verbose: sims_to_display = 2 display = -sims_to_display if simulations_created > sims_to_display else -simulations_created logger.info(" ") logger.info("Simulation{} created:\n" "----------------------".format( "s" if simulations_created > 1 else "")) for sim in self.experiment.simulations[display:]: logger.info("- Simulation {}".format(sim.id)) logger.info( json.dumps(sim.tags, indent=2, cls=GeneralEncoder, sort_keys=True)) logger.info(" ") if simulations_created > sims_to_display: logger.info("... and %s more" % (simulations_created + display))
from scipy.special import gammaln #sys.path.append(os.path.abspath('Single_Node_Sites')) from outbreakindividualdengue import add_OutbreakIndivisualDengue from dtk.utils.core.DTKConfigBuilder import DTKConfigBuilder from dtk.vector.study_sites import configure_site from scipy.stats import uniform from simtools.SetupParser import SetupParser from ColombiaChikvTSSite_Department_admin2_nodes import ColombiaTSSite import numpy as np import scipy as sp site_name = sys.argv[1] SetupParser.default_block = "HPC" with SetupParser.TemporarySetup("LOCAL") as setp: input_dir = setp.get('input_root') # @@ should be imported from elsewhere. PENDING until decided on structure for this. def foi_time(times, widths, heights, days_sim, node_cnt): foi_out = [ np.repeat(0, repeats=len(days_sim)) for ii in range(0, node_cnt) ] for ss in range(0, node_cnt): for yy in range(0, len(times[ss])): times[ss][yy] = times[ss][yy] * len(days_sim) foi_out[ss] = \ foi_out[ss] + \
def __init__(self, names): self.regression_path = os.path.join(SetupParser().get('dll_root'), '..', '..', 'Regression') self.mod_generator = (self.add_test(n) for n in names)
from simtools.SetupParser import SetupParser from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory from examples.magude_multinode.build_cb import build_project_cb from examples.magude_multinode.interventions import add_all_interventions from examples.magude_multinode.reports import add_all_reports # Run parameters: run_priority = "Highest" run_coreset = "emod_32cores" experiment_name = "CoreMagude_Test" if __name__ == '__main__': cb = build_project_cb() add_all_interventions(cb, sim_start_date="2010-01-01") add_all_reports(cb) # Start the simulation SetupParser.default_block = "HPC" SetupParser.init() SetupParser.set("HPC", "priority", run_priority) SetupParser.set("HPC", "node_group", run_coreset) exp_manager = ExperimentManagerFactory.init() exp_manager.run_simulations(config_builder=cb, exp_name=experiment_name)
def test_reinitialization_not_allowed(self): SetupParser.init(selected_block="LOCAL") kwargs = {"selected_block": "LOCAL"} self.assertRaises(SetupParser.AlreadyInitialized, SetupParser.init, **kwargs)
def _get_path(self, path_type): return self.paths.get(path_type, None) or SetupParser.get( self.SETUP_MAPPING[path_type])
def test_verify_user_in_default_block(self): SetupParser.init(selected_block='LOCAL') self.assertTrue( 'user' in SetupParser.singleton.setup.defaults().keys()) self.assertNotEqual(SetupParser.singleton.setup.defaults()['user'], None)
from dtk.utils.core.DTKConfigBuilder import DTKConfigBuilder from dtk.utils.builders.sweep import GenericSweepBuilder from dtk.vector.species import set_species_param from dtk.generic.serialization import add_SerializationTimesteps from dtk.generic.climate import set_climate_constant from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory from simtools.SetupParser import SetupParser if __name__ == "__main__": expname = 'Matsari_closed_loop__burnin' sim_duration = 365 * 70 num_seeds = 1 SetupParser('HPC') cb = DTKConfigBuilder.from_defaults('MALARIA_SIM') builder = GenericSweepBuilder.from_dict({'Run_Number': range(num_seeds)}) set_climate_constant(cb) set_species_param( cb, 'gambiae', 'Larval_Habitat_Types', { "LINEAR_SPLINE": { "Capacity_Distribution_Per_Year": { "Times": [ 0.0, 30.417, 60.833, 91.25, 121.667, 152.083, 182.5,
def tearDown(self): os.chdir(self.cwd) SetupParser._uninit()
num_params / 2. * math.log(math.pi))) optimtool = OptimTool( params, constrain_sample, # <-- WILL NOT BE SAVED IN ITERATION STATE mu_r= r, # <-- radius for numerical derivatve. CAREFUL not to go too small with integer parameters sigma_r=r / 10., # <-- stdev of radius center_repeats= 2, # <-- Number of times to replicate the center (current guess). Nice to compare intrinsic to extrinsic noise samples_per_iteration= 4 # 32 # <-- Samples per iteration, includes center repeats. Actual number of sims run is this number times number of sites. ) calib_manager = CalibManager( name='MalariaPackageTest', # <-- Please customize this name config_builder=cb, map_sample_to_model_input_fn=map_sample_to_model_input, sites=sites, next_point=optimtool, sim_runs_per_param_set=3, # <-- Replicates max_iterations=1, # <-- Iterations plotters=plotters) run_calib_args = {"calib_manager": calib_manager} if __name__ == "__main__": SetupParser.init() cm = run_calib_args["calib_manager"] cm.run_calibration()
def test_verify_block_selection(self): SetupParser.init(selected_block='LOCAL') self.assertEqual(SetupParser.selected_block, 'LOCAL')
def sync(args, unknownArgs): """ Sync COMPS db with local db """ # Create a default HPC setup parser with SetupParser.TemporarySetup(temporary_block='HPC') as sp: endpoint = sp.get('server_endpoint') user = sp.get('user') COMPS_login(endpoint) exp_to_save = list() exp_deleted = 0 # Retrieve all the experiment id from COMPS for the current user exp_ids = get_experiment_ids_for_user(user) # Test the experiments present in the local DB to make sure they still exist in COMPS for exp in DataStore.get_experiments(): if exp.location == "HPC": if exp.exp_id not in exp_ids: # The experiment doesnt exist on COMPS anymore -> delete from local DataStore.delete_experiment(exp) exp_deleted += 1 # Consider experiment id option exp_id = args.exp_id if args.exp_id else None exp_name = args.exp_name if args.exp_name else None user = args.user if args.user else user if exp_name: experiments = get_experiments_by_name(exp_name, user) for experiment_data in experiments: experiment = COMPS_experiment_to_local_db( exp_id=str(experiment_data.id), endpoint=endpoint, verbose=True, save_new_experiment=False) if experiment: exp_to_save.append(experiment) elif exp_id: # Create a new experiment experiment = COMPS_experiment_to_local_db(exp_id=exp_id, endpoint=endpoint, verbose=True, save_new_experiment=False) # The experiment needs to be saved if experiment: exp_to_save.append(experiment) else: # By default only get experiments created in the last month # day_limit = args.days if args.days else day_limit_default day_limit = 30 today = datetime.date.today() limit_date = today - datetime.timedelta(days=int(day_limit)) # For each of them, check if they are in the db for exp in get_experiments_per_user_and_date(user, limit_date): # Create a new experiment experiment = COMPS_experiment_to_local_db( exp_id=str(exp.id), endpoint=endpoint, save_new_experiment=False) # The experiment needs to be saved if experiment: exp_to_save.append(experiment) # Save the experiments if any if len(exp_to_save) > 0 and exp_deleted == 0: DataStore.batch_save_experiments(exp_to_save) logger.info( "The following experiments have been added to the database:") logger.info("\n".join(["- " + str(exp) for exp in exp_to_save])) logger.info("%s experiments have been updated in the DB." % len(exp_to_save)) logger.info("%s experiments have been deleted from the DB." % exp_deleted) else: print("The database was already up to date.") # Start overseer BaseExperimentManager.check_overseer()
def test_regression_959(self): """ Improper 'type' inheritance in simtools.ini Test for https://github.com/InstituteforDiseaseModeling/dtk-tools/issues/959 """ SetupParser.init(selected_block='AM', setup_file=os.path.join(self.input_path, '959', 'simtools1.ini'), is_testing=True) self.assertEqual(SetupParser.get('base_collection_id_dll'), "in AM") SetupParser._uninit() SetupParser.init(selected_block='AM', setup_file=os.path.join(self.input_path, '959', 'simtools2.ini'), is_testing=True) self.assertEqual(SetupParser.get('base_collection_id_dll'), "in HPC") SetupParser._uninit() SetupParser.init(selected_block='AM', setup_file=os.path.join(self.input_path, '959', 'simtools3.ini'), is_testing=True) self.assertEqual(SetupParser.get('base_collection_id_dll'), "in HPC") SetupParser._uninit()
from dtk.tools.climate.ClimateGenerator import ClimateGenerator from dtk.tools.climate.WeatherNode import WeatherNode from dtk.tools.demographics.DemographicsFile import DemographicsFile from simtools.SetupParser import SetupParser # Set up the paths current_dir = os.path.dirname(os.path.realpath(__file__)) output_path = os.path.join(current_dir, 'output') intermediate_dir = os.path.join(current_dir, 'intermediate', 'climate') # Make sure we have directory created if not os.path.exists(intermediate_dir): os.makedirs(intermediate_dir) if not os.path.exists(output_path): os.makedirs(output_path) # Get a setup SetupParser.init('HPC') # Create the 2 nodes we need to pull weather for node_1001 = WeatherNode(lon=27.6, lat=-17.05, name='Node 1001', pop=1000) node_others = WeatherNode(lon=28, lat=-16.5, name='Others', pop=1000) nodes = [node_1001, node_others] # Create the file dg = DemographicsFile(nodes) climate_demog = os.path.join(intermediate_dir, 'climate_demog.json') dg.generate_file(climate_demog) cg = ClimateGenerator(demographics_file_path=climate_demog, work_order_path=os.path.join(intermediate_dir, 'wo.json'), climate_files_output_path=intermediate_dir,
def generate_climate_files(self): # see InputDataWorker for other work options self.wo = InputDataWorker( demographics_file_path=self.demographics_file_path, wo_output_path=self.work_order_path, project_info=self.climate_project, start_year=self.start_year, num_years=self.num_years, resolution=self.resolution, idRef=self.idRef) # login to COMPS (if not already logged in) to submit climate files generation work order self.wo.wo_2_json() from COMPS.Data.WorkItem import WorkerOrPluginKey, WorkItemState from COMPS.Data import QueryCriteria from COMPS.Data import WorkItem, WorkItemFile from COMPS.Data import AssetCollection workerkey = WorkerOrPluginKey(name='InputDataWorker', version='1.0.0.0_RELEASE') wi = WorkItem('dtk-tools InputDataWorker WorkItem', workerkey, SetupParser.get('environment')) wi.set_tags({ 'dtk-tools': None, 'WorkItem type': 'InputDataWorker dtk-tools' }) with open(self.work_order_path, 'rb') as workorder_file: # wi.AddWorkOrder(workorder_file.read()) wi.add_work_order(data=workorder_file.read()) with open(self.demographics_file_path, 'rb') as demog_file: wi.add_file(WorkItemFile( os.path.basename(self.demographics_file_path), 'Demographics', ''), data=demog_file.read()) wi.save() print("Created request for climate files generation.") print("Commissioning...") wi.commission() while wi.state not in (WorkItemState.Succeeded, WorkItemState.Failed, WorkItemState.Canceled): om('Waiting for climate generation to complete (current state: ' + str(wi.state) + ')', style='flushed') time.sleep(5) wi.refresh() print("Climate files SUCCESSFULLY generated") # Get the collection with our files collections = wi.get_related_asset_collections() collection_id = collections[0].id comps_collection = AssetCollection.get( collection_id, query_criteria=QueryCriteria().select_children('assets')) # Get the files if len(comps_collection.assets) > 0: print("Found output files:") for asset in comps_collection.assets: print("- %s (%s)" % (asset.file_name, file_size(asset.length))) print("\nDownloading to %s..." % self.climate_files_output_path) # Download the collection download_asset_collection(comps_collection, self.climate_files_output_path) # return filenames; this use of re in conjunction w/ glob is not great; consider refactor rain_bin_re = os.path.abspath(self.climate_files_output_path + '/*rain*.bin') humidity_bin_re = os.path.abspath(self.climate_files_output_path + '/*humidity*.bin') temperature_bin_re = os.path.abspath( self.climate_files_output_path + '/*temperature*.bin') rain_file_name = os.path.basename(glob.glob(rain_bin_re)[0]) humidity_file_name = os.path.basename( glob.glob(humidity_bin_re)[0]) temperature_file_name = os.path.basename( glob.glob(temperature_bin_re)[0]) print('Climate files SUCCESSFULLY stored.') return rain_file_name, temperature_file_name, humidity_file_name else: print('No output files found')