import pandas as pd from calibtool.IterationState import IterationState from calibtool.utils import StatusPoint from simtools.DataAccess.DataStore import DataStore from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory from simtools.ModBuilder import ModBuilder, ModFn from simtools.SetupParser import SetupParser from simtools.Utilities.COMPSUtilities import COMPS_login from simtools.Utilities.Encoding import NumpyEncoder from simtools.Utilities.Experiments import validate_exp_name, retrieve_experiment from simtools.Utilities.General import init_logging from simtools.Utilities import verbose_timedelta logger = init_logging("Calibration") class SampleIndexWrapper(object): """ Wrapper for a SimConfigBuilder-modifying function to add metadata on sample-point index when called in a iteration over sample points """ __name__ = "SampleIndex" def __init__(self, map_sample_to_model_input_fn): self.map_sample_to_model_input_fn = map_sample_to_model_input_fn def __call__(self, cb, idx, *args, **kwargs): params_dict = self.map_sample_to_model_input_fn(cb, *args, **kwargs) params_dict.update(cb.set_param('__sample_index__', idx))
import collections import itertools from COMPS.Data.Simulation import SimulationState from simtools.DataAccess.DataStore import DataStore from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory from simtools.SetupParser import SetupParser from simtools.Utilities.Encoding import GeneralEncoder from simtools.Utilities.General import init_logging from simtools.Utilities.LocalOS import LocalOS from simtools.Utilities.Experiments import retrieve_experiment, retrieve_simulation logger = init_logging('AnalyzeManager') class AnalyzeManager: def __init__(self, exp_list=None, sim_list=None, analyzers=None, working_dir=None, force_analyze=False, verbose=False, create_dir_map=False, record_finalize_results=False, finalize_results_filename=None): self.experiments = [] self.simulations = [] self.parsers = [] self.analyzers = [] self.experiments_simulations = {} self.verbose = verbose self.force_analyze = force_analyze self.create_dir_map = create_dir_map
import math from simtools.Utilities.CacheEnabled import CacheEnabled from simtools.Utilities.Encoding import GeneralEncoder from simtools.Utilities.General import init_logging, get_tools_revision, animation logger = init_logging('ExperimentManager') import copy import json import multiprocessing import os import subprocess import time from abc import ABCMeta, abstractmethod from collections import Counter import fasteners from simtools.DataAccess.DataStore import DataStore, chunks from simtools.ModBuilder import SingleSimulationBuilder from simtools.Monitor import SimulationMonitor from simtools.SetupParser import SetupParser from simtools.Utilities.Experiments import validate_exp_name from simtools.Utilities.General import is_running from simtools.Utilities.LocalOS import LocalOS current_dir = os.path.dirname(os.path.realpath(__file__)) from COMPS.Data.Simulation import SimulationState import sys
import datetime import json from operator import or_ from simtools.DataAccess import session_scope from simtools.DataAccess.Schema import Experiment, Simulation from sqlalchemy.orm import joinedload from simtools.Utilities.Encoding import GeneralEncoder from simtools.Utilities.General import init_logging, remove_null_values logger = init_logging('DataAccess') from COMPS.Data.Simulation import SimulationState class ExperimentDataStore: @classmethod def create_experiment(cls, **kwargs): logger.debug("Create Experiment") if 'date_created' not in kwargs: kwargs['date_created'] = datetime.datetime.now() return Experiment(**kwargs) @classmethod def get_experiment(cls, exp_id, current_session=None): logger.debug("Get Experiment") with session_scope(current_session) as session: # Get the experiment # Also load the associated simulations eagerly experiment = session.query(Experiment).options( joinedload('simulations').joinedload('experiment')) \ .filter(Experiment.exp_id == exp_id).one_or_none()
import json from collections import Counter from simtools.DataAccess.DataStore import DataStore from simtools.Utilities.COMPSUtilities import sims_from_suite_id, sims_from_experiment_id, COMPS_login from simtools.Utilities.General import init_logging, retry_function from COMPS.Data.Simulation import SimulationState logger = init_logging('Monitor') class SimulationMonitor: """ A class to monitor the status of simulations in the local DB. """ def __init__(self, exp_id): logger.debug("Create a DB Monitor with exp_id=%s" % exp_id) self.exp_id = exp_id def query(self): logger.debug("Query the DB Monitor for Experiment %s" % self.exp_id) states, msgs = {}, {} experiment = DataStore.get_experiment(self.exp_id) if not experiment or not experiment.simulations: return states, msgs for sim in experiment.simulations: states[ sim. id] = sim.status if sim.status else SimulationState.CommissionRequested msgs[sim.id] = sim.message if sim.message else ""
import os from multiprocessing import Process from COMPS.Data import Experiment, Configuration, Priority, Suite from simtools.ExperimentManager.BaseExperimentManager import BaseExperimentManager from simtools.SetupParser import SetupParser from simtools.SimulationCreator.COMPSSimulationCreator import COMPSSimulationCreator from simtools.Utilities.COMPSCache import COMPSCache from simtools.Utilities.COMPSUtilities import experiment_is_running, COMPS_login, get_semaphore, \ get_simulation_by_id from simtools.Utilities.General import init_logging, timestamp logger = init_logging("COMPSExperimentManager") class CompsExperimentManager(BaseExperimentManager): """ Extends the LocalExperimentManager to manage DTK simulations through COMPSAccess wrappers e.g. creation of Simulation, Experiment, Suite objects """ location = 'HPC' def __init__(self, experiment=None, config_builder=None): # Ensure we use the SetupParser environment of the experiment if it already exists super().__init__(experiment, config_builder) temp_block = experiment.selected_block if experiment else SetupParser.selected_block temp_path = experiment.setup_overlay_file if experiment else None self.endpoint = experiment.endpoint if experiment else None with SetupParser.TemporarySetup(temporary_block=temp_block,
import os from COMPS.Data.AssetCollection import AssetCollection as COMPSAssetCollection from simtools.AssetManager.AssetCollection import AssetCollection from simtools.AssetManager.FileList import FileList from simtools.SetupParser import SetupParser from simtools.Utilities.COMPSUtilities import get_asset_collection from simtools.Utilities.General import init_logging logger = init_logging("SimulationAssets") class SimulationAssets(object): """ This class represents a set of AssetCollection objects that together define all files needed by a simulation. """ class InvalidCollection(Exception): pass class NotPrepared(Exception): pass EXE = 'exe' DLL = 'dll' INPUT = 'input' LOCAL = 'local' MASTER = 'master' PYTHON = 'python' COLLECTION_TYPES = [DLL, EXE, INPUT, PYTHON] SETUP_MAPPING = {DLL: 'dll_root', EXE: 'exe_path', INPUT: 'input_root', PYTHON: 'python_path'} def __init__(self): self.collections = {}
import zipfile from simtools.Utilities.General import init_logging, get_md5, retry_function logger = init_logging('Utils') import os import re import shutil from COMPS.Data import Experiment, AssetCollection from COMPS.Data import QueryCriteria from COMPS.Data import Simulation from COMPS.Data import Suite from COMPS import Client from COMPS.Data.Simulation import SimulationState from simtools.SetupParser import SetupParser path_translations = {} def translate_COMPS_path(path): """ Transform a COMPS path into fully qualified path. Supports: - $COMPS_PATH('BIN') - $COMPS_PATH('USER') - $COMPS_PATH('PUBLIC') - $COMPS_PATH('INPUT') - $COMPS_PATH('HOME')
import json import os import fasteners from simtools.Utilities.BetterConfigParser import BetterConfigParser from simtools.Utilities.LocalOS import LocalOS from simtools.Utilities.General import init_logging current_dir = os.path.dirname(os.path.realpath(__file__)) logger = init_logging("SetupParser") class SetupParserMeta(type): # The center of the SetupParser universe, the singleton. singleton = None initialized = False # redirect all attribute calls to the singleton once initialized def __getattr__(cls, item): if cls.initialized: return getattr(cls.singleton, item) else: return None @classmethod @fasteners.interprocess_locked( os.path.join(current_dir, '.setup_parser_init_lock')) def init(cls, selected_block=None, **kwargs): """ Initialize the SetupParser singleton :param selected_block:
import os import shlex import subprocess import time from simtools.DataAccess.DataStore import DataStore from simtools.SimulationRunner.BaseSimulationRunner import BaseSimulationRunner from simtools.Utilities.General import init_logging, is_running logger = init_logging("LocalRunner") from COMPS.Data.Simulation import SimulationState class LocalSimulationRunner(BaseSimulationRunner): """ Run one simulation. """ def __init__(self, simulation, experiment, thread_queue): super(LocalSimulationRunner, self).__init__(experiment) self.queue = thread_queue # used to limit the number of concurrently running simulations self.simulation = simulation self.sim_dir = self.simulation.get_path() if self.check_state() == SimulationState.Created: self.run() else: self.queue.get() if self.simulation.status not in (SimulationState.Failed, SimulationState.Succeeded, SimulationState.Canceled): self.monitor() def run(self): try:
from simtools.DataAccess.LoggingDataStore import LoggingDataStore from simtools.ExperimentManager.BaseExperimentManager import BaseExperimentManager from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory from simtools.SetupParser import SetupParser from simtools.Utilities.COMPSUtilities import get_experiments_per_user_and_date, get_experiments_by_name, COMPS_login, \ get_experiment_ids_for_user from simtools.Utilities.DiskSpaceUsage import DiskSpaceUsage from simtools.Utilities.Experiments import COMPS_experiment_to_local_db, retrieve_experiment from simtools.Utilities.General import nostdout, get_tools_revision, retrieve_item from COMPS.Data.Simulation import SimulationState from simtools.Utilities.GitHub.GitHub import GitHub, DTKGitHub import simtools.Utilities.Initialization as init from simtools.DataAccess.Schema import Experiment, Simulation from simtools.Utilities.General import init_logging logger = init_logging('Commands') def builtinAnalyzers(): analyzers = { 'time_series': TimeseriesAnalyzer(select_function=sample_selection(), group_function=group_by_name('_site_'), plot_function=plot_grouped_lines), 'vector_species': VectorSpeciesAnalyzer(select_function=sample_selection(), group_function=group_by_name('_site_')), } return analyzers
# Add the tools to the path from multiprocessing import Manager sys.path.append(os.path.abspath('..')) import threading import time import traceback from collections import OrderedDict from datetime import datetime from simtools.DataAccess.DataStore import DataStore from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory from simtools.SetupParser import SetupParser from simtools.Utilities.General import init_logging logger = init_logging('Overseer') def LogCleaner(): # Get the last time a cleanup happened last_cleanup = DataStore.get_setting('last_log_cleanup') if not last_cleanup or (datetime.today() - datetime.strptime(last_cleanup.value.split(' ')[0],'%Y-%m-%d')).days > 1: # Do the cleanup from simtools.DataAccess.LoggingDataStore import LoggingDataStore LoggingDataStore.log_cleanup() DataStore.save_setting(DataStore.create_setting(key='last_log_cleanup', value=datetime.today())) if __name__ == "__main__": logger.debug('Start Overseer pid: %d' % os.getpid())
import csv from datetime import datetime from dtk.generic.demographics import distribution_types from dtk.tools.demographics.Node import Node, nodeid_from_lat_lon from simtools.Utilities.General import init_logging logger = init_logging('DemographicsGenerator') class InvalidResolution(BaseException): pass class DemographicsGenerator: """ Generates demographics file based on population input file. The population input file is csv with structure node_label*, lat, lon, pop* *-ed columns are optional """ # mapping of requested arcsecond resolution -> demographic metadata arcsecond resolution. # All Hash values must be integers. CUSTOM_RESOLUTION = 'custom' DEFAULT_RESOLUTION = 30 VALID_RESOLUTIONS = {30: 30, 250: 250, CUSTOM_RESOLUTION: 30} def __init__(self,
import time from COMPS.Data.Simulation import SimulationState from simtools.DataAccess.DataStore import DataStore from simtools.ExperimentManager.CompsExperimentManager import CompsExperimentManager from simtools.Monitor import CompsSimulationMonitor from simtools.SimulationRunner.BaseSimulationRunner import BaseSimulationRunner from simtools.Utilities.COMPSUtilities import experiment_needs_commission, COMPS_login from simtools.Utilities.General import init_logging logger = init_logging('Runner') class COMPSSimulationRunner(BaseSimulationRunner): def __init__(self, experiment, comps_experiment): logger.debug('Create COMPSSimulationRunner with experiment: %s' % experiment.id) super(COMPSSimulationRunner, self).__init__(experiment) # Check if we need to commission COMPS_login(experiment.endpoint) self.comps_experiment = comps_experiment if experiment_needs_commission(self.comps_experiment): logger.debug('COMPS - Start Commissioning for experiment %s' % self.experiment.id) # Commission the experiment self.comps_experiment.commission() self.monitor()
from dtk.interventions.si_initial_seeding import si_campaign from dtk.interventions.sir_initial_seeding import sir_campaign from dtk.interventions.sir_vaccinations_a_initial_seeding import sir_vaccinations_a_campaign from dtk.interventions.sir_vaccinations_b_initial_seeding import sir_vaccinations_b_campaign from dtk.interventions.sir_vaccinations_c_initial_seeding import sir_vaccinations_c_campaign from dtk.interventions.sirs_initial_seeding import sirs_campaign from dtk.interventions.sis_initial_seeding import sis_campaign from dtk.utils.Campaign.utils.CampaignManager import CampaignManager from dtk.utils.Campaign.utils.RawCampaignObject import RawCampaignObject from dtk.utils.parsers.JSON import json2dict from dtk.utils.reports.CustomReport import format as format_reports from simtools.SimConfigBuilder import SimConfigBuilder from simtools.Utilities.Encoding import NumpyEncoder from simtools.Utilities.General import init_logging logger = init_logging('ConfigBuilder') class DTKConfigBuilder(SimConfigBuilder): """ A class for building, modifying, and writing required configuration files for a DTK simulation. There are four ways to create a DTKConfigBuilder: 1. From a set of defaults, with the :py:func:`from_defaults` class method. 2. From existing files, with the :py:func:`from_files` class method. 3. From a default config/campaign by calling the constructor without arguments. 4. From a custom config and/or campaign by calling the constructor with the ``config`` and ``campaign`` arguments. Arguments:
from multiprocessing import Process from simtools.Utilities.General import init_logging logger = init_logging("LocalExperimentManager") import os import re import shutil import signal from datetime import datetime from simtools.ExperimentManager.BaseExperimentManager import BaseExperimentManager from simtools.SimulationCreator.LocalSimulationCreator import LocalSimulationCreator from simtools.SimulationRunner.LocalRunner import LocalSimulationRunner from simtools.Utilities.General import is_running from COMPS.Data.Simulation import SimulationState class LocalExperimentManager(BaseExperimentManager): """ Manages the creation, submission, status, parsing, and analysis of local experiments, i.e. collections of related simulations """ location = 'LOCAL' @property def experiment(self): return self._experiment @experiment.setter
import os import sys from importlib import import_module from simtools.SetupParser import SetupParser from simtools.Utilities.General import init_logging logger = init_logging('Initialization') def load_config_module(config_name): # Support of relative paths config_name = config_name.replace('\\', '/') if '/' in config_name: splitted = config_name.split('/')[:-1] sys.path.append(os.path.join(os.getcwd(), *splitted)) else: sys.path.append(os.getcwd()) module_name = os.path.splitext(os.path.basename(config_name))[0] try: return import_module(module_name) except ImportError as e: e.args = ("'%s' during loading module '%s' in %s files: %s." % (e.msg, module_name, os.getcwd(), os.listdir(os.getcwd())), ) raise e def initialize_SetupParser_from_args(args, unknownArgs): # determine the selected environment