import pandas as pd
from calibtool.IterationState import IterationState
from calibtool.utils import StatusPoint
from simtools.DataAccess.DataStore import DataStore
from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory
from simtools.ModBuilder import ModBuilder, ModFn
from simtools.SetupParser import SetupParser
from simtools.Utilities.COMPSUtilities import COMPS_login
from simtools.Utilities.Encoding import NumpyEncoder
from simtools.Utilities.Experiments import validate_exp_name, retrieve_experiment
from simtools.Utilities.General import init_logging
from simtools.Utilities import verbose_timedelta
logger = init_logging("Calibration")
class SampleIndexWrapper(object):
"""
Wrapper for a SimConfigBuilder-modifying function to add metadata
on sample-point index when called in a iteration over sample points
"""
__name__ = "SampleIndex"
def __init__(self, map_sample_to_model_input_fn):
self.map_sample_to_model_input_fn = map_sample_to_model_input_fn
def __call__(self, cb, idx, *args, **kwargs):
params_dict = self.map_sample_to_model_input_fn(cb, *args, **kwargs)
params_dict.update(cb.set_param('__sample_index__', idx))
import collections
import itertools
from COMPS.Data.Simulation import SimulationState
from simtools.DataAccess.DataStore import DataStore
from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory
from simtools.SetupParser import SetupParser
from simtools.Utilities.Encoding import GeneralEncoder
from simtools.Utilities.General import init_logging
from simtools.Utilities.LocalOS import LocalOS
from simtools.Utilities.Experiments import retrieve_experiment, retrieve_simulation
logger = init_logging('AnalyzeManager')
class AnalyzeManager:
def __init__(self, exp_list=None, sim_list=None, analyzers=None, working_dir=None, force_analyze=False, verbose=False,
create_dir_map=False, record_finalize_results=False, finalize_results_filename=None):
self.experiments = []
self.simulations = []
self.parsers = []
self.analyzers = []
self.experiments_simulations = {}
self.verbose = verbose
self.force_analyze = force_analyze
self.create_dir_map = create_dir_map
import math
from simtools.Utilities.CacheEnabled import CacheEnabled
from simtools.Utilities.Encoding import GeneralEncoder
from simtools.Utilities.General import init_logging, get_tools_revision, animation
logger = init_logging('ExperimentManager')
import copy
import json
import multiprocessing
import os
import subprocess
import time
from abc import ABCMeta, abstractmethod
from collections import Counter
import fasteners
from simtools.DataAccess.DataStore import DataStore, chunks
from simtools.ModBuilder import SingleSimulationBuilder
from simtools.Monitor import SimulationMonitor
from simtools.SetupParser import SetupParser
from simtools.Utilities.Experiments import validate_exp_name
from simtools.Utilities.General import is_running
from simtools.Utilities.LocalOS import LocalOS
current_dir = os.path.dirname(os.path.realpath(__file__))
from COMPS.Data.Simulation import SimulationState
import sys
import datetime
import json
from operator import or_
from simtools.DataAccess import session_scope
from simtools.DataAccess.Schema import Experiment, Simulation
from sqlalchemy.orm import joinedload
from simtools.Utilities.Encoding import GeneralEncoder
from simtools.Utilities.General import init_logging, remove_null_values
logger = init_logging('DataAccess')
from COMPS.Data.Simulation import SimulationState
class ExperimentDataStore:
@classmethod
def create_experiment(cls, **kwargs):
logger.debug("Create Experiment")
if 'date_created' not in kwargs:
kwargs['date_created'] = datetime.datetime.now()
return Experiment(**kwargs)
@classmethod
def get_experiment(cls, exp_id, current_session=None):
logger.debug("Get Experiment")
with session_scope(current_session) as session:
# Get the experiment
# Also load the associated simulations eagerly
experiment = session.query(Experiment).options(
joinedload('simulations').joinedload('experiment')) \
.filter(Experiment.exp_id == exp_id).one_or_none()
import json
from collections import Counter
from simtools.DataAccess.DataStore import DataStore
from simtools.Utilities.COMPSUtilities import sims_from_suite_id, sims_from_experiment_id, COMPS_login
from simtools.Utilities.General import init_logging, retry_function
from COMPS.Data.Simulation import SimulationState
logger = init_logging('Monitor')
class SimulationMonitor:
"""
A class to monitor the status of simulations in the local DB.
"""
def __init__(self, exp_id):
logger.debug("Create a DB Monitor with exp_id=%s" % exp_id)
self.exp_id = exp_id
def query(self):
logger.debug("Query the DB Monitor for Experiment %s" % self.exp_id)
states, msgs = {}, {}
experiment = DataStore.get_experiment(self.exp_id)
if not experiment or not experiment.simulations:
return states, msgs
for sim in experiment.simulations:
states[
sim.
id] = sim.status if sim.status else SimulationState.CommissionRequested
msgs[sim.id] = sim.message if sim.message else ""
import os
from multiprocessing import Process
from COMPS.Data import Experiment, Configuration, Priority, Suite
from simtools.ExperimentManager.BaseExperimentManager import BaseExperimentManager
from simtools.SetupParser import SetupParser
from simtools.SimulationCreator.COMPSSimulationCreator import COMPSSimulationCreator
from simtools.Utilities.COMPSCache import COMPSCache
from simtools.Utilities.COMPSUtilities import experiment_is_running, COMPS_login, get_semaphore, \
get_simulation_by_id
from simtools.Utilities.General import init_logging, timestamp
logger = init_logging("COMPSExperimentManager")
class CompsExperimentManager(BaseExperimentManager):
"""
Extends the LocalExperimentManager to manage DTK simulations through COMPSAccess wrappers
e.g. creation of Simulation, Experiment, Suite objects
"""
location = 'HPC'
def __init__(self, experiment=None, config_builder=None):
# Ensure we use the SetupParser environment of the experiment if it already exists
super().__init__(experiment, config_builder)
temp_block = experiment.selected_block if experiment else SetupParser.selected_block
temp_path = experiment.setup_overlay_file if experiment else None
self.endpoint = experiment.endpoint if experiment else None
with SetupParser.TemporarySetup(temporary_block=temp_block,
import os
from COMPS.Data.AssetCollection import AssetCollection as COMPSAssetCollection
from simtools.AssetManager.AssetCollection import AssetCollection
from simtools.AssetManager.FileList import FileList
from simtools.SetupParser import SetupParser
from simtools.Utilities.COMPSUtilities import get_asset_collection
from simtools.Utilities.General import init_logging
logger = init_logging("SimulationAssets")
class SimulationAssets(object):
"""
This class represents a set of AssetCollection objects that together define all files needed by a simulation.
"""
class InvalidCollection(Exception): pass
class NotPrepared(Exception): pass
EXE = 'exe'
DLL = 'dll'
INPUT = 'input'
LOCAL = 'local'
MASTER = 'master'
PYTHON = 'python'
COLLECTION_TYPES = [DLL, EXE, INPUT, PYTHON]
SETUP_MAPPING = {DLL: 'dll_root', EXE: 'exe_path', INPUT: 'input_root', PYTHON: 'python_path'}
def __init__(self):
self.collections = {}
import zipfile
from simtools.Utilities.General import init_logging, get_md5, retry_function
logger = init_logging('Utils')
import os
import re
import shutil
from COMPS.Data import Experiment, AssetCollection
from COMPS.Data import QueryCriteria
from COMPS.Data import Simulation
from COMPS.Data import Suite
from COMPS import Client
from COMPS.Data.Simulation import SimulationState
from simtools.SetupParser import SetupParser
path_translations = {}
def translate_COMPS_path(path):
"""
Transform a COMPS path into fully qualified path.
Supports:
- $COMPS_PATH('BIN')
- $COMPS_PATH('USER')
- $COMPS_PATH('PUBLIC')
- $COMPS_PATH('INPUT')
- $COMPS_PATH('HOME')
import json
import os
import fasteners
from simtools.Utilities.BetterConfigParser import BetterConfigParser
from simtools.Utilities.LocalOS import LocalOS
from simtools.Utilities.General import init_logging
current_dir = os.path.dirname(os.path.realpath(__file__))
logger = init_logging("SetupParser")
class SetupParserMeta(type):
# The center of the SetupParser universe, the singleton.
singleton = None
initialized = False
# redirect all attribute calls to the singleton once initialized
def __getattr__(cls, item):
if cls.initialized:
return getattr(cls.singleton, item)
else:
return None
@classmethod
@fasteners.interprocess_locked(
os.path.join(current_dir, '.setup_parser_init_lock'))
def init(cls, selected_block=None, **kwargs):
"""
Initialize the SetupParser singleton
:param selected_block:
import os
import shlex
import subprocess
import time
from simtools.DataAccess.DataStore import DataStore
from simtools.SimulationRunner.BaseSimulationRunner import BaseSimulationRunner
from simtools.Utilities.General import init_logging, is_running
logger = init_logging("LocalRunner")
from COMPS.Data.Simulation import SimulationState
class LocalSimulationRunner(BaseSimulationRunner):
"""
Run one simulation.
"""
def __init__(self, simulation, experiment, thread_queue):
super(LocalSimulationRunner, self).__init__(experiment)
self.queue = thread_queue # used to limit the number of concurrently running simulations
self.simulation = simulation
self.sim_dir = self.simulation.get_path()
if self.check_state() == SimulationState.Created:
self.run()
else:
self.queue.get()
if self.simulation.status not in (SimulationState.Failed, SimulationState.Succeeded, SimulationState.Canceled):
self.monitor()
def run(self):
try:
from simtools.DataAccess.LoggingDataStore import LoggingDataStore
from simtools.ExperimentManager.BaseExperimentManager import BaseExperimentManager
from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory
from simtools.SetupParser import SetupParser
from simtools.Utilities.COMPSUtilities import get_experiments_per_user_and_date, get_experiments_by_name, COMPS_login, \
get_experiment_ids_for_user
from simtools.Utilities.DiskSpaceUsage import DiskSpaceUsage
from simtools.Utilities.Experiments import COMPS_experiment_to_local_db, retrieve_experiment
from simtools.Utilities.General import nostdout, get_tools_revision, retrieve_item
from COMPS.Data.Simulation import SimulationState
from simtools.Utilities.GitHub.GitHub import GitHub, DTKGitHub
import simtools.Utilities.Initialization as init
from simtools.DataAccess.Schema import Experiment, Simulation
from simtools.Utilities.General import init_logging
logger = init_logging('Commands')
def builtinAnalyzers():
analyzers = {
'time_series':
TimeseriesAnalyzer(select_function=sample_selection(),
group_function=group_by_name('_site_'),
plot_function=plot_grouped_lines),
'vector_species':
VectorSpeciesAnalyzer(select_function=sample_selection(),
group_function=group_by_name('_site_')),
}
return analyzers
# Add the tools to the path
from multiprocessing import Manager
sys.path.append(os.path.abspath('..'))
import threading
import time
import traceback
from collections import OrderedDict
from datetime import datetime
from simtools.DataAccess.DataStore import DataStore
from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory
from simtools.SetupParser import SetupParser
from simtools.Utilities.General import init_logging
logger = init_logging('Overseer')
def LogCleaner():
# Get the last time a cleanup happened
last_cleanup = DataStore.get_setting('last_log_cleanup')
if not last_cleanup or (datetime.today() - datetime.strptime(last_cleanup.value.split(' ')[0],'%Y-%m-%d')).days > 1:
# Do the cleanup
from simtools.DataAccess.LoggingDataStore import LoggingDataStore
LoggingDataStore.log_cleanup()
DataStore.save_setting(DataStore.create_setting(key='last_log_cleanup', value=datetime.today()))
if __name__ == "__main__":
logger.debug('Start Overseer pid: %d' % os.getpid())
import csv
from datetime import datetime
from dtk.generic.demographics import distribution_types
from dtk.tools.demographics.Node import Node, nodeid_from_lat_lon
from simtools.Utilities.General import init_logging
logger = init_logging('DemographicsGenerator')
class InvalidResolution(BaseException):
pass
class DemographicsGenerator:
"""
Generates demographics file based on population input file.
The population input file is csv with structure
node_label*, lat, lon, pop*
*-ed columns are optional
"""
# mapping of requested arcsecond resolution -> demographic metadata arcsecond resolution.
# All Hash values must be integers.
CUSTOM_RESOLUTION = 'custom'
DEFAULT_RESOLUTION = 30
VALID_RESOLUTIONS = {30: 30, 250: 250, CUSTOM_RESOLUTION: 30}
def __init__(self,
import time
from COMPS.Data.Simulation import SimulationState
from simtools.DataAccess.DataStore import DataStore
from simtools.ExperimentManager.CompsExperimentManager import CompsExperimentManager
from simtools.Monitor import CompsSimulationMonitor
from simtools.SimulationRunner.BaseSimulationRunner import BaseSimulationRunner
from simtools.Utilities.COMPSUtilities import experiment_needs_commission, COMPS_login
from simtools.Utilities.General import init_logging
logger = init_logging('Runner')
class COMPSSimulationRunner(BaseSimulationRunner):
def __init__(self, experiment, comps_experiment):
logger.debug('Create COMPSSimulationRunner with experiment: %s' %
experiment.id)
super(COMPSSimulationRunner, self).__init__(experiment)
# Check if we need to commission
COMPS_login(experiment.endpoint)
self.comps_experiment = comps_experiment
if experiment_needs_commission(self.comps_experiment):
logger.debug('COMPS - Start Commissioning for experiment %s' %
self.experiment.id)
# Commission the experiment
self.comps_experiment.commission()
self.monitor()
from dtk.interventions.si_initial_seeding import si_campaign
from dtk.interventions.sir_initial_seeding import sir_campaign
from dtk.interventions.sir_vaccinations_a_initial_seeding import sir_vaccinations_a_campaign
from dtk.interventions.sir_vaccinations_b_initial_seeding import sir_vaccinations_b_campaign
from dtk.interventions.sir_vaccinations_c_initial_seeding import sir_vaccinations_c_campaign
from dtk.interventions.sirs_initial_seeding import sirs_campaign
from dtk.interventions.sis_initial_seeding import sis_campaign
from dtk.utils.Campaign.utils.CampaignManager import CampaignManager
from dtk.utils.Campaign.utils.RawCampaignObject import RawCampaignObject
from dtk.utils.parsers.JSON import json2dict
from dtk.utils.reports.CustomReport import format as format_reports
from simtools.SimConfigBuilder import SimConfigBuilder
from simtools.Utilities.Encoding import NumpyEncoder
from simtools.Utilities.General import init_logging
logger = init_logging('ConfigBuilder')
class DTKConfigBuilder(SimConfigBuilder):
"""
A class for building, modifying, and writing
required configuration files for a DTK simulation.
There are four ways to create a DTKConfigBuilder:
1. From a set of defaults, with the :py:func:`from_defaults` class method.
2. From existing files, with the :py:func:`from_files` class method.
3. From a default config/campaign by calling the constructor without arguments.
4. From a custom config and/or campaign by calling the constructor with the ``config`` and ``campaign`` arguments.
Arguments:
from multiprocessing import Process
from simtools.Utilities.General import init_logging
logger = init_logging("LocalExperimentManager")
import os
import re
import shutil
import signal
from datetime import datetime
from simtools.ExperimentManager.BaseExperimentManager import BaseExperimentManager
from simtools.SimulationCreator.LocalSimulationCreator import LocalSimulationCreator
from simtools.SimulationRunner.LocalRunner import LocalSimulationRunner
from simtools.Utilities.General import is_running
from COMPS.Data.Simulation import SimulationState
class LocalExperimentManager(BaseExperimentManager):
"""
Manages the creation, submission, status, parsing, and analysis
of local experiments, i.e. collections of related simulations
"""
location = 'LOCAL'
@property
def experiment(self):
return self._experiment
@experiment.setter
import os
import sys
from importlib import import_module
from simtools.SetupParser import SetupParser
from simtools.Utilities.General import init_logging
logger = init_logging('Initialization')
def load_config_module(config_name):
# Support of relative paths
config_name = config_name.replace('\\', '/')
if '/' in config_name:
splitted = config_name.split('/')[:-1]
sys.path.append(os.path.join(os.getcwd(), *splitted))
else:
sys.path.append(os.getcwd())
module_name = os.path.splitext(os.path.basename(config_name))[0]
try:
return import_module(module_name)
except ImportError as e:
e.args = ("'%s' during loading module '%s' in %s files: %s." %
(e.msg, module_name, os.getcwd(), os.listdir(os.getcwd())), )
raise e
def initialize_SetupParser_from_args(args, unknownArgs):
# determine the selected environment
