def translate_COMPS_path(path):
"""
Transform a COMPS path into fully qualified path.
Supports:
- $COMPS_PATH('BIN')
- $COMPS_PATH('USER')
- $COMPS_PATH('PUBLIC')
- $COMPS_PATH('INPUT')
- $COMPS_PATH('HOME')
Query the COMPS Java client with the current environment to get the correct path.
:param path: The COMPS path to transform
:param setup: The setup to find user and environment
:return: The absolute path
"""
from COMPS import Client
# Create the regexp
regexp = re.search('.*(\$COMPS_PATH\((\w+)\)).*', path)
# If no COMPS variable found -> return the path untouched
if not regexp:
return path
# Retrieve the variable to translate
groups = regexp.groups()
comps_variable = groups[1]
# Is the path already cached
if comps_variable in path_translations:
abs_path = path_translations[comps_variable]
else:
with SetupParser.TemporarySetup('HPC') as setup:
# Prepare the variables we will need
environment = setup.get('environment')
# Q uery COMPS to get the path corresponding to the variable
COMPS_login(setup.get('server_endpoint'))
abs_path = Client.auth_manager().get_environment_macros(
environment)[groups[1]]
# Cache
path_translations[comps_variable] = abs_path
# Replace and return
with SetupParser.TemporarySetup() as setup:
user = setup.get('user')
return path.replace(groups[0], abs_path).replace("$(User)", user)
def __init__(self, experiment=None, config_builder=None):
# Ensure we use the SetupParser environment of the experiment if it already exists
super().__init__(experiment, config_builder)
temp_block = experiment.selected_block if experiment else SetupParser.selected_block
temp_path = experiment.setup_overlay_file if experiment else None
self.endpoint = experiment.endpoint if experiment else None
with SetupParser.TemporarySetup(temporary_block=temp_block,
temporary_path=temp_path) as setup:
self.comps_sims_to_batch = int(
setup.get(parameter='sims_per_thread'))
self.endpoint = self.endpoint or setup.get(
parameter='server_endpoint')
COMPS_login(self.endpoint)
self.asset_service = True
self.runner_thread = None
self.sims_to_create = []
self.commissioners = []
self.save_semaphore = get_semaphore()
# If we pass an experiment, retrieve it from COMPS
if self.experiment:
self.comps_experiment = COMPSCache.experiment(
self.experiment.exp_id)
def set_base_collection(self, collection_type, collection):
# Make sure we have the good inputs
if collection_type not in self.COLLECTION_TYPES and collection_type != self.MASTER:
raise self.InvalidCollection("Collection type %s is not supported..." % collection_type)
if not collection:
raise self.InvalidCollection("No collection provided in set_input_collection.")
# If the collection given is not already an AssetCollection -> retrieve
if not isinstance(collection, COMPSAssetCollection):
collection_id = collection
with SetupParser.TemporarySetup('HPC'):
collection = get_asset_collection(collection_id)
if not collection:
raise self.InvalidCollection("The input collection '%s' provided could not be found on COMPS." % collection_id)
if collection_type == self.MASTER:
self.master_collection = AssetCollection(base_collection=collection)
for t in self.SETUP_MAPPING: self.SETUP_MAPPING[t] = ""
else:
self.base_collections[collection_type] = AssetCollection(base_collection=collection)
# For the DLL filter out the exe as we usually have exe+dll in the same collection
if collection_type == self.DLL:
self.base_collections[self.DLL].asset_files_to_use = [a for a in self.base_collections[self.DLL].asset_files_to_use if not a.file_name.endswith('exe')]
def retrieve_experiment(exp_id, sync_if_missing=True, verbose=False, force_update=False):
"""
Retrieve an experiment in the local database based on its id.
Can call a sync if missing if the flag is true.
:param exp_id: Id of the experiment to retrieve
:param sync_if_missing: Should we try to sync if not present?
:return: The experiment found
"""
if not exp_id: raise Exception("Trying to retrieve an experiment without providing an experiment ID")
from uuid import UUID
if isinstance(exp_id,UUID): exp_id = str(exp_id)
# If we dont force the update -> look first in the DB
exp = DataStore.get_experiment(exp_id) or DataStore.get_most_recent_experiment(exp_id)
if exp:
# If we have an experiment and we want to force the update -> delete it
if not force_update:
return exp
else:
DataStore.delete_experiment(exp)
if not sync_if_missing:
raise Exception('Experiment %s not found in the local database and sync disabled.' % exp_id)
logger.info('Experiment with id %s not found in local database, trying sync.' % exp_id)
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
exp = COMPS_experiment_to_local_db(exp_id, endpoint, verbose)
if exp: return exp
raise Exception("Experiment '%s' could not be retrieved." % exp_id)
def diskspace(args, unknownArgs):
# Create a default HPC setup parser
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
COMPS_login(endpoint)
# default to the login user
user = sp.get('user')
if args.users is None or len(args.users) == 0:
args.users = [user]
# query and display the disk information
DiskSpaceUsage.display(args.users, args.top, args.save, args.refresh)
def __init__(self,
exp_list=None,
sim_list=None,
analyzers=None,
working_dir=None,
force_analyze=False,
verbose=False,
create_dir_map=False):
self.experiments = []
self.simulations = []
self.parsers = []
self.analyzers = []
self.experiments_simulations = {}
self.verbose = verbose
self.force_analyze = force_analyze
self.create_dir_map = create_dir_map
self.parse = True
self.working_dir = working_dir or os.getcwd()
with SetupParser.TemporarySetup() as sp:
self.maxThreadSemaphore = multiprocessing.Semaphore(
int(sp.get('max_threads', 16)))
# If no experiment is specified, retrieve the most recent as a convenience
if exp_list == 'latest':
exp_list = DataStore.get_most_recent_experiment()
# Initial adding of experiments
if exp_list:
exp_list = exp_list if isinstance(
exp_list, collections.Iterable) and not isinstance(
exp_list, str) else [exp_list]
for exp in exp_list:
self.add_experiment(exp)
# Initial adding of the simulations
if sim_list:
sim_list = sim_list if isinstance(
sim_list, collections.Iterable) else [sim_list]
for sim in sim_list:
self.add_simulation(sim)
# Initial adding of the analyzers
if analyzers:
analyzer_list = analyzers if isinstance(
analyzers, collections.Iterable) else [analyzers]
for a in analyzer_list:
self.add_analyzer(a)
def __init__(self,
exp_list=None,
sim_list=None,
analyzers=None,
working_dir=None,
force_analyze=False,
verbose=True):
super().__init__()
self.analyzers = []
self.experiments = set()
self.simulations = {}
self.ignored_simulations = {}
with SetupParser.TemporarySetup() as sp:
self.max_threads = min(os.cpu_count(),
int(sp.get('max_threads', 16)))
self.verbose = verbose
self.force_analyze = force_analyze
self.working_dir = working_dir or os.getcwd()
# If no experiment is specified, retrieve the most recent as a convenience
if exp_list == 'latest':
exp_list = DataStore.get_most_recent_experiment()
# Initial adding of experiments
if exp_list:
exp_list = exp_list if isinstance(
exp_list, collections.Iterable) and not isinstance(
exp_list, str) else [exp_list]
for exp in exp_list:
self.add_experiment(exp)
# Initial adding of the simulations
if sim_list:
sim_list = sim_list if isinstance(
sim_list, collections.Iterable) else [sim_list]
for sim in sim_list:
self.add_simulation(sim)
# Initial adding of the analyzers
if analyzers:
analyzer_list = analyzers if isinstance(
analyzers, collections.Iterable) else [analyzers]
for a in analyzer_list:
self.add_analyzer(a)
# Initialize the cache
self.cache = self.initialize_cache(shards=self.max_threads)
def retrieve_object(obj_id, obj_type, sync_if_missing=True, verbose=False, force_update=False):
"""
Retrieve an object (Experiment, Simulation) in the local database based on its id.
Can call a sync if missing if the flag is true
:param obj_id: Id of the object to retrieve
:param obj_type: Type of the object to retrieve
:param sync_if_missing: Should we try to sync if not present?
:return: The experiment found
#TODO: SHould also support suites
"""
typename = obj_type.__name__
if not obj_id: raise ValueError("Trying to retrieve an object (%s) without providing an ID" % typename)
# Try a hit in the DB
if obj_type == Experiment:
obj = DataStore.get_experiment(obj_id)
elif obj_type == Simulation:
obj = DataStore.get_simulation(obj_id)
if obj:
# If we have an experiment and we want to force the update -> delete it
if not force_update:
return obj
else:
if obj_type == Experiment:
DataStore.delete_experiment(obj_id)
elif obj_type == Simulation:
DataStore.delete_simulation(obj_id)
if not sync_if_missing:
raise Exception('%s %s not found in the local database and sync disabled.' % (typename, obj_id))
logger.info('%s with id %s not found in local database, trying sync.' % (typename, obj_id))
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
if obj_type == Experiment:
obj = COMPS_experiment_to_local_db(obj_id, endpoint, verbose)
elif obj_type == Simulation:
obj =None
if obj: return obj
raise Exception("%s '%s' could not be retrieved." % (typename, obj_id))
def retrieve_simulation(sim_id, sync_if_missing=True, verbose=False, force_update=False):
"""
Retrieve a simulation in the local database based on its id.
Can call a sync if missing if the flag is true.
:param sim_id: Id of the simulation to retrieve
:param sync_if_missing: Should we try to sync if not present?
:return: The simulation found
"""
from simtools.Utilities.COMPSUtilities import get_simulation_by_id
if not sim_id: raise Exception("Trying to retrieve a simulation without providing an simulation ID")
from uuid import UUID
if isinstance(sim_id, UUID): sim_id = str(sim_id)
# If we dont force the update -> look first in the DB
sim = DataStore.get_simulation(sim_id)
if sim:
# If we have a simulation and we want to force the update -> delete it
if not force_update:
return sim
else:
DataStore.delete_simulation(sim)
if not sync_if_missing:
raise Exception('Simulation %s not found in the local database and sync disabled.' % sim_id)
logger.info('Simulation with id %s not found in local database, trying sync.' % sim_id)
csim = get_simulation_by_id(sim_id)
if csim:
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
COMPS_experiment_to_local_db(csim.experiment_id, endpoint, verbose)
sim = DataStore.get_simulation(sim_id)
if sim: return sim
raise Exception("Simulation '%s' could not be retrieved." % sim_id)
def get_experiment_info(experiment, cache):
"""
Adds the experiment information for a given experiment to the cache:
- raw_size: the size in bytes
- size: the formatted size (in KB, MB or GB)
- sims: the number of simulations
This function is used by the process pool to parallelize the retrieval of experiment info
:param experiment: The experiment to analyze
"""
if experiment.id in cache and not DiskSpaceUsage.FORCE_REFRESH:
return
# Login to COMPS
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
COMPS_login(endpoint)
# Try to get the simulations
try:
simulations = experiment.get_simulations(
query_criteria=QueryCriteria().select(['id']).select_children(
['hpc_jobs']))
except KeyError:
# No simulations found or error -> set None
cache.set(experiment.id, None)
return
# Calculate the size
size = sum(s.hpc_jobs[0].output_directory_size for s in simulations
if s.hpc_jobs)
# Set the info for this particular experiment in the cache
cache.set(
experiment.id,
ExperimentInfo(experiment.id, experiment.name, experiment.owner,
size, len(simulations)))
from scipy.special import gammaln
#sys.path.append(os.path.abspath('Single_Node_Sites'))
from outbreakindividualdengue import add_OutbreakIndivisualDengue
from dtk.utils.core.DTKConfigBuilder import DTKConfigBuilder
from dtk.vector.study_sites import configure_site
from scipy.stats import uniform
from simtools.SetupParser import SetupParser
from ColombiaChikvTSSite_Department_admin2_nodes import ColombiaTSSite
import numpy as np
import scipy as sp
site_name = sys.argv[1]
SetupParser.default_block = "HPC"
with SetupParser.TemporarySetup("LOCAL") as setp:
input_dir = setp.get('input_root')
# @@ should be imported from elsewhere. PENDING until decided on structure for this.
def foi_time(times, widths, heights, days_sim, node_cnt):
foi_out = [
np.repeat(0, repeats=len(days_sim)) for ii in range(0, node_cnt)
]
for ss in range(0, node_cnt):
for yy in range(0, len(times[ss])):
times[ss][yy] = times[ss][yy] * len(days_sim)
foi_out[ss] = \
foi_out[ss] + \
def sync(args, unknownArgs):
"""
Sync COMPS db with local db
"""
# Create a default HPC setup parser
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
user = sp.get('user')
COMPS_login(endpoint)
exp_to_save = list()
exp_deleted = 0
# Retrieve all the experiment id from COMPS for the current user
exp_ids = get_experiment_ids_for_user(user)
# Test the experiments present in the local DB to make sure they still exist in COMPS
for exp in DataStore.get_experiments():
if exp.location == "HPC":
if exp.exp_id not in exp_ids:
# The experiment doesnt exist on COMPS anymore -> delete from local
DataStore.delete_experiment(exp)
exp_deleted += 1
# Consider experiment id option
exp_id = args.exp_id if args.exp_id else None
exp_name = args.exp_name if args.exp_name else None
user = args.user if args.user else user
if exp_name:
experiments = get_experiments_by_name(exp_name, user)
for experiment_data in experiments:
experiment = COMPS_experiment_to_local_db(
exp_id=str(experiment_data.id),
endpoint=endpoint,
verbose=True,
save_new_experiment=False)
if experiment:
exp_to_save.append(experiment)
elif exp_id:
# Create a new experiment
experiment = COMPS_experiment_to_local_db(exp_id=exp_id,
endpoint=endpoint,
verbose=True,
save_new_experiment=False)
# The experiment needs to be saved
if experiment:
exp_to_save.append(experiment)
else:
# By default only get experiments created in the last month
# day_limit = args.days if args.days else day_limit_default
day_limit = 30
today = datetime.date.today()
limit_date = today - datetime.timedelta(days=int(day_limit))
# For each of them, check if they are in the db
for exp in get_experiments_per_user_and_date(user, limit_date):
# Create a new experiment
experiment = COMPS_experiment_to_local_db(
exp_id=str(exp.id),
endpoint=endpoint,
save_new_experiment=False)
# The experiment needs to be saved
if experiment:
exp_to_save.append(experiment)
# Save the experiments if any
if len(exp_to_save) > 0 and exp_deleted == 0:
DataStore.batch_save_experiments(exp_to_save)
logger.info(
"The following experiments have been added to the database:")
logger.info("\n".join(["- " + str(exp) for exp in exp_to_save]))
logger.info("%s experiments have been updated in the DB." %
len(exp_to_save))
logger.info("%s experiments have been deleted from the DB." %
exp_deleted)
else:
print("The database was already up to date.")
# Start overseer
BaseExperimentManager.check_overseer()
def link(args, unknownArgs):
"""
Open browser to the COMPS Experiment/Simulation with ID or name provided
:param args:
:param unknownArgs:
:return:
"""
# get input from commands line
input_id = args.Id
# default: consider the latest experiment
if input_id is None:
latest = DataStore.get_most_recent_experiment()
input_id = latest.exp_id
try:
comps_item = retrieve_item(input_id)
except:
print('Nothing was found for {}'.format(input_id))
exit()
# check item type
id_type = ''
location = 'LOCAL'
if isinstance(comps_item, Experiment):
item_id = comps_item.exp_id
id_type = 'exp'
location = comps_item.location
elif isinstance(comps_item, Simulation):
item_id = comps_item.id
exp_id = comps_item.experiment_id
id_type = 'sim'
# retrieve location
exp = DataStore.get_experiment(exp_id)
location = exp.location
else:
print('No Experiment or Simulation was found on COMPS for {}'.format(
input_id))
exit()
# make sure it exists on COMPS
if location == 'LOCAL':
print('Item is on LOCAL not on COMPS.')
exit()
# open browser to COMPS Experiment/Simulation
import webbrowser
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
url = ''
if id_type == 'exp':
url = '%s/#explore/Experiments?filters=Id=%s&offset=0&selectedId=%s' % (
endpoint, item_id, item_id)
elif id_type == 'sim':
url = '%s/#explore/Simulations?filters=Id=%s&mode=list&orderby=DateCreated+desc&count=50&offset=0&layout=512C56&selectedId=%s' % (
endpoint, item_id, item_id)
# Open URL in new browser window
webbrowser.open_new(url) # opens in default browser
