def COMPS_login(endpoint):
try:
am = Client.auth_manager()
except:
Client.login(endpoint)
return Client
def createSimDirectoryMap_suite(sname):
#from simtools.OutputParser import CompsDTKOutputParser
from COMPS.Data import Suite, QueryCriteria
from simtools.Utilities.Experiments import retrieve_experiment
from COMPS import Client
Client.login('https://comps.idmod.org')
ste = Suite.get(sname)
expts = ste.get_experiments()
tagstemp = []
simidstemp = []
for exp in expts:
expt = retrieve_experiment(str(exp.id))
tagstemp.append([x.tags for x in expt.simulations])
simidstemp.append([x.id for x in expt.simulations])
tags = [item for sublist in tagstemp for item in sublist]
simids = [item for sublist in simidstemp for item in sublist]
df = pd.DataFrame(tags)
df['simid'] = simids
"""
sdf = get_status(expname)
if not (sdf['status'].str.contains('Succeeded')).all() :
print 'Warning: not all jobs in %s succeeded' % expname, ':',
print len(sdf[sdf['status'] != 'Succeeded']), 'unsucceeded'
df = pd.merge(left=df, right=sdf, on='simid')
df = df[df['status'] == 'Succeeded']
e = Experiment.get(expt.exp_id)
sims = e.get_simulations(QueryCriteria().select(['id', 'state']).where('state=Succeeded').select_children('hpc_jobs'))
sdf = pd.DataFrame( { 'simid' : [x.id for x in sims],
'outpath' : [x.hpc_jobs[-1].working_directory for x in sims] } )
sdf['simid'] = sdf['simid'].astype(unicode)
df = pd.merge(left=df, right=sdf, on='simid')
else :
"""
df['outpath'] = pd.Series([x.get_path() for x in expt.simulations])
return df
def createSimDirectoryMap(expname):
#from simtools.OutputParser import CompsDTKOutputParser
# from COMPS.Data import Suite, QueryCriteria
# from COMPS import Client
# Client.login('https://comps.idmod.org')
# ste = Suite.get(expname)
# expts = ste.get_experiments()
#
# for exp in expts:
# expt = get_expt(str(exp.id), is_name)
# if 'df' not in locals():
# df = pd.DataFrame( [x.tags for x in expt.simulations] )
# df['simid'] = pd.Series( [x.id for x in expt.simulations])
Client.login('https://comps.idmod.org')
expt = retrieve_experiment(expname)
# expt = retrieve_experiment(expname)
df = pd.DataFrame( [x.tags for x in expt.simulations] )
df['simid'] = pd.Series( [x.id for x in expt.simulations])
"""
sdf = get_status(expname)
if not (sdf['status'].str.contains('Succeeded')).all() :
print 'Warning: not all jobs in %s succeeded' % expname, ':',
print len(sdf[sdf['status'] != 'Succeeded']), 'unsucceeded'
df = pd.merge(left=df, right=sdf, on='simid')
df = df[df['status'] == 'Succeeded']
e = Experiment.get(expt.exp_id)
sims = e.get_simulations(QueryCriteria().select(['id', 'state']).where('state=Succeeded').select_children('hpc_jobs'))
sdf = pd.DataFrame( { 'simid' : [x.id for x in sims],
'outpath' : [x.hpc_jobs[-1].working_directory for x in sims] } )
sdf['simid'] = sdf['simid'].astype(unicode)
df = pd.merge(left=df, right=sdf, on='simid')
else :
"""
df['outpath'] = pd.Series([x.get_path() for x in expt.simulations])
return df
def translate_COMPS_path(path):
"""
Transform a COMPS path into fully qualified path.
Supports:
- $COMPS_PATH('BIN')
- $COMPS_PATH('USER')
- $COMPS_PATH('PUBLIC')
- $COMPS_PATH('INPUT')
- $COMPS_PATH('HOME')
Query the COMPS Java client with the current environment to get the correct path.
:param path: The COMPS path to transform
:param setup: The setup to find user and environment
:return: The absolute path
"""
from COMPS import Client
# Create the regexp
regexp = re.search('.*(\$COMPS_PATH\((\w+)\)).*', path)
# If no COMPS variable found -> return the path untouched
if not regexp:
return path
# Retrieve the variable to translate
groups = regexp.groups()
comps_variable = groups[1]
# Is the path already cached
if comps_variable in path_translations:
abs_path = path_translations[comps_variable]
else:
with SetupParser.TemporarySetup('HPC') as setup:
# Prepare the variables we will need
environment = setup.get('environment')
# Q uery COMPS to get the path corresponding to the variable
COMPS_login(setup.get('server_endpoint'))
abs_path = Client.auth_manager().get_environment_macros(
environment)[groups[1]]
# Cache
path_translations[comps_variable] = abs_path
# Replace and return
with SetupParser.TemporarySetup() as setup:
user = setup.get('user')
return path.replace(groups[0], abs_path).replace("$(User)", user)
from simtools.Utilities.COMPSUtilities import get_experiment_by_id
from COMPS import Client
from COMPS.Data import QueryCriteria
Client.login('https://comps.idmod.org')
exp_list = ['ea1e506d-25a7-e811-a2c0-c4346bcb7275', '96583fdd-27a7-e811-a2c0-c4346bcb7275',
'cb60414a-fda7-e811-a2c0-c4346bcb7275', '7e7edfb2-fda7-e811-a2c0-c4346bcb7275',
'30c13b96-9ba8-e811-a2c0-c4346bcb7275', 'f626a243-9ca8-e811-a2c0-c4346bcb7275',
'b95874b8-9ca8-e811-a2c0-c4346bcb7275', '64ef6572-9da8-e811-a2c0-c4346bcb7275',
'b9314306-9fa8-e811-a2c0-c4346bcb7275', '7bf00e89-44a9-e811-a2c0-c4346bcb7275',
'521a5f3f-5faa-e811-a2c0-c4346bcb7275', 'f5ffd5d3-5faa-e811-a2c0-c4346bcb7275']
rural_mult = [1.0, 1.0, 1.5, 1.5, 1.0, 1.0, 1.5, 1.5, 1.0, 1.0, 1.0, 1.0]
coverage = [0, 0.5, 0, 0.5, 0, 0.5, 0, 0.5, 0, 0.5, 0, 0.5]
migration = [.2, .2, .2, .2, .2, .2, .2, .2, .02, .02, .002, .002]
AgeAtVacc = [270, 270, 270, 270, 180, 180, 180, 180, 270, 270, 270, 270]
for ii in range(len(exp_list)):
exp = get_experiment_by_id(exp_list[ii], query_criteria=QueryCriteria().select_children(["tags"]))
exp.merge_tags({'Rural_Infectivity_Multiplier': rural_mult[ii], 'META_Campaign_Coverage': coverage[ii], 'META_Migration': migration[ii], 'MCV1_Dose_Days': AgeAtVacc[ii]})
from simtools.ModBuilder import ModBuilder, ModFn
from simtools.ExperimentManager.ExperimentManagerFactory import ExperimentManagerFactory
from dtk.utils.core.DTKConfigBuilder import DTKConfigBuilder
from malaria.interventions.malaria_drug_campaigns import add_drug_campaign
from core_setup.build_cb import *
from core_setup.interventions import add_all_interventions, preload_intervention_csvs, add_intervention_combos, \
add_mda, add_hs, add_itn, add_irs, add_pure_ivm_mda
from core_setup.reports import add_all_reports
from gridded_sims.calib.calib_helpers import set_ento
from COMPS import Client
Client.login("comps.idmod.org")
def draw_from_burnin(cb, burnin_sim_id):
sim = retrieve_simulation(burnin_sim_id)
serialized_file_path = sim.get_path()
cb.update_params({
'Serialized_Population_Path': os.path.join(serialized_file_path, 'output'),
'Serialized_Population_Filenames': ['state-19710-000.dtk', 'state-19710-001.dtk']
})
def iver_sweep(cb, intervention_df_dict, vc_pack, drug_pack, vc_coverage, mda_coverage, ivm_duration, restricted_ivm=False):
add_hs(cb, intervention_df_dict["hs"])
@property
def hoststring(self):
return self._hoststring
def get_auth_token(self):
if self._auth_token is None:
with open('token.txt', 'r') as t:
self._auth_token = t.read()
return self.__comps_auth_token_key, self._auth_token
if __name__ == "__main__":
compshost = 'comps2.idmod.org'
Client._Client__auth_manager = AutoAuthManager(compshost)
Client.login('https://' + compshost)
if sys.platform == "win32":
full_path = os.path.join(LIB_PATH, 'lib', 'site-packages')
else:
full_path = os.path.join(LIB_PATH, 'lib',
'python{}'.format(sys.version[:3]),
'site-packages')
print("Adding {} to the system path".format(full_path))
if not os.path.exists(full_path):
os.makedirs(full_path)
sys.path.insert(1, full_path)
tb = None
try:
install_packages_from_requirements('Assets/model_requirements.txt',
sys.path)
cleanup_files(full_path)