def query(self):
logger.debug("Query the HPC Monitor for Experiment %s" % self.exp_id)
COMPS_login(self.server_endpoint)
if self.suite_id:
sims = sims_from_suite_id(self.suite_id)
elif self.exp_id:
sims = sims_from_experiment_id(self.exp_id)
else:
raise Exception(
'Unable to monitor COMPS simulations as metadata contains no Suite or Experiment ID:\n'
'(Suite ID: %s, Experiment ID:%s)' %
(self.suite_id, self.exp_id))
states, msgs = {}, {}
for sim in sims:
id_string = str(sim.id)
states[
id_string] = sim.state # this is already a SimulationState object
msgs[id_string] = ''
logger.debug("States returned")
logger.debug(
json.dumps(Counter([st.name for st in states.values()]), indent=3))
return states, msgs
def __init__(self, experiment=None, config_builder=None):
# Ensure we use the SetupParser environment of the experiment if it already exists
super().__init__(experiment, config_builder)
temp_block = experiment.selected_block if experiment else SetupParser.selected_block
temp_path = experiment.setup_overlay_file if experiment else None
self.endpoint = experiment.endpoint if experiment else None
with SetupParser.TemporarySetup(temporary_block=temp_block,
temporary_path=temp_path) as setup:
self.comps_sims_to_batch = int(
setup.get(parameter='sims_per_thread'))
self.endpoint = self.endpoint or setup.get(
parameter='server_endpoint')
COMPS_login(self.endpoint)
self.asset_service = True
self.runner_thread = None
self.sims_to_create = []
self.commissioners = []
self.save_semaphore = get_semaphore()
# If we pass an experiment, retrieve it from COMPS
if self.experiment:
self.comps_experiment = COMPSCache.experiment(
self.experiment.exp_id)
def cleanup(self):
"""
Cleanup the current calibration
- Delete the result directory
- If LOCAL -> also delete the simulations
"""
try:
calib_data = self.read_calib_data()
except Exception:
logger.info('Calib data cannot be read -> skip')
calib_data = None
if calib_data:
with SetupParser.TemporaryBlock(calib_data['selected_block']):
# Retrieve suite ids and iter_count
suites = calib_data.get('suites')
iter_count = calib_data.get('iteration')
# Kill
self.kill()
# Delete the simulations too
logger.info('Cleaning up calibration %s' % self.name)
for i in range(0, iter_count + 1):
# Get the iteration cache
iteration_cache = os.path.join(self.name, 'iter%d' % i,
'IterationState.json')
if not os.path.exists(iteration_cache):
break
# Retrieve the iteration state
it = IterationState.from_file(iteration_cache)
# Create the associated experiment manager and ask for deletion
try:
exp_mgr = ExperimentManagerFactory.from_experiment(
DataStore.get_experiment(it.experiment_id))
exp_mgr.hard_delete()
except:
continue
# Delete all HPC suites (the local suites are only carried by experiments)
for suite in suites:
if suite['type'] == "HPC":
logger.info('Delete COMPS suite %s' % suite['id'])
COMPS_login(SetupParser.get('server_endpoint'))
from simtools.Utilities.COMPSUtilities import delete_suite
delete_suite(suite['id'])
# Then delete the whole directory
calib_dir = os.path.abspath(self.name)
if os.path.exists(calib_dir):
try:
shutil.rmtree(calib_dir)
except OSError:
logger.error("Failed to delete %s" % calib_dir)
logger.error(
"Try deleting the folder manually before retrying the calibration."
)
def hard_delete(self):
"""
Delete data for experiment and marks the server entity for deletion.
"""
# Mark experiment for deletion in COMPS.
COMPS_login(self.endpoint)
self.comps_experiment.delete()
# Delete in the DB
from simtools.DataAccess.DataStore import DataStore
DataStore.delete_experiment(self.experiment)
def diskspace(args, unknownArgs):
# Create a default HPC setup parser
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
COMPS_login(endpoint)
# default to the login user
user = sp.get('user')
if args.users is None or len(args.users) == 0:
args.users = [user]
# query and display the disk information
DiskSpaceUsage.display(args.users, args.top, args.save, args.refresh)
def add_experiment(self, experiment):
from simtools.DataAccess.Schema import Experiment
from simtools.Utilities.COMPSUtilities import COMPS_login
if not isinstance(experiment, Experiment):
experiment = retrieve_experiment(experiment)
if experiment not in self.experiments:
self.experiments.add(experiment)
if experiment.location == "HPC":
COMPS_login(experiment.endpoint)
COMPSCache.load_experiment(experiment.exp_id)
self.filter_simulations(experiment.simulations)
def __init__(self, experiment, comps_experiment):
logger.debug('Create COMPSSimulationRunner with experiment: %s' %
experiment.id)
super(COMPSSimulationRunner, self).__init__(experiment)
# Check if we need to commission
COMPS_login(experiment.endpoint)
self.comps_experiment = comps_experiment
if experiment_needs_commission(self.comps_experiment):
logger.debug('COMPS - Start Commissioning for experiment %s' %
self.experiment.id)
# Commission the experiment
self.comps_experiment.commission()
self.monitor()
def create_experiment(self,
experiment_name,
experiment_id=None,
suite_id=None):
# Also create the experiment in COMPS to get the ID
COMPS_login(SetupParser.get('server_endpoint'))
experiment_name = self.clean_experiment_name(experiment_name)
subdirectory = experiment_name[
0:self.MAX_SUBDIRECTORY_LENGTH] + '_' + timestamp()
config = Configuration(
environment_name=SetupParser.get('environment'),
simulation_input_args=self.commandline.Options,
working_directory_root=os.path.join(SetupParser.get('sim_root'),
subdirectory),
executable_path=self.commandline.Executable,
node_group_name=SetupParser.get('node_group'),
maximum_number_of_retries=int(SetupParser.get('num_retries')),
priority=Priority[SetupParser.get('priority')],
min_cores=self.config_builder.get_param('Num_Cores', 1)
if self.config_builder else 1,
max_cores=self.config_builder.get_param('Num_Cores', 1)
if self.config_builder else 1,
exclusive=self.config_builder.get_param('Exclusive', False)
if self.config_builder else False)
e = Experiment(name=experiment_name,
configuration=config,
suite_id=suite_id)
# Add tags if present
if self.experiment_tags: e.set_tags(self.experiment_tags)
e.save()
# Store in our object
self.comps_experiment = e
# Also add it to the cache
COMPSCache.add_experiment_to_cache(e)
# Create experiment in the base class
super(CompsExperimentManager,
self).create_experiment(experiment_name, str(e.id), suite_id)
# Set some extra stuff
self.experiment.endpoint = self.endpoint
def rescue_expts(expname) :
from COMPS.Data import Experiment, QueryCriteria
from simtools.Utilities.COMPSUtilities import COMPS_login
from simtools.Utilities.Experiments import COMPS_experiment_to_local_db
experiment_name = "%%" + expname + "%%"
endpoint = 'https://comps.idmod.org'
user = '******'
def get_experiments_by_name(name,user):
return Experiment.get(query_criteria=QueryCriteria().where(['name~%s' % name, 'owner=%s' % user]))
COMPS_login(endpoint)
for experiment_data in get_experiments_by_name(experiment_name,user):
# Create a new experiment
print(experiment_data)
experiment = COMPS_experiment_to_local_db(exp_id=str(experiment_data.id),
endpoint=endpoint,
verbose=False,
save_new_experiment=True)
def setUp(self):
SetupParser._uninit()
current_dir = os.path.dirname(os.path.realpath(__file__))
SetupParser.init(selected_block=self.SELECTED_BLOCK,
setup_file=os.path.join(current_dir, 'input',
'am_simtools.ini'))
COMPS_login(SetupParser.get('server_endpoint'))
self.existing_collection = AssetCollection(
base_collection=get_asset_collection(self.EXISTING_COLLECTION_ID))
self.existing_collection.prepare(location='HPC')
self.existing_COMPS_asset_files = self.existing_collection.asset_files_to_use
# a FileList object
dir = os.path.join(self.INPUT_DIR, 'files')
files = [os.path.join(dir, f) for f in os.listdir(dir)]
root = os.path.dirname(os.path.dirname(files[0]))
files = [
os.path.join(
os.path.split(os.path.dirname(f))[1], os.path.basename(f))
for f in files
]
self.local_files = FileList(root=root, files_in_root=files)
# take one away
files = [
os.path.join(f.relative_path, f.file_name)
for f in self.local_files.files
if f.file_name != 'file1_REMOTE_ONLY'
]
# add some more
dir = os.path.join(self.INPUT_DIR, 'additional_files')
additional_files = [os.path.join(dir, f) for f in os.listdir(dir)]
additional_files = [
os.path.join(
os.path.split(os.path.dirname(f))[1], os.path.basename(f))
for f in additional_files
]
files += additional_files
self.local_files_plus_minus = FileList(root=root, files_in_root=files)
def get_experiment_info(experiment, cache):
"""
Adds the experiment information for a given experiment to the cache:
- raw_size: the size in bytes
- size: the formatted size (in KB, MB or GB)
- sims: the number of simulations
This function is used by the process pool to parallelize the retrieval of experiment info
:param experiment: The experiment to analyze
"""
if experiment.id in cache and not DiskSpaceUsage.FORCE_REFRESH:
return
# Login to COMPS
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
COMPS_login(endpoint)
# Try to get the simulations
try:
simulations = experiment.get_simulations(
query_criteria=QueryCriteria().select(['id']).select_children(
['hpc_jobs']))
except KeyError:
# No simulations found or error -> set None
cache.set(experiment.id, None)
return
# Calculate the size
size = sum(s.hpc_jobs[0].output_directory_size for s in simulations
if s.hpc_jobs)
# Set the info for this particular experiment in the cache
cache.set(
experiment.id,
ExperimentInfo(experiment.id, experiment.name, experiment.owner,
size, len(simulations)))
def __init__(self,
selected_block=None,
setup_file=None,
commissioning_directory=None,
overrides={},
is_testing=False,
old_style_instantiation=None):
"""
This should only be used to initialize the SetupParser singleton.
Provides access to the selected_block of the resultant ini config overlay. Overlays are performed by merging
the selected overlay ini file onto the default ini file.
Overlay file selection priority:
1. setup_file, if provided
2. current/local working directory ini file, if it exists
3. ini in commissioning_directory, if provided
4. No overlay (defaults only)
If the selected_block cannot be found in the resultant ini overlay, an exception will be thrown.
:param selected_block: The current block we want to use
:param setup_file: If provided, this ini file will overlay the default UNLESS commissioning_directory was
also provided.
:param commissioning_directory: If provided, the ini file within it will always overlay the default ini file.
:param overrides: The values in this dict supersede those returned by the ConfigParser object in .get()
:param is_testing: Allows bypassing of interactive login to COMPS if True. No login attempt is made in this case.
"""
if old_style_instantiation is None:
self.old_style_instantiation(
selected_block=selected_block,
setup_file=setup_file,
commissioning_directory=commissioning_directory,
overrides=overrides,
is_testing=is_testing)
self.selected_block = (selected_block or self.default_block).upper()
self.setup_file = setup_file
self.commissioning_directory = commissioning_directory
self.commissioning_file = os.path.join(
commissioning_directory,
self.ini_filename) if commissioning_directory else None
local_file = os.path.join(os.getcwd(), self.ini_filename)
self.local_file = local_file if os.path.exists(local_file) else None
self.overrides = overrides
# Identify which ini file will overlay the default ini, if any, and verify the selected file's existence
if not os.path.exists(self.default_file):
raise self.MissingIniFile(
"Default ini file does not exist: %s . Please run 'python setup.py' again."
% self.default_file)
self.overlay_path = self._select_and_verify_overlay(
local_file=self.local_file,
provided_file=self.setup_file,
commissioning_file=self.commissioning_file)
# Load the default file
self.setup = self.config_parser_from_file(self.default_file)
# Apply the overlay if one was found
if self.overlay_path:
overlay = self.config_parser_from_file(self.overlay_path)
else:
overlay = None
self.setup = self._overlay_setup_and_resolve_inheritance(
overlay, self.setup)
# Verify that we have the requested block in our overlain result
if not self.setup.has_section(self.selected_block):
raise self.MissingIniBlock(
"Selected block: %s does not exist in ini file overlay.\nOverlay path: %s"
% (self.selected_block, self.overlay_path))
# If the selected block is type=HPC, take care of HPC initialization
if self.setup.get(self.selected_block,
'type') == "HPC" and not is_testing:
from simtools.Utilities.COMPSUtilities import COMPS_login
COMPS_login(self.setup.get(self.selected_block, 'server_endpoint'))
def pre_creation(self):
# Call login now (even if we are already logged in, we need to call login to initialize the COMPSAccess Client)
COMPS_login(self.server_endpoint)
def cancel_experiment(self):
super(CompsExperimentManager, self).cancel_experiment()
COMPS_login(self.endpoint)
if self.comps_experiment and experiment_is_running(
self.comps_experiment):
self.comps_experiment.cancel()
def retrieve_data(simulation, analyzers, cache):
from simtools.Analysis.AnalyzeManager import EXCEPTION_KEY
# Filter first and get the filenames from filtered analysis
filtered_analysis = [a for a in analyzers if a.filter(simulation)]
filenames = set(itertools.chain(*(a.filenames for a in filtered_analysis)))
# We dont have anything to do :)
if not filenames or not filtered_analysis:
cache.set(simulation.id, None)
return
# The byte_arrays will associate filename with content
byte_arrays = {}
try:
if simulation.experiment.location == "HPC":
COMPS_login(simulation.experiment.endpoint)
COMPS_simulation = COMPSCache.simulation(simulation.id)
assets = [path for path in filenames if path.lower().startswith("assets")]
transient = [path for path in filenames if not path.lower().startswith("assets")]
if transient:
byte_arrays.update(dict(zip(transient, COMPS_simulation.retrieve_output_files(paths=transient))))
if assets:
byte_arrays.update(get_asset_files_for_simulation_id(simulation.id, paths=assets, remove_prefix='Assets'))
else:
byte_arrays = []
for filename in filenames:
path = os.path.join(simulation.get_path(), filename)
with open(path, 'rb') as output_file:
byte_arrays[filename] = output_file.read()
except:
tb = traceback.format_exc()
cache.set(EXCEPTION_KEY, "An exception has been raised during data retrieval.\n"
"Simulation: {} \n"
"Analyzers: {}\n"
"Files: {}\n"
"\n{}".format(simulation, ", ".join(analyzers), ", ".join(filenames), tb))
return
# Selected data will be a dict with analyzer.uid => data
selected_data = {}
for analyzer in filtered_analysis:
# If the analyzer needs the parsed data, parse
if analyzer.parse:
data = {filename:SimulationOutputParser.parse(filename, content)
for filename, content in byte_arrays.items()}
else:
# If the analyzer doesnt wish to parse, give the raw data
data = byte_arrays
# Retrieve the selected data for the given analyzer
try:
selected_data[analyzer.uid] = analyzer.select_simulation_data(data, simulation)
except:
tb = traceback.format_exc()
cache.set(EXCEPTION_KEY, "An exception has been raised during data processing.\n"
"Simulation: {} \n"
"Analyzer: {}\n"
"\n{}".format(simulation, analyzer, tb))
return
# Store in the cache
cache.set(simulation.id, selected_data)
# Create a new AnalyzeManager and add experiment and analyzer
am = AnalyzeManager(verbose=False)
for em in experiments:
am.add_experiment(em.experiment)
analyzer = DownloadAnalyzerTPI(
filenames=['output\\ReportHIVByAgeAndGender.csv'],
TPI_tag="TPI",
ignore_TPI=False,
REP_tag="TPI",
ignore_REP=True,
output_path=output_directory)
am.add_analyzer(analyzer)
# Make sure we refresh our set of experiments
for e in experiments:
e.refresh_experiment()
COMPS_login(SetupParser.get("server_endpoint"))
am.analyze()
# If we are not done we wait for 30 sec, if we are done we leave
if not finished:
print("Waiting 30 seconds")
time.sleep(30)
else:
break
def generate_immune_overlays(self):
'''
generate immune overlays; the json file format pointed to by self.immunity_burnin_meta_file_path is
{
exp_id1:path to experiment meta json file created in the folder indicated by sim_root in dtk_setup.cfg
exp_id2: --||--,
...
}
'''
# group nodes by parameter
self.group_nodes_by_params()
with open('nodes_by_param.json', 'w') as p_f:
json.dump(self.nodes_by_params, p_f, indent=4)
# maintain a list of overlays' file paths so that they can be used by the spatial simulation later?
overlays_list = []
# open file containing paths to immune initialization burnin sweep meta files
with open(self.immunity_burnin_meta_file_path, 'r') as imm_f:
immun_meta_exps = json.load(imm_f)
i = 0
for exp_id, exp_path in immun_meta_exps.items():
with open(os.path.abspath(exp_path), 'r') as meta_f:
exp_meta = json.load(meta_f)
# find out if experiment has been run locally or remotely on hpc;
# that determines the simulation output folder location and structure
exp_location_type = exp_meta['location']
exp_name = exp_meta['exp_name']
# do we have a helper function that determines the output directory of a simulation given simulation and experiment IDs (working both for COMPs and local)?
sim_dir_map = None
if exp_location_type == 'HPC':
from simtools.SetupParser import SetupParser
sp = SetupParser('HPC')
om("Pulling immunization data from COMPs.")
om("This requires a login.")
COMPS_login(sp.get('server_endpoint'))
om("Login success!")
sim_dir_map = parser.createSimDirectoryMap(exp_id)
# iterate through the experiments simulations;
for sim_id, sim_record in exp_meta['sims'].items():
sim_output_path = ''
#for each simulation get the values of parameters relevant to the immune initialization burnin (i.e. the parameters
# given by the keys in self.nodes_params)
# the mapping between nodes and immune overlays below could be done more efficiently, but for the typical number of parameters the
# mapping below should work fine
# get the parameter keys in the right order (see group_nodes_by_params(self) and get_params_key(self...);
# the set of relevant parameters is the same across all nodes, so take the one from the first node
node_params = self.nodes_params.values().next().keys()
param_values = []
for param in node_params:
if param in sim_record:
param_values.append(sim_record[param])
# get nodes associated with this set of parameters
if self.get_params_key(
param_values) in self.nodes_by_params:
params_key = self.get_params_key(param_values)
nodes = self.nodes_by_params[params_key]
# for each simulation get its output immunity report
if exp_location_type == 'LOCAL':
sim_output_path = os.path.join(
exp_meta['sim_root'],
exp_name + '_' + exp_meta['exp_id'],
sim_id, 'output')
elif exp_location_type == 'HPC':
sim_output_path = os.path.join(
sim_dir_map[sim_id], 'output')
immunity_report_file_path = os.path.join(
sim_output_path,
'MalariaImmunityReport_AnnualAverage.json')
# generate immune overlay
from dtk.tools.demographics.createimmunelayer import \
immune_init_from_custom_output_for_spatial as immune_init
immune_overlay_json = immune_init(
{
"Metadata": {
"Author": "dtk-tools",
"IdReference":
"Gridded world grump30arcsec",
"NodeCount": len(nodes)
},
"Nodes": nodes
},
immunity_report_file_path,
)
# save compiled overlay
overlay_file_name = params_key + exp_name + '.json'
overlay_file_path = os.path.join(
self.immune_overlays_path, overlay_file_name)
with open(overlay_file_path, 'w') as imo_f:
json.dump(immune_overlay_json, imo_f)
CompileDemographics(overlay_file_path,
forceoverwrite=True)
# add overlay location to overlay list
overlays_list.append(overlay_file_name)
print(
str(len(overlays_list)) +
" immune initialization overlay files processed successfully!"
)
return overlays_list
def sync(args, unknownArgs):
"""
Sync COMPS db with local db
"""
# Create a default HPC setup parser
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
user = sp.get('user')
COMPS_login(endpoint)
exp_to_save = list()
exp_deleted = 0
# Retrieve all the experiment id from COMPS for the current user
exp_ids = get_experiment_ids_for_user(user)
# Test the experiments present in the local DB to make sure they still exist in COMPS
for exp in DataStore.get_experiments():
if exp.location == "HPC":
if exp.exp_id not in exp_ids:
# The experiment doesnt exist on COMPS anymore -> delete from local
DataStore.delete_experiment(exp)
exp_deleted += 1
# Consider experiment id option
exp_id = args.exp_id if args.exp_id else None
exp_name = args.exp_name if args.exp_name else None
user = args.user if args.user else user
if exp_name:
experiments = get_experiments_by_name(exp_name, user)
for experiment_data in experiments:
experiment = COMPS_experiment_to_local_db(
exp_id=str(experiment_data.id),
endpoint=endpoint,
verbose=True,
save_new_experiment=False)
if experiment:
exp_to_save.append(experiment)
elif exp_id:
# Create a new experiment
experiment = COMPS_experiment_to_local_db(exp_id=exp_id,
endpoint=endpoint,
verbose=True,
save_new_experiment=False)
# The experiment needs to be saved
if experiment:
exp_to_save.append(experiment)
else:
# By default only get experiments created in the last month
# day_limit = args.days if args.days else day_limit_default
day_limit = 30
today = datetime.date.today()
limit_date = today - datetime.timedelta(days=int(day_limit))
# For each of them, check if they are in the db
for exp in get_experiments_per_user_and_date(user, limit_date):
# Create a new experiment
experiment = COMPS_experiment_to_local_db(
exp_id=str(exp.id),
endpoint=endpoint,
save_new_experiment=False)
# The experiment needs to be saved
if experiment:
exp_to_save.append(experiment)
# Save the experiments if any
if len(exp_to_save) > 0 and exp_deleted == 0:
DataStore.batch_save_experiments(exp_to_save)
logger.info(
"The following experiments have been added to the database:")
logger.info("\n".join(["- " + str(exp) for exp in exp_to_save]))
logger.info("%s experiments have been updated in the DB." %
len(exp_to_save))
logger.info("%s experiments have been deleted from the DB." %
exp_deleted)
else:
print("The database was already up to date.")
# Start overseer
BaseExperimentManager.check_overseer()
coverage=coverage,
rate=0.15,
drugname="AL")
return {
'intervention': intervention,
'%s_coverage' % intervention: coverage
}
if __name__ == "__main__":
SetupParser.init()
# collect site-specific data to pass to builder functions
COMPS_login("https://comps.idmod.org")
sites = pd.read_csv("site_details.csv")
print("finding collection ids and vector details")
site_input_dir = os.path.join("input", "sites", "all")
with open("species_details.json") as f:
species_details = json.loads(f.read())
if asset_exp_id:
print("retrieving asset experiment")
asset_expt = retrieve_experiment(asset_exp_id)
template_asset = asset_expt.simulations[0].tags
cb.set_exe_collection(template_asset["exe_collection_id"])
cb.set_dll_collection(template_asset["dll_collection_id"])
cb.set_input_collection(template_asset["input_collection_id"])
def COMPS_experiment_to_local_db(exp_id, endpoint, verbose=False, save_new_experiment=True):
"""
Return a DB object representing an experiment coming from COMPS.
This function saves the newly retrieved experiment in the DB by default but this behavior can be changed by switching
the save_new_experiment parameter allowing to return an experiment object and save later with a batch for example.
:param exp_id:
:param endpoint:
:param verbose:
:param save_new_experiment:
:return:
"""
# Make sure we are logged in
COMPS_login(endpoint)
#Ensure exp_id is a string
exp_id = str(exp_id)
# IF the experiment already exists and
experiment = DataStore.get_experiment(exp_id)
if experiment and experiment.is_done():
if verbose:
print("Experiment ('%s') already exists in local db." % exp_id)
# Do not bother with finished experiments
return None
from COMPS.Data import QueryCriteria
try:
query_criteria = QueryCriteria().select_children('tags')
exp_comps = get_experiment_by_id(exp_id, query_criteria) or get_experiments_by_name(exp_id, query_criteria)[-1]
except:
if verbose:
print("The experiment ('%s') doesn't exist in COMPS." % exp_id)
return None
# Case: experiment doesn't exist in local db
if not experiment:
# Cast the creation_date
experiment = DataStore.create_experiment(exp_id=str(exp_comps.id),
suite_id=str(exp_comps.suite_id) if exp_comps.suite_id else None,
exp_name=exp_comps.name,
tags=exp_comps.tags,
date_created=utc_to_local(exp_comps.date_created).replace(tzinfo=None),
location='HPC',
selected_block='HPC',
endpoint=endpoint)
# Note: experiment may be new or comes from local db
# Get associated simulations of the experiment
sims = exp_comps.get_simulations(QueryCriteria().select(['id', 'state', 'date_created']).select_children('tags'))
# Skip empty experiments or experiments that have the same number of sims
if len(sims) == 0 or len(sims) == len(experiment.simulations):
if verbose:
if len(sims) == 0:
print("Skip empty experiment ('%s')." % exp_id)
elif len(sims) == len(experiment.simulations):
print("Skip experiment ('%s') since local one has the same number of simulations." % exp_id)
return None
# Go through the sims and create them
for sim in sims:
# Cast the simulation tags
# Create the simulation
simulation = DataStore.create_simulation(id=str(sim.id),
status=sim.state, # this is already a SimulationState object
tags={tag:cast_number(val) for tag,val in sim.tags.items()},
date_created=utc_to_local(sim.date_created).replace(tzinfo=None))
# Add to the experiment
experiment.simulations.append(simulation)
# Save it to the DB
if save_new_experiment: DataStore.save_experiment(experiment, verbose=verbose)
return experiment
