def retrieve_experiment(exp_id, sync_if_missing=True, verbose=False, force_update=False):
"""
Retrieve an experiment in the local database based on its id.
Can call a sync if missing if the flag is true.
:param exp_id: Id of the experiment to retrieve
:param sync_if_missing: Should we try to sync if not present?
:return: The experiment found
"""
if not exp_id: raise Exception("Trying to retrieve an experiment without providing an experiment ID")
from uuid import UUID
if isinstance(exp_id,UUID): exp_id = str(exp_id)
# If we dont force the update -> look first in the DB
exp = DataStore.get_experiment(exp_id) or DataStore.get_most_recent_experiment(exp_id)
if exp:
# If we have an experiment and we want to force the update -> delete it
if not force_update:
return exp
else:
DataStore.delete_experiment(exp)
if not sync_if_missing:
raise Exception('Experiment %s not found in the local database and sync disabled.' % exp_id)
logger.info('Experiment with id %s not found in local database, trying sync.' % exp_id)
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
exp = COMPS_experiment_to_local_db(exp_id, endpoint, verbose)
if exp: return exp
raise Exception("Experiment '%s' could not be retrieved." % exp_id)
def commission_iteration(self, next_params):
"""
Commission an experiment of simulations constructed from a list of combinations of
random seeds, calibration sites, and the next sample points.
Cache the relevant experiment and simulation information to the IterationState.
"""
if self.simulations:
logger.info(
'Reloading simulation data from cached iteration (%s) state.' %
self.iteration)
self.exp_manager = ExperimentManagerFactory.from_experiment(
DataStore.get_experiment(self.experiment_id))
else:
self.exp_manager = ExperimentManagerFactory.init()
# use passed in function to create exp_builder
exp_builder = self.exp_builder_func(next_params)
self.exp_manager.run_simulations(
config_builder=self.config_builder,
exp_name='%s_iter%d' % (self.calibration_name, self.iteration),
exp_builder=exp_builder,
suite_id=self.suite_id)
self.simulations = self.exp_manager.experiment.toJSON(
)['simulations']
self.experiment_id = self.exp_manager.experiment.exp_id
self.save()
def cleanup(self):
"""
Cleanup the current calibration
- Delete the result directory
- If LOCAL -> also delete the simulations
"""
try:
calib_data = self.read_calib_data()
except Exception:
logger.info('Calib data cannot be read -> skip')
calib_data = None
if calib_data:
with SetupParser.TemporaryBlock(calib_data['selected_block']):
# Retrieve suite ids and iter_count
suites = calib_data.get('suites')
iter_count = calib_data.get('iteration')
# Kill
self.kill()
# Delete the simulations too
logger.info('Cleaning up calibration %s' % self.name)
for i in range(0, iter_count + 1):
# Get the iteration cache
iteration_cache = os.path.join(self.name, 'iter%d' % i,
'IterationState.json')
if not os.path.exists(iteration_cache):
break
# Retrieve the iteration state
it = IterationState.from_file(iteration_cache)
# Create the associated experiment manager and ask for deletion
try:
exp_mgr = ExperimentManagerFactory.from_experiment(
DataStore.get_experiment(it.experiment_id))
exp_mgr.hard_delete()
except:
continue
# Delete all HPC suites (the local suites are only carried by experiments)
for suite in suites:
if suite['type'] == "HPC":
logger.info('Delete COMPS suite %s' % suite['id'])
COMPS_login(SetupParser.get('server_endpoint'))
from simtools.Utilities.COMPSUtilities import delete_suite
delete_suite(suite['id'])
# Then delete the whole directory
calib_dir = os.path.abspath(self.name)
if os.path.exists(calib_dir):
try:
shutil.rmtree(calib_dir)
except OSError:
logger.error("Failed to delete %s" % calib_dir)
logger.error(
"Try deleting the folder manually before retrying the calibration."
)
def done_commissioning(self):
self.experiment = DataStore.get_experiment(self.experiment.exp_id)
for sim in self.experiment.simulations:
if not sim.status or sim.status in [
SimulationState.CommissionRequested,
SimulationState.Created
]:
return False
return True
def get_experiment_from_iteration(self, iteration=None, force_metadata=False):
"""
Retrieve experiment for a given iteration
"""
exp = None
# Only check iteration for resume cases
if force_metadata:
iteration = self.adjust_iteration(iteration)
it = self.read_iteration_data(iteration)
exp = DataStore.get_experiment(it.experiment_id)
return exp
def query(self):
logger.debug("Query the DB Monitor for Experiment %s" % self.exp_id)
states, msgs = {}, {}
experiment = DataStore.get_experiment(self.exp_id)
if not experiment or not experiment.simulations:
return states, msgs
for sim in experiment.simulations:
states[
sim.
id] = sim.status if sim.status else SimulationState.CommissionRequested
msgs[sim.id] = sim.message if sim.message else ""
logger.debug("States returned")
logger.debug(Counter(states.values()))
return states, msgs
def retrieve_object(obj_id, obj_type, sync_if_missing=True, verbose=False, force_update=False):
"""
Retrieve an object (Experiment, Simulation) in the local database based on its id.
Can call a sync if missing if the flag is true
:param obj_id: Id of the object to retrieve
:param obj_type: Type of the object to retrieve
:param sync_if_missing: Should we try to sync if not present?
:return: The experiment found
#TODO: SHould also support suites
"""
typename = obj_type.__name__
if not obj_id: raise ValueError("Trying to retrieve an object (%s) without providing an ID" % typename)
# Try a hit in the DB
if obj_type == Experiment:
obj = DataStore.get_experiment(obj_id)
elif obj_type == Simulation:
obj = DataStore.get_simulation(obj_id)
if obj:
# If we have an experiment and we want to force the update -> delete it
if not force_update:
return obj
else:
if obj_type == Experiment:
DataStore.delete_experiment(obj_id)
elif obj_type == Simulation:
DataStore.delete_simulation(obj_id)
if not sync_if_missing:
raise Exception('%s %s not found in the local database and sync disabled.' % (typename, obj_id))
logger.info('%s with id %s not found in local database, trying sync.' % (typename, obj_id))
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
if obj_type == Experiment:
obj = COMPS_experiment_to_local_db(obj_id, endpoint, verbose)
elif obj_type == Simulation:
obj =None
if obj: return obj
raise Exception("%s '%s' could not be retrieved." % (typename, obj_id))
def test_run(self):
input_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'input')
model_file = os.path.join(input_path, 'dummy_model.py')
local_manager = ExperimentManagerFactory.from_model(
model_file, 'LOCAL')
local_manager.run_simulations(
config_builder=PythonConfigBuilder.from_defaults('sleep'),
exp_builder=RunNumberSweepBuilder(self.nsims))
self.assertEqual(local_manager.experiment.exp_name, 'test')
experiment = local_manager.experiment
local_manager = ExperimentManagerFactory.from_experiment(
experiment=DataStore.get_experiment(experiment.exp_id))
states, msgs = local_manager.get_simulation_status()
self.assertListEqual(states.values(),
[SimulationState.Created] * self.nsims)
local_manager.hard_delete()
import time
time.sleep(3)
def load_experiment_from_iteration(self, iteration=None):
"""
Load experiment for a given or the latest iteration
"""
if iteration is None:
# restore the existing calibration data
calib_data = self.read_calib_data()
# Get the last iteration
latest_iteration = calib_data.get('iteration', None)
else:
latest_iteration = iteration
try:
# Restore IterationState
it = IterationState.from_file(os.path.join(self.name, 'iter%d' % latest_iteration, 'IterationState.json'))
# Get experiment by id
return DataStore.get_experiment(it.experiment_id)
except:
return None
def refresh_experiment(self):
self.check_overseer()
# Refresh the experiment
self.experiment = DataStore.get_experiment(self.experiment.exp_id)
def link(args, unknownArgs):
"""
Open browser to the COMPS Experiment/Simulation with ID or name provided
:param args:
:param unknownArgs:
:return:
"""
# get input from commands line
input_id = args.Id
# default: consider the latest experiment
if input_id is None:
latest = DataStore.get_most_recent_experiment()
input_id = latest.exp_id
try:
comps_item = retrieve_item(input_id)
except:
print('Nothing was found for {}'.format(input_id))
exit()
# check item type
id_type = ''
location = 'LOCAL'
if isinstance(comps_item, Experiment):
item_id = comps_item.exp_id
id_type = 'exp'
location = comps_item.location
elif isinstance(comps_item, Simulation):
item_id = comps_item.id
exp_id = comps_item.experiment_id
id_type = 'sim'
# retrieve location
exp = DataStore.get_experiment(exp_id)
location = exp.location
else:
print('No Experiment or Simulation was found on COMPS for {}'.format(
input_id))
exit()
# make sure it exists on COMPS
if location == 'LOCAL':
print('Item is on LOCAL not on COMPS.')
exit()
# open browser to COMPS Experiment/Simulation
import webbrowser
with SetupParser.TemporarySetup(temporary_block='HPC') as sp:
endpoint = sp.get('server_endpoint')
url = ''
if id_type == 'exp':
url = '%s/#explore/Experiments?filters=Id=%s&offset=0&selectedId=%s' % (
endpoint, item_id, item_id)
elif id_type == 'sim':
url = '%s/#explore/Simulations?filters=Id=%s&mode=list&orderby=DateCreated+desc&count=50&offset=0&layout=512C56&selectedId=%s' % (
endpoint, item_id, item_id)
# Open URL in new browser window
webbrowser.open_new(url) # opens in default browser
def COMPS_experiment_to_local_db(exp_id, endpoint, verbose=False, save_new_experiment=True):
"""
Return a DB object representing an experiment coming from COMPS.
This function saves the newly retrieved experiment in the DB by default but this behavior can be changed by switching
the save_new_experiment parameter allowing to return an experiment object and save later with a batch for example.
:param exp_id:
:param endpoint:
:param verbose:
:param save_new_experiment:
:return:
"""
# Make sure we are logged in
COMPS_login(endpoint)
#Ensure exp_id is a string
exp_id = str(exp_id)
# IF the experiment already exists and
experiment = DataStore.get_experiment(exp_id)
if experiment and experiment.is_done():
if verbose:
print("Experiment ('%s') already exists in local db." % exp_id)
# Do not bother with finished experiments
return None
from COMPS.Data import QueryCriteria
try:
query_criteria = QueryCriteria().select_children('tags')
exp_comps = get_experiment_by_id(exp_id, query_criteria) or get_experiments_by_name(exp_id, query_criteria)[-1]
except:
if verbose:
print("The experiment ('%s') doesn't exist in COMPS." % exp_id)
return None
# Case: experiment doesn't exist in local db
if not experiment:
# Cast the creation_date
experiment = DataStore.create_experiment(exp_id=str(exp_comps.id),
suite_id=str(exp_comps.suite_id) if exp_comps.suite_id else None,
exp_name=exp_comps.name,
tags=exp_comps.tags,
date_created=utc_to_local(exp_comps.date_created).replace(tzinfo=None),
location='HPC',
selected_block='HPC',
endpoint=endpoint)
# Note: experiment may be new or comes from local db
# Get associated simulations of the experiment
sims = exp_comps.get_simulations(QueryCriteria().select(['id', 'state', 'date_created']).select_children('tags'))
# Skip empty experiments or experiments that have the same number of sims
if len(sims) == 0 or len(sims) == len(experiment.simulations):
if verbose:
if len(sims) == 0:
print("Skip empty experiment ('%s')." % exp_id)
elif len(sims) == len(experiment.simulations):
print("Skip experiment ('%s') since local one has the same number of simulations." % exp_id)
return None
# Go through the sims and create them
for sim in sims:
# Cast the simulation tags
# Create the simulation
simulation = DataStore.create_simulation(id=str(sim.id),
status=sim.state, # this is already a SimulationState object
tags={tag:cast_number(val) for tag,val in sim.tags.items()},
date_created=utc_to_local(sim.date_created).replace(tzinfo=None))
# Add to the experiment
experiment.simulations.append(simulation)
# Save it to the DB
if save_new_experiment: DataStore.save_experiment(experiment, verbose=verbose)
return experiment
