def retrieve_experiment(exp_id, sync_if_missing=True, verbose=False, force_update=False): """ Retrieve an experiment in the local database based on its id. Can call a sync if missing if the flag is true. :param exp_id: Id of the experiment to retrieve :param sync_if_missing: Should we try to sync if not present? :return: The experiment found """ if not exp_id: raise Exception("Trying to retrieve an experiment without providing an experiment ID") from uuid import UUID if isinstance(exp_id,UUID): exp_id = str(exp_id) # If we dont force the update -> look first in the DB exp = DataStore.get_experiment(exp_id) or DataStore.get_most_recent_experiment(exp_id) if exp: # If we have an experiment and we want to force the update -> delete it if not force_update: return exp else: DataStore.delete_experiment(exp) if not sync_if_missing: raise Exception('Experiment %s not found in the local database and sync disabled.' % exp_id) logger.info('Experiment with id %s not found in local database, trying sync.' % exp_id) with SetupParser.TemporarySetup(temporary_block='HPC') as sp: endpoint = sp.get('server_endpoint') exp = COMPS_experiment_to_local_db(exp_id, endpoint, verbose) if exp: return exp raise Exception("Experiment '%s' could not be retrieved." % exp_id)
def commission_iteration(self, next_params): """ Commission an experiment of simulations constructed from a list of combinations of random seeds, calibration sites, and the next sample points. Cache the relevant experiment and simulation information to the IterationState. """ if self.simulations: logger.info( 'Reloading simulation data from cached iteration (%s) state.' % self.iteration) self.exp_manager = ExperimentManagerFactory.from_experiment( DataStore.get_experiment(self.experiment_id)) else: self.exp_manager = ExperimentManagerFactory.init() # use passed in function to create exp_builder exp_builder = self.exp_builder_func(next_params) self.exp_manager.run_simulations( config_builder=self.config_builder, exp_name='%s_iter%d' % (self.calibration_name, self.iteration), exp_builder=exp_builder, suite_id=self.suite_id) self.simulations = self.exp_manager.experiment.toJSON( )['simulations'] self.experiment_id = self.exp_manager.experiment.exp_id self.save()
def cleanup(self): """ Cleanup the current calibration - Delete the result directory - If LOCAL -> also delete the simulations """ try: calib_data = self.read_calib_data() except Exception: logger.info('Calib data cannot be read -> skip') calib_data = None if calib_data: with SetupParser.TemporaryBlock(calib_data['selected_block']): # Retrieve suite ids and iter_count suites = calib_data.get('suites') iter_count = calib_data.get('iteration') # Kill self.kill() # Delete the simulations too logger.info('Cleaning up calibration %s' % self.name) for i in range(0, iter_count + 1): # Get the iteration cache iteration_cache = os.path.join(self.name, 'iter%d' % i, 'IterationState.json') if not os.path.exists(iteration_cache): break # Retrieve the iteration state it = IterationState.from_file(iteration_cache) # Create the associated experiment manager and ask for deletion try: exp_mgr = ExperimentManagerFactory.from_experiment( DataStore.get_experiment(it.experiment_id)) exp_mgr.hard_delete() except: continue # Delete all HPC suites (the local suites are only carried by experiments) for suite in suites: if suite['type'] == "HPC": logger.info('Delete COMPS suite %s' % suite['id']) COMPS_login(SetupParser.get('server_endpoint')) from simtools.Utilities.COMPSUtilities import delete_suite delete_suite(suite['id']) # Then delete the whole directory calib_dir = os.path.abspath(self.name) if os.path.exists(calib_dir): try: shutil.rmtree(calib_dir) except OSError: logger.error("Failed to delete %s" % calib_dir) logger.error( "Try deleting the folder manually before retrying the calibration." )
def done_commissioning(self): self.experiment = DataStore.get_experiment(self.experiment.exp_id) for sim in self.experiment.simulations: if not sim.status or sim.status in [ SimulationState.CommissionRequested, SimulationState.Created ]: return False return True
def get_experiment_from_iteration(self, iteration=None, force_metadata=False): """ Retrieve experiment for a given iteration """ exp = None # Only check iteration for resume cases if force_metadata: iteration = self.adjust_iteration(iteration) it = self.read_iteration_data(iteration) exp = DataStore.get_experiment(it.experiment_id) return exp
def query(self): logger.debug("Query the DB Monitor for Experiment %s" % self.exp_id) states, msgs = {}, {} experiment = DataStore.get_experiment(self.exp_id) if not experiment or not experiment.simulations: return states, msgs for sim in experiment.simulations: states[ sim. id] = sim.status if sim.status else SimulationState.CommissionRequested msgs[sim.id] = sim.message if sim.message else "" logger.debug("States returned") logger.debug(Counter(states.values())) return states, msgs
def retrieve_object(obj_id, obj_type, sync_if_missing=True, verbose=False, force_update=False): """ Retrieve an object (Experiment, Simulation) in the local database based on its id. Can call a sync if missing if the flag is true :param obj_id: Id of the object to retrieve :param obj_type: Type of the object to retrieve :param sync_if_missing: Should we try to sync if not present? :return: The experiment found #TODO: SHould also support suites """ typename = obj_type.__name__ if not obj_id: raise ValueError("Trying to retrieve an object (%s) without providing an ID" % typename) # Try a hit in the DB if obj_type == Experiment: obj = DataStore.get_experiment(obj_id) elif obj_type == Simulation: obj = DataStore.get_simulation(obj_id) if obj: # If we have an experiment and we want to force the update -> delete it if not force_update: return obj else: if obj_type == Experiment: DataStore.delete_experiment(obj_id) elif obj_type == Simulation: DataStore.delete_simulation(obj_id) if not sync_if_missing: raise Exception('%s %s not found in the local database and sync disabled.' % (typename, obj_id)) logger.info('%s with id %s not found in local database, trying sync.' % (typename, obj_id)) with SetupParser.TemporarySetup(temporary_block='HPC') as sp: endpoint = sp.get('server_endpoint') if obj_type == Experiment: obj = COMPS_experiment_to_local_db(obj_id, endpoint, verbose) elif obj_type == Simulation: obj =None if obj: return obj raise Exception("%s '%s' could not be retrieved." % (typename, obj_id))
def test_run(self): input_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'input') model_file = os.path.join(input_path, 'dummy_model.py') local_manager = ExperimentManagerFactory.from_model( model_file, 'LOCAL') local_manager.run_simulations( config_builder=PythonConfigBuilder.from_defaults('sleep'), exp_builder=RunNumberSweepBuilder(self.nsims)) self.assertEqual(local_manager.experiment.exp_name, 'test') experiment = local_manager.experiment local_manager = ExperimentManagerFactory.from_experiment( experiment=DataStore.get_experiment(experiment.exp_id)) states, msgs = local_manager.get_simulation_status() self.assertListEqual(states.values(), [SimulationState.Created] * self.nsims) local_manager.hard_delete() import time time.sleep(3)
def load_experiment_from_iteration(self, iteration=None): """ Load experiment for a given or the latest iteration """ if iteration is None: # restore the existing calibration data calib_data = self.read_calib_data() # Get the last iteration latest_iteration = calib_data.get('iteration', None) else: latest_iteration = iteration try: # Restore IterationState it = IterationState.from_file(os.path.join(self.name, 'iter%d' % latest_iteration, 'IterationState.json')) # Get experiment by id return DataStore.get_experiment(it.experiment_id) except: return None
def refresh_experiment(self): self.check_overseer() # Refresh the experiment self.experiment = DataStore.get_experiment(self.experiment.exp_id)
def link(args, unknownArgs): """ Open browser to the COMPS Experiment/Simulation with ID or name provided :param args: :param unknownArgs: :return: """ # get input from commands line input_id = args.Id # default: consider the latest experiment if input_id is None: latest = DataStore.get_most_recent_experiment() input_id = latest.exp_id try: comps_item = retrieve_item(input_id) except: print('Nothing was found for {}'.format(input_id)) exit() # check item type id_type = '' location = 'LOCAL' if isinstance(comps_item, Experiment): item_id = comps_item.exp_id id_type = 'exp' location = comps_item.location elif isinstance(comps_item, Simulation): item_id = comps_item.id exp_id = comps_item.experiment_id id_type = 'sim' # retrieve location exp = DataStore.get_experiment(exp_id) location = exp.location else: print('No Experiment or Simulation was found on COMPS for {}'.format( input_id)) exit() # make sure it exists on COMPS if location == 'LOCAL': print('Item is on LOCAL not on COMPS.') exit() # open browser to COMPS Experiment/Simulation import webbrowser with SetupParser.TemporarySetup(temporary_block='HPC') as sp: endpoint = sp.get('server_endpoint') url = '' if id_type == 'exp': url = '%s/#explore/Experiments?filters=Id=%s&offset=0&selectedId=%s' % ( endpoint, item_id, item_id) elif id_type == 'sim': url = '%s/#explore/Simulations?filters=Id=%s&mode=list&orderby=DateCreated+desc&count=50&offset=0&layout=512C56&selectedId=%s' % ( endpoint, item_id, item_id) # Open URL in new browser window webbrowser.open_new(url) # opens in default browser
def COMPS_experiment_to_local_db(exp_id, endpoint, verbose=False, save_new_experiment=True): """ Return a DB object representing an experiment coming from COMPS. This function saves the newly retrieved experiment in the DB by default but this behavior can be changed by switching the save_new_experiment parameter allowing to return an experiment object and save later with a batch for example. :param exp_id: :param endpoint: :param verbose: :param save_new_experiment: :return: """ # Make sure we are logged in COMPS_login(endpoint) #Ensure exp_id is a string exp_id = str(exp_id) # IF the experiment already exists and experiment = DataStore.get_experiment(exp_id) if experiment and experiment.is_done(): if verbose: print("Experiment ('%s') already exists in local db." % exp_id) # Do not bother with finished experiments return None from COMPS.Data import QueryCriteria try: query_criteria = QueryCriteria().select_children('tags') exp_comps = get_experiment_by_id(exp_id, query_criteria) or get_experiments_by_name(exp_id, query_criteria)[-1] except: if verbose: print("The experiment ('%s') doesn't exist in COMPS." % exp_id) return None # Case: experiment doesn't exist in local db if not experiment: # Cast the creation_date experiment = DataStore.create_experiment(exp_id=str(exp_comps.id), suite_id=str(exp_comps.suite_id) if exp_comps.suite_id else None, exp_name=exp_comps.name, tags=exp_comps.tags, date_created=utc_to_local(exp_comps.date_created).replace(tzinfo=None), location='HPC', selected_block='HPC', endpoint=endpoint) # Note: experiment may be new or comes from local db # Get associated simulations of the experiment sims = exp_comps.get_simulations(QueryCriteria().select(['id', 'state', 'date_created']).select_children('tags')) # Skip empty experiments or experiments that have the same number of sims if len(sims) == 0 or len(sims) == len(experiment.simulations): if verbose: if len(sims) == 0: print("Skip empty experiment ('%s')." % exp_id) elif len(sims) == len(experiment.simulations): print("Skip experiment ('%s') since local one has the same number of simulations." % exp_id) return None # Go through the sims and create them for sim in sims: # Cast the simulation tags # Create the simulation simulation = DataStore.create_simulation(id=str(sim.id), status=sim.state, # this is already a SimulationState object tags={tag:cast_number(val) for tag,val in sim.tags.items()}, date_created=utc_to_local(sim.date_created).replace(tzinfo=None)) # Add to the experiment experiment.simulations.append(simulation) # Save it to the DB if save_new_experiment: DataStore.save_experiment(experiment, verbose=verbose) return experiment