def query(self): logger.debug("Query the HPC Monitor for Experiment %s" % self.exp_id) COMPS_login(self.server_endpoint) if self.suite_id: sims = sims_from_suite_id(self.suite_id) elif self.exp_id: sims = sims_from_experiment_id(self.exp_id) else: raise Exception( 'Unable to monitor COMPS simulations as metadata contains no Suite or Experiment ID:\n' '(Suite ID: %s, Experiment ID:%s)' % (self.suite_id, self.exp_id)) states, msgs = {}, {} for sim in sims: id_string = str(sim.id) states[ id_string] = sim.state # this is already a SimulationState object msgs[id_string] = '' logger.debug("States returned") logger.debug( json.dumps(Counter([st.name for st in states.values()]), indent=3)) return states, msgs
def __init__(self, experiment=None, config_builder=None): # Ensure we use the SetupParser environment of the experiment if it already exists super().__init__(experiment, config_builder) temp_block = experiment.selected_block if experiment else SetupParser.selected_block temp_path = experiment.setup_overlay_file if experiment else None self.endpoint = experiment.endpoint if experiment else None with SetupParser.TemporarySetup(temporary_block=temp_block, temporary_path=temp_path) as setup: self.comps_sims_to_batch = int( setup.get(parameter='sims_per_thread')) self.endpoint = self.endpoint or setup.get( parameter='server_endpoint') COMPS_login(self.endpoint) self.asset_service = True self.runner_thread = None self.sims_to_create = [] self.commissioners = [] self.save_semaphore = get_semaphore() # If we pass an experiment, retrieve it from COMPS if self.experiment: self.comps_experiment = COMPSCache.experiment( self.experiment.exp_id)
def cleanup(self): """ Cleanup the current calibration - Delete the result directory - If LOCAL -> also delete the simulations """ try: calib_data = self.read_calib_data() except Exception: logger.info('Calib data cannot be read -> skip') calib_data = None if calib_data: with SetupParser.TemporaryBlock(calib_data['selected_block']): # Retrieve suite ids and iter_count suites = calib_data.get('suites') iter_count = calib_data.get('iteration') # Kill self.kill() # Delete the simulations too logger.info('Cleaning up calibration %s' % self.name) for i in range(0, iter_count + 1): # Get the iteration cache iteration_cache = os.path.join(self.name, 'iter%d' % i, 'IterationState.json') if not os.path.exists(iteration_cache): break # Retrieve the iteration state it = IterationState.from_file(iteration_cache) # Create the associated experiment manager and ask for deletion try: exp_mgr = ExperimentManagerFactory.from_experiment( DataStore.get_experiment(it.experiment_id)) exp_mgr.hard_delete() except: continue # Delete all HPC suites (the local suites are only carried by experiments) for suite in suites: if suite['type'] == "HPC": logger.info('Delete COMPS suite %s' % suite['id']) COMPS_login(SetupParser.get('server_endpoint')) from simtools.Utilities.COMPSUtilities import delete_suite delete_suite(suite['id']) # Then delete the whole directory calib_dir = os.path.abspath(self.name) if os.path.exists(calib_dir): try: shutil.rmtree(calib_dir) except OSError: logger.error("Failed to delete %s" % calib_dir) logger.error( "Try deleting the folder manually before retrying the calibration." )
def hard_delete(self): """ Delete data for experiment and marks the server entity for deletion. """ # Mark experiment for deletion in COMPS. COMPS_login(self.endpoint) self.comps_experiment.delete() # Delete in the DB from simtools.DataAccess.DataStore import DataStore DataStore.delete_experiment(self.experiment)
def diskspace(args, unknownArgs): # Create a default HPC setup parser with SetupParser.TemporarySetup(temporary_block='HPC') as sp: endpoint = sp.get('server_endpoint') COMPS_login(endpoint) # default to the login user user = sp.get('user') if args.users is None or len(args.users) == 0: args.users = [user] # query and display the disk information DiskSpaceUsage.display(args.users, args.top, args.save, args.refresh)
def add_experiment(self, experiment): from simtools.DataAccess.Schema import Experiment from simtools.Utilities.COMPSUtilities import COMPS_login if not isinstance(experiment, Experiment): experiment = retrieve_experiment(experiment) if experiment not in self.experiments: self.experiments.add(experiment) if experiment.location == "HPC": COMPS_login(experiment.endpoint) COMPSCache.load_experiment(experiment.exp_id) self.filter_simulations(experiment.simulations)
def __init__(self, experiment, comps_experiment): logger.debug('Create COMPSSimulationRunner with experiment: %s' % experiment.id) super(COMPSSimulationRunner, self).__init__(experiment) # Check if we need to commission COMPS_login(experiment.endpoint) self.comps_experiment = comps_experiment if experiment_needs_commission(self.comps_experiment): logger.debug('COMPS - Start Commissioning for experiment %s' % self.experiment.id) # Commission the experiment self.comps_experiment.commission() self.monitor()
def create_experiment(self, experiment_name, experiment_id=None, suite_id=None): # Also create the experiment in COMPS to get the ID COMPS_login(SetupParser.get('server_endpoint')) experiment_name = self.clean_experiment_name(experiment_name) subdirectory = experiment_name[ 0:self.MAX_SUBDIRECTORY_LENGTH] + '_' + timestamp() config = Configuration( environment_name=SetupParser.get('environment'), simulation_input_args=self.commandline.Options, working_directory_root=os.path.join(SetupParser.get('sim_root'), subdirectory), executable_path=self.commandline.Executable, node_group_name=SetupParser.get('node_group'), maximum_number_of_retries=int(SetupParser.get('num_retries')), priority=Priority[SetupParser.get('priority')], min_cores=self.config_builder.get_param('Num_Cores', 1) if self.config_builder else 1, max_cores=self.config_builder.get_param('Num_Cores', 1) if self.config_builder else 1, exclusive=self.config_builder.get_param('Exclusive', False) if self.config_builder else False) e = Experiment(name=experiment_name, configuration=config, suite_id=suite_id) # Add tags if present if self.experiment_tags: e.set_tags(self.experiment_tags) e.save() # Store in our object self.comps_experiment = e # Also add it to the cache COMPSCache.add_experiment_to_cache(e) # Create experiment in the base class super(CompsExperimentManager, self).create_experiment(experiment_name, str(e.id), suite_id) # Set some extra stuff self.experiment.endpoint = self.endpoint
def rescue_expts(expname) : from COMPS.Data import Experiment, QueryCriteria from simtools.Utilities.COMPSUtilities import COMPS_login from simtools.Utilities.Experiments import COMPS_experiment_to_local_db experiment_name = "%%" + expname + "%%" endpoint = 'https://comps.idmod.org' user = '******' def get_experiments_by_name(name,user): return Experiment.get(query_criteria=QueryCriteria().where(['name~%s' % name, 'owner=%s' % user])) COMPS_login(endpoint) for experiment_data in get_experiments_by_name(experiment_name,user): # Create a new experiment print(experiment_data) experiment = COMPS_experiment_to_local_db(exp_id=str(experiment_data.id), endpoint=endpoint, verbose=False, save_new_experiment=True)
def setUp(self): SetupParser._uninit() current_dir = os.path.dirname(os.path.realpath(__file__)) SetupParser.init(selected_block=self.SELECTED_BLOCK, setup_file=os.path.join(current_dir, 'input', 'am_simtools.ini')) COMPS_login(SetupParser.get('server_endpoint')) self.existing_collection = AssetCollection( base_collection=get_asset_collection(self.EXISTING_COLLECTION_ID)) self.existing_collection.prepare(location='HPC') self.existing_COMPS_asset_files = self.existing_collection.asset_files_to_use # a FileList object dir = os.path.join(self.INPUT_DIR, 'files') files = [os.path.join(dir, f) for f in os.listdir(dir)] root = os.path.dirname(os.path.dirname(files[0])) files = [ os.path.join( os.path.split(os.path.dirname(f))[1], os.path.basename(f)) for f in files ] self.local_files = FileList(root=root, files_in_root=files) # take one away files = [ os.path.join(f.relative_path, f.file_name) for f in self.local_files.files if f.file_name != 'file1_REMOTE_ONLY' ] # add some more dir = os.path.join(self.INPUT_DIR, 'additional_files') additional_files = [os.path.join(dir, f) for f in os.listdir(dir)] additional_files = [ os.path.join( os.path.split(os.path.dirname(f))[1], os.path.basename(f)) for f in additional_files ] files += additional_files self.local_files_plus_minus = FileList(root=root, files_in_root=files)
def get_experiment_info(experiment, cache): """ Adds the experiment information for a given experiment to the cache: - raw_size: the size in bytes - size: the formatted size (in KB, MB or GB) - sims: the number of simulations This function is used by the process pool to parallelize the retrieval of experiment info :param experiment: The experiment to analyze """ if experiment.id in cache and not DiskSpaceUsage.FORCE_REFRESH: return # Login to COMPS with SetupParser.TemporarySetup(temporary_block='HPC') as sp: endpoint = sp.get('server_endpoint') COMPS_login(endpoint) # Try to get the simulations try: simulations = experiment.get_simulations( query_criteria=QueryCriteria().select(['id']).select_children( ['hpc_jobs'])) except KeyError: # No simulations found or error -> set None cache.set(experiment.id, None) return # Calculate the size size = sum(s.hpc_jobs[0].output_directory_size for s in simulations if s.hpc_jobs) # Set the info for this particular experiment in the cache cache.set( experiment.id, ExperimentInfo(experiment.id, experiment.name, experiment.owner, size, len(simulations)))
def __init__(self, selected_block=None, setup_file=None, commissioning_directory=None, overrides={}, is_testing=False, old_style_instantiation=None): """ This should only be used to initialize the SetupParser singleton. Provides access to the selected_block of the resultant ini config overlay. Overlays are performed by merging the selected overlay ini file onto the default ini file. Overlay file selection priority: 1. setup_file, if provided 2. current/local working directory ini file, if it exists 3. ini in commissioning_directory, if provided 4. No overlay (defaults only) If the selected_block cannot be found in the resultant ini overlay, an exception will be thrown. :param selected_block: The current block we want to use :param setup_file: If provided, this ini file will overlay the default UNLESS commissioning_directory was also provided. :param commissioning_directory: If provided, the ini file within it will always overlay the default ini file. :param overrides: The values in this dict supersede those returned by the ConfigParser object in .get() :param is_testing: Allows bypassing of interactive login to COMPS if True. No login attempt is made in this case. """ if old_style_instantiation is None: self.old_style_instantiation( selected_block=selected_block, setup_file=setup_file, commissioning_directory=commissioning_directory, overrides=overrides, is_testing=is_testing) self.selected_block = (selected_block or self.default_block).upper() self.setup_file = setup_file self.commissioning_directory = commissioning_directory self.commissioning_file = os.path.join( commissioning_directory, self.ini_filename) if commissioning_directory else None local_file = os.path.join(os.getcwd(), self.ini_filename) self.local_file = local_file if os.path.exists(local_file) else None self.overrides = overrides # Identify which ini file will overlay the default ini, if any, and verify the selected file's existence if not os.path.exists(self.default_file): raise self.MissingIniFile( "Default ini file does not exist: %s . Please run 'python setup.py' again." % self.default_file) self.overlay_path = self._select_and_verify_overlay( local_file=self.local_file, provided_file=self.setup_file, commissioning_file=self.commissioning_file) # Load the default file self.setup = self.config_parser_from_file(self.default_file) # Apply the overlay if one was found if self.overlay_path: overlay = self.config_parser_from_file(self.overlay_path) else: overlay = None self.setup = self._overlay_setup_and_resolve_inheritance( overlay, self.setup) # Verify that we have the requested block in our overlain result if not self.setup.has_section(self.selected_block): raise self.MissingIniBlock( "Selected block: %s does not exist in ini file overlay.\nOverlay path: %s" % (self.selected_block, self.overlay_path)) # If the selected block is type=HPC, take care of HPC initialization if self.setup.get(self.selected_block, 'type') == "HPC" and not is_testing: from simtools.Utilities.COMPSUtilities import COMPS_login COMPS_login(self.setup.get(self.selected_block, 'server_endpoint'))
def pre_creation(self): # Call login now (even if we are already logged in, we need to call login to initialize the COMPSAccess Client) COMPS_login(self.server_endpoint)
def cancel_experiment(self): super(CompsExperimentManager, self).cancel_experiment() COMPS_login(self.endpoint) if self.comps_experiment and experiment_is_running( self.comps_experiment): self.comps_experiment.cancel()
def retrieve_data(simulation, analyzers, cache): from simtools.Analysis.AnalyzeManager import EXCEPTION_KEY # Filter first and get the filenames from filtered analysis filtered_analysis = [a for a in analyzers if a.filter(simulation)] filenames = set(itertools.chain(*(a.filenames for a in filtered_analysis))) # We dont have anything to do :) if not filenames or not filtered_analysis: cache.set(simulation.id, None) return # The byte_arrays will associate filename with content byte_arrays = {} try: if simulation.experiment.location == "HPC": COMPS_login(simulation.experiment.endpoint) COMPS_simulation = COMPSCache.simulation(simulation.id) assets = [path for path in filenames if path.lower().startswith("assets")] transient = [path for path in filenames if not path.lower().startswith("assets")] if transient: byte_arrays.update(dict(zip(transient, COMPS_simulation.retrieve_output_files(paths=transient)))) if assets: byte_arrays.update(get_asset_files_for_simulation_id(simulation.id, paths=assets, remove_prefix='Assets')) else: byte_arrays = [] for filename in filenames: path = os.path.join(simulation.get_path(), filename) with open(path, 'rb') as output_file: byte_arrays[filename] = output_file.read() except: tb = traceback.format_exc() cache.set(EXCEPTION_KEY, "An exception has been raised during data retrieval.\n" "Simulation: {} \n" "Analyzers: {}\n" "Files: {}\n" "\n{}".format(simulation, ", ".join(analyzers), ", ".join(filenames), tb)) return # Selected data will be a dict with analyzer.uid => data selected_data = {} for analyzer in filtered_analysis: # If the analyzer needs the parsed data, parse if analyzer.parse: data = {filename:SimulationOutputParser.parse(filename, content) for filename, content in byte_arrays.items()} else: # If the analyzer doesnt wish to parse, give the raw data data = byte_arrays # Retrieve the selected data for the given analyzer try: selected_data[analyzer.uid] = analyzer.select_simulation_data(data, simulation) except: tb = traceback.format_exc() cache.set(EXCEPTION_KEY, "An exception has been raised during data processing.\n" "Simulation: {} \n" "Analyzer: {}\n" "\n{}".format(simulation, analyzer, tb)) return # Store in the cache cache.set(simulation.id, selected_data)
# Create a new AnalyzeManager and add experiment and analyzer am = AnalyzeManager(verbose=False) for em in experiments: am.add_experiment(em.experiment) analyzer = DownloadAnalyzerTPI( filenames=['output\\ReportHIVByAgeAndGender.csv'], TPI_tag="TPI", ignore_TPI=False, REP_tag="TPI", ignore_REP=True, output_path=output_directory) am.add_analyzer(analyzer) # Make sure we refresh our set of experiments for e in experiments: e.refresh_experiment() COMPS_login(SetupParser.get("server_endpoint")) am.analyze() # If we are not done we wait for 30 sec, if we are done we leave if not finished: print("Waiting 30 seconds") time.sleep(30) else: break
def generate_immune_overlays(self): ''' generate immune overlays; the json file format pointed to by self.immunity_burnin_meta_file_path is { exp_id1:path to experiment meta json file created in the folder indicated by sim_root in dtk_setup.cfg exp_id2: --||--, ... } ''' # group nodes by parameter self.group_nodes_by_params() with open('nodes_by_param.json', 'w') as p_f: json.dump(self.nodes_by_params, p_f, indent=4) # maintain a list of overlays' file paths so that they can be used by the spatial simulation later? overlays_list = [] # open file containing paths to immune initialization burnin sweep meta files with open(self.immunity_burnin_meta_file_path, 'r') as imm_f: immun_meta_exps = json.load(imm_f) i = 0 for exp_id, exp_path in immun_meta_exps.items(): with open(os.path.abspath(exp_path), 'r') as meta_f: exp_meta = json.load(meta_f) # find out if experiment has been run locally or remotely on hpc; # that determines the simulation output folder location and structure exp_location_type = exp_meta['location'] exp_name = exp_meta['exp_name'] # do we have a helper function that determines the output directory of a simulation given simulation and experiment IDs (working both for COMPs and local)? sim_dir_map = None if exp_location_type == 'HPC': from simtools.SetupParser import SetupParser sp = SetupParser('HPC') om("Pulling immunization data from COMPs.") om("This requires a login.") COMPS_login(sp.get('server_endpoint')) om("Login success!") sim_dir_map = parser.createSimDirectoryMap(exp_id) # iterate through the experiments simulations; for sim_id, sim_record in exp_meta['sims'].items(): sim_output_path = '' #for each simulation get the values of parameters relevant to the immune initialization burnin (i.e. the parameters # given by the keys in self.nodes_params) # the mapping between nodes and immune overlays below could be done more efficiently, but for the typical number of parameters the # mapping below should work fine # get the parameter keys in the right order (see group_nodes_by_params(self) and get_params_key(self...); # the set of relevant parameters is the same across all nodes, so take the one from the first node node_params = self.nodes_params.values().next().keys() param_values = [] for param in node_params: if param in sim_record: param_values.append(sim_record[param]) # get nodes associated with this set of parameters if self.get_params_key( param_values) in self.nodes_by_params: params_key = self.get_params_key(param_values) nodes = self.nodes_by_params[params_key] # for each simulation get its output immunity report if exp_location_type == 'LOCAL': sim_output_path = os.path.join( exp_meta['sim_root'], exp_name + '_' + exp_meta['exp_id'], sim_id, 'output') elif exp_location_type == 'HPC': sim_output_path = os.path.join( sim_dir_map[sim_id], 'output') immunity_report_file_path = os.path.join( sim_output_path, 'MalariaImmunityReport_AnnualAverage.json') # generate immune overlay from dtk.tools.demographics.createimmunelayer import \ immune_init_from_custom_output_for_spatial as immune_init immune_overlay_json = immune_init( { "Metadata": { "Author": "dtk-tools", "IdReference": "Gridded world grump30arcsec", "NodeCount": len(nodes) }, "Nodes": nodes }, immunity_report_file_path, ) # save compiled overlay overlay_file_name = params_key + exp_name + '.json' overlay_file_path = os.path.join( self.immune_overlays_path, overlay_file_name) with open(overlay_file_path, 'w') as imo_f: json.dump(immune_overlay_json, imo_f) CompileDemographics(overlay_file_path, forceoverwrite=True) # add overlay location to overlay list overlays_list.append(overlay_file_name) print( str(len(overlays_list)) + " immune initialization overlay files processed successfully!" ) return overlays_list
def sync(args, unknownArgs): """ Sync COMPS db with local db """ # Create a default HPC setup parser with SetupParser.TemporarySetup(temporary_block='HPC') as sp: endpoint = sp.get('server_endpoint') user = sp.get('user') COMPS_login(endpoint) exp_to_save = list() exp_deleted = 0 # Retrieve all the experiment id from COMPS for the current user exp_ids = get_experiment_ids_for_user(user) # Test the experiments present in the local DB to make sure they still exist in COMPS for exp in DataStore.get_experiments(): if exp.location == "HPC": if exp.exp_id not in exp_ids: # The experiment doesnt exist on COMPS anymore -> delete from local DataStore.delete_experiment(exp) exp_deleted += 1 # Consider experiment id option exp_id = args.exp_id if args.exp_id else None exp_name = args.exp_name if args.exp_name else None user = args.user if args.user else user if exp_name: experiments = get_experiments_by_name(exp_name, user) for experiment_data in experiments: experiment = COMPS_experiment_to_local_db( exp_id=str(experiment_data.id), endpoint=endpoint, verbose=True, save_new_experiment=False) if experiment: exp_to_save.append(experiment) elif exp_id: # Create a new experiment experiment = COMPS_experiment_to_local_db(exp_id=exp_id, endpoint=endpoint, verbose=True, save_new_experiment=False) # The experiment needs to be saved if experiment: exp_to_save.append(experiment) else: # By default only get experiments created in the last month # day_limit = args.days if args.days else day_limit_default day_limit = 30 today = datetime.date.today() limit_date = today - datetime.timedelta(days=int(day_limit)) # For each of them, check if they are in the db for exp in get_experiments_per_user_and_date(user, limit_date): # Create a new experiment experiment = COMPS_experiment_to_local_db( exp_id=str(exp.id), endpoint=endpoint, save_new_experiment=False) # The experiment needs to be saved if experiment: exp_to_save.append(experiment) # Save the experiments if any if len(exp_to_save) > 0 and exp_deleted == 0: DataStore.batch_save_experiments(exp_to_save) logger.info( "The following experiments have been added to the database:") logger.info("\n".join(["- " + str(exp) for exp in exp_to_save])) logger.info("%s experiments have been updated in the DB." % len(exp_to_save)) logger.info("%s experiments have been deleted from the DB." % exp_deleted) else: print("The database was already up to date.") # Start overseer BaseExperimentManager.check_overseer()
coverage=coverage, rate=0.15, drugname="AL") return { 'intervention': intervention, '%s_coverage' % intervention: coverage } if __name__ == "__main__": SetupParser.init() # collect site-specific data to pass to builder functions COMPS_login("https://comps.idmod.org") sites = pd.read_csv("site_details.csv") print("finding collection ids and vector details") site_input_dir = os.path.join("input", "sites", "all") with open("species_details.json") as f: species_details = json.loads(f.read()) if asset_exp_id: print("retrieving asset experiment") asset_expt = retrieve_experiment(asset_exp_id) template_asset = asset_expt.simulations[0].tags cb.set_exe_collection(template_asset["exe_collection_id"]) cb.set_dll_collection(template_asset["dll_collection_id"]) cb.set_input_collection(template_asset["input_collection_id"])
def COMPS_experiment_to_local_db(exp_id, endpoint, verbose=False, save_new_experiment=True): """ Return a DB object representing an experiment coming from COMPS. This function saves the newly retrieved experiment in the DB by default but this behavior can be changed by switching the save_new_experiment parameter allowing to return an experiment object and save later with a batch for example. :param exp_id: :param endpoint: :param verbose: :param save_new_experiment: :return: """ # Make sure we are logged in COMPS_login(endpoint) #Ensure exp_id is a string exp_id = str(exp_id) # IF the experiment already exists and experiment = DataStore.get_experiment(exp_id) if experiment and experiment.is_done(): if verbose: print("Experiment ('%s') already exists in local db." % exp_id) # Do not bother with finished experiments return None from COMPS.Data import QueryCriteria try: query_criteria = QueryCriteria().select_children('tags') exp_comps = get_experiment_by_id(exp_id, query_criteria) or get_experiments_by_name(exp_id, query_criteria)[-1] except: if verbose: print("The experiment ('%s') doesn't exist in COMPS." % exp_id) return None # Case: experiment doesn't exist in local db if not experiment: # Cast the creation_date experiment = DataStore.create_experiment(exp_id=str(exp_comps.id), suite_id=str(exp_comps.suite_id) if exp_comps.suite_id else None, exp_name=exp_comps.name, tags=exp_comps.tags, date_created=utc_to_local(exp_comps.date_created).replace(tzinfo=None), location='HPC', selected_block='HPC', endpoint=endpoint) # Note: experiment may be new or comes from local db # Get associated simulations of the experiment sims = exp_comps.get_simulations(QueryCriteria().select(['id', 'state', 'date_created']).select_children('tags')) # Skip empty experiments or experiments that have the same number of sims if len(sims) == 0 or len(sims) == len(experiment.simulations): if verbose: if len(sims) == 0: print("Skip empty experiment ('%s')." % exp_id) elif len(sims) == len(experiment.simulations): print("Skip experiment ('%s') since local one has the same number of simulations." % exp_id) return None # Go through the sims and create them for sim in sims: # Cast the simulation tags # Create the simulation simulation = DataStore.create_simulation(id=str(sim.id), status=sim.state, # this is already a SimulationState object tags={tag:cast_number(val) for tag,val in sim.tags.items()}, date_created=utc_to_local(sim.date_created).replace(tzinfo=None)) # Add to the experiment experiment.simulations.append(simulation) # Save it to the DB if save_new_experiment: DataStore.save_experiment(experiment, verbose=verbose) return experiment