def results(event, production, file, hash=None): """ Fetch or list the results of a production. """ if config.get("ledger", "engine") == "gitlab": _, repository = connect_gitlab() gitlab_event = gitlab.find_events(repository, subset=event) event = gitlab_event[0].event_object elif config.get("ledger", "engine") == "yamlfile": ledger = Ledger(config.get("ledger", "location")) event = ledger.get_event(event) production = [production_o for production_o in event.productions if production_o.name == production][0] store = Store(root=config.get("storage", "results_store")) if not file: try: items = store.manifest.list_resources(event.name, production.name).items() click.secho(f"{'Resource':30} {'Hash':32} {'UUID':32}") click.secho("-"*96) for resource, details in items: click.secho(f"{resource:30} {details['hash']:32} {details['uuid']:32}") except KeyError: click.secho("There are no results for this production.") else: try: click.echo(store.fetch_file(event, production, file, hash)) except FileNotFoundError: click.secho(f"{file} could not be found for this production.")
def build(event): """ Create the run configuration files for a given event for jobs which are ready to run. If no event is specified then all of the events will be processed. """ server, repository = connect_gitlab() events = gitlab.find_events(repository, milestone=config.get("olivaw", "milestone"), subset=[event], update=False) for event in events: click.echo(f"Working on {event.title}") logger = logging.AsimovLogger(event=event.event_object) ready_productions = event.event_object.get_all_latest() for production in ready_productions: click.echo(f"\tWorking on production {production.name}") if production.status in { "running", "stuck", "wait", "finished", "uploaded", "cancelled", "stopped" }: continue try: configuration = production.get_configuration() except ValueError: try: rundir = config.get("general", "rundir_default") production.make_config(f"{production.name}.ini") click.echo(f"Production config {production.name} created.") logger.info("Run configuration created.", production=production) try: event.event_object.repository.add_file( f"{production.name}.ini", os.path.join(f"{production.category}", f"{production.name}.ini")) logger.info( "Configuration committed to event repository.", production=production) except Exception as e: logger.error( f"Configuration could not be committed to repository.\n{e}", production=production) except DescriptionException as e: logger.error("Run configuration failed", production=production, channels=["file", "mattermost"])
def checkifo(event): server, repository = connect_gitlab() gitlab_events = gitlab.find_events(repository, subset=event) for event in gitlab_events: if "event time" not in event.event_object.meta: print(f"Time not found {event.event_object.name}") time = event.event_object.meta['event time'] gpsstart = time - 600 gpsend = time + 600 bits = ['Bit 0', 'Bit 1', 'Bit 2'] active_ifo = [] for ifo in ["L1", "H1", "V1"]: frametypes = event.event_object.meta['data']['frame-types'] urls = find_urls(site=f"{ifo[0]}", frametype=frametypes[ifo], gpsstart=gpsstart, gpsend=gpsend) datacache = Cache.from_urls(urls) if len(datacache) == 0: print(f"No {ifo} data found.") continue if "state vector" in event.meta: state_vector_channel = event.meta['state vector'] else: state_vector_channel = ast.literal_eval( config.get("data", "state-vector")) state = gwpy.timeseries.StateVector.read( datacache, state_vector_channel[ifo], start=gpsstart, end=gpsend, pad= 0 # padding data so that errors are not raised even if found data are not continuous. ) if not np.issubdtype(state.dtype, np.unsignedinteger): # if data are not unsigned integers, cast to them now so that # we can determine the bit content for the flags state = state.astype( "uint32", casting="unsafe", subok=True, copy=False, ) flags = state.to_dqflags() segments = flags[bits[0]].active for bit in bits: segments -= ~flags[bit].active if len(segments) > 0: active_ifo += [ifo] print(event.event_object.name) if event.event_object.meta['interferometers'] != active_ifo: print(f"Gitlab data\t{event.event_object.meta['interferometers']}") print(f"Recommended IFOS\t{active_ifo}") event.event_object.meta['interferometers'] = active_ifo event.update_data()
def resultslinks(event, update, root): """ Find all available results for a given event. """ server, repository = connect_gitlab() events = gitlab.find_events(repository, milestone=config.get("olivaw", "milestone"), subset=[event], update=update, repo=False) for event in events: click.secho(f"{event.title}") logger = logging.AsimovLogger(event=event.event_object) for production in event.productions: try: for result, meta in production.results().items(): print( f"{production.event.name}/{production.name}/{result}, {production.results(result)}" ) pathlib.Path( os.path.join(root, production.event.name, production.name)).mkdir(parents=True, exist_ok=True) os.symlink( f"{production.results(result)}", f"{root}/{production.event.name}/{production.name}/{result.split('/')[-1]}" ) except AttributeError: pass
def status(event): """ Provide a simple summary of the status of a given event. Arguments --------- name : str, optional The name of the event. """ server, repository = connect_gitlab() events = gitlab.find_events(repository, milestone=config.get("olivaw", "milestone"), subset=[event], update=False, repo=False) for event in events: click.secho(f"{event.title:30}", bold=True) if len(event.event_object.meta['productions']) > 0: click.secho("\tProductions", bold=True) for production in event.event_object.meta['productions']: click.echo(f"\t\t{list(production.keys())[0]}") if len(event.event_object.get_all_latest()) > 0: click.secho("\tJobs waiting", bold=True) waiting = event.event_object.get_all_latest() for awaiting in waiting: click.echo(f"\t\t{awaiting.name}\t{awaiting.status}")
def audit(event): """ Conduct an audit of the contents of production ini files against the production ledger. Parameters ---------- event : str, optional The event to be checked. Optional; if the event isn't provided all events will be audited. """ if isinstance(event, str): event = [event] _, repository = connect_gitlab() gitlab_events = gitlab.find_events(repository, subset=event, update=False, repo=True) for production in gitlab_events[0].productions: category = config.get("general", "calibration_directory") config_file = os.path.join(production.event.repository.directory, category, f"{production.name}.ini") pipe = known_pipelines[production.pipeline.lower()](production, category) click.echo(pipe.read_ini(config_file))
def make_config(self, filename, template_directory=None): """ Make the configuration file for this production. Parameters ---------- filename : str The location at which the config file should be saved. template_directory : str, optional The path to the directory containing the pipeline config templates. Defaults to the directory specified in the asimov configuration file. """ if "template" in self.meta: template = f"{self.meta['template']}.ini" else: template = f"{self.pipeline}.ini" try: template_directory = config.get("templating", "directory") template_file = os.path.join(f"{template_directory}", template) except: from pkg_resources import resource_filename template_file = resource_filename("asimov", f'configs/{template}') config_dict = {s: dict(config.items(s)) for s in config.sections()} with open(template_file, "r") as template_file: liq = Liquid(template_file.read()) rendered = liq.render(production=self, config=config) with open(filename, "w") as output_file: output_file.write(rendered)
def create(event, pipeline, family, comment, needs, template, status, approximant): """ Add a new production to an event. """ if config.get("ledger", "engine") == "gitlab": _, repository = connect_gitlab() gitlab_event = gitlab.find_events(repository, subset=event) event = gitlab_event[0].event_object elif config.get("ledger", "engine") == "yamlfile": ledger = Ledger(config.get("ledger", "location")) event = ledger.get_event(event) # event_prods = event.productions names = [production.name for production in event_prods] family_entries = [int(name.split(family)[1]) for name in names if family in name] # if "bayeswave" in needs: bw_entries = [production.name for production in event_prods if ("bayeswave" in production.pipeline.lower()) and (production.review.status not in {"REJECTED", "DEPRECATED"})] needs = bw_entries # production = {"comment": comment, "pipeline": pipeline, "status": status} if needs: production['needs'] = needs if template: production['template'] = template if approximant: production['approximant'] = approximant if len(family_entries)>0: number = max(family_entries)+1 else: number = 0 production_dict = {f"{family}{number}": production} production = Production.from_dict(production_dict, event=event) # click.echo(production) event.add_production(production) if config.get("ledger", "engine") == "gitlab": gitlab_event[0].update_data() elif config.get("ledger", "engine") == "yamlfile": ledger.events[event.name] = event.to_dict() ledger.save()
def store_results(self): """ Store the PE Summary results """ files = [ f"{self.production.name}_pesummary.dat", "posterior_samples.h5", f"{self.production.name}_skymap.fits" ] for filename in files: results = os.path.join(config.get('general', 'webroot'), self.production.event.name, self.production.name, "results", "samples", filename) store = Store(root=config.get("storage", "directory")) store.add_file(self.production.event.name, self.production.name, file=results)
def __init__(self, production, category=None): super(BayesWave, self).__init__(production, category) self.logger = logger = logging.AsimovLogger(event=production.event) if not production.pipeline.lower() == "bayeswave": raise PipelineException try: self.category = config.get("general", "category") except: self.category = "C01_offline" self.logger.info("Assuming C01_offline calibration.")
def connect_gitlab(configs=None): """ Connect to the gitlab server. Returns ------- server : `Gitlab` The gitlab server. repository: `Gitlab.project` The gitlab project. """ if configs: config = configs else: from asimov import config server = gitlab.gitlab.Gitlab('https://git.ligo.org', private_token=config.get("gitlab", "token")) repository = server.projects.get( config.get("olivaw", "tracking_repository")) return server, repository
def results(self, filename=None, handle=False, hash=None): store = Store(root=config.get("storage", "results_store")) if not filename: try: items = store.manifest.list_resources(self.event.name, self.name) return items except KeyError: return None elif handle: return open(store.fetch_file(self.event.name, self.name, filename, hash), "r") else: return store.fetch_file(self.event.name, self.name, filename, hash=hash)
def from_url(cls, url, name, directory=None, update=False): """ Clone a git repository into a working directory, then create an EventRepo object for it. Parameters ---------- url : str The URL of the git repository name : str The name for the git repository (probably the event name) directory : str, optional The location to store the cloned repository. If this value isn't provided the repository is cloned into the /tmp directory. update : bool Flag to determine if the repository is updated when loaded. Defaults to False. """ if not directory: tmp = config.get("general", "git_default") directory = f"{tmp}/{name}" pathlib.Path(directory).mkdir(parents=True, exist_ok=True) # Replace an https address with an ssh address if "https" in url: url = url.replace("https://", "git@") final = "/".join(url.split("/")[1:]) start = url.split("/")[0] url = f"{start}:{final}" try: repo = git.Repo.clone_from(url, directory) repo.git.execute(["git", "lfs", "install"]) repo.git.execute(["git", "lfs", "fetch"]) repo.git.execute(["git", "lfs", "pull"]) except git.exc.GitCommandError: repo = git.Repo(directory) try: repo.git.stash() except git.exc.GitCommandError: pass if update: try: repo.remotes[0].pull() except git.exc.GitCommandError: pass return cls(directory, url, update=update)
def submit(event, update): """ Submit the run configuration files for a given event for jobs which are ready to run. If no event is specified then all of the events will be processed. """ server, repository = connect_gitlab() events = gitlab.find_events(repository, milestone=config.get("olivaw", "milestone"), subset=[event], update=update) for event in events: logger = logging.AsimovLogger(event=event.event_object) ready_productions = event.event_object.get_all_latest() for production in ready_productions: if production.status.lower() in { "running", "stuck", "wait", "processing", "uploaded", "finished", "manual", "cancelled", "stopped" }: continue if production.status.lower() == "restart": if production.pipeline.lower() in known_pipelines: pipe = known_pipelines[production.pipeline.lower()]( production, "C01_offline") pipe.clean() pipe.submit_dag() else: #try: # configuration = production.get_configuration() #except ValueError as e: # #build(event) # logger.error(f"Error while trying to submit a configuration. {e}", production=production, channels="gitlab") if production.pipeline.lower() in known_pipelines: pipe = known_pipelines[production.pipeline.lower()]( production, "C01_offline") try: pipe.build_dag() except PipelineException: logger.error( "The pipeline failed to build a DAG file.", production=production) try: pipe.submit_dag() production.status = "running" except PipelineException as e: production.status = "stuck" logger.error( f"The pipeline failed to submit the DAG file to the cluster. {e}", production=production)
def upload_prod(self, production, rundir, preferred=False, category="C01_offline", rootdir="public_html/LVC/projects/O3/C01/", rename=False): """ Upload the results of a PE job to the event repostory. Parameters ---------- category : str, optional The category of the job. Defaults to "C01_offline". production : str The production name. rundir : str The run directory of the PE job. """ preferred_list = ["--preferred", "--append_preferred"] web_path = os.path.join(os.path.expanduser("~"), *rootdir.split("/"), self.event, production) # TODO Make this generic if rename: prod_name = rename else: prod_name = production command = [ config.get("pesummary", "location"), "--event", self.event, "--exp", prod_name, "--rundir", rundir, "--webdir", web_path, "--edit_homepage_table" ] if preferred: command += preferred_list dagman = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, err = dagman.communicate() if err or not "master -> master" in str(out): raise ValueError(f"Sample upload failed.\n{out}\n{err}") else: return out
def results(event, update): """ Find all available results for a given event. """ server, repository = connect_gitlab() events = gitlab.find_events(repository, milestone=config.get("olivaw", "milestone"), subset=[event], update=update, repo=False) for event in events: click.secho(f"{event.title}") logger = logging.AsimovLogger(event=event.event_object) for production in event.productions: try: for result, meta in production.results().items(): print( f"{production.event.name}/{production.name}/{result}, {production.results(result)}" ) except: pass
def collect_assets(self): """ Collect the assets for this job and commit them to the event repository. Since this job also generates the PSDs these should be added to the production ledger. """ results_dir = glob.glob(f"{self.production.rundir}/trigtime_*")[0] sample_rate = self.production.meta['quality']['sample-rate'] store = Store(root=config.get("storage", "directory")) for det in self.production.meta['interferometers']: asset = os.path.join(results_dir, "post", "clean", f"glitch_median_PSD_forLI_{det}.dat") destination = os.path.join(self.category, "psds", str(sample_rate), f"{det}-psd.dat") try: self.production.event.repository.add_file(asset, destination) except Exception as e: error = PipelineLogger(f"There was a problem committing the PSD for {det} to the repository.\n\n{e}", issue=self.production.event.issue_object, production=self.production.name) error.submit_comment() # Add to the event ledger if "psds" not in self.production.event.meta: self.production.event.meta['psds'] = {} if sample_rate not in self.production.event.meta['psds']: self.production.event.meta['psds'][sample_rate] = {} self.production.event.meta['psds'][sample_rate][det] = destination # Let's have a better way of doing this self.production.event.issue_object.update_data() copyfile(asset, f"{det}-{sample_rate}-psd.dat") try: store.add_file(self.production.event.name, self.production.name, file = f"{det}-{sample_rate}-psd.dat") except Exception as e: error = PipelineLogger(f"There was a problem committing the PSD for {det} to the store.\n\n{e}", issue=self.production.event.issue_object, production=self.production.name) error.submit_comment()
def before_submit(self): """ Convert the text-based PSD to an XML psd if the xml doesn't exist already. """ event = self.production.event category = config.get("general", "calibration_directory") current = os.getcwd() if len(self.production.get_psds("xml")) == 0: for ifo in self.production.meta['interferometers']: os.chdir(f"{event.repository.directory}/") sample = self.production.meta['quality']['sample-rate'] self._convert_psd(self.production.meta['psds'][sample][ifo], ifo) asset = f"{ifo.upper()}-psd.xml.gz" git_location = os.path.join(category, "psds") self.production.event.repository.add_file( asset, os.path.join(git_location, str(sample), f"psd_{ifo}.xml.gz"), commit_message=f"Added the xml format PSD for {ifo}.") os.chdir(current)
def supress_psd(self, ifo, fmin, fmax): """ Suppress portions of a PSD. Author: Carl-Johan Haster - August 2020 (Updated for asimov by Daniel Williams - November 2020 """ store = Store(root=config.get("storage", "directory")) sample_rate = self.production.meta['quality']['sample-rate'] orig_PSD_file = np.genfromtxt(os.path.join(self.production.event.repository.directory, self.category, "psds", str(sample_rate), f"{ifo}-psd.dat")) freq = orig_PSD_file[:,0] PSD = orig_PSD_file[:,1] suppression_region = np.logical_and(np.greater_equal(freq, fmin), np.less_equal(freq, fmax)) #Suppress the PSD in this region PSD[suppression_region] = 1. new_PSD = np.vstack([freq, PSD]).T asset = f"{ifo}-psd.dat" np.savetxt(asset, new_PSD, fmt='%+.5e') destination = os.path.join(self.category, "psds", str(sample_rate), f"{ifo}-psd.dat") try: self.production.event.repository.add_file(asset, destination)#, message=f"Added the supresed {ifo} PSD") except Exception as e: raise PipelineException(f"There was a problem committing the suppresed PSD for {ifo} to the repository.\n\n{e}", issue=self.production.event.issue_object, production=self.production.name) copyfile(asset, f"{ifo}-{sample_rate}-psd-suppresed.dat") try: store.add_file(self.production.event.name, self.production.name, file = f"{ifo}-{sample_rate}-psd-suppresed.dat") except AlreadyPresentException: pass self.logger.info(f"PSD supression applied to {ifo} between {fmin} and {fmax}")
def get_gracedb(self, gfile, destination): """ Get a file from Gracedb, and store it in the event repository. Parameters ---------- gfile : str The name of the gracedb file, e.g. `coinc.xml`. destination : str The location in the repository for this file. """ gid = self.meta['gid'] client = GraceDb(service_url=config.get("gracedb", "url")) file_obj = client.files(gid, gfile) with open("download.file", "w") as dest_file: dest_file.write(file_obj.read().decode()) self.repository.add_file("download.file", destination, commit_message = f"Downloaded {gfile} from GraceDB")
def ledger(event, yaml_f): """ Return the ledger for a given event. If no event is specified then the entire production ledger is returned. """ server, repository = connect_gitlab() events = gitlab.find_events(repository, milestone=config.get("olivaw", "milestone"), subset=[event], update=False, repo=False) total = [] for event in events: total.append(yaml.safe_load(event.event_object.to_yaml())) click.echo(yaml.dump(total)) if yaml_f: with open(yaml_f, "w") as f: f.write(yaml.dump(total))
def run_pesummary(self): """ Run PESummary on the results of this job. """ psds = self.production.psds calibration = [ os.path.join(self.production.event.repository.directory, cal) for cal in self.production.meta['calibration'].values() ] configfile = self.production.event.repository.find_prods( self.production.name, self.category)[0] command = [ "--webdir", os.path.join(config.get('general', 'webroot'), self.production.event.name, self.production.name, "results"), "--labels", self.production.name, "--gw", "--cosmology", config.get('pesummary', 'cosmology'), "--redshift_method", config.get('pesummary', 'redshift'), "--nsamples_for_skymap", config.get('pesummary', 'skymap_samples'), "--evolve_spins", "True", "--multi_process", "4", "--regenerate", "mass_1_source mass_2_source chirp_mass_source total_mass_source final_mass_source final_mass_source_non_evolved radiated_energy", "--config", os.path.join(self.production.event.repository.directory, self.category, configfile) ] # Samples command += ["--samples"] command += self.samples() # Calibration information command += ["--calibration"] command += calibration # PSDs command += ["--psd"] command += psds.values() self.logger.info( f"Submitted PE summary run. Command: {config.get('pesummary', 'executable')} {' '.join(command)}", production=self.production, channels=['file']) hostname_job = htcondor.Submit({ "executable": config.get("pesummary", "executable"), "arguments": " ".join(command), "accounting_group": config.get("pipelines", "accounting"), "output": f"{self.production.rundir}/pesummary.out", "error": f"{self.production.rundir}/pesummary.err", "log": f"{self.production.rundir}/pesummary.log", "request_cpus": "4", "getenv": "true", "batch-name": f"PESummary/{self.production.event.name}/{self.production.name}", "request_memory": "8192MB", "request_disk": "8192MB", }) schedulers = htcondor.Collector().locate( htcondor.DaemonTypes.Schedd, config.get("condor", "scheduler")) schedd = htcondor.Schedd(schedulers) with schedd.transaction() as txn: cluster_id = hostname_job.queue(txn) return cluster_id
def build_dag(self, user=None): """ Construct a DAG file in order to submit a production to the condor scheduler using util_RIFT_pseudo_pipe.py Parameters ---------- production : str The production name. user : str The user accounting tag which should be used to run the job. Raises ------ PipelineException Raised if the construction of the DAG fails. Notes ----- In order to assemble the pipeline the RIFT runner requires additional production metadata: at least the l_max value. An example RIFT production specification would then look something like: :: - Prod3: rundir: tests/tmp/s000000xx/Prod3 pipeline: rift approximant: IMRPhenomPv3 lmax: 2 cip jobs: 5 # This is optional, and will default to 3 bootstrap: Prod1 bootstrap fmin: 20 needs: Prod1 comment: RIFT production run. status: wait """ cwd = os.getcwd() #os.chdir(self.production.event.meta['working directory']) #os.chdir(os.path.join(self.production.event.repository.directory, # self.category)) if self.production.event.repository: gps_file = self.production.get_timefile() coinc_file = self.production.get_coincfile() coinc_file = os.path.join( self.production.event.repository.directory, "C01_offline", coinc_file) ini = self.production.get_configuration().ini_loc ini = os.path.join(self.production.event.repository.directory, "C01_offline", ini) else: gps_file = "gpstime.txt" ini = os.path.join(self.production.event.meta['working directory'], f"{self.production.name}.ini") coinc_file = os.path.join(cwd, "coinc.xml") if self.production.get_meta("user"): user = self.production.get_meta("user") else: user = config.get("condor", "user") self.production.set_meta("user", user) os.environ['LIGO_USER_NAME'] = f"{user}" os.environ[ 'LIGO_ACCOUNTING'] = f"{config.get('pipelines', 'accounting')}" try: calibration = config.get("general", "calibration") except: calibration = "C01" approximant = self.production.meta['approximant'] #ini.save() if self.production.rundir: rundir = os.path.relpath(self.production.rundir, os.getcwd()) else: rundir = os.path.join(os.path.expanduser("~"), self.production.event.name, self.production.name) self.production.rundir = rundir #lmax = self.production.meta['priors']['amp order'] if "lmax" in self.production.meta: lmax = self.production.meta['lmax'] elif "HM" in self.production.meta['approximant']: lmax = 4 else: lmax = 2 if "cip jobs" in self.production.meta: cip = self.production.meta['cip jobs'] else: cip = 3 command = [ os.path.join(config.get("pipelines", "environment"), "bin", "util_RIFT_pseudo_pipe.py"), "--use-coinc", coinc_file, "--l-max", f"{lmax}", "--calibration", f"{calibration}", "--add-extrinsic", "--approx", f"{approximant}", "--cip-explode-jobs", str(cip), "--use-rundir", rundir, "--ile-force-gpu", "--use-ini", ini ] print(" ".join(command)) # If a starting frequency is specified, add it if "start-frequency" in self.production.meta: command += [ "--fmin-template", self.production.quality['start-frequency'] ] self.logger.info(" ".join(command), production=self.production) # Placeholder LI grid bootstrapping; conditional on it existing and location specification if self.bootstrap: if self.bootstrap == "manual": if self.production.event.repository: bootstrap_file = os.path.join( self.production.event.repository.directory, "C01_offline", f"{self.production.name}_bootstrap.xml.gz") else: bootstrap_file = "{self.production.name}_bootstrap.xml.gz" else: raise PipelineException( f"Unable to find the bootstrapping production for {self.production.name}.", issue=self.production.event.issue_object, production=self.production.name) command += ["--manual-initial-grid", bootstrap_file] self.logger.info(command, production=self.production) os.chdir(self.production.event.meta['working directory']) pipe = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, err = pipe.communicate() if err: self.production.status = "stuck" if hasattr(self.production.event, "issue_object"): self.logger.info(out, production=self.production) self.logger.error(err, production=self.production) raise PipelineException( f"DAG file could not be created.\n{command}\n{out}\n\n{err}", issue=self.production.event.issue_object, production=self.production.name) else: self.logger.info(out, production=self.production) self.logger.error(err, production=self.production) raise PipelineException( f"DAG file could not be created.\n{command}\n{out}\n\n{err}", production=self.production.name) else: if self.production.event.repository: os.chdir(self.production.rundir) for psdfile in self.production.get_psds("xml"): ifo = psdfile.split("/")[-1].split("_")[1].split(".")[0] os.system(f"cp {psdfile} {ifo}-psd.xml.gz") #os.system("cat *_local.cache > local.cache") if hasattr(self.production.event, "issue_object"): return PipelineLogger( message=out, issue=self.production.event.issue_object, production=self.production.name) else: return PipelineLogger(message=out, production=self.production.name)
def clone(location): import venv import pathlib import shutil working = "working" results = "results" remote_config = os.path.join(location, "asimov.conf") config = configparser.ConfigParser() config.read([remote_config]) click.echo(f'Cloning {config.get("project", "name")}') root = os.path.join( os.getcwd(), config.get("project", "name").lower().replace(" ", "-")) pathlib.Path(root).mkdir(parents=True, exist_ok=True) os.chdir(root) config.set("project", "root", root) # Make the virtual environment #builder = venv.EnvBuilder(system_site_packages=False, # clear=False, # symlinks=False, # upgrade=False, # with_pip=True, # prompt=f"Asimov {project_name}") #builder.create("environment") #config.set("general", "environment", "environment") # Make the working directory #shutil.copytree(os.path.join(config.get("general", "rundir_default"), working) #config.set("general", "rundir_default", working) # Make the git directory #pathlib.Path(checkouts).mkdir(parents=True, exist_ok=True) #config.set("general", "git_default", checkouts) # Copy the results store #shutil.copyfile(os.path.join(location, config.get("storage", "results_store")), results) shutil.copytree( os.path.join(location, config.get("storage", "results_store")), results) config.set("storage", "results_store", results) # Make the ledger if config.get("ledger", "engine") == "yamlfile": shutil.copyfile( os.path.join(location, config.get("ledger", "location")), "ledger.yml") elif config.get("ledger", "engine") == "gitlab": _, repository = connect_gitlab(config) events = gitlab.find_events(repository, update=False, subset=[None], label=config.get("gitlab", "event_label"), repo=False) total = [] for event in events: total.append(yaml.safe_load(event.event_object.to_yaml())) with open("ledger.yml", "w") as f: f.write(yaml.dump(total)) config.set("ledger", "engine", "yamlfile") config.set("ledger", "location", "ledger.yml") with open("asimov.conf", "w") as config_file: config.write(config_file)
def build_dag(self, psds=None, user=None, clobber_psd=False): """ Construct a DAG file in order to submit a production to the condor scheduler using LALInferencePipe. Parameters ---------- production : str The production name. psds : dict, optional The PSDs which should be used for this DAG. If no PSDs are provided the PSD files specified in the ini file will be used instead. user : str The user accounting tag which should be used to run the job. Raises ------ PipelineException Raised if the construction of the DAG fails. """ # Change to the location of the ini file. os.chdir( os.path.join(self.production.event.repository.directory, self.category)) gps_file = self.production.get_timefile() ini = self.production.get_configuration() if not user: if self.production.get_meta("user"): user = self.productevenion.get_meta("user") else: user = ini._get_user() self.production.set_meta("user", user) ini.update_accounting(user) if 'queue' in self.production.meta: queue = self.production.meta['queue'] else: queue = 'Priority_PE' ini.set_queue(queue) ini.save() if self.production.rundir: rundir = self.production.rundir else: rundir = os.path.join(os.path.expanduser("~"), self.production.event.name, self.production.name) self.production.rundir = rundir #os.mkdir(self.production.rundir, exist_ok=True) command = [ os.path.join(config.get("pipelines", "environment"), "bin", "lalinference_pipe"), "-g", f"{gps_file}", "-r", self.production.rundir, ini.ini_loc ] self.logger.info(" ".join(command)) pipe = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, err = pipe.communicate() if err or "Successfully created DAG file." not in str(out): self.production.status = "stuck" if hasattr(self.production.event, "issue_object"): self.logger.error( f"DAG file could not be created.\n{command}\n{out}\n\n{err}" ) raise PipelineException( f"DAG file could not be created.\n{command}\n{out}\n\n{err}", issue=self.production.event.issue_object, production=self.production.name) else: self.logger.error( f"DAG file could not be created.\n{command}\n{out}\n\n{err}" ) raise PipelineException( f"DAG file could not be created.\n{command}\n{out}\n\n{err}", production=self.production.name) else: self.logger.info("DAG created") if hasattr(self.production.event, "issue_object"): return PipelineLogger(message=out, issue=self.production.event.issue_object, production=self.production.name) else: return PipelineLogger(message=out, production=self.production.name)
def create(name, oldname=None, gid=None, superevent=None, repo=None): """ Create a new event record in the ledger.. Parameters ---------- superevent : str The ID of the superevent to be used from GraceDB name : str The name of the event to be recorded in the issue tracker names : path, optional The path to the name file which maps between old and new super event IDs oldname: str, optional The old name of the event. """ import pathlib if gid or superevent: from ligo.gracedb.rest import GraceDb, HTTPError client = GraceDb(service_url=config.get("gracedb", "url")) r = client.ping() if superevent: data = client.superevent(superevent).json() event_data = client.event(data['preferred_event']).json() gid = data['preferred_event'] interferometers = event_data['instruments'].split(",") elif gid: event_data = client.event(gid).json() interferometers = event_data['instruments'].split(",") else: event_data = None interferometers = [] if gid or superevent: event_url = f"{config.get('gracedb', 'url')}/events/{gid}/view/" if not repo: repo = None #repo = f"[email protected]:pe/O3/{name}" event = Event( name=name, repository=repo, calibration={}, interferometers=interferometers, ) if oldname: event.meta['old superevent'] = oldname if gid: event.meta['event time'] = event_data['gpstime'] event.meta['gid'] = gid working_dir = os.path.join(config.get('general', 'rundir_default'), name) event.meta['working directory'] = working_dir pathlib.Path(working_dir).mkdir(parents=True, exist_ok=True) if config.get("ledger", "engine") == "gitlab": _, repository = connect_gitlab() from pkg_resources import resource_filename issue_template = resource_filename('asimov', 'gitlabissue.md') gitlab.EventIssue.create_issue(repository, event, issue_template=issue_template) elif config.get("ledger", "engine") == "yamlfile": ledger = Ledger(config.get("ledger", "location")) ledger.add_event(event) ledger.save()
def __init__(self, event, name, status, pipeline, comment=None, **kwargs): self.event = event self.name = name if status: self.status_str = status.lower() else: self.status_str = "none" self.pipeline = pipeline.lower() self.comment = comment self.meta = deepcopy(self.event.meta) if "productions" in self.meta: self.meta.pop("productions") self.meta = update(self.meta, kwargs) if "review" in self.meta: self.review = Review.from_dict(self.meta['review'], production=self) self.meta.pop("review") else: self.review = Review() # Check that the upper frequency is included, otherwise calculate it if "quality" in self.meta: if ("high-frequency" not in self.meta['quality']) and ("sample-rate" in self.meta['quality']): # Account for the PSD roll-off with the 0.875 factor self.meta['quality']['high-frequency'] = int(0.875 * self.meta['quality']['sample-rate']/2) # Get the data quality recommendations if 'quality' in self.event.meta: self.quality = self.event.meta['quality'] else: self.quality = {} if ('quality' in self.meta): if ('quality' in kwargs): self.meta['quality'].update(kwargs['quality']) self.quality = self.meta['quality'] if ('quality' in self.meta) and ("event time" in self.meta): if "segment start" not in self.meta['quality']: self.meta['quality']['segment start'] = self.meta['event time'] - self.meta['quality']['segment-length'] + 2 self.event.meta['quality']['segment start'] = self.meta['quality']['segment start'] # Gather the appropriate prior data for this production if 'priors' in self.meta: self.priors = self.meta['priors'] # Need to fetch the correct PSDs for this sample rate if 'psds' in self.meta: if self.quality['sample-rate'] in self.meta['psds']: self.psds = self.meta['psds'][self.quality['sample-rate']] else: self.psds = {} else: self.psds = {} for ifo, psd in self.psds.items(): if self.event.repository: self.psds[ifo] = os.path.join(self.event.repository.directory, psd) else: self.psds[ifo] = psd self.category = config.get("general", "calibration_directory") if "needs" in self.meta: self.dependencies = self._process_dependencies(self.meta['needs']) else: self.dependencies = None
""" import os import asimov from asimov.event import Event, DescriptionException from asimov import config from asimov import gitlab from asimov import config import numpy as np import yaml import click server = gitlab.gitlab.Gitlab(config.get("gitlab", "url"), private_token=config.get("gitlab", "token")) repository = server.projects.get(config.get("olivaw", "tracking_repository")) def find_calibrations(time): with open("LLO_calibs.txt") as llo_file: data_llo = llo_file.read().split("\n") data_llo = [ datum for datum in data_llo if datum[-16:] == "FinalResults.txt" ] times_llo = { int(datum.split("GPSTime_")[1].split("_C01")[0]): datum for datum in data_llo }
import gwpy.segments from gwpy.segments import DataQualityFlag from gwdatafind import find_urls from glue.lal import Cache import collections.abc state_vector_channel = {"L1": "L1:DCS-CALIB_STATE_VECTOR_C01", "H1": "H1:DCS-CALIB_STATE_VECTOR_C01", "V1": "V1:DQ_ANALYSIS_STATE_VECTOR"} frametypes= {"L1": "L1_HOFT_CLEAN_SUB60HZ_C01", "H1": "H1_HOFT_CLEAN_SUB60HZ_C01", "V1": "V1Online"} server = gitlab.gitlab.Gitlab(config.get("gitlab", "url"), private_token=config.get("gitlab", "token")) repository = server.projects.get(config.get("olivaw", "tracking_repository")) # # Get GraceDB stuff # from ligo.gracedb.rest import GraceDb, HTTPError client = GraceDb(service_url=config.get("gracedb", "url")) r = client.ping() #superevent_iterator = client.superevents('O3B_CBC_CATALOG') #superevent_ids = [superevent['superevent_id'] for superevent in superevent_iterator] CALIBRATION_NOTE = """ ## Calibration envelopes
def build_dag(self, user=None): """ Construct a DAG file in order to submit a production to the condor scheduler using bayeswave_pipe Parameters ---------- production : str The production name. user : str The user accounting tag which should be used to run the job. Raises ------ PipelineException Raised if the construction of the DAG fails. """ if self.production.event.repository: os.chdir(os.path.join(self.production.event.repository.directory, self.category)) try: gps_file = self.production.get_timefile() except AsimovFileNotFound: if "event time" in self.production.meta: gps_time = self.production.get_meta("event time") with open("gpstime.txt", "w") as f: f.write(str(gps_time)) gps_file = os.path.join(f"{self.production.category}", f"gpstime.txt") self.production.event.repository.add_file(f"gpstime.txt", gps_file) else: raise PipelineException("Cannot find the event time.") else: gps_time = self.production.get_meta("event time") with open("gpstime.txt", "w") as f: f.write(str(gps_time)) gps_file = os.path.join("gpstime.txt") if self.production.event.repository: ini = self.production.get_configuration() if not user: if self.production.get_meta("user"): user = self.production.get_meta("user") else: user = ini._get_user() self.production.set_meta("user", user) ini.update_accounting(user) if 'queue' in self.production.meta: queue = self.production.meta['queue'] else: queue = 'Priority_PE' ini.set_queue(queue) ini.save() ini = ini.ini_loc else: ini = f"{self.production.name}.ini" if self.production.rundir: rundir = self.production.rundir else: rundir = os.path.join(os.path.expanduser("~"), self.production.event.name, self.production.name) self.production.rundir = rundir try: pathlib.Path(directory).mkdir(parents=True, exist_ok=False) except: pass gps_time = self.production.get_meta("event time") pipe_cmd = os.path.join(config.get("pipelines", "environment"), "bin", "bayeswave_pipe") command = [pipe_cmd, #"-l", f"{gps_file}", f"--trigger-time={gps_time}", "-r", self.production.rundir, ini ] self.logger.info(" ".join(command)) pipe = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, err = pipe.communicate() if "To submit:" not in str(out): self.production.status = "stuck" if "issue_object" in self.production.event: raise PipelineException(f"DAG file could not be created.\n{command}\n{out}\n\n{err}", issue=self.production.event.issue_object, production=self.production.name) else: raise PipelineException(f"DAG file could not be created.\n{command}\n{out}\n\n{err}", production=self.production.name) else: if hasattr(self.production.event, "issue_object"): return PipelineLogger(message=out, issue=self.production.event.issue_object, production=self.production.name) else: return PipelineLogger(message=out, production=self.production.name)