def results(event, production, file, hash=None):
"""
Fetch or list the results of a production.
"""
if config.get("ledger", "engine") == "gitlab":
_, repository = connect_gitlab()
gitlab_event = gitlab.find_events(repository, subset=event)
event = gitlab_event[0].event_object
elif config.get("ledger", "engine") == "yamlfile":
ledger = Ledger(config.get("ledger", "location"))
event = ledger.get_event(event)
production = [production_o for production_o in event.productions if production_o.name == production][0]
store = Store(root=config.get("storage", "results_store"))
if not file:
try:
items = store.manifest.list_resources(event.name, production.name).items()
click.secho(f"{'Resource':30} {'Hash':32} {'UUID':32}")
click.secho("-"*96)
for resource, details in items:
click.secho(f"{resource:30} {details['hash']:32} {details['uuid']:32}")
except KeyError:
click.secho("There are no results for this production.")
else:
try:
click.echo(store.fetch_file(event, production, file, hash))
except FileNotFoundError:
click.secho(f"{file} could not be found for this production.")
def build(event):
"""
Create the run configuration files for a given event for jobs which are ready to run.
If no event is specified then all of the events will be processed.
"""
server, repository = connect_gitlab()
events = gitlab.find_events(repository,
milestone=config.get("olivaw", "milestone"),
subset=[event],
update=False)
for event in events:
click.echo(f"Working on {event.title}")
logger = logging.AsimovLogger(event=event.event_object)
ready_productions = event.event_object.get_all_latest()
for production in ready_productions:
click.echo(f"\tWorking on production {production.name}")
if production.status in {
"running", "stuck", "wait", "finished", "uploaded",
"cancelled", "stopped"
}:
continue
try:
configuration = production.get_configuration()
except ValueError:
try:
rundir = config.get("general", "rundir_default")
production.make_config(f"{production.name}.ini")
click.echo(f"Production config {production.name} created.")
logger.info("Run configuration created.",
production=production)
try:
event.event_object.repository.add_file(
f"{production.name}.ini",
os.path.join(f"{production.category}",
f"{production.name}.ini"))
logger.info(
"Configuration committed to event repository.",
production=production)
except Exception as e:
logger.error(
f"Configuration could not be committed to repository.\n{e}",
production=production)
except DescriptionException as e:
logger.error("Run configuration failed",
production=production,
channels=["file", "mattermost"])
def checkifo(event):
server, repository = connect_gitlab()
gitlab_events = gitlab.find_events(repository, subset=event)
for event in gitlab_events:
if "event time" not in event.event_object.meta:
print(f"Time not found {event.event_object.name}")
time = event.event_object.meta['event time']
gpsstart = time - 600
gpsend = time + 600
bits = ['Bit 0', 'Bit 1', 'Bit 2']
active_ifo = []
for ifo in ["L1", "H1", "V1"]:
frametypes = event.event_object.meta['data']['frame-types']
urls = find_urls(site=f"{ifo[0]}",
frametype=frametypes[ifo],
gpsstart=gpsstart,
gpsend=gpsend)
datacache = Cache.from_urls(urls)
if len(datacache) == 0:
print(f"No {ifo} data found.")
continue
if "state vector" in event.meta:
state_vector_channel = event.meta['state vector']
else:
state_vector_channel = ast.literal_eval(
config.get("data", "state-vector"))
state = gwpy.timeseries.StateVector.read(
datacache,
state_vector_channel[ifo],
start=gpsstart,
end=gpsend,
pad=
0 # padding data so that errors are not raised even if found data are not continuous.
)
if not np.issubdtype(state.dtype, np.unsignedinteger):
# if data are not unsigned integers, cast to them now so that
# we can determine the bit content for the flags
state = state.astype(
"uint32",
casting="unsafe",
subok=True,
copy=False,
)
flags = state.to_dqflags()
segments = flags[bits[0]].active
for bit in bits:
segments -= ~flags[bit].active
if len(segments) > 0: active_ifo += [ifo]
print(event.event_object.name)
if event.event_object.meta['interferometers'] != active_ifo:
print(f"Gitlab data\t{event.event_object.meta['interferometers']}")
print(f"Recommended IFOS\t{active_ifo}")
event.event_object.meta['interferometers'] = active_ifo
event.update_data()
def resultslinks(event, update, root):
"""
Find all available results for a given event.
"""
server, repository = connect_gitlab()
events = gitlab.find_events(repository,
milestone=config.get("olivaw", "milestone"),
subset=[event],
update=update,
repo=False)
for event in events:
click.secho(f"{event.title}")
logger = logging.AsimovLogger(event=event.event_object)
for production in event.productions:
try:
for result, meta in production.results().items():
print(
f"{production.event.name}/{production.name}/{result}, {production.results(result)}"
)
pathlib.Path(
os.path.join(root, production.event.name,
production.name)).mkdir(parents=True,
exist_ok=True)
os.symlink(
f"{production.results(result)}",
f"{root}/{production.event.name}/{production.name}/{result.split('/')[-1]}"
)
except AttributeError:
pass
def status(event):
"""
Provide a simple summary of the status of a given event.
Arguments
---------
name : str, optional
The name of the event.
"""
server, repository = connect_gitlab()
events = gitlab.find_events(repository,
milestone=config.get("olivaw", "milestone"),
subset=[event],
update=False,
repo=False)
for event in events:
click.secho(f"{event.title:30}", bold=True)
if len(event.event_object.meta['productions']) > 0:
click.secho("\tProductions", bold=True)
for production in event.event_object.meta['productions']:
click.echo(f"\t\t{list(production.keys())[0]}")
if len(event.event_object.get_all_latest()) > 0:
click.secho("\tJobs waiting", bold=True)
waiting = event.event_object.get_all_latest()
for awaiting in waiting:
click.echo(f"\t\t{awaiting.name}\t{awaiting.status}")
def audit(event):
"""
Conduct an audit of the contents of production ini files
against the production ledger.
Parameters
----------
event : str, optional
The event to be checked.
Optional; if the event isn't provided all events will be audited.
"""
if isinstance(event, str):
event = [event]
_, repository = connect_gitlab()
gitlab_events = gitlab.find_events(repository,
subset=event,
update=False,
repo=True)
for production in gitlab_events[0].productions:
category = config.get("general", "calibration_directory")
config_file = os.path.join(production.event.repository.directory,
category, f"{production.name}.ini")
pipe = known_pipelines[production.pipeline.lower()](production,
category)
click.echo(pipe.read_ini(config_file))
def make_config(self, filename, template_directory=None):
"""
Make the configuration file for this production.
Parameters
----------
filename : str
The location at which the config file should be saved.
template_directory : str, optional
The path to the directory containing the pipeline config templates.
Defaults to the directory specified in the asimov configuration file.
"""
if "template" in self.meta:
template = f"{self.meta['template']}.ini"
else:
template = f"{self.pipeline}.ini"
try:
template_directory = config.get("templating", "directory")
template_file = os.path.join(f"{template_directory}", template)
except:
from pkg_resources import resource_filename
template_file = resource_filename("asimov", f'configs/{template}')
config_dict = {s: dict(config.items(s)) for s in config.sections()}
with open(template_file, "r") as template_file:
liq = Liquid(template_file.read())
rendered = liq.render(production=self, config=config)
with open(filename, "w") as output_file:
output_file.write(rendered)
def create(event, pipeline, family, comment, needs, template, status, approximant):
"""
Add a new production to an event.
"""
if config.get("ledger", "engine") == "gitlab":
_, repository = connect_gitlab()
gitlab_event = gitlab.find_events(repository, subset=event)
event = gitlab_event[0].event_object
elif config.get("ledger", "engine") == "yamlfile":
ledger = Ledger(config.get("ledger", "location"))
event = ledger.get_event(event)
#
event_prods = event.productions
names = [production.name for production in event_prods]
family_entries = [int(name.split(family)[1]) for name in names if family in name]
#
if "bayeswave" in needs:
bw_entries = [production.name for production in event_prods
if ("bayeswave" in production.pipeline.lower())
and (production.review.status not in {"REJECTED", "DEPRECATED"})]
needs = bw_entries
#
production = {"comment": comment, "pipeline": pipeline, "status": status}
if needs:
production['needs'] = needs
if template:
production['template'] = template
if approximant:
production['approximant'] = approximant
if len(family_entries)>0:
number = max(family_entries)+1
else:
number = 0
production_dict = {f"{family}{number}": production}
production = Production.from_dict(production_dict, event=event)
#
click.echo(production)
event.add_production(production)
if config.get("ledger", "engine") == "gitlab":
gitlab_event[0].update_data()
elif config.get("ledger", "engine") == "yamlfile":
ledger.events[event.name] = event.to_dict()
ledger.save()
def store_results(self):
"""
Store the PE Summary results
"""
files = [
f"{self.production.name}_pesummary.dat", "posterior_samples.h5",
f"{self.production.name}_skymap.fits"
]
for filename in files:
results = os.path.join(config.get('general', 'webroot'),
self.production.event.name,
self.production.name, "results", "samples",
filename)
store = Store(root=config.get("storage", "directory"))
store.add_file(self.production.event.name,
self.production.name,
file=results)
def __init__(self, production, category=None):
super(BayesWave, self).__init__(production, category)
self.logger = logger = logging.AsimovLogger(event=production.event)
if not production.pipeline.lower() == "bayeswave":
raise PipelineException
try:
self.category = config.get("general", "category")
except:
self.category = "C01_offline"
self.logger.info("Assuming C01_offline calibration.")
def connect_gitlab(configs=None):
"""
Connect to the gitlab server.
Returns
-------
server : `Gitlab`
The gitlab server.
repository: `Gitlab.project`
The gitlab project.
"""
if configs:
config = configs
else:
from asimov import config
server = gitlab.gitlab.Gitlab('https://git.ligo.org',
private_token=config.get("gitlab", "token"))
repository = server.projects.get(
config.get("olivaw", "tracking_repository"))
return server, repository
def results(self, filename=None, handle=False, hash=None):
store = Store(root=config.get("storage", "results_store"))
if not filename:
try:
items = store.manifest.list_resources(self.event.name, self.name)
return items
except KeyError:
return None
elif handle:
return open(store.fetch_file(self.event.name, self.name, filename, hash), "r")
else:
return store.fetch_file(self.event.name, self.name, filename, hash=hash)
def from_url(cls, url, name, directory=None, update=False):
"""
Clone a git repository into a working directory,
then create an EventRepo object for it.
Parameters
----------
url : str
The URL of the git repository
name : str
The name for the git repository (probably the event name)
directory : str, optional
The location to store the cloned repository.
If this value isn't provided the repository is
cloned into the /tmp directory.
update : bool
Flag to determine if the repository is updated when loaded.
Defaults to False.
"""
if not directory:
tmp = config.get("general", "git_default")
directory = f"{tmp}/{name}"
pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
# Replace an https address with an ssh address
if "https" in url:
url = url.replace("https://", "git@")
final = "/".join(url.split("/")[1:])
start = url.split("/")[0]
url = f"{start}:{final}"
try:
repo = git.Repo.clone_from(url, directory)
repo.git.execute(["git", "lfs", "install"])
repo.git.execute(["git", "lfs", "fetch"])
repo.git.execute(["git", "lfs", "pull"])
except git.exc.GitCommandError:
repo = git.Repo(directory)
try:
repo.git.stash()
except git.exc.GitCommandError:
pass
if update:
try:
repo.remotes[0].pull()
except git.exc.GitCommandError:
pass
return cls(directory, url, update=update)
def submit(event, update):
"""
Submit the run configuration files for a given event for jobs which are ready to run.
If no event is specified then all of the events will be processed.
"""
server, repository = connect_gitlab()
events = gitlab.find_events(repository,
milestone=config.get("olivaw", "milestone"),
subset=[event],
update=update)
for event in events:
logger = logging.AsimovLogger(event=event.event_object)
ready_productions = event.event_object.get_all_latest()
for production in ready_productions:
if production.status.lower() in {
"running", "stuck", "wait", "processing", "uploaded",
"finished", "manual", "cancelled", "stopped"
}:
continue
if production.status.lower() == "restart":
if production.pipeline.lower() in known_pipelines:
pipe = known_pipelines[production.pipeline.lower()](
production, "C01_offline")
pipe.clean()
pipe.submit_dag()
else:
#try:
# configuration = production.get_configuration()
#except ValueError as e:
# #build(event)
# logger.error(f"Error while trying to submit a configuration. {e}", production=production, channels="gitlab")
if production.pipeline.lower() in known_pipelines:
pipe = known_pipelines[production.pipeline.lower()](
production, "C01_offline")
try:
pipe.build_dag()
except PipelineException:
logger.error(
"The pipeline failed to build a DAG file.",
production=production)
try:
pipe.submit_dag()
production.status = "running"
except PipelineException as e:
production.status = "stuck"
logger.error(
f"The pipeline failed to submit the DAG file to the cluster. {e}",
production=production)
def upload_prod(self,
production,
rundir,
preferred=False,
category="C01_offline",
rootdir="public_html/LVC/projects/O3/C01/",
rename=False):
"""
Upload the results of a PE job to the event repostory.
Parameters
----------
category : str, optional
The category of the job.
Defaults to "C01_offline".
production : str
The production name.
rundir : str
The run directory of the PE job.
"""
preferred_list = ["--preferred", "--append_preferred"]
web_path = os.path.join(os.path.expanduser("~"), *rootdir.split("/"),
self.event,
production) # TODO Make this generic
if rename:
prod_name = rename
else:
prod_name = production
command = [
config.get("pesummary",
"location"), "--event", self.event, "--exp", prod_name,
"--rundir", rundir, "--webdir", web_path, "--edit_homepage_table"
]
if preferred:
command += preferred_list
dagman = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = dagman.communicate()
if err or not "master -> master" in str(out):
raise ValueError(f"Sample upload failed.\n{out}\n{err}")
else:
return out
def results(event, update):
"""
Find all available results for a given event.
"""
server, repository = connect_gitlab()
events = gitlab.find_events(repository,
milestone=config.get("olivaw", "milestone"),
subset=[event],
update=update,
repo=False)
for event in events:
click.secho(f"{event.title}")
logger = logging.AsimovLogger(event=event.event_object)
for production in event.productions:
try:
for result, meta in production.results().items():
print(
f"{production.event.name}/{production.name}/{result}, {production.results(result)}"
)
except:
pass
def collect_assets(self):
"""
Collect the assets for this job and commit them to the event repository.
Since this job also generates the PSDs these should be added to the production ledger.
"""
results_dir = glob.glob(f"{self.production.rundir}/trigtime_*")[0]
sample_rate = self.production.meta['quality']['sample-rate']
store = Store(root=config.get("storage", "directory"))
for det in self.production.meta['interferometers']:
asset = os.path.join(results_dir, "post", "clean", f"glitch_median_PSD_forLI_{det}.dat")
destination = os.path.join(self.category, "psds", str(sample_rate), f"{det}-psd.dat")
try:
self.production.event.repository.add_file(asset, destination)
except Exception as e:
error = PipelineLogger(f"There was a problem committing the PSD for {det} to the repository.\n\n{e}",
issue=self.production.event.issue_object,
production=self.production.name)
error.submit_comment()
# Add to the event ledger
if "psds" not in self.production.event.meta:
self.production.event.meta['psds'] = {}
if sample_rate not in self.production.event.meta['psds']:
self.production.event.meta['psds'][sample_rate] = {}
self.production.event.meta['psds'][sample_rate][det] = destination
# Let's have a better way of doing this
self.production.event.issue_object.update_data()
copyfile(asset, f"{det}-{sample_rate}-psd.dat")
try:
store.add_file(self.production.event.name, self.production.name,
file = f"{det}-{sample_rate}-psd.dat")
except Exception as e:
error = PipelineLogger(f"There was a problem committing the PSD for {det} to the store.\n\n{e}",
issue=self.production.event.issue_object,
production=self.production.name)
error.submit_comment()
def before_submit(self):
"""
Convert the text-based PSD to an XML psd if the xml doesn't exist already.
"""
event = self.production.event
category = config.get("general", "calibration_directory")
current = os.getcwd()
if len(self.production.get_psds("xml")) == 0:
for ifo in self.production.meta['interferometers']:
os.chdir(f"{event.repository.directory}/")
sample = self.production.meta['quality']['sample-rate']
self._convert_psd(self.production.meta['psds'][sample][ifo],
ifo)
asset = f"{ifo.upper()}-psd.xml.gz"
git_location = os.path.join(category, "psds")
self.production.event.repository.add_file(
asset,
os.path.join(git_location, str(sample),
f"psd_{ifo}.xml.gz"),
commit_message=f"Added the xml format PSD for {ifo}.")
os.chdir(current)
def supress_psd(self, ifo, fmin, fmax):
"""
Suppress portions of a PSD.
Author: Carl-Johan Haster - August 2020
(Updated for asimov by Daniel Williams - November 2020
"""
store = Store(root=config.get("storage", "directory"))
sample_rate = self.production.meta['quality']['sample-rate']
orig_PSD_file = np.genfromtxt(os.path.join(self.production.event.repository.directory, self.category, "psds", str(sample_rate), f"{ifo}-psd.dat"))
freq = orig_PSD_file[:,0]
PSD = orig_PSD_file[:,1]
suppression_region = np.logical_and(np.greater_equal(freq, fmin), np.less_equal(freq, fmax))
#Suppress the PSD in this region
PSD[suppression_region] = 1.
new_PSD = np.vstack([freq, PSD]).T
asset = f"{ifo}-psd.dat"
np.savetxt(asset, new_PSD, fmt='%+.5e')
destination = os.path.join(self.category, "psds", str(sample_rate), f"{ifo}-psd.dat")
try:
self.production.event.repository.add_file(asset, destination)#, message=f"Added the supresed {ifo} PSD")
except Exception as e:
raise PipelineException(f"There was a problem committing the suppresed PSD for {ifo} to the repository.\n\n{e}",
issue=self.production.event.issue_object,
production=self.production.name)
copyfile(asset, f"{ifo}-{sample_rate}-psd-suppresed.dat")
try:
store.add_file(self.production.event.name, self.production.name,
file = f"{ifo}-{sample_rate}-psd-suppresed.dat")
except AlreadyPresentException:
pass
self.logger.info(f"PSD supression applied to {ifo} between {fmin} and {fmax}")
def get_gracedb(self, gfile, destination):
"""
Get a file from Gracedb, and store it in the event repository.
Parameters
----------
gfile : str
The name of the gracedb file, e.g. `coinc.xml`.
destination : str
The location in the repository for this file.
"""
gid = self.meta['gid']
client = GraceDb(service_url=config.get("gracedb", "url"))
file_obj = client.files(gid, gfile)
with open("download.file", "w") as dest_file:
dest_file.write(file_obj.read().decode())
self.repository.add_file("download.file", destination,
commit_message = f"Downloaded {gfile} from GraceDB")
def ledger(event, yaml_f):
"""
Return the ledger for a given event.
If no event is specified then the entire production ledger is returned.
"""
server, repository = connect_gitlab()
events = gitlab.find_events(repository,
milestone=config.get("olivaw", "milestone"),
subset=[event],
update=False,
repo=False)
total = []
for event in events:
total.append(yaml.safe_load(event.event_object.to_yaml()))
click.echo(yaml.dump(total))
if yaml_f:
with open(yaml_f, "w") as f:
f.write(yaml.dump(total))
def run_pesummary(self):
"""
Run PESummary on the results of this job.
"""
psds = self.production.psds
calibration = [
os.path.join(self.production.event.repository.directory, cal)
for cal in self.production.meta['calibration'].values()
]
configfile = self.production.event.repository.find_prods(
self.production.name, self.category)[0]
command = [
"--webdir",
os.path.join(config.get('general',
'webroot'), self.production.event.name,
self.production.name, "results"), "--labels",
self.production.name, "--gw", "--cosmology",
config.get('pesummary', 'cosmology'), "--redshift_method",
config.get('pesummary', 'redshift'), "--nsamples_for_skymap",
config.get('pesummary', 'skymap_samples'), "--evolve_spins",
"True", "--multi_process", "4", "--regenerate",
"mass_1_source mass_2_source chirp_mass_source total_mass_source final_mass_source final_mass_source_non_evolved radiated_energy",
"--config",
os.path.join(self.production.event.repository.directory,
self.category, configfile)
]
# Samples
command += ["--samples"]
command += self.samples()
# Calibration information
command += ["--calibration"]
command += calibration
# PSDs
command += ["--psd"]
command += psds.values()
self.logger.info(
f"Submitted PE summary run. Command: {config.get('pesummary', 'executable')} {' '.join(command)}",
production=self.production,
channels=['file'])
hostname_job = htcondor.Submit({
"executable":
config.get("pesummary", "executable"),
"arguments":
" ".join(command),
"accounting_group":
config.get("pipelines", "accounting"),
"output":
f"{self.production.rundir}/pesummary.out",
"error":
f"{self.production.rundir}/pesummary.err",
"log":
f"{self.production.rundir}/pesummary.log",
"request_cpus":
"4",
"getenv":
"true",
"batch-name":
f"PESummary/{self.production.event.name}/{self.production.name}",
"request_memory":
"8192MB",
"request_disk":
"8192MB",
})
schedulers = htcondor.Collector().locate(
htcondor.DaemonTypes.Schedd, config.get("condor", "scheduler"))
schedd = htcondor.Schedd(schedulers)
with schedd.transaction() as txn:
cluster_id = hostname_job.queue(txn)
return cluster_id
def build_dag(self, user=None):
"""
Construct a DAG file in order to submit a production to the
condor scheduler using util_RIFT_pseudo_pipe.py
Parameters
----------
production : str
The production name.
user : str
The user accounting tag which should be used to run the job.
Raises
------
PipelineException
Raised if the construction of the DAG fails.
Notes
-----
In order to assemble the pipeline the RIFT runner requires additional
production metadata: at least the l_max value.
An example RIFT production specification would then look something like:
::
- Prod3:
rundir: tests/tmp/s000000xx/Prod3
pipeline: rift
approximant: IMRPhenomPv3
lmax: 2
cip jobs: 5 # This is optional, and will default to 3
bootstrap: Prod1
bootstrap fmin: 20
needs: Prod1
comment: RIFT production run.
status: wait
"""
cwd = os.getcwd()
#os.chdir(self.production.event.meta['working directory'])
#os.chdir(os.path.join(self.production.event.repository.directory,
# self.category))
if self.production.event.repository:
gps_file = self.production.get_timefile()
coinc_file = self.production.get_coincfile()
coinc_file = os.path.join(
self.production.event.repository.directory, "C01_offline",
coinc_file)
ini = self.production.get_configuration().ini_loc
ini = os.path.join(self.production.event.repository.directory,
"C01_offline", ini)
else:
gps_file = "gpstime.txt"
ini = os.path.join(self.production.event.meta['working directory'],
f"{self.production.name}.ini")
coinc_file = os.path.join(cwd, "coinc.xml")
if self.production.get_meta("user"):
user = self.production.get_meta("user")
else:
user = config.get("condor", "user")
self.production.set_meta("user", user)
os.environ['LIGO_USER_NAME'] = f"{user}"
os.environ[
'LIGO_ACCOUNTING'] = f"{config.get('pipelines', 'accounting')}"
try:
calibration = config.get("general", "calibration")
except:
calibration = "C01"
approximant = self.production.meta['approximant']
#ini.save()
if self.production.rundir:
rundir = os.path.relpath(self.production.rundir, os.getcwd())
else:
rundir = os.path.join(os.path.expanduser("~"),
self.production.event.name,
self.production.name)
self.production.rundir = rundir
#lmax = self.production.meta['priors']['amp order']
if "lmax" in self.production.meta:
lmax = self.production.meta['lmax']
elif "HM" in self.production.meta['approximant']:
lmax = 4
else:
lmax = 2
if "cip jobs" in self.production.meta:
cip = self.production.meta['cip jobs']
else:
cip = 3
command = [
os.path.join(config.get("pipelines", "environment"), "bin",
"util_RIFT_pseudo_pipe.py"), "--use-coinc",
coinc_file, "--l-max", f"{lmax}", "--calibration",
f"{calibration}", "--add-extrinsic", "--approx", f"{approximant}",
"--cip-explode-jobs",
str(cip), "--use-rundir", rundir, "--ile-force-gpu", "--use-ini",
ini
]
print(" ".join(command))
# If a starting frequency is specified, add it
if "start-frequency" in self.production.meta:
command += [
"--fmin-template", self.production.quality['start-frequency']
]
self.logger.info(" ".join(command), production=self.production)
# Placeholder LI grid bootstrapping; conditional on it existing and location specification
if self.bootstrap:
if self.bootstrap == "manual":
if self.production.event.repository:
bootstrap_file = os.path.join(
self.production.event.repository.directory,
"C01_offline",
f"{self.production.name}_bootstrap.xml.gz")
else:
bootstrap_file = "{self.production.name}_bootstrap.xml.gz"
else:
raise PipelineException(
f"Unable to find the bootstrapping production for {self.production.name}.",
issue=self.production.event.issue_object,
production=self.production.name)
command += ["--manual-initial-grid", bootstrap_file]
self.logger.info(command, production=self.production)
os.chdir(self.production.event.meta['working directory'])
pipe = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = pipe.communicate()
if err:
self.production.status = "stuck"
if hasattr(self.production.event, "issue_object"):
self.logger.info(out, production=self.production)
self.logger.error(err, production=self.production)
raise PipelineException(
f"DAG file could not be created.\n{command}\n{out}\n\n{err}",
issue=self.production.event.issue_object,
production=self.production.name)
else:
self.logger.info(out, production=self.production)
self.logger.error(err, production=self.production)
raise PipelineException(
f"DAG file could not be created.\n{command}\n{out}\n\n{err}",
production=self.production.name)
else:
if self.production.event.repository:
os.chdir(self.production.rundir)
for psdfile in self.production.get_psds("xml"):
ifo = psdfile.split("/")[-1].split("_")[1].split(".")[0]
os.system(f"cp {psdfile} {ifo}-psd.xml.gz")
#os.system("cat *_local.cache > local.cache")
if hasattr(self.production.event, "issue_object"):
return PipelineLogger(
message=out,
issue=self.production.event.issue_object,
production=self.production.name)
else:
return PipelineLogger(message=out,
production=self.production.name)
def clone(location):
import venv
import pathlib
import shutil
working = "working"
results = "results"
remote_config = os.path.join(location, "asimov.conf")
config = configparser.ConfigParser()
config.read([remote_config])
click.echo(f'Cloning {config.get("project", "name")}')
root = os.path.join(
os.getcwd(),
config.get("project", "name").lower().replace(" ", "-"))
pathlib.Path(root).mkdir(parents=True, exist_ok=True)
os.chdir(root)
config.set("project", "root", root)
# Make the virtual environment
#builder = venv.EnvBuilder(system_site_packages=False,
# clear=False,
# symlinks=False,
# upgrade=False,
# with_pip=True,
# prompt=f"Asimov {project_name}")
#builder.create("environment")
#config.set("general", "environment", "environment")
# Make the working directory
#shutil.copytree(os.path.join(config.get("general", "rundir_default"), working)
#config.set("general", "rundir_default", working)
# Make the git directory
#pathlib.Path(checkouts).mkdir(parents=True, exist_ok=True)
#config.set("general", "git_default", checkouts)
# Copy the results store
#shutil.copyfile(os.path.join(location, config.get("storage", "results_store")), results)
shutil.copytree(
os.path.join(location, config.get("storage", "results_store")),
results)
config.set("storage", "results_store", results)
# Make the ledger
if config.get("ledger", "engine") == "yamlfile":
shutil.copyfile(
os.path.join(location, config.get("ledger", "location")),
"ledger.yml")
elif config.get("ledger", "engine") == "gitlab":
_, repository = connect_gitlab(config)
events = gitlab.find_events(repository,
update=False,
subset=[None],
label=config.get("gitlab", "event_label"),
repo=False)
total = []
for event in events:
total.append(yaml.safe_load(event.event_object.to_yaml()))
with open("ledger.yml", "w") as f:
f.write(yaml.dump(total))
config.set("ledger", "engine", "yamlfile")
config.set("ledger", "location", "ledger.yml")
with open("asimov.conf", "w") as config_file:
config.write(config_file)
def build_dag(self, psds=None, user=None, clobber_psd=False):
"""
Construct a DAG file in order to submit a production to the
condor scheduler using LALInferencePipe.
Parameters
----------
production : str
The production name.
psds : dict, optional
The PSDs which should be used for this DAG. If no PSDs are
provided the PSD files specified in the ini file will be used
instead.
user : str
The user accounting tag which should be used to run the job.
Raises
------
PipelineException
Raised if the construction of the DAG fails.
"""
# Change to the location of the ini file.
os.chdir(
os.path.join(self.production.event.repository.directory,
self.category))
gps_file = self.production.get_timefile()
ini = self.production.get_configuration()
if not user:
if self.production.get_meta("user"):
user = self.productevenion.get_meta("user")
else:
user = ini._get_user()
self.production.set_meta("user", user)
ini.update_accounting(user)
if 'queue' in self.production.meta:
queue = self.production.meta['queue']
else:
queue = 'Priority_PE'
ini.set_queue(queue)
ini.save()
if self.production.rundir:
rundir = self.production.rundir
else:
rundir = os.path.join(os.path.expanduser("~"),
self.production.event.name,
self.production.name)
self.production.rundir = rundir
#os.mkdir(self.production.rundir, exist_ok=True)
command = [
os.path.join(config.get("pipelines", "environment"), "bin",
"lalinference_pipe"), "-g", f"{gps_file}", "-r",
self.production.rundir, ini.ini_loc
]
self.logger.info(" ".join(command))
pipe = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = pipe.communicate()
if err or "Successfully created DAG file." not in str(out):
self.production.status = "stuck"
if hasattr(self.production.event, "issue_object"):
self.logger.error(
f"DAG file could not be created.\n{command}\n{out}\n\n{err}"
)
raise PipelineException(
f"DAG file could not be created.\n{command}\n{out}\n\n{err}",
issue=self.production.event.issue_object,
production=self.production.name)
else:
self.logger.error(
f"DAG file could not be created.\n{command}\n{out}\n\n{err}"
)
raise PipelineException(
f"DAG file could not be created.\n{command}\n{out}\n\n{err}",
production=self.production.name)
else:
self.logger.info("DAG created")
if hasattr(self.production.event, "issue_object"):
return PipelineLogger(message=out,
issue=self.production.event.issue_object,
production=self.production.name)
else:
return PipelineLogger(message=out,
production=self.production.name)
def create(name, oldname=None, gid=None, superevent=None, repo=None):
"""
Create a new event record in the ledger..
Parameters
----------
superevent : str
The ID of the superevent to be used from GraceDB
name : str
The name of the event to be recorded in the issue tracker
names : path, optional
The path to the name file which maps between old and new super event IDs
oldname: str, optional
The old name of the event.
"""
import pathlib
if gid or superevent:
from ligo.gracedb.rest import GraceDb, HTTPError
client = GraceDb(service_url=config.get("gracedb", "url"))
r = client.ping()
if superevent:
data = client.superevent(superevent).json()
event_data = client.event(data['preferred_event']).json()
gid = data['preferred_event']
interferometers = event_data['instruments'].split(",")
elif gid:
event_data = client.event(gid).json()
interferometers = event_data['instruments'].split(",")
else:
event_data = None
interferometers = []
if gid or superevent:
event_url = f"{config.get('gracedb', 'url')}/events/{gid}/view/"
if not repo:
repo = None
#repo = f"[email protected]:pe/O3/{name}"
event = Event(
name=name,
repository=repo,
calibration={},
interferometers=interferometers,
)
if oldname:
event.meta['old superevent'] = oldname
if gid:
event.meta['event time'] = event_data['gpstime']
event.meta['gid'] = gid
working_dir = os.path.join(config.get('general', 'rundir_default'), name)
event.meta['working directory'] = working_dir
pathlib.Path(working_dir).mkdir(parents=True, exist_ok=True)
if config.get("ledger", "engine") == "gitlab":
_, repository = connect_gitlab()
from pkg_resources import resource_filename
issue_template = resource_filename('asimov', 'gitlabissue.md')
gitlab.EventIssue.create_issue(repository,
event,
issue_template=issue_template)
elif config.get("ledger", "engine") == "yamlfile":
ledger = Ledger(config.get("ledger", "location"))
ledger.add_event(event)
ledger.save()
def __init__(self, event, name, status, pipeline, comment=None, **kwargs):
self.event = event
self.name = name
if status:
self.status_str = status.lower()
else:
self.status_str = "none"
self.pipeline = pipeline.lower()
self.comment = comment
self.meta = deepcopy(self.event.meta)
if "productions" in self.meta:
self.meta.pop("productions")
self.meta = update(self.meta, kwargs)
if "review" in self.meta:
self.review = Review.from_dict(self.meta['review'], production=self)
self.meta.pop("review")
else:
self.review = Review()
# Check that the upper frequency is included, otherwise calculate it
if "quality" in self.meta:
if ("high-frequency" not in self.meta['quality']) and ("sample-rate" in self.meta['quality']):
# Account for the PSD roll-off with the 0.875 factor
self.meta['quality']['high-frequency'] = int(0.875 * self.meta['quality']['sample-rate']/2)
# Get the data quality recommendations
if 'quality' in self.event.meta:
self.quality = self.event.meta['quality']
else:
self.quality = {}
if ('quality' in self.meta):
if ('quality' in kwargs):
self.meta['quality'].update(kwargs['quality'])
self.quality = self.meta['quality']
if ('quality' in self.meta) and ("event time" in self.meta):
if "segment start" not in self.meta['quality']:
self.meta['quality']['segment start'] = self.meta['event time'] - self.meta['quality']['segment-length'] + 2
self.event.meta['quality']['segment start'] = self.meta['quality']['segment start']
# Gather the appropriate prior data for this production
if 'priors' in self.meta:
self.priors = self.meta['priors']
# Need to fetch the correct PSDs for this sample rate
if 'psds' in self.meta:
if self.quality['sample-rate'] in self.meta['psds']:
self.psds = self.meta['psds'][self.quality['sample-rate']]
else:
self.psds = {}
else:
self.psds = {}
for ifo, psd in self.psds.items():
if self.event.repository:
self.psds[ifo] = os.path.join(self.event.repository.directory, psd)
else:
self.psds[ifo] = psd
self.category = config.get("general", "calibration_directory")
if "needs" in self.meta:
self.dependencies = self._process_dependencies(self.meta['needs'])
else:
self.dependencies = None
"""
import os
import asimov
from asimov.event import Event, DescriptionException
from asimov import config
from asimov import gitlab
from asimov import config
import numpy as np
import yaml
import click
server = gitlab.gitlab.Gitlab(config.get("gitlab", "url"),
private_token=config.get("gitlab", "token"))
repository = server.projects.get(config.get("olivaw", "tracking_repository"))
def find_calibrations(time):
with open("LLO_calibs.txt") as llo_file:
data_llo = llo_file.read().split("\n")
data_llo = [
datum for datum in data_llo if datum[-16:] == "FinalResults.txt"
]
times_llo = {
int(datum.split("GPSTime_")[1].split("_C01")[0]): datum
for datum in data_llo
}
import gwpy.segments
from gwpy.segments import DataQualityFlag
from gwdatafind import find_urls
from glue.lal import Cache
import collections.abc
state_vector_channel = {"L1": "L1:DCS-CALIB_STATE_VECTOR_C01",
"H1": "H1:DCS-CALIB_STATE_VECTOR_C01",
"V1": "V1:DQ_ANALYSIS_STATE_VECTOR"}
frametypes= {"L1": "L1_HOFT_CLEAN_SUB60HZ_C01",
"H1": "H1_HOFT_CLEAN_SUB60HZ_C01",
"V1": "V1Online"}
server = gitlab.gitlab.Gitlab(config.get("gitlab", "url"), private_token=config.get("gitlab", "token"))
repository = server.projects.get(config.get("olivaw", "tracking_repository"))
#
# Get GraceDB stuff
#
from ligo.gracedb.rest import GraceDb, HTTPError
client = GraceDb(service_url=config.get("gracedb", "url"))
r = client.ping()
#superevent_iterator = client.superevents('O3B_CBC_CATALOG')
#superevent_ids = [superevent['superevent_id'] for superevent in superevent_iterator]
CALIBRATION_NOTE = """
## Calibration envelopes
def build_dag(self, user=None):
"""
Construct a DAG file in order to submit a production to the
condor scheduler using bayeswave_pipe
Parameters
----------
production : str
The production name.
user : str
The user accounting tag which should be used to run the job.
Raises
------
PipelineException
Raised if the construction of the DAG fails.
"""
if self.production.event.repository:
os.chdir(os.path.join(self.production.event.repository.directory,
self.category))
try:
gps_file = self.production.get_timefile()
except AsimovFileNotFound:
if "event time" in self.production.meta:
gps_time = self.production.get_meta("event time")
with open("gpstime.txt", "w") as f:
f.write(str(gps_time))
gps_file = os.path.join(f"{self.production.category}", f"gpstime.txt")
self.production.event.repository.add_file(f"gpstime.txt", gps_file)
else:
raise PipelineException("Cannot find the event time.")
else:
gps_time = self.production.get_meta("event time")
with open("gpstime.txt", "w") as f:
f.write(str(gps_time))
gps_file = os.path.join("gpstime.txt")
if self.production.event.repository:
ini = self.production.get_configuration()
if not user:
if self.production.get_meta("user"):
user = self.production.get_meta("user")
else:
user = ini._get_user()
self.production.set_meta("user", user)
ini.update_accounting(user)
if 'queue' in self.production.meta:
queue = self.production.meta['queue']
else:
queue = 'Priority_PE'
ini.set_queue(queue)
ini.save()
ini = ini.ini_loc
else:
ini = f"{self.production.name}.ini"
if self.production.rundir:
rundir = self.production.rundir
else:
rundir = os.path.join(os.path.expanduser("~"),
self.production.event.name,
self.production.name)
self.production.rundir = rundir
try:
pathlib.Path(directory).mkdir(parents=True, exist_ok=False)
except:
pass
gps_time = self.production.get_meta("event time")
pipe_cmd = os.path.join(config.get("pipelines", "environment"), "bin", "bayeswave_pipe")
command = [pipe_cmd,
#"-l", f"{gps_file}",
f"--trigger-time={gps_time}",
"-r", self.production.rundir,
ini
]
self.logger.info(" ".join(command))
pipe = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = pipe.communicate()
if "To submit:" not in str(out):
self.production.status = "stuck"
if "issue_object" in self.production.event:
raise PipelineException(f"DAG file could not be created.\n{command}\n{out}\n\n{err}",
issue=self.production.event.issue_object,
production=self.production.name)
else:
raise PipelineException(f"DAG file could not be created.\n{command}\n{out}\n\n{err}",
production=self.production.name)
else:
if hasattr(self.production.event, "issue_object"):
return PipelineLogger(message=out,
issue=self.production.event.issue_object,
production=self.production.name)
else:
return PipelineLogger(message=out,
production=self.production.name)
