def upload(self, fname, psds, low_frequency_cutoff, testing=True):
"""Upload this trigger to gracedb
Parameters
----------
fname: str
The name to give the xml file associated with this trigger
pds: dict of pybc.types.FrequencySeries
A ifo keyed dictionary of psds to be uploaded in association
with this trigger.
low_frequency_cutoff: float
The low frequency cutoff of the psds.
testing: bool
Switch to determine if the upload should be sent to gracedb as a
test trigger (True) or a production trigger (False)
"""
from ligo.gracedb.rest import GraceDb
import lal
import lal.series
self.save(fname)
if testing:
group = 'Test'
else:
group = 'CBC'
gracedb = GraceDb()
r = gracedb.createEvent(group, "pycbc", fname, "AllSky").json()
logging.info("Uploaded event %s.", r["graceid"])
if self.is_hardware_injection:
gracedb.writeLabel(r['graceid'], 'INJ')
logging.info("Tagging event %s as an injection", r["graceid"])
# Convert our psds to the xml psd format.
# FIXME: we should not use lal.series!!!
psds_lal = {}
for ifo in psds:
psd = psds[ifo]
kmin = int(low_frequency_cutoff / psd.delta_f)
fseries = lal.CreateREAL8FrequencySeries(
"psd", psd.epoch, low_frequency_cutoff, psd.delta_f,
lal.StrainUnit**2 / lal.HertzUnit,
len(psd) - kmin)
fseries.data.data = psd.numpy()[kmin:] / pycbc.DYN_RANGE_FAC**2.0
psds_lal[ifo] = fseries
psd_xmldoc = lal.series.make_psd_xmldoc(psds_lal)
ligolw_utils.write_filename(psd_xmldoc, "tmp_psd.xml.gz", gz=True)
gracedb.writeLog(r["graceid"],
"PyCBC PSD estimate from the time of event",
"psd.xml.gz",
open("tmp_psd.xml.gz", "rb").read(), "psd").json()
logging.info("Uploaded file psd.xml.gz to event %s.", r["graceid"])
def upload(self, fname, psds, low_frequency_cutoff, testing=True):
"""Upload this trigger to gracedb
Parameters
----------
fname: str
The name to give the xml file associated with this trigger
pds: dict of pybc.types.FrequencySeries
A ifo keyed dictionary of psds to be uploaded in association
with this trigger.
low_frequency_cutoff: float
The low frequency cutoff of the psds.
testing: bool
Switch to determine if the upload should be sent to gracedb as a
test trigger (True) or a production trigger (False)
"""
from ligo.gracedb.rest import GraceDb
import lal
import lal.series
self.save(fname)
if testing:
group = 'Test'
else:
group = 'CBC'
gracedb = GraceDb()
r = gracedb.createEvent(group, "pycbc", fname, "AllSky").json()
logging.info("Uploaded event %s.", r["graceid"])
if self.is_hardware_injection:
gracedb.writeLabel(r['graceid'], 'INJ')
logging.info("Tagging event %s as an injection", r["graceid"])
# Convert our psds to the xml psd format.
# FIXME: we should not use lal.series!!!
psds_lal = {}
for ifo in psds:
psd = psds[ifo]
kmin = int(low_frequency_cutoff / psd.delta_f)
fseries = lal.CreateREAL8FrequencySeries(
"psd", psd.epoch, low_frequency_cutoff, psd.delta_f,
lal.StrainUnit**2 / lal.HertzUnit, len(psd) - kmin)
fseries.data.data = psd.numpy()[kmin:] / pycbc.DYN_RANGE_FAC ** 2.0
psds_lal[ifo] = fseries
psd_xmldoc = lal.series.make_psd_xmldoc(psds_lal)
ligolw_utils.write_filename(psd_xmldoc, "tmp_psd.xml.gz", gz=True)
gracedb.writeLog(r["graceid"],
"PyCBC PSD estimate from the time of event",
"psd.xml.gz", open("tmp_psd.xml.gz", "rb").read(),
"psd").json()
logging.info("Uploaded file psd.xml.gz to event %s.", r["graceid"])
if 'rank' in filename:
rank_filenames.append(filename)
if 'fap' in filename:
fap_filenames.append(filename)
rank_filenames.sort()
fap_filenames.sort()
if (not rank_filenames) or (
not fap_filenames): # we couldn't find either rank or fap files
# exit gracefully
if opts.verbose:
print "no iDQ timeseries for %s at %s" % (opts.classifier, ifo)
if not opts.skip_gracedb_upload:
gracedb.writeLog(opts.gracedb_id,
message="No iDQ timeseries for %s at %s" %
(opts.classifier, ifo),
tagname=idq.tagnames)
sys.exit(0)
#===================================================================================================
### process FAP files
if opts.verbose:
print "reading fap timeseries from:"
for filename in fap_filenames:
print '\t' + filename
#fig = isp.plt.figure(figsize=isp.rank_timeseries_figsize)
#rnk_ax = fig.add_axes(isp.rank_timeseries_axpos)
#fap_ax = rnk_ax.twinx()
fig = isp.plt.figure(figsize=isp.rank_splittimeseries_figsize)
def upload(self, fname, gracedb_server=None, testing=True,
extra_strings=None):
"""Upload this trigger to gracedb
Parameters
----------
fname: str
The name to give the xml file associated with this trigger
gracedb_server: string, optional
URL to the GraceDB web API service for uploading the event.
If omitted, the default will be used.
testing: bool
Switch to determine if the upload should be sent to gracedb as a
test trigger (True) or a production trigger (False).
"""
from ligo.gracedb.rest import GraceDb
import matplotlib
matplotlib.use('Agg')
import pylab
# first of all, make sure the event is saved on disk
# as GraceDB operations can fail later
self.save(fname)
if self.snr_series is not None:
if fname.endswith('.xml.gz'):
snr_series_fname = fname.replace('.xml.gz', '.hdf')
else:
snr_series_fname = fname.replace('.xml', '.hdf')
snr_series_plot_fname = snr_series_fname.replace('.hdf',
'_snr.png')
psd_series_plot_fname = snr_series_fname.replace('.hdf',
'_psd.png')
pylab.figure()
for ifo in sorted(self.snr_series):
curr_snrs = self.snr_series[ifo]
curr_snrs.save(snr_series_fname, group='%s/snr' % ifo)
pylab.plot(curr_snrs.sample_times, abs(curr_snrs),
c=ifo_color(ifo), label=ifo)
if ifo in self.ifos:
snr = self.coinc_results['foreground/%s/%s' %
(ifo, 'snr')]
endt = self.coinc_results['foreground/%s/%s' %
(ifo, 'end_time')]
pylab.plot([endt], [snr], c=ifo_color(ifo), marker='x')
pylab.legend()
pylab.xlabel('GPS time (s)')
pylab.ylabel('SNR')
pylab.savefig(snr_series_plot_fname)
pylab.close()
pylab.figure()
for ifo in sorted(self.snr_series):
# Undo dynamic range factor
curr_psd = self.psds[ifo].astype(numpy.float64)
curr_psd /= pycbc.DYN_RANGE_FAC ** 2.0
curr_psd.save(snr_series_fname, group='%s/psd' % ifo)
# Can't plot log(0) so start from point 1
pylab.loglog(curr_psd.sample_frequencies[1:],
curr_psd[1:]**0.5, c=ifo_color(ifo), label=ifo)
pylab.legend()
pylab.xlim([10, 1300])
pylab.ylim([3E-24, 1E-20])
pylab.xlabel('Frequency (Hz)')
pylab.ylabel('ASD')
pylab.savefig(psd_series_plot_fname)
pylab.close()
if self.probabilities is not None:
prob_fname = fname.replace('.xml.gz', '_probs.json')
prob_plot_fname = prob_fname.replace('.json', '.png')
prob_plot = {k: v for (k, v) in self.probabilities.items()
if v != 0.0}
labels, sizes = zip(*prob_plot.items())
colors = [source_color(label) for label in labels]
fig, ax = pylab.subplots()
ax.pie(sizes, labels=labels, colors=colors, autopct='%1.1f%%',
textprops={'fontsize': 15})
ax.axis('equal')
fig.savefig(prob_plot_fname)
pylab.close()
gid = None
try:
# try connecting to GraceDB
gracedb = GraceDb(gracedb_server) \
if gracedb_server is not None else GraceDb()
# create GraceDB event
group = 'Test' if testing else 'CBC'
r = gracedb.createEvent(group, "pycbc", fname, "AllSky").json()
gid = r["graceid"]
logging.info("Uploaded event %s", gid)
if self.is_hardware_injection:
gracedb.writeLabel(gid, 'INJ')
logging.info("Tagging event %s as an injection", gid)
# upload PSDs. Note that the PSDs are already stored in the
# original event file and we just upload a copy of that same file
# here. This keeps things as they were in O2 and can be removed
# after updating the follow-up infrastructure
psd_fname = 'psd.xml.gz' if fname.endswith('.gz') else 'psd.xml'
gracedb.writeLog(gid, "PyCBC PSD estimate from the time of event",
psd_fname, open(fname, "rb").read(), "psd")
logging.info("Uploaded PSDs for event %s", gid)
# add info for tracking code version
version_str = 'Using PyCBC version {}{} at {}'
version_str = version_str.format(
pycbc_version.version,
' (release)' if pycbc_version.release else '',
os.path.dirname(pycbc.__file__))
gracedb.writeLog(gid, version_str)
extra_strings = [] if extra_strings is None else extra_strings
for text in extra_strings:
gracedb.writeLog(gid, text, tag_name=['analyst_comments'])
# upload SNR series in HDF format and plots
if self.snr_series is not None:
gracedb.writeLog(gid, 'SNR timeseries HDF file upload',
filename=snr_series_fname)
gracedb.writeLog(gid, 'SNR timeseries plot upload',
filename=snr_series_plot_fname,
tag_name=['background'],
displayName=['SNR timeseries'])
gracedb.writeLog(gid, 'PSD plot upload',
filename=psd_series_plot_fname,
tag_name=['psd'], displayName=['PSDs'])
# upload source probabilities in json format and plot
if self.probabilities is not None:
gracedb.writeLog(gid, 'source probabilities JSON file upload',
filename=prob_fname, tag_name=['em_follow'])
logging.info('Uploaded source probabilities for event %s', gid)
gracedb.writeLog(gid, 'source probabilities plot upload',
filename=prob_plot_fname,
tag_name=['em_follow'])
logging.info('Uploaded source probabilities pie chart for '
'event %s', gid)
except Exception as exc:
logging.error('Something failed during the upload/annotation of '
'event %s on GraceDB. The event may not have been '
'uploaded!', fname)
logging.error(str(exc))
return gid
def parseAlert(queue, queueByGraceID, alert, t0, config):
'''
the way approval_processorMP digests lvalerts
--> check if this alert is a command and delegate to parseCommand
1) instantiates GraceDB client
2) pulls childConfig settings
3) makes sure we have the logger
4) get lvalert specifics
5) ensure we have the event_dict for the graceid = lvalert['uid']
6) take proper action depending on the lvalert info coming in and currentstate of the event_dict
'''
#-------------------------------------------------------------------
# process commands sent via lvalert_commandMP
#-------------------------------------------------------------------
if alert['uid'] == 'command': ### this is a command message!
return parseCommand(queue, queueByGraceID, alert,
t0) ### delegate to parseCommand and return
#-------------------------------------------------------------------
# extract relevant config parameters and set up necessary data structures
#-------------------------------------------------------------------
# instantiate GraceDB client from the childConfig
client = config.get('general', 'client')
g = GraceDb(client)
# get other childConfig settings; save in configdict
voeventerror_email = config.get('general', 'voeventerror_email')
force_all_internal = config.get('general', 'force_all_internal')
preliminary_internal = config.get('general', 'preliminary_internal')
forgetmenow_timeout = config.getfloat('general', 'forgetmenow_timeout')
approval_processorMPfiles = config.get('general',
'approval_processorMPfiles')
hardware_inj = config.get('labelCheck', 'hardware_inj')
wait_for_hardware_inj = config.getfloat('labelCheck',
'wait_for_hardware_inj')
default_farthresh = config.getfloat('farCheck', 'default_farthresh')
time_duration = config.getfloat('injectionCheck', 'time_duration')
humanscimons = config.get('operator_signoffCheck', 'humanscimons')
### extract options about advocates
advocates = config.get('advocate_signoffCheck', 'advocates')
advocate_text = config.get('advocate_signoffCheck', 'advocate_text')
advocate_email = config.get('advocate_signoffCheck', 'advocate_email')
### extract options for GRB alerts
em_coinc_text = config.get('GRB_alerts', 'em_coinc_text')
coinc_text = config.get('GRB_alerts', 'coinc_text')
grb_email = config.get('GRB_alerts', 'grb_email')
notification_text = config.get('GRB_alerts', 'notification_text')
### extract options about idq
ignore_idq = config.get('idq_joint_fapCheck', 'ignore_idq')
default_idqthresh = config.getfloat('idq_joint_fapCheck',
'default_idqthresh')
idq_pipelines = config.get('idq_joint_fapCheck', 'idq_pipelines')
idq_pipelines = idq_pipelines.replace(' ', '')
idq_pipelines = idq_pipelines.split(',')
skymap_ignore_list = config.get('have_lvem_skymapCheck',
'skymap_ignore_list')
### set up configdict (passed to local data structure: eventDicts)
configdict = makeConfigDict(config)
# set up logging
### FIXME: why not open the logger each time parseAlert is called?
### that would allow you to better control which loggers are necessary and minimize the number of open files.
### it also minimizes the possibility of something accidentally being written to loggers because they were left open.
### what's more, this is a natural place to set up multiple loggers, one for all data and one for data pertaining only to this graceid
global logger
if globals().has_key('logger'): # check to see if we have logger
logger = globals()['logger']
else: # if not, set one up
logger = loadLogger(config)
logger.info(
'\n{0} ************ approval_processorMP.log RESTARTED ************\n'
.format(convertTime()))
#-------------------------------------------------------------------
# extract relevant info about this alert
#-------------------------------------------------------------------
# get alert specifics and event_dict information
graceid = alert['uid']
alert_type = alert['alert_type']
description = alert['description']
filename = alert['file']
#-------------------------------------------------------------------
# ensure we have an event_dict and ForgetMeNow tracking this graceid
#-------------------------------------------------------------------
if alert_type == 'new': ### new event -> we must first create event_dict and set up ForgetMeNow queue item for G events
### create event_dict
event_dict = EventDict(
) # create a new instance of EventDict class which is a blank event_dict
if is_external_trigger(
alert) == True: # this is an external GRB trigger
event_dict.grb_trigger_setup(
alert['object'], graceid, g, config, logger
) # populate this event_dict with grb trigger info from lvalert
else:
event_dict.setup(
alert['object'], graceid, configdict, g, config, logger
) # populate this event_dict with information from lvalert
eventDicts[
graceid] = event_dict # add the instance to the global eventDicts
eventDictionaries[
graceid] = event_dict.data # add the dictionary to the global eventDictionaries
### ForgetMeNow queue item
item = ForgetMeNow(t0, forgetmenow_timeout, graceid, eventDicts, queue,
queueByGraceID, logger)
queue.insert(item) # add queue item to the overall queue
### set up queueByGraceID
newSortedQueue = utils.SortedQueue(
) # create sorted queue for event candidate
newSortedQueue.insert(
item) # put ForgetMeNow queue item into the sorted queue
queueByGraceID[
item.
graceid] = newSortedQueue # add queue item to the queueByGraceID
saveEventDicts(
approval_processorMPfiles
) # trying to see if expirationtime is updated from None
message = '{0} -- {1} -- Created event dictionary for {1}.'.format(
convertTime(), graceid)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Created event dictionary.',
tagname='em_follow')
else:
pass
else: ### not a new alert -> we may already be tracking this graceid
if eventDicts.has_key(graceid): ### we're already tracking it
# get event_dict with expirationtime key updated for the rest of parseAlert
event_dict = eventDicts[graceid]
# find ForgetMeNow corresponding to this graceid and update expiration time
for item in queueByGraceID[graceid]:
if item.name == ForgetMeNow.name: # selects the queue item that is a ForgetMeNow instance
item.setExpiration(t0) # updates the expirationtime key
queue.resort(
) ### may be expensive, but is needed to guarantee that queue remains sorted
queueByGraceID[graceid].resort()
break
else: ### we couldn't find a ForgetMeNow for this event! Something is wrong!
os.system(
'echo \'ForgetMeNow KeyError\' | mail -s \'ForgetMeNow KeyError {0}\' {1}'
.format(graceid, advocate_email))
raise KeyError(
'could not find ForgetMeNow for %s' % graceid
) ### Reed thinks this is necessary as a safety net.
### we want the process to terminate if things are not set up correctly to force us to fix it
else: # event_dict for event candidate does not exist. we need to create it with up-to-date information
event_dict = EventDict(
) # create a new instance of the EventDict class which is a blank event_dict
if is_external_trigger(alert) == True:
event_dict.grb_trigger_setup(
g.events(graceid).next(), graceid, g, config, logger)
else:
event_dict.setup(
g.events(graceid).next(), graceid, configdict, g, config,
logger
) # fill in event_dict using queried event candidate dictionary
event_dict.update(
) # update the event_dict with signoffs and iDQ info
eventDicts[
graceid] = event_dict # add this instance to the global eventDicts
eventDictionaries[
graceid] = event_dict.data # add the dictionary to the global eventDictionaries
# create ForgetMeNow queue item and add to overall queue and queueByGraceID
item = ForgetMeNow(t0, forgetmenow_timeout, graceid, eventDicts,
queue, queueByGraceID, logger)
queue.insert(item) # add queue item to the overall queue
### set up queueByGraceID
newSortedQueue = utils.SortedQueue(
) # create sorted queue for new event candidate
newSortedQueue.insert(
item) # put ForgetMeNow queue item into the sorted queue
queueByGraceID[
item.
graceid] = newSortedQueue # add queue item to the queueByGraceID
message = '{0} -- {1} -- Created event dictionary for {1}.'.format(
convertTime(), graceid)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Created event dictionary.',
tagname='em_follow')
else:
pass
#--------------------
# ignore alerts that are not relevant, like simulation or MDC events
#--------------------
# if the graceid starts with 'M' for MDCs or 'S' for Simulation, ignore
if re.match('M', graceid) or re.match(
'S',
graceid): ### FIXME: we want to make this a config-file option!
message = '{0} -- {1} -- Mock data challenge or simulation. Ignoring.'.format(
convertTime(), graceid)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Mock data challenge or simulation. Ignoring.',
tagname='em_follow')
else:
pass
saveEventDicts(approval_processorMPfiles)
return 0
#--------------------
# take care of external GRB triggers
#--------------------
if is_external_trigger(
alert
) == True: # for now, grouped everything related to external triggers together below
# if it's not a log message updating us about possible coincidence with gravitational-waves OR labels OR json file uploads we are not interested
if alert_type == 'label':
record_label(event_dict.data, description)
if alert_type == 'update':
# is this a comment containing coinc info that needs to be parsed?
if 'comment' in alert['object'].keys():
comment = alert['object']['comment']
if 'Significant event in on-source' in comment: # got comment structure from Dipongkar
coinc_pipeline, coinc_fap = record_coinc_info(
event_dict.data, comment, alert, logger)
# begin creating the dictionary that will turn into json file
message_dict = {}
# populate text field for the GCN circular-to-be
message_dict['message'] = coinc_text.format(
graceid, coinc_fap)
message_dict['loaded_to_gracedb'] = 0
# make json string and file
message_dict = json.dumps(message_dict)
tmpfile = open('/tmp/coinc_{0}.json'.format(graceid), 'w')
tmpfile.write(message_dict)
tmpfile.close()
# make sure to load with a comment that we look for to check off that it's been loaded into gracedb
# was it an online or offline pipeline?
if 'Online' in coinc_pipeline:
event_dict.data['grb_online_json'] = message_dict
g.writeLog(
graceid,
'GRB-GW Coincidence JSON file: grb_online_json',
'/tmp/coinc_{0}.json'.format(graceid),
tagname='em_follow')
elif 'Offline' in coinc_pipeline:
event_dict.data['grb_offline_json'] = message_dict
g.writeLog(
graceid,
'GRB-GW Coincidence JSON file: grb_offline_json',
'/tmp/coinc_{0}.json'.format(graceid),
tagname='em_follow')
os.remove('/tmp/coinc_{0}.json'.format(graceid))
### alert via email
os.system(
'echo \{0}\' | mail -s \'Coincidence JSON created for {1}\' {2}'
.format(notification_text, graceid, grb_email))
# is this the json file loaded into GraceDb?
if 'GRB-GW Coincidence JSON file' in comment:
# if it is, find out which type of json it was and then message_dict['loaded_to_gracedb'] = 1
json_type = re.findall('file: (.*)', comment)[0]
message_dict = event_dict.data[json_type]
message_dict = json.loads(
message_dict) # converts string to dictionary
message_dict['loaded_to_gracedb'] = 1
# when we send to observers, message_dict['sent_to_observers'] = 1
else:
pass
saveEventDicts(approval_processorMPfiles)
return 0
#--------------------
# Appending which checks must be satisfied in preliminary_to_initial state before moving on
#--------------------
if humanscimons == 'yes':
preliminary_to_initial.append('operator_signoffCheck')
if advocates == 'yes':
preliminary_to_initial.append('advocate_signoffCheck')
#--------------------
# update information based on the alert_type
# includes extracting information from the alert
# may also include generating VOEvents and issuing them
#--------------------
# actions for each alert_type
currentstate = event_dict.data[
'currentstate'] ### actions depend on the current state
### NOTE: we handle alert_type=="new" above as well and this conditional is slightly redundant...
if alert_type == 'new':
#----------------
### pass event through PipelineThrottle
#----------------
### check if a PipelineThrottle exists for this node
group = event_dict.data['group']
pipeline = event_dict.data['pipeline']
search = event_dict.data['search']
key = generate_ThrottleKey(group, pipeline, search=search)
if queueByGraceID.has_key(key): ### a throttle already exists
if len(queueByGraceID[key]) > 1:
raise ValueError(
'too many QueueItems in SortedQueue for pipelineThrottle key=%s'
% key)
item = queueByGraceID[key][
0] ### we expect there to be only one item in this SortedQueue
else: ### we need to make a throttle!
# pull PipelineThrottle parameters from the config
if config.has_section(key):
throttleWin = config.getfloat(key, 'throttleWin')
targetRate = config.getfloat(key, 'targetRate')
requireManualReset = config.get(key, 'requireManualReset')
conf = config.getfloat(key, 'conf')
else:
throttleWin = config.getfloat('default_PipelineThrottle',
'throttleWin')
targetRate = config.getfloat('default_PipelineThrottle',
'targetRate')
requireManualReset = config.get('default_PipelineThrottle',
'requireManualReset')
conf = config.getfloat('default_PipelineThrottle', 'conf')
item = PipelineThrottle(t0,
throttleWin,
targetRate,
group,
pipeline,
search=search,
requireManualReset=False,
conf=0.9,
graceDB_url=client)
queue.insert(item) ### add to overall queue
newSortedQueue = utils.SortedQueue(
) # create sorted queue for event candidate
newSortedQueue.insert(
item) # put ForgetMeNow queue item into the sorted queue
queueByGraceID[
item.
graceid] = newSortedQueue # add queue item to the queueByGraceID
item.addEvent(graceid, t0) ### add new event to throttle
### this takes care of labeling in gracedb as necessary
if item.isThrottled():
### send some warning message?
return 0 ### we're done here because we're ignoring this event -> exit from parseAlert
# #----------------
# ### pass data to Grouper
# #----------------
# raise Warning("Grouper is not implemented yet! we're currently using a temporate groupTag and prototype code")
# '''
# need to extract groupTag from group_pipeline[_search] mapping.
# These associations should be specified in the config file, so we'll have to specify this somehow.
# probably just a "Grouper" section, with (option = value) pairs that look like (groupTag = nodeA nodeB nodeC ...)
# '''
# groupTag = 'TEMPORARY'
# ### check to see if Grouper exists for this groupTag
# if queueByGraceID.has_key(groupTag): ### at least one Grouper already exists
# ### determine if any of the existing Groupers are still accepting new triggers
# for item in queueByGraceID[groupTag]:
# if item.isOpen():
# break ### this Grouper is still open, so we'll just use it
# else: ### no Groupers are open, so we need to create one
# item = Grouper(t0, grouperWin, groupTag, eventDicts, graceDB_url=client) ### create the actual QueueItem
# queue.insert( item ) ### insert it in the overall queue
# newSortedQueue = utils.SortedQueue() ### set up the SortedQueue for queueByGraceID
# newSortedQueue.insert(item)
# queueByGraceID[groupTag] = newSortedQueue
# else: ### we need to make a Grouper
# grouperWin = config.getfloat('grouper', 'grouperWin')
# item = Grouper(t0, grouperWin, groupTag, eventDicts, graceDB_url=client) ### create the actual QueueItem
# queue.insert( item ) ### insert it in the overall queue
# newSortedQueue = utils.SortedQueue() ### set up the SortedQueue for queueByGraceID
# newSortedQueue.insert(item)
# queueByGraceID[groupTag] = newSortedQueue
# item.addEvent( graceid ) ### add this graceid to the item
return 0 ### we're done here. When Grouper makes a decision, we'll tick through the rest of the processes with a "selected" label
elif alert_type == 'label':
record_label(event_dict.data, description)
if description == 'PE_READY': ### PE_READY label was just applied. We may need to send an update alert
message = '{0} -- {1} -- Sending update VOEvent.'.format(
convertTime(), graceid)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(
graceid,
'AP: Received PE_READY label. Sending update VOEvent.',
tagname='em_follow')
process_alert(event_dict.data, 'update', g, config, logger)
else:
pass
message = '{0} -- {1} -- State: {2} --> complete.'.format(
convertTime(), graceid, currentstate)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: State: {0} --> complete.'.format(currentstate),
tagname='em_follow')
event_dict.data['currentstate'] = 'complete'
else:
pass
elif description == 'EM_READY': ### EM_READY label was just applied. We may need to send an initial alert
message = '{0} -- {1} -- Sending initial VOEvent.'.format(
convertTime(), graceid)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(
graceid,
'AP: Received EM_READY label. Sending initial VOEvent.',
tagname='em_follow')
process_alert(event_dict.data, 'initial', g, config, logger)
else:
pass
message = '{0} -- {1} -- State: {2} --> initial_to_update.'.format(
convertTime(), graceid, currentstate)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: State: {0} --> initial_to_update.'.format(
currentstate),
tagname='em_follow')
event_dict.data['currentstate'] = 'initial_to_update'
else:
pass
elif description == "EM_Throttled": ### the event is throttled and we need to turn off all processing for it
event_dict.data[
'currentstate'] = 'throttled' ### update current state
### check if we need to send retractions
voevents = event_dict.data['voevents']
if len(voevents) > 0:
if 'retraction' not in sorted(voevents)[-1]:
# there are existing VOEvents we've sent, but no retraction alert
process_alert(event_dict.data, 'retraction', g, config,
logger)
### update ForgetMeNow expiration to handle all the clean-up?
### we probably do NOT want to change the clean-up schedule because we'll still likely receive a lot of alerts about this guy
### therefore, we just retain the local data and ignore him, rather than erasing the local data and having to query to reconstruct it repeatedly as new alerts come in
# for item in queueByGraceID[graceid]: ### update expiration of the ForgetMeNow so it is immediately processed next.
# if item.name == ForgetMeNow.name:
# time.setExpiration(-np.infty )
# ### FIXME: this can break the order in SortedQueue's. We need to pop and reinsert or call a manual resort
# queue.resort() ### may be expensive but is needed to guarantee that queue remains sorted
# queueByGraceID[graceid].resort()
# break
# else:
# raise ValueError('could not find ForgetMeNow QueueItem for graceid=%s'%graceid)
elif description == "EM_Selected": ### this event was selected by a Grouper
raise NotImplementedError(
'write logic to handle \"Selected\" labels')
elif description == "EM_Superseded": ### this event was superceded by another event within Grouper
raise NotImplementedError(
'write logic to handle \"Superseded" labels')
elif (
checkLabels(description.split(), config) > 0
): ### some other label was applied. We may need to issue a retraction notice.
event_dict.data['currentstate'] = 'rejected'
### check to see if we need to send a retraction
voevents = event_dict.data['voevents']
if len(voevents) > 0:
if 'retraction' not in sorted(voevents[-1]):
# there are existing VOEvents we've sent, but no retraction alert
process_alert(event_dict.data, 'retraction', g, config,
logger)
saveEventDicts(
approval_processorMPfiles) ### save the updated eventDict to disk
return 0
### FIXME: Reed left off commenting here...
elif alert_type == 'update':
# first the case that we have a new lvem skymap
if (filename.endswith('.fits.gz') or filename.endswith('.fits')):
if 'lvem' in alert['object'][
'tag_names']: # we only care about skymaps tagged lvem for sharing with MOU partners
submitter = alert['object']['issuer'][
'display_name'] # in the past, we used to care who submitted skymaps; keeping this functionality just in case
record_skymap(event_dict.data, filename, submitter, logger)
else:
pass
# interested in iDQ information or other updates
else:
if 'comment' in alert['object'].keys():
comment = alert['object']['comment']
if re.match(
'minimum glitch-FAP', comment
): # looking to see if it's iDQ glitch-FAP information
record_idqvalues(event_dict.data, comment, logger)
elif re.match(
'resent VOEvent', comment
): # looking to see if another running instance of approval_processorMP sent a VOEvent
response = re.findall(
r'resent VOEvent (.*) in (.*)',
comment) # extracting which VOEvent was re-sent
event_dict.data[response[0][1]].append(response[0][0])
saveEventDicts(approval_processorMPfiles)
elif 'EM-Bright probabilities computed from detection pipeline' in comment: # got comment structure from Shaon G.
record_em_bright(event_dict.data, comment, logger)
elif 'Temporal coincidence with external trigger' in comment: # got comment structure from Alex U.
exttrig, coinc_far = record_coinc_info(
event_dict.data, comment, alert, logger)
# create dictionary that will become json file
message_dict = {}
grb_instrument = eventDictionaries[exttrig]['pipeline']
message_dict['message'] = em_coinc_text.format(
exttrig, grb_instrument, graceid, coinc_far)
message_dict['loaded_to_gracedb'] = 0
message_dict = json.dumps(message_dict)
# update event dictionaries for both the gw and external trigger
eventDictionaries[exttrig][
'em_coinc_json'] = message_dict # this updates the external trigger event_dict.data
event_dict.data[
'em_coinc_json'] = message_dict # this updates the gw trigger event_dict.data
# load json file to the gw gracedb page
tmpfile = open('/tmp/coinc_{0}.json'.format(graceid), 'w')
tmpfile.write(message_dict)
tmpfile.close()
g.writeLog(graceid,
'GRB-GW Coincidence JSON file: em_coinc_json',
'/tmp/coinc_{0}.json'.format(graceid),
tagname='em_follow')
os.remove('/tmp/coinc_{0}.json'.format(graceid))
# load json file to the external trigger page
tmpfile = open('/tmp/coinc_{0}.json'.format(exttrig), 'w')
tmpfile.write(message_dict)
tmpfile.close()
g.writeLog(exttrig,
'GRB-GW Coincidence JSON file: em_coinc_json',
'/tmp/coinc_{0}.json'.format(exttrig),
tagname='em_follow')
os.remove('/tmp/coinc_{0}.json'.format(exttrig))
### alert via email
os.system(
'echo \{0}\' | mail -s \'Coincidence JSON created for {1}\' {2}'
.format(notification_text, exttrig, grb_email))
saveEventDicts(approval_processorMPfiles)
elif 'GRB-GW Coincidence JSON file' in comment: # this is the comment that accompanies a loaded coinc json file
message_dict = event_dict.data['em_coinc_json']
message_dict = json.loads(
message_dict) # converts string to dictionary
message_dict['loaded_to_gracedb'] = 1
saveEventDicts(approval_processorMPfiles)
else:
pass
elif alert_type == 'signoff':
signoff_object = alert['object']
record_signoff(event_dict.data, signoff_object)
#---------------------------------------------
# run checks specific to currentstate of the event candidate
#---------------------------------------------
passedcheckcount = 0
if currentstate == 'new_to_preliminary':
time.sleep(
wait_for_hardware_inj
) #this is for those cases where we dont have the INJ label right away
queried_dict = g.events(graceid).next() #query gracedb for the graceid
event_dict.data['labels'] = queried_dict['labels'].keys(
) #get the latest labels before running checks
for Check in new_to_preliminary:
eval('event_dict.{0}()'.format(Check))
checkresult = event_dict.data[Check + 'result']
if checkresult == None:
pass
elif checkresult == False:
# because in 'new_to_preliminary' state, no need to apply DQV label
message = '{0} -- {1} -- Failed {2} in currentstate: {3}.'.format(
convertTime(), graceid, Check, currentstate)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Failed {0} in currentstate: {1}.'.format(
Check, currentstate),
tagname='em_follow')
else:
pass
message = '{0} -- {1} -- State: {2} --> rejected.'.format(
convertTime(), graceid, currentstate)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(
graceid,
'AP: State: {0} --> rejected.'.format(currentstate),
tagname='em_follow')
event_dict.data['currentstate'] = 'rejected'
else:
pass
saveEventDicts(approval_processorMPfiles)
return 0
elif checkresult == True:
passedcheckcount += 1
if passedcheckcount == len(new_to_preliminary):
message = '{0} -- {1} -- Passed all {2} checks.'.format(
convertTime(), graceid, currentstate)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Passed all {0} checks.'.format(currentstate),
tagname='em_follow')
else:
pass
message = '{0} -- {1} -- Sending preliminary VOEvent.'.format(
convertTime(), graceid)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Sending preliminary VOEvent.',
tagname='em_follow')
process_alert(event_dict.data, 'preliminary', g, config,
logger)
else:
pass
message = '{0} -- {1} -- State: {2} --> preliminary_to_initial.'.format(
convertTime(), graceid, currentstate)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: State: {0} --> preliminary_to_initial.'.format(
currentstate),
tagname='em_follow')
event_dict.data['currentstate'] = 'preliminary_to_initial'
else:
pass
labels = event_dict.data['labels']
# notify the operators if we haven't previously processed this event
instruments = event_dict.data['instruments']
for instrument in instruments:
if instrument in str(labels):
pass
else:
message = '{0} -- {1} -- Labeling {2}OPS.'.format(
convertTime(), graceid, instrument)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Labeling {0}OPS.'.format(instrument),
tagname='em_follow')
g.writeLabel(graceid, '{0}OPS'.format(instrument))
else:
pass
# notify the advocates if we haven't previously processed this event
if 'ADV' in str(labels):
pass
else:
message = '{0} -- {1} -- Labeling ADVREQ.'.format(
convertTime(), graceid)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Labeling ADVREQ.',
tagname='em_follow')
g.writeLabel(graceid, 'ADVREQ')
os.system(
'echo \'{0}\' | mail -s \'{1} passed criteria for follow-up\' {2}'
.format(advocate_text, graceid, advocate_email))
# expose event to LV-EM
url_perm_base = g.service_url + urllib.quote(
'events/{0}/perms/gw-astronomy:LV-EM:Observers/'.
format(graceid))
for perm in ['view', 'change']:
url = url_perm_base + perm
#g.put(url)
else:
pass
saveEventDicts(approval_processorMPfiles)
return 0
elif currentstate == 'preliminary_to_initial':
for Check in preliminary_to_initial:
eval('event_dict.{0}()'.format(Check))
checkresult = event_dict.data[Check + 'result']
if checkresult == None:
pass
elif checkresult == False:
message = '{0} -- {1} -- Failed {2} in currentstate: {3}.'.format(
convertTime(), graceid, Check, currentstate)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Failed {0} in currentstate: {1}.'.format(
Check, currentstate),
tagname='em_follow')
else:
pass
message = '{0} -- {1} -- State: {2} --> rejected.'.format(
convertTime(), graceid, currentstate)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(
graceid,
'AP: State: {0} --> rejected.'.format(currentstate),
tagname='em_follow')
event_dict.data['currentstate'] = 'rejected'
else:
pass
# need to set DQV label so long as it isn't the operator_signoffCheck or advocate_signoffCheck
if 'signoffCheck' in Check:
message = '{0} -- {1} -- Not labeling DQV because signoffCheck is separate from explicit data quality checks.'.format(
convertTime(), graceid)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(
graceid,
'AP: Not labeling DQV because signoffCheck is separate from explicit data quality checks.',
tagname='em_follow')
else:
pass
else:
message = '{0} -- {1} -- Labeling DQV.'.format(
convertTime(), graceid)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Labeling DQV.',
tagname='em_follow')
g.writeLabel(graceid, 'DQV')
else:
pass
saveEventDicts(approval_processorMPfiles)
return 0
elif checkresult == True:
passedcheckcount += 1
if Check == 'have_lvem_skymapCheck': # we want to send skymaps out as quickly as possible, even if humans have not vetted the event
process_alert(
event_dict.data, 'preliminary', g, config, logger
) # if it turns out we've sent this alert with this skymap before, the process_alert function will just not send this repeat
if passedcheckcount == len(preliminary_to_initial):
message = '{0} -- {1} -- Passed all {2} checks.'.format(
convertTime(), graceid, currentstate)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Passed all {0} checks.'.format(currentstate),
tagname='em_follow')
else:
pass
message = '{0} -- {1} -- Labeling EM_READY.'.format(
convertTime(), graceid)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Labeling EM_READY.',
tagname='em_follow')
g.writeLabel(graceid, 'EM_READY')
else:
pass
saveEventDicts(approval_processorMPfiles)
return 0
elif currentstate == 'initial_to_update':
for Check in initial_to_update:
eval('event_dict.{0}()'.format(Check))
checkresult = event_dict.data[Check + 'result']
if checkresult == None:
pass
elif checkresult == False:
# need to set DQV label
message = '{0} -- {1} -- Failed {2} in currentstate: {3}.'.format(
convertTime(), graceid, Check, currentstate)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Failed {0} in currentstate: {1}.'.format(
Check, currentstate),
tagname='em_follow')
else:
pass
message = '{0} -- {1} -- State: {2} --> rejected.'.format(
convertTime(), graceid, currentstate)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(
graceid,
'AP: State: {0} --> rejected.'.format(currentstate),
tagname='em_follow')
event_dict.data['currentstate'] = 'rejected'
else:
pass
message = '{0} -- {1} -- Labeling DQV.'.format(
convertTime(), graceid)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Labeling DQV.',
tagname='em_follow')
g.writeLabel(graceid, 'DQV')
else:
pass
saveEventDicts(approval_processorMPfiles)
return 0
elif checkresult == True:
passedcheckcount += 1
if passedcheckcount == len(initial_to_update):
message = '{0} -- {1} -- Passed all {2} checks.'.format(
convertTime(), graceid, currentstate)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Passed all {0} checks.'.format(currentstate),
tagname='em_follow')
else:
pass
message = '{0} -- {1} -- Labeling PE_READY.'.format(
convertTime(), graceid)
if loggerCheck(event_dict.data, message) == False:
logger.info(message)
g.writeLog(graceid,
'AP: Labeling PE_READY.',
tagname='em_follow')
g.writeLabel(graceid, 'PE_READY')
else:
pass
saveEventDicts(approval_processorMPfiles)
return 0
else:
saveEventDicts(approval_processorMPfiles)
return 0
if dictionary[event]['FAR'] <= FAR_thresh:
#if FAR threshold is exceeded, write trigtimes and timeslides to files
e_new += 1
rr_trigtimes.write('%s\n'%dictionary[event]['gpstime'])
rr_timeslides.write('%s\n'%" ".join([dictionary[event]['timeslides'][ifo] for ifo in ifos]))
#If event is 0-lag, write to GraceDb if enabled
if lag == '0lag' and gdb_flag:
#Save dictionary as json file
dic_path = rundir+'/GDB/%s.json'%('%s-%s'%(dictionary[event]['gpstime'],event))
with open(dic_path, 'wt') as fp:
json.dump(dictionary[event], fp)
#Upload dictionary to GraceDb
response = gdb.createEvent('Burst','LIB',dic_path, search='AllSky', filecontents=None)
#Parse GraceDb ID
response = json.loads(response.read())
gid = str(response["graceid"])
#Update GraceDb log with post-proc pages
# gdb.writeLog(gid, message="Preliminary results: f_0 = %s, Q = %s"%(dictionary[event]['frequency'],dictionary[event]['quality']))
gdb.writeLog(gid, message="Preliminary results: BSN = %s, BCI = %s, oSNR = %s"%(dictionary[event]['BSN'],dictionary[event]['BCI'],dictionary[event]['Omicron SNR']))
gdb.writeLog(gid, message="Follow-up results will be written: https://ldas-jobs.ligo.caltech.edu/~ryan.lynch/%s/%s/followup/%s/%s/%s/posplots.html"%(lib_label,lag,'%s_%s_%s'%("".join(ifos),actual_start,stride-overlap),'%s-%s'%(dictionary[event]['gpstime'],e_new),"".join(ifos)))
rr_trigtimes.close()
rr_timeslides.close()
#run pipeline to make dag for reruns
os.system('%s/lalinference_2ndpipe_beta.py %s/runfiles/LIB_%s_reruns_beta.ini -r %s/LIB_%s_rr/ -p /usr1/ryan.lynch/logs/ -g %s/PostProc/LIB_trigs/LIB_%s_times_rr_%s.txt'%(infodir, rundir, lag, rundir, lag, rundir, lag, "".join(ifos)))
for filename in all_files:
if opts.classifier == idq.extract_fap_name(filename): # and ifo in filename: ### this last bit not needed?
if 'rank' in filename:
rank_filenames.append(filename)
if 'fap' in filename:
fap_filenames.append(filename)
rank_filenames.sort()
fap_filenames.sort()
if (not rank_filenames) or (not fap_filenames): # we couldn't find either rank or fap files
# exit gracefully
if opts.verbose:
print "no iDQ timeseries for %s at %s"%(opts.classifier, ifo)
if not opts.skip_gracedb_upload:
gracedb.writeLog(opts.gracedb_id, message="No iDQ timeseries for %s at %s"%(opts.classifier, ifo))
sys.exit(0)
#===================================================================================================
### process FAP files
if opts.verbose:
print "reading fap timeseries from:"
for filename in fap_filenames:
print '\t' + filename
#fig = isp.plt.figure(figsize=isp.rank_timeseries_figsize)
#rnk_ax = fig.add_axes(isp.rank_timeseries_axpos)
#fap_ax = rnk_ax.twinx()
fig = isp.plt.figure(figsize=isp.rank_splittimeseries_figsize)
rnk_ax = fig.add_axes(isp.rank_splittimeseries_axpos)
# ## Find relevant files
# #########################################
if opts.verbose:
print 'Finding relevant *glitch*.xml files'
gchxml_filenames = sorted([filename for filename in
idq.get_all_files_in_range(opts.input_dir,
opts.start, opts.end, pad=0, suffix='.xml.gz'
) if opts.classifier in filename.split('/'
)[-1] and 'glitch' in filename
and opts.ifo in filename])
if not gchxml_filenames:
# no files found, print the message, and exit
if not opts.skip_gracedb_upload:
gracedb.writeLog(opts.gracedb_id, message="No iDQ glitch tables data from "+opts.classifier+" available for the candidate at "+opts.ifo)
print "No glitch files found, exiting."
sys.exit(0)
if opts.verbose:
print "Found:"
for filename in gchxml_filenames:
print '\t' + filename
# ##############################################################
# ## Load and Merge xml files using in-memory sqlite database
# ##############################################################
#
# load files into database
connection, cursor = idq_tables_dbutils.load_xml_files_into_database(\
# get start and end time for our look-up routines
#========================
event_gps_time = float(gdb_entry['gpstime'])
gps_start = event_gps_time - time_before
gps_end = event_gps_time + time_after
plotting_gps_start = event_gps_time - plotting_time_before
plotting_gps_end = event_gps_time + plotting_time_after
performance_gps_start = event_gps_time - performance_time_before
performance_gps_end = event_gps_time + performance_time_after
logger.info("Started searching for iDQ information within [%.3f, %.3f] at %s"%(gps_start, gps_end, ifo))
if not options.skip_gracedb_upload:
gracedb.writeLog(gdb_id, message="Started searching for iDQ information within [%.3f, %.3f] at %s"%(gps_start, gps_end, ifo))
#=================================================
# LOGIC for waiting for idq data
#=================================================
### figure out if we need to wait for time to pass
wait = gps_end - (idq.nowgps()+delay)
if wait > 0:
logger.info("waiting %.2f seconds until we pass gps=%.3f"%(wait, gps_end))
time.sleep(wait)
if options.realtime_log:
### now we need to parse the realtime log to figure out where the realtime job is
logger.info("parsing %s to extract idq-realtime state"%options.realtime_log)
realtime_log = open(options.realtime_log, "r") ### open realtime log for reading
#=================================================
### read in the config file
if opts.verbose:
print "reading config from : %s\nand setting up schedule of checks"%(configfile)
config = ConfigParser.SafeConfigParser()
config.read(configfile)
### set up the schedule of checks
schedule = checks.config_to_schedule( config, event_type, verbose=opts.verbose )
### annotate gracedb with list of scheduled checks
if opts.annotate_gracedb:
log = "event_supervisor scheduled to check: %s"%(", ".join([description for dt, foo, kwargs, email, description in schedule]))
gracedb.writeLog( gdb_id, log )
### perform the scheduled checks
if opts.verbose:
print "performing schedule"
#to = time.time() ### start time of our checking proceedures
to = time.mktime(time.strptime(gdb_entry['created'], '%Y-%m-%d %H:%M:%S %Z')) ### parse creation time from GraceDB
for dt, foo, kwargs, email, description in schedule:
### check current time stamp
wait = dt - (time.time()-to)
if wait > 0:
if opts.verbose:
print "\nwaiting %.3f seconds before performing : %s\n"%(wait, description)
time.sleep( wait )
def upload(
self,
fname,
psds,
low_frequency_cutoff,
testing=True,
extra_strings=None,
):
"""Upload this trigger to gracedb
Parameters
----------
fname: str
The name to give the xml file associated with this trigger
pds: dict of pybc.types.FrequencySeries
A ifo keyed dictionary of psds to be uploaded in association
with this trigger.
low_frequency_cutoff: float
The low frequency cutoff of the psds.
testing: bool
Switch to determine if the upload should be sent to gracedb as a
test trigger (True) or a production trigger (False)
"""
from ligo.gracedb.rest import GraceDb
# first of all, make sure the event and PSDs are saved on disk
# as GraceDB operations can fail later
self.save(fname)
psds_lal = {}
for ifo in psds:
psd = psds[ifo]
kmin = int(low_frequency_cutoff / psd.delta_f)
fseries = lal.CreateREAL8FrequencySeries(
"psd", psd.epoch, low_frequency_cutoff, psd.delta_f,
lal.StrainUnit**2 / lal.HertzUnit,
len(psd) - kmin)
fseries.data.data = psd.numpy()[kmin:] / pycbc.DYN_RANGE_FAC**2.0
psds_lal[ifo] = fseries
psd_xmldoc = make_psd_xmldoc(psds_lal)
psd_xml_path = os.path.splitext(fname)[0] + '-psd.xml.gz'
ligolw_utils.write_filename(psd_xmldoc, psd_xml_path, gz=True)
if self.upload_snr_series:
snr_series_fname = os.path.splitext(fname)[0] + '.hdf'
for ifo in self.snr_series:
self.snr_series[ifo].save(snr_series_fname,
group='%s/snr' % ifo)
self.snr_series_psd[ifo].save(snr_series_fname,
group='%s/psd' % ifo)
# try connecting to GraceDB
try:
gracedb = GraceDb(self.gracedb_server) \
if self.gracedb_server is not None else GraceDb()
except Exception as exc:
logging.error('Cannot connect to GraceDB')
logging.error(str(exc))
logging.error('Carrying on, but event %s will NOT be uploaded!',
fname)
return None
# create GraceDB event
group = 'Test' if testing else 'CBC'
try:
r = gracedb.createEvent(group, "pycbc", fname, "AllSky").json()
except Exception as exc:
logging.error('Cannot create GraceDB event')
logging.error(str(exc))
logging.error('Carrying on, but event %s will NOT be uploaded!',
fname)
return None
logging.info("Uploaded event %s", r["graceid"])
if self.is_hardware_injection:
try:
gracedb.writeLabel(r['graceid'], 'INJ')
except Exception as exc:
logging.error("Cannot tag event %s as an injection",
r["graceid"])
logging.error(str(exc))
logging.info("Tagging event %s as an injection", r["graceid"])
# upload PSDs
try:
gracedb.writeLog(r["graceid"],
"PyCBC PSD estimate from the time of event",
"psd.xml.gz",
open(psd_xml_path, "rb").read(), "psd").json()
except Exception as exc:
logging.error("Cannot upload PSDs for event %s", r["graceid"])
logging.error(str(exc))
logging.info("Uploaded PSDs for event %s", r["graceid"])
# add other tags and comments
try:
gracedb.writeLog(
r["graceid"],
"Using PyCBC code hash %s" % pycbc_version.git_hash).json()
extra_strings = [] if extra_strings is None else extra_strings
for text in extra_strings:
gracedb.writeLog(r["graceid"], text).json()
except Exception as exc:
logging.error("Cannot write comments for event %s", r["graceid"])
logging.error(str(exc))
# upload SNR series in HDF format
if self.upload_snr_series:
try:
gracedb.writeFile(r['graceid'], snr_series_fname)
except Exception as exc:
logging.error("Cannot upload HDF SNR series for event %s",
r["graceid"])
logging.error(str(exc))
return r['graceid']
snglfits.make_confidence_regions(
verbose=opts.verbose) ### make confidence region stuff
snglfits.make_antenna_patterns(
verbose=opts.verbose) ### compute antenna pattern statistics
#-----------
### generate final html document
htmlname = snglfits.write(verbose=opts.verbose)
### upload to GraceDb
if not opts.skip_gracedb_upload:
if opts.verbose:
print "uploading %s to GraceDb(%s)" % (htmlname, opts.graceDbURL)
gdb = GraceDb(opts.graceDbURL)
gdbdir = os.path.join(opts.graceDbURL, '..', 'events', opts.graceid,
'files')
fitsbase = os.path.basename(fits)
gdb.writeLog(
opts.graceid,
message=
'skymap summary for <a href="%s">%s</a> can be found <a href=\"%s\">here</a>'
% (os.path.join(gdbdir, fitsbase), fitsbase,
os.path.join(gdbdir, os.path.basename(htmlname))),
filename=htmlname,
tagname=fits2html.standard_tagname + opts.graceDb_tagname +
opts.graceDb_html_tagname)
#========================
# get start and end time for our look-up routines
#========================
event_gps_time = float(gdb_entry['gpstime'])
gps_start = event_gps_time - time_before
gps_end = event_gps_time + time_after
plotting_gps_start = event_gps_time - plotting_time_before
plotting_gps_end = event_gps_time + plotting_time_after
logger.info("Started searching for iDQ information within [%.3f, %.3f] at %s" %
(gps_start, gps_end, ifo))
gracedb.writeLog(
gdb_id,
message="Started searching for iDQ information within [%.3f, %.3f] at %s" %
(gps_start, gps_end, ifo))
#=================================================
# LOGIC for waiting for idq data
#=================================================
### figure out if we need to wait for time to pass
wait = gps_end - (idq.nowgps() + delay)
if wait > 0:
logger.info("waiting %.2f seconds until we pass gps=%.3f" %
(wait, gps_end))
time.sleep(wait)
### now we need to parse the realtime log to figure out where the realtime job is
realtime_logname = config.get("general", "realtime_log")
logger.info("parsing %s to extract idq-realtime state" % realtime_logname)
def upload(self, fname, psds, low_frequency_cutoff,
testing=True,
extra_strings=None,
):
"""Upload this trigger to gracedb
Parameters
----------
fname: str
The name to give the xml file associated with this trigger
pds: dict of pybc.types.FrequencySeries
A ifo keyed dictionary of psds to be uploaded in association
with this trigger.
low_frequency_cutoff: float
The low frequency cutoff of the psds.
testing: bool
Switch to determine if the upload should be sent to gracedb as a
test trigger (True) or a production trigger (False)
"""
from ligo.gracedb.rest import GraceDb
# first of all, make sure the event and PSDs are saved on disk
# as GraceDB operations can fail later
self.save(fname)
psds_lal = {}
for ifo in psds:
psd = psds[ifo]
kmin = int(low_frequency_cutoff / psd.delta_f)
fseries = lal.CreateREAL8FrequencySeries(
"psd", psd.epoch, low_frequency_cutoff, psd.delta_f,
lal.StrainUnit**2 / lal.HertzUnit, len(psd) - kmin)
fseries.data.data = psd.numpy()[kmin:] / pycbc.DYN_RANGE_FAC ** 2.0
psds_lal[ifo] = fseries
psd_xmldoc = make_psd_xmldoc(psds_lal)
psd_xml_path = os.path.splitext(fname)[0] + '-psd.xml.gz'
ligolw_utils.write_filename(psd_xmldoc, psd_xml_path, gz=True)
if self.upload_snr_series:
snr_series_fname = os.path.splitext(fname)[0] + '.hdf'
for ifo in self.snr_series:
self.snr_series[ifo].save(snr_series_fname,
group='%s/snr' % ifo)
self.snr_series_psd[ifo].save(snr_series_fname,
group='%s/psd' % ifo)
# try connecting to GraceDB
try:
gracedb = GraceDb(self.gracedb_server) \
if self.gracedb_server is not None else GraceDb()
except Exception as exc:
logging.error('Cannot connect to GraceDB')
logging.error(str(exc))
logging.error('Carrying on, but event %s will NOT be uploaded!', fname)
return None
# create GraceDB event
group = 'Test' if testing else 'CBC'
try:
r = gracedb.createEvent(group, "pycbc", fname, "AllSky").json()
except Exception as exc:
logging.error('Cannot create GraceDB event')
logging.error(str(exc))
logging.error('Carrying on, but event %s will NOT be uploaded!', fname)
return None
logging.info("Uploaded event %s", r["graceid"])
if self.is_hardware_injection:
try:
gracedb.writeLabel(r['graceid'], 'INJ')
except Exception as exc:
logging.error("Cannot tag event %s as an injection", r["graceid"])
logging.error(str(exc))
logging.info("Tagging event %s as an injection", r["graceid"])
# upload PSDs
try:
gracedb.writeLog(r["graceid"],
"PyCBC PSD estimate from the time of event",
"psd.xml.gz", open(psd_xml_path, "rb").read(),
"psd").json()
except Exception as exc:
logging.error("Cannot upload PSDs for event %s", r["graceid"])
logging.error(str(exc))
logging.info("Uploaded PSDs for event %s", r["graceid"])
# add other tags and comments
try:
gracedb.writeLog(r["graceid"],
"Using PyCBC code hash %s" % pycbc_version.git_hash).json()
extra_strings = [] if extra_strings is None else extra_strings
for text in extra_strings:
gracedb.writeLog(r["graceid"], text).json()
except Exception as exc:
logging.error("Cannot write comments for event %s", r["graceid"])
logging.error(str(exc))
# upload SNR series in HDF format
if self.upload_snr_series:
try:
gracedb.writeFile(r['graceid'], snr_series_fname)
except Exception as exc:
logging.error("Cannot upload HDF SNR series for event %s",
r["graceid"])
logging.error(str(exc))
return r['graceid']
<tr><td rowspan=2>bandwidth (Hz)</td><td>H1</td><td align=right>{bwH}</td><td align=right>{bwHlow}</td><td align=right>{bwHhigh}</td></tr> \
<tr><td>L1</td><td align=right>{bwL}</td><td align=right>{bwLlow}</td><td align=right>{bwLhigh}</td></tr> \
<tr><td rowspan=2>duration (s)</td><td>H1</td><td align=right>{durH}</td><td align=right>{durHlow}</td><td align=right>{durHhigh}</td></tr> \
<tr><td>L1</td><td align=right>{durL}</td><td align=right>{durLlow}</td><td align=right>{durLhigh}</td></tr></table> \
'.format(freqH=freq[0][0],freqHlow=freq[0][1],freqHhigh=freq[0][2],freqL=freq[1][0],freqLlow=freq[1][1],freqLhigh=freq[1][2],bwH=bandwidth[0][0],bwHlow=bandwidth[0][1],bwHhigh=bandwidth[0][2],bwL=bandwidth[1][0],bwLlow=bandwidth[1][1],bwLhigh=bandwidth[1][2],durH=dur_c[0],durL=dur_c[1],durHlow=dur_low[0],durLlow=dur_low[1],durHhigh=dur_high[0],durLhigh=dur_high[1])
BFtable = '<table> \
<tr><th colspan=2>BWB Bayes Factors</th></tr> \
<tr><td>lnBSG</td><td align=right>{BSG}+/-{errBSG}</td></tr> \
<tr><td>lnBSN</td><td align=right>{BSN}+/-{errBSN}</td></tr> \
</table>'.format(BSG=BSG,BSN=BSN,errBSG=err_SG,errBSN=err_SN)
# Sky map
skyname = glob.glob('skymap*.fits')[0]
os.system('cp {sky} BW_skymap.fits'.format(sky=skyname)) # (change name so it's clear on GraceDB which skymap is ours)
#skytag = ["sky_loc","lvem"]
skytag = ["sky_loc"]
# Actually send info to gracedb
gracedb = GraceDb()
#gracedb.writeLog(graceid, "BayesWave Skymap image", filename='plots/skymap.png', tagname='sky_loc')
gracedb.writeLog(graceid, "BayesWave skymap FITS", filename='BW_skymap.fits', tagname=skytag)
gracedb.writeLog(graceid, "<a href='https://ldas-jobs.ligo.caltech.edu/~meg.millhouse/O1/zero_lag/job_{0}'>BWB Follow-up results</a>".format(graceid), tagname='pe')
gracedb.writeLog(graceid,paramtable,tagname='pe')
gracedb.writeLog(graceid,BFtable,tagname='pe')
os.chdir('..')
os.system('cp -r '+dirname+' /home/meg.millhouse/public_html/O1/zero_lag/job_'+graceid)
for filename in all_files:
if opts.classifier == idq.extract_fap_name(filename): # and opts.ifo in filename: ### this last bit not needed?
if 'rank' in filename:
rank_filenames.append(filename)
if 'fap' in filename:
fap_filenames.append(filename)
rank_filenames.sort()
fap_filenames.sort()
if (not rank_filenames) or (not fap_filenames): # we couldn't find either rank or fap files
# exit gracefully
if opts.verbose:
print "no iDQ timeseries for %s at %s"%(opts.classifier, opts.ifo)
if not opts.skip_gracedb_upload:
gracedb.writeLog(opts.gracedb_id, message="No iDQ timeseries for %s at %s"%(opts.classifier, opts.ifo))
sys.exit(0)
#=================================================
# define plot
fig = plt.figure()
r_ax = plt.subplot(1, 1, 1)
f_ax = r_ax.twinx()
f_ax.set_yscale('log') # this may be fragile if fap=0 for all points in the plot. That's super rare, so maybe we don't have to worry about it?
r_ax.set_title(opts.ifo)
#=================================================
# RANK
#=================================================
# #########################################
# ## Find relevant files
# #########################################
if opts.verbose:
print 'Finding relevant *glitch*.xml files'
gchxml_filenames = sorted([filename for filename in
idq.get_all_files_in_range(opts.input_dir, opts.start, opts.end, pad=0, suffix='.xml.gz')
if opts.classifier == idq.extract_xml_name(filename)
and 'glitch' in filename
and opts.ifo in filename])
if not gchxml_filenames:
# no files found, print the message, and exit
if not opts.skip_gracedb_upload:
gracedb.writeLog(opts.gracedb_id, message="No iDQ glitch tables data from "+opts.classifier+" available for the candidate at "+opts.ifo)
print "No glitch files found, exiting."
sys.exit(0)
if opts.verbose:
print "Found:"
for filename in gchxml_filenames:
print '\t' + filename
# ##############################################################
# ## Load and Merge xml files using in-memory sqlite database
# ##############################################################
#
# load files into database
connection, cursor = idq_tables_dbutils.load_xml_files_into_database(\
dt = [event.livetime(segs[fapThr][0])/T for fapThr in opts.FAPthr]
maxFAP = [segs[fapThr][1] for fapThr in opts.FAPthr]
### write json for calibration check
jsonfilename = idq.gdb_calib_json( gdbdir, ifo, opts.classifier, filetag, opts.start, opts.end-opts.start )
if opts.verbose:
print " %s"%jsonfilename
file_obj = open(jsonfilename, "w")
file_obj.write( json.dumps( {opts.classifier:{'nominal FAP':opts.FAPthr, 'maximum reported FAP':maxFAP, 'observed deadtime':dt, 'duration':T} } ) )
file_obj.close()
if not opts.skip_gracedb_upload:
message = "iDQ calibration sanity check for %s at %s within [%.3f, %.3f]"%(opts.classifier, ifo, opts.start, opts.end)
if opts.verbose:
print " "+message
gracedb.writeLog( opts.gracedb_id, message=message, filename=jsonfilename )
### plot calibration check
fig = isp.plt.figure()
ax = fig.add_axes( isp.default_axpos )
if np.any(np.array(maxFAP)>0) and np.any(np.array(dt)>0):
ax.loglog(maxFAP, dt, marker='o', linestyle='none')
else:
ax.plot(maxFAP, dt, marker='o', linestyle='none')
ax.set_xlabel('Nominal FAP')
ax.set_ylabel('Observed deadtime')
ax.grid(True)
xmin = 1e-4
<tr><td>lnBSN</td><td align=right>{BSN}+/-{errBSN}</td></tr> \
</table>'.format(BSG=BSG, BSN=BSN, errBSG=err_SG, errBSN=err_SN)
# Sky map
skyname = glob.glob('skymap*.fits')[0]
os.system('cp {sky} BW_skymap.fits'.format(sky=skyname)
) # (change name so it's clear on GraceDB which skymap is ours)
#skytag = ["sky_loc","lvem"]
skytag = ["sky_loc"]
# Actually send info to gracedb
gracedb = GraceDb()
#gracedb.writeLog(graceid, "BayesWave Skymap image", filename='plots/skymap.png', tagname='sky_loc')
gracedb.writeLog(graceid,
"BayesWave skymap FITS",
filename='BW_skymap.fits',
tagname=skytag)
gracedb.writeLog(
graceid,
"<a href='https://ldas-jobs.ligo.caltech.edu/~meg.millhouse/O1/zero_lag/job_{0}'>BWB Follow-up results</a>"
.format(graceid),
tagname='pe')
gracedb.writeLog(graceid, paramtable, tagname='pe')
gracedb.writeLog(graceid, BFtable, tagname='pe')
os.chdir('..')
os.system('cp -r ' + dirname +
' /home/meg.millhouse/public_html/O1/zero_lag/job_' + graceid)
if 'rank' in filename:
rank_filenames.append(filename)
if 'fap' in filename:
fap_filenames.append(filename)
rank_filenames.sort()
fap_filenames.sort()
if (not rank_filenames) or (
not fap_filenames): # we couldn't find either rank or fap files
# exit gracefully
if opts.verbose:
print "no iDQ timeseries for %s at %s" % (opts.classifier, opts.ifo)
if not opts.skip_gracedb_upload:
gracedb.writeLog(opts.gracedb_id,
message="No iDQ timeseries for %s at %s" %
(opts.classifier, opts.ifo))
sys.exit(0)
#=================================================
# define plot
fig = plt.figure()
r_ax = plt.subplot(1, 1, 1)
f_ax = r_ax.twinx()
f_ax.set_yscale(
'log'
) # this may be fragile if fap=0 for all points in the plot. That's super rare, so maybe we don't have to worry about it?
r_ax.set_title(opts.ifo)
if event['pipeline'].lower() not in allowed_pipelines:
print " not allowed to label this pipeline"
sys.exit(1)
logs = gdb.logs(gid).json()['log']
result = dict((ifo, 1) for ifo in ifos)
for log in logs:
comment = log['comment']
if "minimum glitch-FAP for" in comment:
gFAP = float(comment.split()[-1])
for ifo in ifos:
if ifo in comment:
result[ifo] = gFAP
break
jFAP = np.prod(result.values()) ### take product of gFAPs for 2 IFOs
if jFAP <= jFAP_thr:
message = "iDQ veto generator computed joint glitch-FAP : %.3e <= %.3e; <b>This is probably a glitch</b> and I am applying a DQV label" % (
jFAP, jFAP_thr)
if annotate_gracedb:
gdb.writeLabel(gid, 'DQV')
gdb.writeLog(gid, message=message, tagname=['data_quality'])
print message
else:
message = "iDQ veto generator computed joint glitch-FAP : %.3e > %.3e; <b>This is probably not a glitch</b> and I am not applying a label" % (
jFAP, jFAP_thr)
if annotate_gracedb:
gdb.writeLog(gid, message=message, tagname=['data_quality'])
print message
def upload(
self,
fname,
psds,
low_frequency_cutoff,
testing=True,
extra_strings=None,
):
"""Upload this trigger to gracedb
Parameters
----------
fname: str
The name to give the xml file associated with this trigger
pds: dict of pybc.types.FrequencySeries
A ifo keyed dictionary of psds to be uploaded in association
with this trigger.
low_frequency_cutoff: float
The low frequency cutoff of the psds.
testing: bool
Switch to determine if the upload should be sent to gracedb as a
test trigger (True) or a production trigger (False)
"""
from ligo.gracedb.rest import GraceDb
self.save(fname)
extra_strings = [] if extra_strings is None else extra_strings
if testing:
group = 'Test'
else:
group = 'CBC'
gracedb = GraceDb()
r = gracedb.createEvent(group, "pycbc", fname, "AllSky").json()
logging.info("Uploaded event %s.", r["graceid"])
if self.is_hardware_injection:
gracedb.writeLabel(r['graceid'], 'INJ')
logging.info("Tagging event %s as an injection", r["graceid"])
psds_lal = {}
for ifo in psds:
psd = psds[ifo]
kmin = int(low_frequency_cutoff / psd.delta_f)
fseries = lal.CreateREAL8FrequencySeries(
"psd", psd.epoch, low_frequency_cutoff, psd.delta_f,
lal.StrainUnit**2 / lal.HertzUnit,
len(psd) - kmin)
fseries.data.data = psd.numpy()[kmin:] / pycbc.DYN_RANGE_FAC**2.0
psds_lal[ifo] = fseries
psd_xmldoc = make_psd_xmldoc(psds_lal)
ligolw_utils.write_filename(psd_xmldoc, "tmp_psd.xml.gz", gz=True)
gracedb.writeLog(r["graceid"],
"PyCBC PSD estimate from the time of event",
"psd.xml.gz",
open("tmp_psd.xml.gz", "rb").read(), "psd").json()
gracedb.writeLog(r["graceid"], "using pycbc code hash %s" %
pycbc_version.git_hash).json()
for text in extra_strings:
gracedb.writeLog(r["graceid"], text).json()
logging.info("Uploaded file psd.xml.gz to event %s.", r["graceid"])
if self.upload_snr_series:
snr_series_fname = fname + '.hdf'
for ifo in self.snr_series:
self.snr_series[ifo].save(snr_series_fname,
group='%s/snr' % ifo)
self.snr_series_psd[ifo].save(snr_series_fname,
group='%s/psd' % ifo)
GraceDb().writeFile(r['graceid'], snr_series_fname)
return r['graceid']
graceid = alert['graceid']
if opts.verbose:
print "found graceid=%s"%graceid
### figure out if this is the
if (alert['alert_type']=='update') and alert['file'] and alert['file'].endswith('js') and ("_los" in alert['file']): ### this could be fragile...
### download file
los = json.loads( gracedb.files( graceid, alert['file'] ).read() )
### iterate through IFO pairs
for ifo_pair, data in los.items():
mi = data['MI'] # mutual information
hj = data['Hj'] # joint entropy
mid = mi / hj # mutual information distance
if config.has_section(ifo_pair): ### assume ifo_pair is alphabetically ordered...
thr = config.getfloat(ifo_pair, 'thr')
if mid > thr:
labels = config.get(ifo_pair, 'labels').split()
gracedb.writeLog( graceid, message='Mutual Information Distance for %s in %s line-of-sight frame is above %.3f! Applying labels: %s'%(alert['file'].split('_los')[0], ifo_pair, thr, ", ".join(labels)))
for label in labels:
gracedb.writeLabel( graceid, label )
else:
gracedb.writeLog( graceid, message="Mutual Information Distnace for %s in %s line-of-sight frame is below %.3f."%(alert['file'].split('_los')[0], ifo_pair, thr, ", ".join(labels)))
else:
if opts.verbose:
print "ignoring..."
def upload(self, fname, gracedb_server=None, testing=True,
extra_strings=None):
"""Upload this trigger to gracedb
Parameters
----------
fname: str
The name to give the xml file associated with this trigger
gracedb_server: string, optional
URL to the GraceDB web API service for uploading the event.
If omitted, the default will be used.
testing: bool
Switch to determine if the upload should be sent to gracedb as a
test trigger (True) or a production trigger (False).
"""
from ligo.gracedb.rest import GraceDb
# first of all, make sure the event is saved on disk
# as GraceDB operations can fail later
self.save(fname)
if self.snr_series is not None:
if fname.endswith('.xml.gz'):
snr_series_fname = fname.replace('.xml.gz', '.hdf')
else:
snr_series_fname = fname.replace('.xml', '.hdf')
for ifo in self.snr_series:
self.snr_series[ifo].save(snr_series_fname,
group='%s/snr' % ifo)
self.psds[ifo].save(snr_series_fname,
group='%s/psd' % ifo)
gid = None
try:
# try connecting to GraceDB
gracedb = GraceDb(gracedb_server) \
if gracedb_server is not None else GraceDb()
# create GraceDB event
group = 'Test' if testing else 'CBC'
r = gracedb.createEvent(group, "pycbc", fname, "AllSky").json()
gid = r["graceid"]
logging.info("Uploaded event %s", gid)
if self.is_hardware_injection:
gracedb.writeLabel(gid, 'INJ')
logging.info("Tagging event %s as an injection", gid)
# upload PSDs. Note that the PSDs are already stored in the
# original event file and we just upload a copy of that same file
# here. This keeps things as they were in O2 and can be removed
# after updating the follow-up infrastructure
psd_fname = 'psd.xml.gz' if fname.endswith('.gz') else 'psd.xml'
gracedb.writeLog(gid, "PyCBC PSD estimate from the time of event",
psd_fname, open(fname, "rb").read(), "psd")
logging.info("Uploaded PSDs for event %s", gid)
# add other tags and comments
gracedb.writeLog(
gid, "Using PyCBC code hash %s" % pycbc_version.git_hash)
extra_strings = [] if extra_strings is None else extra_strings
for text in extra_strings:
gracedb.writeLog(gid, text)
# upload SNR series in HDF format
if self.snr_series is not None:
gracedb.writeFile(gid, snr_series_fname)
except Exception as exc:
logging.error('Something failed during the upload/annotation of '
'event %s on GraceDB. The event may not have been '
'uploaded!', fname)
logging.error(str(exc))
return gid
def upload(self,
fname,
gracedb_server=None,
testing=True,
extra_strings=None,
search='AllSky'):
"""Upload this trigger to gracedb
Parameters
----------
fname: str
The name to give the xml file associated with this trigger
gracedb_server: string, optional
URL to the GraceDB web API service for uploading the event.
If omitted, the default will be used.
testing: bool
Switch to determine if the upload should be sent to gracedb as a
test trigger (True) or a production trigger (False).
search: str
String going into the "search" field of the GraceDB event.
"""
from ligo.gracedb.rest import GraceDb
import matplotlib
matplotlib.use('Agg')
import pylab as pl
# first of all, make sure the event is saved on disk
# as GraceDB operations can fail later
self.save(fname)
gid = None
try:
# try connecting to GraceDB
gracedb = GraceDb(gracedb_server) \
if gracedb_server is not None else GraceDb()
# create GraceDB event
group = 'Test' if testing else 'CBC'
r = gracedb.createEvent(group, "pycbc", fname, search).json()
gid = r["graceid"]
logging.info("Uploaded event %s", gid)
if self.is_hardware_injection:
gracedb.writeLabel(gid, 'INJ')
logging.info("Tagging event %s as an injection", gid)
# add info for tracking code version
gracedb_tag_with_version(gracedb, gid)
extra_strings = [] if extra_strings is None else extra_strings
for text in extra_strings:
gracedb.writeLog(gid, text, tag_name=['analyst_comments'])
except Exception as exc:
logging.error(
'Something failed during the upload/annotation of '
'event %s on GraceDB. The event may not have been '
'uploaded!', fname)
logging.error(str(exc))
# plot the SNR timeseries and noise PSDs
if self.snr_series is not None:
if fname.endswith('.xml.gz'):
snr_series_fname = fname.replace('.xml.gz', '.hdf')
else:
snr_series_fname = fname.replace('.xml', '.hdf')
snr_series_plot_fname = snr_series_fname.replace(
'.hdf', '_snr.png')
asd_series_plot_fname = snr_series_fname.replace(
'.hdf', '_asd.png')
pl.figure()
ref_time = int(self.merger_time)
for ifo in sorted(self.snr_series):
curr_snrs = self.snr_series[ifo]
curr_snrs.save(snr_series_fname, group='%s/snr' % ifo)
pl.plot(curr_snrs.sample_times - ref_time,
abs(curr_snrs),
c=ifo_color(ifo),
label=ifo)
if ifo in self.ifos:
base = 'foreground/{}/'.format(ifo)
snr = self.coinc_results[base + 'snr']
mt = (self.coinc_results[base + 'end_time'] +
self.time_offset)
pl.plot([mt - ref_time], [snr],
c=ifo_color(ifo),
marker='x')
pl.legend()
pl.xlabel('GPS time from {:d} (s)'.format(ref_time))
pl.ylabel('SNR')
pl.savefig(snr_series_plot_fname)
pl.close()
generate_asd_plot(self.psds, asd_series_plot_fname)
# Additionally save the PSDs into the snr_series file
for ifo in sorted(self.psds):
# Undo dynamic range factor
curr_psd = self.psds[ifo].astype(numpy.float64)
curr_psd /= pycbc.DYN_RANGE_FAC**2.0
curr_psd.save(snr_series_fname, group='%s/psd' % ifo)
if self.probabilities is not None:
prob_fname = fname.replace('.xml.gz', '_probs.json')
prob_plot_fname = prob_fname.replace('.json', '.png')
prob_plot = {
k: v
for (k, v) in self.probabilities.items() if v != 0.0
}
labels, sizes = zip(*prob_plot.items())
colors = [source_color(label) for label in labels]
fig, ax = pl.subplots()
ax.pie(sizes,
labels=labels,
colors=colors,
autopct='%1.1f%%',
textprops={'fontsize': 15})
ax.axis('equal')
fig.savefig(prob_plot_fname)
pl.close()
# upload SNR series in HDF format and plots
if gid is not None and self.snr_series is not None:
try:
gracedb.writeLog(gid,
'SNR timeseries HDF file upload',
filename=snr_series_fname)
gracedb.writeLog(gid,
'SNR timeseries plot upload',
filename=snr_series_plot_fname,
tag_name=['background'],
displayName=['SNR timeseries'])
gracedb.writeLog(gid,
'ASD plot upload',
filename=asd_series_plot_fname,
tag_name=['psd'],
displayName=['ASDs'])
except Exception as exc:
logging.error('Failed to upload plots for %s', gid)
logging.error(str(exc))
# upload source probabilities in JSON format and plot
if gid is not None and self.probabilities is not None:
try:
gracedb.writeLog(gid,
'Source probabilities JSON file upload',
filename=prob_fname,
tag_name=['em_follow'])
logging.info('Uploaded source probabilities for event %s', gid)
gracedb.writeLog(gid,
'Source probabilities plot upload',
filename=prob_plot_fname,
tag_name=['em_follow'])
logging.info(
'Uploaded source probabilities pie chart for '
'event %s', gid)
except Exception as exc:
logging.error(
'Failed to upload source probability results for %s', gid)
logging.error(str(exc))
return gid
json.dumps({
opts.classifier: {
'nominal FAP': opts.FAPthr,
'maximum reported FAP': maxFAP,
'observed deadtime': dt,
'duration': T
}
}))
file_obj.close()
if not opts.skip_gracedb_upload:
message = "iDQ calibration sanity check for %s at %s within [%.3f, %.3f]" % (
opts.classifier, ifo, opts.start, opts.end)
if opts.verbose:
print " " + message
gracedb.writeLog(opts.gracedb_id, message=message, filename=jsonfilename)
### plot calibration check
fig = isp.plt.figure()
ax = fig.add_axes(isp.default_axpos)
if np.any(np.array(maxFAP) > 0) and np.any(np.array(dt) > 0):
ax.loglog(maxFAP, dt, marker='o', linestyle='none')
else:
ax.plot(maxFAP, dt, marker='o', linestyle='none')
ax.set_xlabel('Nominal FAP')
ax.set_ylabel('Observed deadtime')
ax.grid(True)
xmin = 1e-4
multfits.make_dT(verbose=opts.verbose) ### make time-delay historgrams, et al
multfits.make_los(verbose=opts.verbose) ### make line-of-sight projections
multfits.make_confidence_regions(
verbose=opts.verbose) ### make confidence region stuff
multfits.make_comparison(
verbose=opts.verbose) ### copmute comparison statistics
#-----------
### generate final html document
htmlname = multfits.write(verbose=opts.verbose)
### upload to GraceDb
if not opts.skip_gracedb_upload:
if opts.verbose:
print "uploading %s to GraceDb(%s)" % (htmlname, opts.graceDbURL)
gdb = GraceDb(opts.graceDbURL)
gdbdir = os.path.join(opts.graceDbURL, '..', 'events', opts.graceid,
'files')
gdb.writeLog(
opts.graceid,
message='comparison of skymaps can be found <a href="%s">here</a>' %
os.path.join(gdbdir, os.path.basename(htmlname)),
filename=htmlname,
tagname=fits2html.standard_tagname + opts.graceDb_tagname +
opts.graceDb_html_tagname)
gps_start = event_gps_time - time_before
gps_end = event_gps_time + time_after
plotting_gps_start = event_gps_time - plotting_time_before
plotting_gps_end = event_gps_time + plotting_time_after
performance_gps_start = event_gps_time - performance_time_before
performance_gps_end = event_gps_time + performance_time_after
logger.info("Started searching for iDQ information within [%.3f, %.3f] at %s" %
(gps_start, gps_end, ifo))
if not options.skip_gracedb_upload:
gracedb.writeLog(
gdb_id,
message=
"Started searching for iDQ information within [%.3f, %.3f] at %s" %
(gps_start, gps_end, ifo),
tagname=idq.tagnames)
#=================================================
# LOGIC for waiting for idq data
#=================================================
### figure out if we need to wait for time to pass
wait = gps_end - (idq.nowgps() + delay)
if wait > 0:
logger.info("waiting %.2f seconds until we pass gps=%.3f" %
(wait, gps_end))
time.sleep(wait)
if options.realtime_log:
### now we need to parse the realtime log to figure out where the realtime job is
if opts.verbose:
print "generating OmegaScans for : %s\n gps : %.6f"%(opts.graceid, gps)
#-------------------------------------------------
### report to GraceDB that we've started follow-up
if upload_or_verbose:
message = "automatic OmegaScans begun for: %s."%(", ".join(chansets))
if not persist:
message = message + " WARNING: we will not track the individual OmegaScan processes to ensure completion"
if opts.verbose:
print message
if opts.upload:
gdb.writeLog( opts.graceid, message=message, tagname=tagname )
### set up frame directory
frmdir = os.path.join(outdir, opts.graceid, "frames")
if os.path.exists(frmdir):
if not opts.force: ### ignore existing data if --force was supplied
raise ValueError( "directory=%s already exists!"%(frmdir) )
else:
os.makedirs( frmdir )
### set up variables that are common for all chansets
username = getuser()
hostname = gethostname()
### iterate through chansets, processing each one separately
if persist:
def upload(self, fname, gracedb_server=None, testing=True,
extra_strings=None):
"""Upload this trigger to gracedb
Parameters
----------
fname: str
The name to give the xml file associated with this trigger
gracedb_server: string, optional
URL to the GraceDB web API service for uploading the event.
If omitted, the default will be used.
testing: bool
Switch to determine if the upload should be sent to gracedb as a
test trigger (True) or a production trigger (False).
"""
from ligo.gracedb.rest import GraceDb
import matplotlib
matplotlib.use('Agg')
import pylab
# first of all, make sure the event is saved on disk
# as GraceDB operations can fail later
self.save(fname)
if self.snr_series is not None:
if fname.endswith('.xml.gz'):
snr_series_fname = fname.replace('.xml.gz', '.hdf')
else:
snr_series_fname = fname.replace('.xml', '.hdf')
snr_series_plot_fname = snr_series_fname.replace('.hdf',
'_snr.png')
psd_series_plot_fname = snr_series_fname.replace('.hdf',
'_psd.png')
pylab.figure()
for ifo in self.snr_series:
curr_snrs = self.snr_series[ifo]
curr_snrs.save(snr_series_fname, group='%s/snr' % ifo)
pylab.plot(curr_snrs.sample_times, abs(curr_snrs),
c=ifo_color(ifo), label=ifo)
if ifo in self.ifos:
snr = self.coinc_results['foreground/%s/%s' %
(ifo, 'snr')]
endt = self.coinc_results['foreground/%s/%s' %
(ifo, 'end_time')]
pylab.plot([endt], [snr], c=ifo_color(ifo), marker='x')
pylab.legend()
pylab.xlabel('GPS time (s)')
pylab.ylabel('SNR')
pylab.savefig(snr_series_plot_fname)
pylab.close()
pylab.figure()
for ifo in self.snr_series:
# Undo dynamic range factor
curr_psd = self.psds[ifo].astype(numpy.float64)
curr_psd /= pycbc.DYN_RANGE_FAC ** 2.0
curr_psd.save(snr_series_fname, group='%s/psd' % ifo)
# Can't plot log(0) so start from point 1
pylab.loglog(curr_psd.sample_frequencies[1:],
curr_psd[1:]**0.5, c=ifo_color(ifo), label=ifo)
pylab.legend()
pylab.xlim([20, 2000])
pylab.ylim([1E-24, 1E-21])
pylab.xlabel('Frequency (Hz)')
pylab.ylabel('ASD')
pylab.savefig(psd_series_plot_fname)
gid = None
try:
# try connecting to GraceDB
gracedb = GraceDb(gracedb_server) \
if gracedb_server is not None else GraceDb()
# create GraceDB event
group = 'Test' if testing else 'CBC'
r = gracedb.createEvent(group, "pycbc", fname, "AllSky").json()
gid = r["graceid"]
logging.info("Uploaded event %s", gid)
if self.is_hardware_injection:
gracedb.writeLabel(gid, 'INJ')
logging.info("Tagging event %s as an injection", gid)
# upload PSDs. Note that the PSDs are already stored in the
# original event file and we just upload a copy of that same file
# here. This keeps things as they were in O2 and can be removed
# after updating the follow-up infrastructure
psd_fname = 'psd.xml.gz' if fname.endswith('.gz') else 'psd.xml'
gracedb.writeLog(gid, "PyCBC PSD estimate from the time of event",
psd_fname, open(fname, "rb").read(), "psd")
logging.info("Uploaded PSDs for event %s", gid)
# add other tags and comments
gracedb.writeLog(
gid, "Using PyCBC code hash %s" % pycbc_version.git_hash)
extra_strings = [] if extra_strings is None else extra_strings
for text in extra_strings:
gracedb.writeLog(gid, text)
# upload SNR series in HDF format and plots
if self.snr_series is not None:
gracedb.writeLog(gid, 'SNR timeseries HDF file upload',
filename=snr_series_fname)
gracedb.writeLog(gid, 'SNR timeseries plot upload',
filename=snr_series_plot_fname,
tag_name=['background'],
displayName=['SNR timeseries'])
gracedb.writeLog(gid, 'PSD plot upload',
filename=psd_series_plot_fname,
tag_name=['psd'], displayName=['PSDs'])
except Exception as exc:
logging.error('Something failed during the upload/annotation of '
'event %s on GraceDB. The event may not have been '
'uploaded!', fname)
logging.error(str(exc))
return gid
print "found %d existing FITS files" % len(localnames)
for filename in localnames:
print " " + filename
#-------------------------------------------------
# BEGIN THE ANALYSIS
#-------------------------------------------------
### iterate over filenames, spawning a snglFITS job for each
for filename in filenames:
if not opts.skip_gracedb_upload:
gracedb.writeLog(
graceid,
message='started skymap summary for <a href="%s">%s</a>' %
(os.path.join(graceDbURL, '..', 'events', graceid, 'files',
filename), filename),
tagname=fits2html.standard_tagname)
#--------------------
### download FITS file
localname = os.path.join(outdir, filename)
if opts.verbose:
print('downloading from %s : %s -> %s' %
(graceDbURL, filename, localname))
file_obj = open(localname, 'w')
file_obj.write(gracedb.files(graceid, filename).read())
file_obj.close()
