def test_dump(self):
"""Check that writing to a file actually works as expected"""
with open(datapaths.swift_bat_grb_pos_v2, 'rb') as f:
packet = vp.load(f)
with tempfile.TemporaryFile(mode='w+b') as f:
vp.dump(packet, f)
def archive_voevent(v, rootdir):
relpath, filename = v.attrib['ivorn'].split('//')[1].split('#')
filename += ".xml"
fullpath = os.path.sep.join((rootdir, relpath, filename))
ps.utils.ensure_dir(fullpath)
with open(fullpath, 'w') as f:
voeventparse.dump(v, f)
def archive_voevent_to_file(v, rootdir):
relpath, filename = v.attrib['ivorn'].split('//')[1].split('#')
filename += ".xml"
fullpath = os.path.sep.join((rootdir, relpath, filename))
ensure_dir(fullpath)
with open(fullpath, 'wb') as f:
voeventparse.dump(v, f)
logger.debug("Wrote voevent {} to {}".format(
v.attrib['ivorn'], fullpath
))
def dummy_send_to_comet_stub(voevent, host='localhost', port=8098):
tf = tempfile.NamedTemporaryFile(delete=False)
print textwrap.dedent("""\
*************
Would have sent a VOEvent to node: {host}:{port};
Copy of XML dumped to: {fname}
*************
""".format(host=host, port=port, fname=tf.name))
voeventparse.dump(voevent, tf)
tf.close()
def dummy_send_to_comet_stub(voevent, host='localhost', port=8098):
tf = tempfile.NamedTemporaryFile(delete=False)
print(textwrap.dedent("""\
*************
Would have sent a VOEvent to node: {host}:{port};
Copy of XML dumped to: {fname}
*************
""".format(host=host, port=port, fname=tf.name)))
voeventparse.dump(voevent, tf)
tf.close()
def dummy_send_to_comet_stub(voevent, host='localhost', port=8098):
tf = tempfile.NamedTemporaryFile(delete=False)
logmsg=textwrap.dedent("""\
*************
Would have sent a VOEvent to node: {host}:{port};
IVORN: {ivorn}
Copy of XML dumped to: {fname}
*************
""".format(host=host, port=port, ivorn=voevent.attrib['ivorn'],
fname=tf.name))
logger.debug(logmsg)
voeventparse.dump(voevent, tf)
tf.close()
def send_voevent(voevent, host='localhost', port=8098):
tf = tempfile.TemporaryFile()
voeventparse.dump(voevent, tf)
tf.seek(0)
# tf.close()
try:
cmd = ['comet-sendvo']
cmd.append('--host=' + host)
cmd.append('--port=' + str(port))
subprocess.check_call(cmd, stdin=tf)
except subprocess.CalledProcessError as e:
logger.error("send_voevent failed")
raise e
def test_initial_case(self):
with open(datapaths.swift_bat_grb_pos_v2) as f:
swift_alert = BatGrb(vp.load(f))
current_utc_time = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
request_status = {
'sent_time': current_utc_time,
'acknowledged': False,
}
v = vo_subs.create_ami_followup_notification(
swift_alert, stream_id=1, request_status=request_status)
vp.assert_valid_as_v2_0(v)
with open('/tmp/test_voevent.xml', 'wb') as f:
vp.dump(v, f)
def test_voevent_generation():
tmpdir = '/tmp/fps_feed_test/gaia'
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
feed = gaia.GaiaFeed()
feed._content = gaia_content
# for feed_id in feed.event_id_data_map.keys()[:10]:
for feed_id in feed.event_id_data_map.keys():
v = feed.generate_voevent(feed_id)
stream_id = feed.feed_id_to_stream_id(feed_id)
vp.assert_valid_as_v2_0(v)
outpath = os.path.join(tmpdir, '{}.xml'.format(stream_id))
with open(outpath, 'wb') as f:
vp.dump(v, f)
def test_assasn_voevent_generation():
tmpdir = '/tmp/fps_feed_test/assasn'
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
feed2 = asassn.AsassnFeed()
feed2._content = asassn_content_2018
for feed_id in list(feed2.event_id_data_map.keys())[:10]:
# for feed_id in feed2.id_row_map.keys():
v = feed2.generate_voevent(feed_id)
stream_id = feed2.feed_id_to_stream_id(feed_id)
vp.assert_valid_as_v2_0(v)
outpath = os.path.join(tmpdir, '{}.xml'.format(stream_id))
with open(outpath, 'wb') as f:
vp.dump(v, f)
def test_initial_case(self):
with open(datapaths.swift_bat_grb_pos_v2) as f:
swift_alert = BatGrb(vp.load(f))
request_status = {'sent_time':datetime.datetime.utcnow(),
'acknowledged':False,
}
v = vo_subs.create_ami_followup_notification(swift_alert,
stream_id=001,
request_status=request_status)
vp.assert_valid_as_v2_0(v)
with open('/tmp/test_voevent.xml', 'w') as f:
vp.dump(v, f)
def send_voevent(voevent, host='localhost', port=8098):
logger.debug("comet-sendvo voevent: {}".format(voevent.attrib['ivorn']))
tf = tempfile.TemporaryFile()
voeventparse.dump(voevent, tf)
tf.seek(0)
# tf.close()
try:
cmd = ['comet-sendvo']
cmd.append('--host=' + host)
cmd.append('--port=' + str(port))
output = subprocess.check_output(cmd, stdin=tf, )
except subprocess.CalledProcessError as e:
logger.error(f"send_voevent failed, output was: {e.output}")
raise e
return output
def parse_from_voevent(voevent):
feed = SwiftFeed(voevent)
events = feed.parse_content_to_event_data_list()
assert len(events) == 1
feed_id = list(feed.event_id_data_map.keys())[0]
voevent = feed.generate_voevent(feed_id)
vp.assert_valid_as_v2_0(voevent)
if True:
# Write to file for desk-checking:
tmpdir = '/tmp/fps_feed_test/swift'
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
outpath = os.path.join(tmpdir, '{}.xml'.format(feed_id))
with open(outpath, 'wb') as f:
vp.dump(voevent, f)
print(("Example voevent output to " + outpath))
return voevent
def save_xml(self, xmlname, force_pretty_print=False):
'''
Check the validity of the voevent xml file and save as xmlname.
'''
# check if the created event is a valid VOEvent v2.0 event
if vp.valid_as_v2_0(self.v):
# save to VOEvent xml
if force_pretty_print:
with open(xmlname, 'w') as f:
if force_pretty_print:
# use xml.dom.minidom to force pretty printing xml
vp.voevent._return_to_standard_xml(self.v)
txt = lxml.etree.tostring(self.v)
f.write(parseString(txt).toprettyxml())
else:
with open(xmlname, 'wb') as f:
# lxml pretty printing often does not work
# workaround use force_pretty_print=True
vp.dump(self.v, f, pretty_print=True)
def gw_retraction(gw_dic, conf_dic, output_dic):
"""
Function to deal with retraction alert
:param gw_dic: dictionary with GW related infos
:param conf_dic: dictionary with configuration infos
:param output_dic: dictionary with output architecture
:return: list of VO event files to be send
"""
# we need to stop all processing of still on-going
# send update to DB -> this will also stop ToO process
# on space segment; to be checked in interface
# they can do it also on their side
# stop processing
# to be done
# send infos to DB
# to be done
tels_ground = conf_dic["Tels_tiling"] + conf_dic["Tels_galaxy"]
print(tels_ground)
# initialize list of vo file
vo_tosend = []
# prepare output files
for tel in tels_ground:
print('Retraction done for candidate ' + gw_dic['GraceID'] + 'and tel ' + tel)
common_dic = create_dictionary(gw_dic, conf_dic["Experiment"], tel, "")
# utils_too.prepare_voe_retraction(tel, voevent, output_dic)
voevent = basic_gw_voevent(gw_dic, conf_dic, common_dic)
# need to create correct file name and save it
file_voevent = utils_too.voevent_name(common_dic)
# dump voevent
with open(output_dic["vopath"] + "/" + file_voevent, 'wb') as fileo:
vp.dump(voevent, fileo)
vo_tosend.append(output_dic["vopath"] + "/" + file_voevent)
return vo_tosend
def send_voevent(voevent, host='localhost', port=8098):
"""
Send a voevent to a broker using the comet-sendvo publishing tool.
Args:
voevent: A voeventparse voevent object.
host (string): IP address or hostname of VOEvent broker.
port (int): Port, default 8098.
"""
tf = tempfile.TemporaryFile()
voeventparse.dump(voevent, tf)
tf.seek(0)
# tf.close()
try:
cmd = ['comet-sendvo']
cmd.append('--host=' + host)
cmd.append('--port=' + str(port))
subprocess.check_call(cmd, stdin=tf)
except subprocess.CalledProcessError as e:
logger.error("send_voevent failed")
raise e
def send_voevent(voevent, host="localhost", port=8098):
"""
Send a voevent to a broker using the comet-sendvo publishing tool.
Args:
voevent: A voeventparse voevent object.
host (string): IP address or hostname of VOEvent broker.
port (int): Port, default 8098.
"""
tf = tempfile.TemporaryFile()
voeventparse.dump(voevent, tf)
tf.seek(0)
# tf.close()
try:
cmd = ["comet-sendvo"]
cmd.append("--host=" + host)
cmd.append("--port=" + str(port))
subprocess.check_call(cmd, stdin=tf)
except subprocess.CalledProcessError as e:
logger.error("send_voevent failed")
raise e
def save_to_tmpfile(voevent):
testpacket_tmpfile_path = "/tmp/fps_alarrm_testpacket.xml"
with open(testpacket_tmpfile_path, 'w') as f:
logger = logging.getLogger()
vp.dump(voevent, f)
logger.debug("Saved packet to " + testpacket_tmpfile_path)
import logging
logging.basicConfig()
example_identity = {
id_keys.address : 'voevent.organization.tld',
id_keys.stream : 'ProjectFooAlerts',
id_keys.shortName : 'ProjectFoo',
id_keys.contactName : "Jo Bloggs",
id_keys.contactEmail : "*****@*****.**",
}
## For testing with 4PiSky Broker:
## (You will need to contact us first about whitelisting your IP address)
# host = 'voevent.4pisky.org'
## For testing locally
## (must have an instance of Comet running and set to receive submissions, see:
## http://comet.transientskp.org/en/1.2.1/usage/broker.html#invoking-comet )
host = 'localhost'
test_packet = fourpiskytools.voevent.create_test_packet(example_identity)
# Dump a copy of the test packet to the current directory for manual inspection
with open('test_packet.xml','wb') as f:
voeventparse.dump(test_packet, f)
fourpiskytools.comet.send_voevent(test_packet, host=host)
# Event generated by some telescope-data processing pipeline...
# Dummy values:
ra, dec = 90.0, -45.0
positional_error = 0.1 #degrees error radius
event_datetime = datetime(year=2000,
month=1,
day=1,
hour=0,
minute=0,
tzinfo=pytz.UTC)
alert_packet = create_basic_location_alert(
example_identity,
# role = vp.definitions.roles.observation,
role=vp.definitions.roles.test,
description="Y2K Bug",
ra=ra,
dec=dec,
err=positional_error,
event_time=event_datetime)
# See voevent-parse docs for detailed example of creating / modifying a packet:
# http://voevent-parse.readthedocs.org/en/0.7.0/examples.html#author-a-new-voevent-packet
# Dump a copy of the packet to the current directory for manual inspection
with open('alert_packet.xml', 'wb') as f:
vp.dump(alert_packet, f)
fourpiskytools.comet.send_voevent(alert_packet, host=host)
print("\n***And your What:***\n")
print(vp.prettystr(v.What))
# You would normally describe or reference your telescope / instrument here:
vp.add_how(v, descriptions='Discovered via 4PiSky',
references=vp.Reference('http://4pisky.org'))
# The 'Why' section is optional, allows for speculation on probable
# astrophysical cause
vp.add_why(v, importance=0.5,
inferences=vp.Inference(probability=0.1,
relation='identified',
name='GRB121212A',
concept='process.variation.burst;em.radio')
)
# We can also cite earlier VOEvents:
vp.add_citations(v,
vp.EventIvorn(
ivorn='ivo://astronomy.physics.science.org/super_exciting_events#101',
cite_type=vp.definitions.cite_types.followup))
# Check everything is schema compliant:
vp.assert_valid_as_v2_0(v)
output_filename = 'new_voevent_example.xml'
with open(output_filename, 'wb') as f:
vp.dump(v, f)
print("Wrote your voevent to ", os.path.abspath(output_filename))
def compute_ground(gw_dic, conf_dic, output_dir, params, map_struct):
"""
Compute observation plan for the different telescopes
:param gw_dic: dictionary with VO event information
:param conf_dic: dictionary with configuration info for running
the process plan (list telescopes, ...)
:param output_dir: output directory where to save the VO event with
observation plan
:param params: dictionary with paraemters needed to start gwemopt
:param map_struct: structure filled by gwemopt when loading skymap
:return: list of VO files created
"""
# initialize list of vo file
vo_tosend = []
# retrieve number of tiles we will use for the large field of view
# telescope ie where will not target specific galaxy only
params["max_nb_tiles"] = np.array(conf_dic["Tels_tiling_tilesnb"])
# Adapt percentage of golden tiles with the 90% skymap size.
# Arbitrary, needs to be optimised!!!
if float(gw_dic["90cr"]) < 60:
params["iterativeOverlap"] = 0.8
params["doIterativeTiling"] = False
params["doPerturbativeTiling"] = False
else:
params["iterativeOverlap"] = 0.2
# move the two next lines to True if you want to optimize on the
# network
params["doIterativeTiling"] = False
params["doPerturbativeTiling"] = False
# Perform observation plan based on gwemopt package here using tiling for large FOV telescopes
# output will be 2 tables : tiling and and list of associated galaxies
atables_tiling = too.Observation_plan_multiple(
conf_dic["Tels_tiling"], gw_dic["Time"], gw_dic["GraceID"], params, map_struct, 'Tiling')
# still need to prepare output files
# loop on the different telescopes to create voevent associated to
# observation plans
for tel_name in enumerate(conf_dic["Tels_tiling"]):
common_dic = create_dictionary(gw_dic, conf_dic["Experiment"], tel_name[1], 'Tiling')
voevent = create_gw_voevent(gw_dic, conf_dic, common_dic, atables_tiling[0][tel_name[1]])
# need to create correct file name and save it
file_voevent = utils_too.voevent_name(common_dic)
# dump voevent
with open(output_dir + "/" + file_voevent, 'wb') as fileo:
vp.dump(voevent, fileo)
vo_tosend.append(output_dir + "/" + file_voevent)
# Perform observation plan based on gwemopt package here using galaxies for small FOV telescopes
# output will be 2 tables : tiling (empty here) and list of associated galaxies
params["galaxy_grade"] = 'S'
params["max_nb_tiles"] = conf_dic["nb_galaxies"]
atables_galaxy = too.Observation_plan_multiple(
conf_dic["Tels_galaxy"], gw_dic["Time"], gw_dic["GraceID"], params, map_struct,
'Galaxy targeting')
# now save VO event for telescopes with galaxies
for tel_name in enumerate(conf_dic["Tels_galaxy"]):
# in case of CWB trigger the tables will be of type None
# but we would like to create empty VO event file nevertheless
# so here we are forcing the obs mode argument to be hard coded
common_dic = create_dictionary(gw_dic, conf_dic["Experiment"], tel_name[1],
'Galaxy')
voevent = create_gw_voevent(gw_dic, conf_dic, common_dic, atables_galaxy[1])
# need to create correct file name and save it
file_voevent = utils_too.voevent_name(common_dic)
# dump voevent
with open(output_dir + "/" + file_voevent, 'wb') as fileo:
vp.dump(voevent, fileo)
vo_tosend.append(output_dir + "/" + file_voevent)
return vo_tosend
# For testing locally
# (must have an instance of Comet running, see:
# http://comet.transientskp.org/en/1.2.1/usage/broker.html#invoking-comet )
host = 'localhost'
# Event generated by some telescope-data processing pipeline...
# Dummy values:
ra,dec = 90.0,-45.0
positional_error = 0.1 #degrees error radius
event_datetime = datetime(year=2000,month=1,day=1,hour=0,minute=0)
alert_packet = create_basic_location_alert(example_identity,
# role = vp.definitions.roles.observation,
role = vp.definitions.roles.test,
description="Y2K Bug",
ra=ra,
dec=dec,
err=positional_error,
event_time=event_datetime)
# See voevent-parse docs for detailed example of creating / modifying a packet:
# http://voevent-parse.readthedocs.org/en/0.7.0/examples.html#author-a-new-voevent-packet
# Dump a copy of the packet to the current directory for manual inspection
with open('alert_packet.xml','w') as f:
vp.dump(alert_packet, f)
fourpiskytools.comet.send_voevent(alert_packet, host=host)
vp.add_why(v)
v.Why.Description = "Fading source on top of 2MASS Galaxy (offset from bulge)"
# ##Check and save##
# Finally - and importantly, as discussed in the [VOEvent notes](http://voevent.readthedocs.org/en/latest/parse.html) - let's make sure that this event is really valid according to our schema:
# In[ ]:
vp.valid_as_v2_0(v)
# Great! We can now save it to disk:
# In[ ]:
with open('gaia.xml', 'w') as f:
vp.dump(v, f)
# And we're all done. You can open the file in your favourite text editor to see what
# we've produced, but note that it probably won't be particularly elegantly
# formatted - an alternative option is to [open it in your browser](my_gaia.xml).
# ##Advanced##
# ###Free-style element authoring###
# Note that if you want to do something that's not part of the standard use-cases addressed by voevent-parse,
# you can always use the underlying lxml.objectify tools to manipulate elements yourself.
# For example - don't like the 'voevent-parse' tag that gets added to your VOEvent Who skeleton? You can delete it:
# In[ ]:
## Before deletion:
## (Enclosed in an if-clause in case this is re-run after the cell below)
def compute_ground(grb_dic, conf_dic, output_dir, params, map_struct):
"""
Compute observation plan for the different telescopes
:param grb_dic: dictionary with VO event information
:param conf_dic: dictionary with configuration info for running
the process plan (list telescopes, ...)
:param output_dir: output directory where to save the VO event with
observation plan
:param params: dictionary with paraemters needed to start gwemopt
:param map_struct: structure filled by gwemopt when loading skymap
:return: list of VO files created
"""
# initialize list of vo file
vo_tosend = []
if (grb_dic["teles"]
== "SWIFT") or grb_dic["Packet_Type"] == 110 or grb_dic[
"Packet_Type"] == 119 or grb_dic["Packet_Type"] == 111:
for tel_name in enumerate(conf_dic["Tels_followup"]):
common_dic = create_dictionary(grb_dic, conf_dic["Experiment"],
tel_name[1], 'Follow-up')
voevent = create_grb_voevent(grb_dic, conf_dic, common_dic, "")
file_voevent = utils_too.voevent_name(common_dic)
# dump voevent
with open(output_dir + "/" + file_voevent, 'wb') as fileo:
vp.dump(voevent, fileo)
vo_tosend.append(output_dir + "/" + file_voevent)
if grb_dic["Packet_Type"] == 112 or grb_dic["Packet_Type"] == 115:
# retrieve number of tiles we will use for the large field of view
# telescope ie where will not target specific galaxy only
params["max_nb_tiles"] = np.array(conf_dic["Tels_tiling_tilesnb"])
# Adapt percentage of golden tiles with the 90% skymap size.
# Arbitrary, needs to be optimised!!!
if float(grb_dic["90cr"]) < 60:
params["iterativeOverlap"] = 0.8
params["doIterativeTiling"] = False
params["doPerturbativeTiling"] = False
else:
params["iterativeOverlap"] = 0.2
# move the two next lines to True if you want to optimize on the
# network
params["doIterativeTiling"] = False
params["doPerturbativeTiling"] = False
# Perform observation plan based on gwemopt package here using tiling for large FOV telescopes
# output will be 2 tables : tiling and and list of associated galaxies
atables_tiling = too.Observation_plan_multiple(conf_dic["Tels_tiling"],
grb_dic["dateobs"],
grb_dic["grbid"],
params, map_struct,
'Tiling')
# still need to prepare output files
# loop on the different telescopes to create voevent associated to
# observation plans
for tel_name in enumerate(conf_dic["Tels_tiling"]):
common_dic = create_dictionary(grb_dic, conf_dic["Experiment"],
tel_name[1], 'Tiling')
voevent = create_grb_voevent(grb_dic, conf_dic, common_dic,
atables_tiling[0][tel_name[1]])
# need to create correct file name and save it
file_voevent = utils_too.voevent_name(common_dic)
# dump voevent
with open(output_dir + "/" + file_voevent, 'wb') as fileo:
vp.dump(voevent, fileo)
vo_tosend.append(output_dir + "/" + file_voevent)
return vo_tosend