def test_initial_case(self):
with open(datapaths.swift_bat_grb_pos_v2) as f:
swift_alert = BatGrb(vp.load(f))
current_utc_time = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
request_status = {
'sent_time': current_utc_time,
'acknowledged': False,
}
v = vo_subs.create_ami_followup_notification(
swift_alert, stream_id=1, request_status=request_status)
vp.assert_valid_as_v2_0(v)
with open('/tmp/test_voevent.xml', 'wb') as f:
vp.dump(v, f)
def test_voevent_generation():
tmpdir = '/tmp/fps_feed_test/gaia'
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
feed = gaia.GaiaFeed()
feed._content = gaia_content
# for feed_id in feed.event_id_data_map.keys()[:10]:
for feed_id in feed.event_id_data_map.keys():
v = feed.generate_voevent(feed_id)
stream_id = feed.feed_id_to_stream_id(feed_id)
vp.assert_valid_as_v2_0(v)
outpath = os.path.join(tmpdir, '{}.xml'.format(stream_id))
with open(outpath, 'wb') as f:
vp.dump(v, f)
def test_assasn_voevent_generation():
tmpdir = '/tmp/fps_feed_test/assasn'
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
feed2 = asassn.AsassnFeed()
feed2._content = asassn_content_2018
for feed_id in list(feed2.event_id_data_map.keys())[:10]:
# for feed_id in feed2.id_row_map.keys():
v = feed2.generate_voevent(feed_id)
stream_id = feed2.feed_id_to_stream_id(feed_id)
vp.assert_valid_as_v2_0(v)
outpath = os.path.join(tmpdir, '{}.xml'.format(stream_id))
with open(outpath, 'wb') as f:
vp.dump(v, f)
def test_initial_case(self):
with open(datapaths.swift_bat_grb_pos_v2) as f:
swift_alert = BatGrb(vp.load(f))
request_status = {'sent_time':datetime.datetime.utcnow(),
'acknowledged':False,
}
v = vo_subs.create_ami_followup_notification(swift_alert,
stream_id=001,
request_status=request_status)
vp.assert_valid_as_v2_0(v)
with open('/tmp/test_voevent.xml', 'w') as f:
vp.dump(v, f)
def test_get_toplevel_params(self):
v = self.blank
p_foo1 = vp.Param(name='foo',
value=42,
unit='bars',
ac=True
)
p_foo2 = vp.Param(name='foo',
value=123,
unit='bars',
ac=True
)
p_noname = vp.Param(name='delete_me', value=111)
param_list = [p_foo1, p_foo2, p_noname]
del p_noname.attrib['name']
v.What.extend(param_list)
# Show flaws in old routine:
with pytest.warns(FutureWarning):
old_style_toplevel_param_dict = vp.pull_params(v)[None]
# print(old_style_toplevel_param_dict)
vp.assert_valid_as_v2_0(v)
# The old 'pull_params' routine will simply drop Params with duplicate
# names:
assert len(old_style_toplevel_param_dict)==(len(param_list) - 1)
none_group = vp.Group([],name=None)
complex_group1 =vp.Group([vp.Param(name='real', value=1.),
vp.Param(name='imag', value=0.5)],
name='complex')
complex_group2 =vp.Group([vp.Param(name='real', value=1.5),
vp.Param(name='imag', value=2.5)],
name='complex')
group_list = [none_group, complex_group1, complex_group2]
v.What.extend(group_list)
vp.assert_valid_as_v2_0(v)
old_style_toplevel_param_dict_w_group = vp.pull_params(v)[None]
# An un-named group will also overshadow top-level Params.
# This is a total fail, even though it's actually in-spec.
assert len(old_style_toplevel_param_dict_w_group)==0
toplevel_params = vp.get_toplevel_params(v)
# .values method behaves like regular dict, one value for each key:
assert len(toplevel_params.values())==(len(param_list) - 1)
# Use .allvalues if you want to see duplicates:
assert len(toplevel_params.allvalues())==len(param_list)
grouped_params = vp.get_grouped_params(v)
assert len(grouped_params.values())==len(group_list) - 1
assert len(grouped_params.allvalues())==len(group_list)
def parse_VOEvent(self, voevent, mapping):
'''
Parse VOEvent xml file.
:param voevent: VOEvent xml file
:param mapping: mapping from mapping.json
:type voevent: lxml.objectify.ObjectifiedElement, str
:type mapping: dict
:returns: mapping (mapping from mapping.json with values filled),
event_type (event_type and citation if applicable)
:rtype: dict, tuple
'''
# load VOEvent xml file
try:
v = vp.load(voevent)
except AttributeError:
f = open(voevent, "rb")
v = vp.load(f)
f.close()
# assert if xml file is a valid VOEvent
vp.assert_valid_as_v2_0(v)
# Check if the event is a new VOEvent
# For a new VOEvent there should be no citations
try:
event_type = (v.xpath('Citations')[0].EventIVORN.attrib['cite'],
v.xpath('Citations')[0].EventIVORN.text)
except IndexError:
event_type = ('new', None)
self.logger.info("Event of of type: {}".format(event_type))
# use the mapping to get required data from VOEvent xml
# if a path is not found in the xml it gets an empty list which is
# removed in the next step
# puts all params into dict param_data[group][param_name]
try:
param_data = vp.get_grouped_params(v)
except AttributeError:
# <What> section is not needed for retractions
param_data = None
for table in mapping.keys(): # iterate over all tables
for idx, item in enumerate(mapping[table]):
# Add values from XML to dictionary
mapping[table][idx]['value'] = self.get_value(
v, param_data, item, event_type)
if item.get('description'):
note = self.get_description(v, item)
if note:
mapping[table][idx]['note'] = note
return mapping, event_type
def test_followup_citation(self):
ref = 'ivo://nasa.gsfc.gcn/SWIFT#BAT_GRB_Pos_532871-729'
vp.add_citations(
self.v,
vp.EventIvorn('ivo://nasa.gsfc.gcn/SWIFT#BAT_GRB_Pos_532871-729',
cite_type=vp.definitions.cite_types.followup))
vp.assert_valid_as_v2_0(self.v)
self.assertEqual(len(self.v.Citations.getchildren()), 1)
# print
# print vp.prettystr(self.v.Citations)
vp.add_citations(
self.v,
vp.EventIvorn('ivo://nasa.gsfc.gcn/SWIFT#BAT_GRB_Pos_532871-730',
cite_type=vp.definitions.cite_types.followup))
self.assertTrue(vp.valid_as_v2_0(self.v))
# print voe.prettystr(self.v.Citations)
self.assertEqual(len(self.v.Citations.getchildren()), 2)
def test_get_toplevel_params(self):
v = self.blank
p_foo1 = vp.Param(name='foo', value=42, unit='bars', ac=True)
p_foo2 = vp.Param(name='foo', value=123, unit='bars', ac=True)
p_noname = vp.Param(name='delete_me', value=111)
param_list = [p_foo1, p_foo2, p_noname]
del p_noname.attrib['name']
v.What.extend(param_list)
# Show flaws in old routine:
with pytest.warns(FutureWarning):
old_style_toplevel_param_dict = vp.pull_params(v)[None]
# print(old_style_toplevel_param_dict)
vp.assert_valid_as_v2_0(v)
# The old 'pull_params' routine will simply drop Params with duplicate
# names:
assert len(old_style_toplevel_param_dict) == (len(param_list) - 1)
none_group = vp.Group([], name=None)
complex_group1 = vp.Group([
vp.Param(name='real', value=1.),
vp.Param(name='imag', value=0.5)
],
name='complex')
complex_group2 = vp.Group([
vp.Param(name='real', value=1.5),
vp.Param(name='imag', value=2.5)
],
name='complex')
group_list = [none_group, complex_group1, complex_group2]
v.What.extend(group_list)
vp.assert_valid_as_v2_0(v)
old_style_toplevel_param_dict_w_group = vp.pull_params(v)[None]
# An un-named group will also overshadow top-level Params.
# This is a total fail, even though it's actually in-spec.
assert len(old_style_toplevel_param_dict_w_group) == 0
toplevel_params = vp.get_toplevel_params(v)
# .values method behaves like regular dict, one value for each key:
assert len(toplevel_params.values()) == (len(param_list) - 1)
# Use .allvalues if you want to see duplicates:
assert len(toplevel_params.allvalues()) == len(param_list)
grouped_params = vp.get_grouped_params(v)
assert len(grouped_params.values()) == len(group_list) - 1
assert len(grouped_params.allvalues()) == len(group_list)
def parse_from_voevent(voevent):
feed = SwiftFeed(voevent)
events = feed.parse_content_to_event_data_list()
assert len(events) == 1
feed_id = list(feed.event_id_data_map.keys())[0]
voevent = feed.generate_voevent(feed_id)
vp.assert_valid_as_v2_0(voevent)
if True:
# Write to file for desk-checking:
tmpdir = '/tmp/fps_feed_test/swift'
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir)
outpath = os.path.join(tmpdir, '{}.xml'.format(feed_id))
with open(outpath, 'wb') as f:
vp.dump(voevent, f)
print(("Example voevent output to " + outpath))
return voevent
def test_followup_citation(self):
ref = 'ivo://nasa.gsfc.gcn/SWIFT#BAT_GRB_Pos_532871-729'
vp.add_citations(self.v,
vp.EventIvorn(
'ivo://nasa.gsfc.gcn/SWIFT#BAT_GRB_Pos_532871-729',
cite_type=vp.definitions.cite_types.followup)
)
vp.assert_valid_as_v2_0(self.v)
self.assertEqual(len(self.v.Citations.getchildren()), 1)
# print
# print vp.prettystr(self.v.Citations)
vp.add_citations(self.v,
vp.EventIvorn(
'ivo://nasa.gsfc.gcn/SWIFT#BAT_GRB_Pos_532871-730',
cite_type=vp.definitions.cite_types.followup)
)
self.assertTrue(vp.valid_as_v2_0(self.v))
# print voe.prettystr(self.v.Citations)
self.assertEqual(len(self.v.Citations.getchildren()), 2)
def test_namespace_variations(self):
# NB, not enclosing root element in a namespace is invalid under schema
# But this has been seen in the past (isolated bug case?)
# Anyway, handled easily enough
with open(datapaths.no_namespace_test_packet, 'rb') as f:
vff = vp.load(f)
self.assertFalse(vp.valid_as_v2_0(vff))
self.assertEqual(vff.tag, 'VOEvent')
self.assertEqual(vff.attrib['ivorn'],
'ivo://com.dc3/dc3.broker#BrokerTest-2014-02-24T15:55:27.72')
with open(datapaths.swift_bat_grb_pos_v2, 'rb') as f:
xml_str = f.read()
xml_str = xml_str.replace(b'voe', b'foobar_ns')
# print xml_str
vfs = vp.loads(xml_str)
vp.assert_valid_as_v2_0(vfs)
self.assertEqual(vfs.tag, 'VOEvent')
self.assertEqual(vfs.attrib['ivorn'],
'ivo://nasa.gsfc.gcn/SWIFT#BAT_GRB_Pos_532871-729')
def test_namespace_variations(self):
# NB, not enclosing root element in a namespace is invalid under schema
# But this has been seen in the past (isolated bug case?)
# Anyway, handled easily enough
with open(datapaths.no_namespace_test_packet, 'rb') as f:
vff = vp.load(f)
self.assertFalse(vp.valid_as_v2_0(vff))
self.assertEqual(vff.tag, 'VOEvent')
self.assertEqual(vff.attrib['ivorn'],
'ivo://com.dc3/dc3.broker#BrokerTest-2014-02-24T15:55:27.72')
with open(datapaths.swift_bat_grb_pos_v2, 'rb') as f:
xml_str = f.read()
xml_str = xml_str.replace(b'voe', b'foobar_ns')
# print xml_str
vfs = vp.loads(xml_str)
vp.assert_valid_as_v2_0(vfs)
self.assertEqual(vfs.tag, 'VOEvent')
self.assertEqual(vfs.attrib['ivorn'],
'ivo://nasa.gsfc.gcn/SWIFT#BAT_GRB_Pos_532871-729')
print("\n***And your What:***\n")
print(vp.prettystr(v.What))
# You would normally describe or reference your telescope / instrument here:
vp.add_how(v, descriptions='Discovered via 4PiSky',
references=vp.Reference('http://4pisky.org'))
# The 'Why' section is optional, allows for speculation on probable
# astrophysical cause
vp.add_why(v, importance=0.5,
inferences=vp.Inference(probability=0.1,
relation='identified',
name='GRB121212A',
concept='process.variation.burst;em.radio')
)
# We can also cite earlier VOEvents:
vp.add_citations(v,
vp.EventIvorn(
ivorn='ivo://astronomy.physics.science.org/super_exciting_events#101',
cite_type=vp.definitions.cite_types.followup))
# Check everything is schema compliant:
vp.assert_valid_as_v2_0(v)
output_filename = 'new_voevent_example.xml'
with open(output_filename, 'wb') as f:
vp.dump(v, f)
print("Wrote your voevent to ", os.path.abspath(output_filename))
def create_voevent(jsonfile=None, deployment=False, **kwargs):
""" template syntax for voeventparse creation of voevent
"""
required = [
'internalname', 'mjds', 'dm', 'width', 'snr', 'ra', 'dec', 'radecerr'
]
preferred = ['fluence', 'p_flux', 'importance', 'dmerr']
# set values
dd = kwargs.copy()
if jsonfile is not None: # as made by caltechdata.set_metadata
for k, v in trigger.items():
if k in required + preferred:
dd[k] = v
assert all([
k in dd for k in required
]), f'Input keys {list(dd.keys())} not complete (requires {required})'
# TODO: set this correctly
dt = time.Time(dd['mjds'], format='mjd').to_datetime(timezone=pytz.utc)
# create voevent instance
role = vp.definitions.roles.observation if deployment else vp.definitions.roles.test
v = vp.Voevent(
stream='', # TODO: check
stream_id=1,
role=role)
vp.set_who(v,
date=datetime.datetime.utcnow(),
author_ivorn="voevent.dsa-110.caltech.org") # TODO: check
vp.set_author(v,
title="DSA-110 Testing Node",
contactName="Casey Law",
contactEmail="*****@*****.**")
params = []
dm = vp.Param(name="dm",
value=str(dd['dm']),
unit="pc/cm^3",
ucd="phys.dispMeasure;em.radio.750-1500MHz",
dataType='float',
ac=True)
dm.Description = 'Dispersion Measure'
params.append(dm)
width = vp.Param(name="width",
value=str(dd['width']),
unit="ms",
ucd="time.duration;src.var.pulse",
dataType='float',
ac=True)
width.Description = 'Temporal width of burst'
params.append(width)
snr = vp.Param(name="snr",
value=str(dd['snr']),
ucd="stat.snr",
dataType='float',
ac=True)
snr.Description = 'Signal to noise ratio'
params.append(snr)
if 'fluence' in dd:
fluence = vp.Param(
name='fluence',
value=str(dd['fluence']),
unit='Jansky ms',
ucd='em.radio.750-1500MHz', # TODO: check
dataType='float',
ac=False)
fluence.Description = 'Fluence'
params.append(fluence)
if 'p_flux' in dd:
p_flux = vp.Param(name='peak_flux',
value=str(dd['p_flux']),
unit='Janskys',
ucd='em.radio.750-1500MHz',
dataType='float',
ac=True)
p_flux.Description = 'Peak Flux'
params.append(p_flux)
if 'dmerr' in dd:
dmerr = vp.Param(name="dm_error",
value=str(dd['dmerr']),
unit="pc/cm^3",
ucd="phys.dispMeasure;em.radio.750-1500MHz",
dataType='float',
ac=True)
dmerr.Description = 'Dispersion Measure error'
params.append(dmerr)
v.What.append(vp.Group(params=params, name='event parameters'))
vp.add_where_when(v,
coords=vp.Position2D(
ra=str(dd['ra']),
dec=str(dd['dec']),
err=str(dd['radecerr']),
units='deg',
system=vp.definitions.sky_coord_system.utc_fk5_geo),
obs_time=dt,
observatory_location='OVRO')
print("\n***Here is your WhereWhen:***\n")
print(vp.prettystr(v.WhereWhen))
print("\n***And your What:***\n")
print(vp.prettystr(v.What))
vp.add_how(v,
descriptions='Discovered with DSA-110',
references=vp.Reference('http://deepsynoptic.org'))
if 'importance' in dd:
vp.add_why(v, importance=str(dd['importance']))
else:
vp.add_why(v)
v.Why.Name = str(dd['internalname'])
vp.assert_valid_as_v2_0(v)
return v
def test_invalid_error_reporting(self):
with self.assertRaises(etree.DocumentInvalid):
v = vp.Voevent(stream='voevent.soton.ac.uk/TEST',
stream_id='001',
role='DeadParrot')
vp.assert_valid_as_v2_0(v)
def test_invalid_error_reporting(self):
with self.assertRaises(etree.DocumentInvalid):
v = vp.Voevent(stream='voevent.soton.ac.uk/TEST',
stream_id='001',
role='DeadParrot')
vp.assert_valid_as_v2_0(v)
def create_grb_voevent(grb_dic, conf_vo, what_com, atable):
"""
Create VO event with observation plan for a given telescope in the network
:param gw_dic: dictionary with GRB infos
:param conf_vo: dictionary to be used to fill header of VO event
:param what_com: dictionary with common infos needed for VO creation
:param atable: astropy table with observation plan and meta data
:return: voevent object
"""
# get telescope name and role, will be used several time
tel_name = what_com['tel_name']
obs_mode = what_com['obs_mode']
voevent = basic_grb_voevent(grb_dic, conf_vo, what_com)
#GBM ground or final position
if grb_dic["Packet_Type"] == 112 or grb_dic["Packet_Type"] == 115:
pixloc = vp.Param(name="Loc_url",
value=str(grb_dic["skymap"]["localization_name"]),
ucd="meta.ref.url",
dataType="string")
pixloc.Description = "URL to retrieve location of the healpix skymap"
voevent.What.append(pixloc)
if tel_name != "":
name_tel = vp.Param(name="Name_tel",
value=str(tel_name),
ucd="instr",
dataType="string")
name_tel.Description = "Name of the telescope used for the observation strategy"
voevent.What.append(name_tel)
if atable and atable[0]:
# now prepare observation plan or galaxies list
fields = objectify.Element("Table", name="Obs_plan")
fields.Description = "Tiles for the observation plan"
# first field is the identificaiton of the field
# it will be different between tiling and galaxy targetting
if obs_mode == "Tiling":
field_id = objectify.SubElement(fields,
"Field",
name="Field_id",
ucd="",
unit="",
dataType="string")
field_id.Description = "ID of the field of FOV"
tel_obsplan = np.transpose(
np.array([
atable['rank_id'], atable['RA'], atable['DEC'],
atable['Prob']
]))
else:
field_id = objectify.SubElement(fields,
"Field",
name="Gal_id",
ucd="",
unit="",
dataType="string")
field_id.Description = "ID of the galaxy"
# For galaxie we will uniformize how we name them based on catalogs
gal_id = utils_too.uniformize_galnames(atable)
tel_obsplan = np.transpose(
np.array([
gal_id, atable['RAJ2000'], atable['DEJ2000'],
atable['S']
]))
# right_asc = objectify.SubElement(fields, "Field", name="RA", ucd="pos.eq.ra ", unit="deg", dataType="float")
right_asc = objectify.SubElement(fields,
"Field",
name="RA",
ucd="pos.eq.ra ",
unit="deg",
dataType="float")
right_asc.Description = "The right ascension at center of fov in equatorial coordinates"
dec = objectify.SubElement(fields,
"Field",
name="Dec",
ucd="pos.eq.ra ",
unit="deg",
dataType="float")
dec.Description = "The declination at center of fov in equatorial coordinates"
os_grade = objectify.SubElement(fields,
"Field",
name="Os_score",
ucd="meta.number",
unit="None",
dataType="float")
os_grade.Description = "Gives the importance of the tile/galaxy to observe"
data = objectify.SubElement(fields, "Data")
# loop on the observation plan
# put a limit to 500 fields otherwise get an error when sending a VO event
for i in np.arange(min(500, len(tel_obsplan))):
table_row = objectify.SubElement(data, "TR")
for j in np.arange(len(tel_obsplan[i])):
# objectify.SubElement(TR, "TD",value=str(Tel_dic["OS"][i][j]))
objectify.SubElement(table_row, 'TD')
table_row.TD[-1] = str(tel_obsplan[i][j])
voevent.What.append(fields)
grb_dic["dateobs"] = grb_dic["dateobs"].replace(tzinfo=pytz.utc)
vp.add_where_when(voevent,
coords=vp.Position2D(
ra=grb_dic["ra"],
dec=grb_dic["dec"],
err=grb_dic["error"],
units='deg',
system=vp.definitions.sky_coord_system.utc_fk5_geo),
obs_time=grb_dic["dateobs"],
observatory_location=grb_dic["inst"])
# Check everything is schema compliant:
vp.assert_valid_as_v2_0(voevent)
return voevent
v.What.foo[-1] = "foo{}".format(i)
print("I have {} foos for you:".format(len(v.What.foo)))
print(vp.prettystr(v.What))
# In[ ]:
# Get rid of all the foo:
del v.What.foo[:]
# Alternatively, you can create elements independently, then append them to a parent (remember how lxml.objectify pretends elements are lists?) - this is occasionally useful if you want to write a function that returns an element, e.g. to create a new [Param](http://voevent-parse.readthedocs.org/en/master/reference.html#voeventparse.misc.Param) (but voevent-parse wraps that up for you already):
# In[ ]:
temp = objectify.Element('NonSchemaCompliantParam',
attrib={'somekey': 'somevalue'})
v.What.append(temp)
print(vp.prettystr(v.What))
# Obviously, these are non-schema compliant elements.
# Don't make up your own format - use Params for storing general data:
# In[ ]:
vp.valid_as_v2_0(v)
# Note that you can get a traceback if you need to figure out why a VOEvent is non-schema-compliant - this will report the first invalid element it comes across:
# In[ ]:
vp.assert_valid_as_v2_0(v)