def _has_bad_coords(root, stream): """ Predicate function encapsulating 'data clean up' filter code. Currently minimal, but these sort of functions tend to grow over time. Problem 1: Some of the GCN packets have an RA /Dec equal to (0,0) in the WhereWhen, and a flag in the What signifying that those are actually dummy co-ords. (This is used for time-stamping an event which is not localised). So, we don't load those positions, to avoid muddying the database corpus. Problem 2: com.dc3/dc3.broker#BrokerTest packets have dummy RA/Dec values, with no units specified. (They're also marked role=test, so it's not such a big deal, but it generates a lot of debug-log churn.) """ if stream == "com.dc3/dc3.broker": return True if not stream.split('/')[0] == 'nasa.gsfc.gcn': return False toplevel_params = vp.get_toplevel_params(root) if "Coords_String" in toplevel_params: if (toplevel_params["Coords_String"]['value'] == "unavailable/inappropriate"): return True return False
def __init__(self, payload): super().__init__(payload) # Get XML param dicts # NB: you can't store these on the Event because they're unpickleable. top_params = vp.get_toplevel_params(self.voevent) group_params = vp.get_grouped_params(self.voevent) # Default params self.notice = EVENT_DICTIONARY[self.packet_type]['notice_type'] self.type = 'GW' self.source = EVENT_DICTIONARY[self.packet_type]['source'] # Get the event ID (e.g. S190510g) self.id = top_params['GraceID']['value'] # Create our own event name (e.g. LVC_S190510g) self.name = '{}_{}'.format(self.source, self.id) # Get info from the VOEvent # See https://emfollow.docs.ligo.org/userguide/content.html#notice-contents self.far = float(top_params['FAR']['value']) self.gracedb_url = top_params['EventPage']['value'] self.instruments = top_params['Instruments']['value'] self.group = top_params['Group']['value'] # CBC or Burst self.pipeline = top_params['Pipeline']['value'] # Get classification probabilities and properties if self.group == 'CBC': classification_dict = group_params.allitems()[1][ 1] # Horrible, but blame XML self.classification = { key: float(classification_dict[key]['value']) for key in classification_dict } properties_dict = group_params.allitems()[2][1] self.properties = { key: float(properties_dict[key]['value']) for key in properties_dict } else: self.classification = {} self.properties = {} # Get skymap URL for group in group_params: if 'skymap_fits' in group_params[group]: self.skymap_url = group_params[group]['skymap_fits']['value'] self.skymap_type = group # Don't download the skymap here, it may well be very large. # Only do it when it's absolutely necessary # These params will only be set once the skymap is downloaded self.distance = np.inf self.distance_error = 0 self.contour_areas = {0.5: None, 0.9: None}
def _get_packet_type(payload): """Get the packet type from a VOEvent payload.""" voevent = vp.loads(payload) top_params = vp.get_toplevel_params(voevent) try: packet_type = int(top_params['Packet_Type']['value']) except KeyError: # If it's a VOEvent but not a GCN it won't have a packet type (e.g. Gaia alerts) packet_type = None return packet_type
def __init__(self, payload): super().__init__(payload) # Get XML param dicts # NB: you can't store these on the Event because they're unpickleable. top_params = vp.get_toplevel_params(self.voevent) # group_params = vp.get_grouped_params(self.voevent) # Default params self.notice = EVENT_DICTIONARY[self.packet_type]['notice_type'] self.type = EVENT_DICTIONARY[self.packet_type]['event_type'] self.source = EVENT_DICTIONARY[self.packet_type]['source'] # Get the run and event ID (e.g. 13311922683750) self.id = top_params['AMON_ID']['value'] # Create our own event name (e.g. ICECUBE_13311922683750) self.name = '{}_{}'.format(self.source, self.id) # Get info from the VOEvent # signalness: the probability this is an astrophysical signal relative to backgrounds self.signalness = float(top_params['signalness']['value']) self.far = float(top_params['FAR']['value']) # Position coordinates self.position = vp.get_event_position(self.voevent) self.coord = SkyCoord(ra=self.position.ra, dec=self.position.dec, unit=self.position.units) self.target = FixedTarget(self.coord) # Position error self.coord_error = Angle(self.position.err, unit=self.position.units) # Systematic error for cascade event is given, so = 0 if self.notice == 'ICECUBE_CASCADE': self.systematic_error = Angle(0, unit='deg') else: self.systematic_error = Angle(.2, unit='deg') self.total_error = Angle(np.sqrt(self.coord_error**2 + self.systematic_error**2), unit='deg') # Enclosed skymap url for CASCADE_EVENT, but others # Get skymap URL if 'skymap_fits' in top_params: self.skymap_url = top_params['skymap_fits']['value'] else: self.skymap_url = None # Don't download the skymap here, it may well be very large. # Only do it when it's absolutely necessary # These params will only be set once the skymap is downloaded self.contour_areas = {0.5: None, 0.9: None}
def test_get_toplevel_params(self): v = self.blank p_foo1 = vp.Param(name='foo', value=42, unit='bars', ac=True ) p_foo2 = vp.Param(name='foo', value=123, unit='bars', ac=True ) p_noname = vp.Param(name='delete_me', value=111) param_list = [p_foo1, p_foo2, p_noname] del p_noname.attrib['name'] v.What.extend(param_list) # Show flaws in old routine: with pytest.warns(FutureWarning): old_style_toplevel_param_dict = vp.pull_params(v)[None] # print(old_style_toplevel_param_dict) vp.assert_valid_as_v2_0(v) # The old 'pull_params' routine will simply drop Params with duplicate # names: assert len(old_style_toplevel_param_dict)==(len(param_list) - 1) none_group = vp.Group([],name=None) complex_group1 =vp.Group([vp.Param(name='real', value=1.), vp.Param(name='imag', value=0.5)], name='complex') complex_group2 =vp.Group([vp.Param(name='real', value=1.5), vp.Param(name='imag', value=2.5)], name='complex') group_list = [none_group, complex_group1, complex_group2] v.What.extend(group_list) vp.assert_valid_as_v2_0(v) old_style_toplevel_param_dict_w_group = vp.pull_params(v)[None] # An un-named group will also overshadow top-level Params. # This is a total fail, even though it's actually in-spec. assert len(old_style_toplevel_param_dict_w_group)==0 toplevel_params = vp.get_toplevel_params(v) # .values method behaves like regular dict, one value for each key: assert len(toplevel_params.values())==(len(param_list) - 1) # Use .allvalues if you want to see duplicates: assert len(toplevel_params.allvalues())==len(param_list) grouped_params = vp.get_grouped_params(v) assert len(grouped_params.values())==len(group_list) - 1 assert len(grouped_params.allvalues())==len(group_list)
def test_get_toplevel_params(self): v = self.blank p_foo1 = vp.Param(name='foo', value=42, unit='bars', ac=True) p_foo2 = vp.Param(name='foo', value=123, unit='bars', ac=True) p_noname = vp.Param(name='delete_me', value=111) param_list = [p_foo1, p_foo2, p_noname] del p_noname.attrib['name'] v.What.extend(param_list) # Show flaws in old routine: with pytest.warns(FutureWarning): old_style_toplevel_param_dict = vp.pull_params(v)[None] # print(old_style_toplevel_param_dict) vp.assert_valid_as_v2_0(v) # The old 'pull_params' routine will simply drop Params with duplicate # names: assert len(old_style_toplevel_param_dict) == (len(param_list) - 1) none_group = vp.Group([], name=None) complex_group1 = vp.Group([ vp.Param(name='real', value=1.), vp.Param(name='imag', value=0.5) ], name='complex') complex_group2 = vp.Group([ vp.Param(name='real', value=1.5), vp.Param(name='imag', value=2.5) ], name='complex') group_list = [none_group, complex_group1, complex_group2] v.What.extend(group_list) vp.assert_valid_as_v2_0(v) old_style_toplevel_param_dict_w_group = vp.pull_params(v)[None] # An un-named group will also overshadow top-level Params. # This is a total fail, even though it's actually in-spec. assert len(old_style_toplevel_param_dict_w_group) == 0 toplevel_params = vp.get_toplevel_params(v) # .values method behaves like regular dict, one value for each key: assert len(toplevel_params.values()) == (len(param_list) - 1) # Use .allvalues if you want to see duplicates: assert len(toplevel_params.allvalues()) == len(param_list) grouped_params = vp.get_grouped_params(v) assert len(grouped_params.values()) == len(group_list) - 1 assert len(grouped_params.allvalues()) == len(group_list)
def decode_common_voevent(voevent): """ Function to decode common part of VO event :param voevent: xml object already parsed :return: dictionary filled with info """ # first init dictionary to store information gw_dic = init_gw_dic() # Fill the dictionary with the xml content # first retreive the parameters toplevel_params = vp.get_toplevel_params(voevent) # fill the info on the sequence gw_dic["Packet_Type"] = int(toplevel_params['Packet_Type']['value']) gw_dic["Pkt_Ser_Num"] = int(toplevel_params['Pkt_Ser_Num']['value']) gw_dic["AlertType"] = toplevel_params['AlertType']['value'] # fill the info on the event gw_dic["GraceID"] = toplevel_params['GraceID']['value'] gw_dic["HardwareInj"] = toplevel_params['HardwareInj']['value'] gw_dic["EventPage"] = toplevel_params['EventPage']['value'] # fill remaining info in case of a non retracted event if gw_dic["Packet_Type"] != 164: gw_dic["Instruments"] = toplevel_params['Instruments']['value'] gw_dic["FAR"] = float(toplevel_params['FAR']['value']) gw_dic["Group"] = toplevel_params['Group']['value'] gw_dic["Pipeline"] = toplevel_params['Pipeline']['value'] # fill skymap info gw_dic["Skymap"] = str(voevent. find(".//Param[@name='skymap_fits']"). attrib['value']) # fill time info isotime = voevent.WhereWhen.ObsDataLocation.\ ObservationLocation.AstroCoords.Time.TimeInstant.ISOTime.text # parse the iso time (string format) to a datetime object # add Z to the end of the ISO time string to specify UTC time zone gw_dic["Time"] = dateutil.parser.parse(isotime+"Z") return gw_dic
def __init__(self, payload): super().__init__(payload) # Get XML param dicts # NB: you can't store these on the Event because they're unpickleable. top_params = vp.get_toplevel_params(self.voevent) # Default params self.notice = EVENT_DICTIONARY[self.packet_type]['notice_type'] self.type = 'GW_RETRACTION' self.source = EVENT_DICTIONARY[self.packet_type]['source'] # Get the event ID (e.g. S190510g) self.id = top_params['GraceID']['value'] # Create our own event name (e.g. LVC_S190510g) self.name = '{}_{}'.format(self.source, self.id) # Get info from the VOEvent # Retractions have far fewer params self.gracedb_url = top_params['EventPage']['value']
def VOEvent_decoder(XML_file): result_qry_json = [] with open(XML_file, 'rb') as f: v = voeventparse.load(f) # Basic attribute access Ivorn = v.attrib['ivorn'] VOE_role = v.attrib['role'] AuthorIVORN = v.Who.AuthorIVORN VOE_date = v.Who.Date Name_sender = v.Who.Author.shortName Phone_sender = v.Who.Author.contactPhone Mail_sender = v.Who.Author.contactEmail # Copying by value, and validation: #print("Original valid as v2.0? ", voeventparse.valid_as_v2_0(v)) #v_copy = copy.copy(v) #print("Copy valid? ", voeventparse.valid_as_v2_0(v_copy)) ####################################################### # And now, parse the VOEvent ####################################################### #c = voeventparse.get_event_position(v) #print("Coords:", c) # ============================================================================= # Retrieve the WHAT Params # ============================================================================= toplevel_params = voeventparse.get_toplevel_params(v) #print("Params:", toplevel_params) #print() for par in toplevel_params: Event_ID = toplevel_params['Event_ID']['value'] Event_type = toplevel_params['Event_type']['value'] Event_inst = toplevel_params['Event_inst']['value'] Loc_url = toplevel_params['Loc_url']['value'] BA_name = toplevel_params['FA']['value'] Prob = toplevel_params['Prob']['value'] Quicklook_url = toplevel_params['Quicklook_url']['value'] Distance = toplevel_params['Distance']['value'] Err_distance = toplevel_params['Err_distance']['value'] fifty_cr_skymap = toplevel_params['50cr_skymap']['value'] ninety_cr_skymap = toplevel_params['90cr_skymap']['value'] FAR = toplevel_params['FAR']['value'] Group = toplevel_params['Group']['value'] Pipeline = toplevel_params['Pipeline']['value'] Obs_req = toplevel_params['Obs_req']['value'] grouped_params = voeventparse.get_grouped_params(v) #print("Group Params:", grouped_params) #print() for par in grouped_params: Event_status = grouped_params['Status']['Event_status']['value'] Revision = grouped_params['Status']['Revision']['value'] Prob_BNS = grouped_params['Classification']['BNS']['value'] Prob_NSBH = grouped_params['Classification']['NSBH']['value'] Prob_BBH = grouped_params['Classification']['BBH']['value'] Prob_Terrestrial = grouped_params['Classification']['Terrestrial'][ 'value'] Prob_NS = grouped_params['Properties']['HasNS']['value'] Prob_EM = grouped_params['Properties']['HasRemnant']['value'] Name_svom_tel = grouped_params['Set_up_OS']['Name_tel']['value'] FOV_svom_tel = grouped_params['Set_up_OS']['FOV']['value'] FOV_coverage_svom_tel = grouped_params['Set_up_OS']['FOV_coverage'][ 'value'] Mag_limit_svom_tel = grouped_params['Set_up_OS']['Mag_limit']['value'] exposure_svom_tel = grouped_params['Set_up_OS']['exposure']['value'] Slew_rate_svom_tel = grouped_params['Set_up_OS']['Slew_rate']['value'] Readout_svom_tel = grouped_params['Set_up_OS']['Readout']['value'] Filters_svom_tel = grouped_params['Set_up_OS']['Filters_tel']['value'] Latitude_svom_tel = grouped_params['Set_up_OS']['Latitude']['value'] Longitude_svom_tel = grouped_params['Set_up_OS']['Longitude']['value'] Elevation_svom_tel = grouped_params['Set_up_OS']['Elevation']['value'] obs_plan_RA_unit = v.What.Table.Field[1].attrib['unit'] obs_plan_dec_unit = v.What.Table.Field[2].attrib['unit'] obs_plan_Grid_ID = [] obs_plan_RA_center = [] obs_plan_dec_center = [] obs_plan_OS_grade = [] for par in v.What.Table.Data.TR: obs_plan_Grid_ID.append(par.TD[0]) obs_plan_RA_center.append(par.TD[1]) obs_plan_dec_center.append(par.TD[2]) obs_plan_OS_grade.append(par.TD[3]) # ============================================================================= # Retrieve the WHERE & WHEN Params # ============================================================================= trigger_Collab = v.WhereWhen.ObsDataLocation.ObservatoryLocation.attrib[ 'id'] AstroCoordSystem = v.WhereWhen.ObsDataLocation.ObservationLocation.AstroCoordSystem.attrib[ 'id'] Time_unit = v.WhereWhen.ObsDataLocation.ObservationLocation.AstroCoords.Time.attrib[ 'unit'] Trigger_date = v.WhereWhen.ObsDataLocation.ObservationLocation.AstroCoords.Time.TimeInstant.ISOTime Trigger_date_jd = Time(Trigger_date + '.00') Trigger_date_jd_start = Trigger_date_jd.jd Trigger_date = str(Trigger_date).replace('T', ' ') Trigger_pos_unit = v.WhereWhen.ObsDataLocation.ObservationLocation.AstroCoords.Position2D.attrib[ 'unit'] Trigger_RA = v.WhereWhen.ObsDataLocation.ObservationLocation.AstroCoords.Position2D.Value2.C1 Trigger_dec = v.WhereWhen.ObsDataLocation.ObservationLocation.AstroCoords.Position2D.Value2.C2 Trigger_poserr = v.WhereWhen.ObsDataLocation.ObservationLocation.AstroCoords.Position2D.Error2Radius # ============================================================================= # Retrieve the WHY Params # ============================================================================= alert_importance = v.Why.attrib['importance'] # ============================================================================= # First retrieve some params in the DB to fill the trigger table # ============================================================================= ID_SVOM_ba_shift = Retrieve_BA_ID(db, Trigger_date_jd_start) ID_external_trigger_type = Retrieve_trigger_type_ID( db, Event_type, VOE_role) ID_external_trigger_telescope = Retrieve_telescope_type_ID(db, Event_inst) result_qry_json = [] d = {} d["Alert_type"] = VOE_role d["Event_ID"] = Event_ID d["Event_type"] = Event_type d["Event_inst"] = Event_inst d["Loc_url"] = Loc_url d["BA_name"] = BA_name d["Prob"] = Prob d["Quicklook_url"] = Quicklook_url d["Distance"] = Distance d["Err_distance"] = Err_distance d["fifty_cr_skymap"] = fifty_cr_skymap d["ninety_cr_skymap"] = ninety_cr_skymap d["FAR"] = FAR d["Group"] = Group d["Pipeline"] = Pipeline d["Obs_req"] = Obs_req d["Event_status"] = Event_status d["Revision"] = Revision d["Prob_BNS"] = Prob_BNS d["Prob_NSBH"] = Prob_NSBH d["Prob_BBH"] = Prob_BBH d["Prob_Terrestrial"] = Prob_Terrestrial d["Prob_NS"] = Prob_NS d["Prob_EM"] = Prob_EM d["Name_svom_tel"] = Name_svom_tel d["FOV_svom_tel"] = FOV_svom_tel d["FOV_coverage_svom_tel"] = FOV_coverage_svom_tel d["Mag_limit_svom_tel"] = Mag_limit_svom_tel d["exposure_svom_tel"] = exposure_svom_tel d["Slew_rate_svom_tel"] = Slew_rate_svom_tel d["Readout_svom_tel"] = Readout_svom_tel d["Filters_svom_tel"] = Filters_svom_tel d["Latitude_svom_tel"] = Latitude_svom_tel d["Longitude_svom_tel"] = Longitude_svom_tel d["Elevation_svom_tel"] = Elevation_svom_tel d["Longitude_svom_tel"] = Longitude_svom_tel d["obs_plan_RA_unit"] = obs_plan_RA_unit d["obs_plan_dec_unit"] = obs_plan_dec_unit d["obs_plan_Grid_ID"] = obs_plan_Grid_ID d["obs_plan_RA_center"] = obs_plan_RA_center d["obs_plan_dec_center"] = obs_plan_dec_center d["obs_plan_OS_grade"] = obs_plan_OS_grade d["trigger_Collab"] = trigger_Collab d["AstroCoordSystem"] = AstroCoordSystem d["Time_unit"] = Time_unit d["Trigger_date"] = str(Trigger_date) d["Trigger_date_jd"] = str(Trigger_date_jd_start) d["Trigger_pos_unit"] = Trigger_pos_unit d["Trigger_RA"] = str(Trigger_RA) d["Trigger_dec"] = str(Trigger_dec) d["Trigger_poserr"] = str(Trigger_poserr) d["alert_importance"] = str(alert_importance) d["ID_SVOM_ba_shift"] = ID_SVOM_ba_shift d["ID_external_trigger_type"] = ID_external_trigger_type d["ID_external_trigger_telescope"] = ID_external_trigger_telescope result_qry_json = json.dumps(d, ensure_ascii=False) return result_qry_json, obs_plan_RA_unit, obs_plan_dec_unit, obs_plan_Grid_ID, obs_plan_RA_center, obs_plan_dec_center, obs_plan_OS_grade
def __init__(self, payload): super().__init__(payload) # Get XML param dicts # NB: you can't store these on the Event because they're unpickleable. top_params = vp.get_toplevel_params(self.voevent) group_params = vp.get_grouped_params(self.voevent) # Default params self.notice = EVENT_DICTIONARY[self.packet_type]['notice_type'] self.type = 'GRB' self.source = EVENT_DICTIONARY[self.packet_type]['source'] # Get the event ID (e.g. 579943502) self.id = top_params['TrigID']['value'] # Create our own event name (e.g. Fermi_579943502) self.name = '{}_{}'.format(self.source, self.id) # Get properties from the VOEvent if self.source == 'Fermi': self.properties = { key: group_params['Trigger_ID'][key]['value'] for key in group_params['Trigger_ID'] if key != 'Long_short' } try: self.duration = group_params['Trigger_ID']['Long_short'][ 'value'] except KeyError: # Some don't have the duration self.duration = 'unknown' elif self.source == 'Swift': self.properties = { key: group_params['Solution_Status'][key]['value'] for key in group_params['Solution_Status'] } for key in self.properties: if self.properties[key] == 'true': self.properties[key] = True elif self.properties[key] == 'false': self.properties[key] = False # Position coordinates self.position = vp.get_event_position(self.voevent) self.coord = SkyCoord(ra=self.position.ra, dec=self.position.dec, unit=self.position.units) self.target = FixedTarget(self.coord) # Position error self.coord_error = Angle(self.position.err, unit=self.position.units) if self.source == 'Fermi': self.systematic_error = Angle(5.6, unit='deg') else: self.systematic_error = Angle(0, unit='deg') self.total_error = Angle(np.sqrt(self.coord_error**2 + self.systematic_error**2), unit='deg') # Galactic coordinates self.gal_lat = self.coord.galactic.b.value galactic_center = SkyCoord(l=0, b=0, unit='deg,deg', frame='galactic') self.gal_dist = self.coord.galactic.separation(galactic_center).value # Try creating the Fermi skymap url # Fermi haven't actually updated their alerts to include the URL to the HEALPix skymap, # but we can try and create it based on the typical location. try: old_url = top_params['LightCurve_URL']['value'] skymap_url = old_url.replace('lc_medres34', 'healpix_all').replace( '.gif', '.fit') self.skymap_url = skymap_url except Exception: # Worth a try, fall back to creating our own self.skymap_url = None # Don't create or download the skymap here, it may well be very large. # Only do it when it's absolutely necessary # These params will only be set once the skymap is downloaded self.contour_areas = {0.5: None, 0.9: None}
# Changing values: v_copy.Who.Author.shortName = 'BillyBob' v_copy.attrib['role'] = voeventparse.definitions.roles.test print("Changes valid? ", voeventparse.valid_as_v2_0(v_copy)) v_copy.attrib['role'] = 'flying circus' print("How about now? ", voeventparse.valid_as_v2_0(v_copy)) print("But the original is ok, because we copied? ", voeventparse.valid_as_v2_0(v)) v.Who.BadPath = "This new attribute certainly won't conform with the schema." assert voeventparse.valid_as_v2_0(v) == False del v.Who.BadPath assert voeventparse.valid_as_v2_0(v) == True ####################################################### # And now, SCIENCE ####################################################### c = voeventparse.get_event_position(v) print("Coords:", c) print() toplevel_params = voeventparse.get_toplevel_params(v) # print("Toplevel Params:") # pp.pprint(toplevel_params.items()) print("Trigger ID:", toplevel_params['TrigID']['value']) grouped_params = voeventparse.get_grouped_params(v) # pp.pprint(grouped_params.allitems()) print( "GRB Identified:", grouped_params['Solution_Status']['GRB_Identified']['value'])