def test_write_with_extra_tags_and_read(self): """ Tests that a QuakeML file with additional custom "extra" tags gets written correctly and that when reading it again the extra tags are parsed correctly. """ filename = os.path.join(self.path, "quakeml_1.2_origin.xml") with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") cat = _read_quakeml(filename) self.assertEqual(len(w), 0) # add some custom tags to first event: # - tag with explicit namespace but no explicit ns abbreviation # - tag without explicit namespace (gets obspy default ns) # - tag with explicit namespace and namespace abbreviation my_extra = AttribDict( {'public': {'value': False, 'namespace': 'http://some-page.de/xmlns/1.0', 'attrib': {'some_attrib': 'some_value', 'another_attrib': 'another_value'}}, 'custom': {'value': 'True', 'namespace': 'http://test.org/xmlns/0.1'}, 'new_tag': {'value': 1234, 'namespace': 'http://test.org/xmlns/0.1'}, 'tX': {'value': UTCDateTime('2013-01-02T13:12:14.600000Z'), 'namespace': 'http://test.org/xmlns/0.1'}, 'dataid': {'namespace': 'http://anss.org/xmlns/catalog/0.1', 'type': 'attribute', 'value': '00999999'}, # some nested tags : 'quantity': {'namespace': 'http://some-page.de/xmlns/1.0', 'attrib': {'attrib1': 'attrib_value1', 'attrib2': 'attrib_value2'}, 'value': { 'my_nested_tag1': { 'namespace': 'http://some-page.de/xmlns/1.0', 'value': 1.23E10}, 'my_nested_tag2': { 'namespace': 'http://some-page.de/xmlns/1.0', 'value': False}}}}) nsmap = {'ns0': 'http://test.org/xmlns/0.1', 'catalog': 'http://anss.org/xmlns/catalog/0.1'} cat[0].extra = my_extra.copy() # insert a pick with an extra field p = Pick() p.extra = {'weight': {'value': 2, 'namespace': 'http://test.org/xmlns/0.1'}} cat[0].picks.append(p) with NamedTemporaryFile() as tf: tmpfile = tf.name # write file cat.write(tmpfile, format='QUAKEML', nsmap=nsmap) # check contents with open(tmpfile, 'rb') as fh: # enforce reproducible attribute orders through write_c14n obj = etree.fromstring(fh.read()).getroottree() buf = io.BytesIO() obj.write_c14n(buf) buf.seek(0, 0) content = buf.read() # check namespace definitions in root element expected = [b'<q:quakeml', b'xmlns:catalog="http://anss.org/xmlns/catalog/0.1"', b'xmlns:ns0="http://test.org/xmlns/0.1"', b'xmlns:ns1="http://some-page.de/xmlns/1.0"', b'xmlns:q="http://quakeml.org/xmlns/quakeml/1.2"', b'xmlns="http://quakeml.org/xmlns/bed/1.2"'] for line in expected: self.assertIn(line, content) # check additional tags expected = [ b'<ns0:custom>True</ns0:custom>', b'<ns0:new_tag>1234</ns0:new_tag>', b'<ns0:tX>2013-01-02T13:12:14.600000Z</ns0:tX>', b'<ns1:public ' b'another_attrib="another_value" ' b'some_attrib="some_value">false</ns1:public>' ] for line in expected: self.assertIn(line, content) # now, read again to test if it's parsed correctly.. cat = _read_quakeml(tmpfile) # when reading.. # - namespace abbreviations should be disregarded # - we always end up with a namespace definition, even if it was # omitted when originally setting the custom tag # - custom namespace abbreviations should attached to Catalog self.assertTrue(hasattr(cat[0], 'extra')) def _tostr(x): if isinstance(x, bool): if x: return str('true') else: return str('false') elif isinstance(x, AttribDict): for key, value in x.items(): x[key].value = _tostr(value['value']) return x else: return str(x) for key, value in my_extra.items(): my_extra[key]['value'] = _tostr(value['value']) self.assertEqual(cat[0].extra, my_extra) self.assertTrue(hasattr(cat[0].picks[0], 'extra')) self.assertEqual( cat[0].picks[0].extra, {'weight': {'value': '2', 'namespace': 'http://test.org/xmlns/0.1'}}) self.assertTrue(hasattr(cat, 'nsmap')) self.assertEqual(getattr(cat, 'nsmap')['ns0'], nsmap['ns0'])
def test_write_with_extra_tags_and_read(self): """ Tests that a QuakeML file with additional custom "extra" tags gets written correctly and that when reading it again the extra tags are parsed correctly. """ filename = os.path.join(self.path, "quakeml_1.2_origin.xml") with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") cat = readQuakeML(filename) self.assertEqual(len(w), 0) # add some custom tags to first event: # - tag with explicit namespace but no explicit ns abbreviation # - tag without explicit namespace (gets obspy default ns) # - tag with explicit namespace and namespace abbreviation my_extra = AttribDict( {'public': {'value': False, 'namespace': r"http://some-page.de/xmlns/1.0", 'attrib': {u"some_attrib": u"some_value", u"another_attrib": u"another_value"}}, 'custom': {'value': u"True", 'namespace': r'http://test.org/xmlns/0.1'}, 'new_tag': {'value': 1234, 'namespace': r"http://test.org/xmlns/0.1"}, 'tX': {'value': UTCDateTime('2013-01-02T13:12:14.600000Z'), 'namespace': r'http://test.org/xmlns/0.1'}, 'dataid': {'namespace': r'http://anss.org/xmlns/catalog/0.1', 'type': 'attribute', 'value': '00999999'}}) nsmap = {"ns0": r"http://test.org/xmlns/0.1", "catalog": r'http://anss.org/xmlns/catalog/0.1'} cat[0].extra = my_extra.copy() # insert a pick with an extra field p = Pick() p.extra = {'weight': {'value': 2, 'namespace': r"http://test.org/xmlns/0.1"}} cat[0].picks.append(p) with NamedTemporaryFile() as tf: tmpfile = tf.name # write file cat.write(tmpfile, format="QUAKEML", nsmap=nsmap) # check contents with open(tmpfile, "r") as fh: content = fh.read() # check namespace definitions in root element expected = ['<q:quakeml', 'xmlns:catalog="http://anss.org/xmlns/catalog/0.1"', 'xmlns:ns0="http://test.org/xmlns/0.1"', 'xmlns:ns1="http://some-page.de/xmlns/1.0"', 'xmlns:q="http://quakeml.org/xmlns/quakeml/1.2"', 'xmlns="http://quakeml.org/xmlns/bed/1.2"'] for line in expected: self.assertTrue(line in content) # check additional tags expected = [ '<ns0:custom>True</ns0:custom>', '<ns0:new_tag>1234</ns0:new_tag>', '<ns0:tX>2013-01-02T13:12:14.600000Z</ns0:tX>', '<ns1:public ' 'another_attrib="another_value" ' 'some_attrib="some_value">false</ns1:public>' ] for lines in expected: self.assertTrue(line in content) # now, read again to test if its parsed correctly.. cat = readQuakeML(tmpfile) # when reading.. # - namespace abbreviations should be disregarded # - we always end up with a namespace definition, even if it was # omitted when originally setting the custom tag # - custom namespace abbreviations should attached to Catalog self.assertTrue(hasattr(cat[0], "extra")) def _tostr(x): if isinstance(x, bool): if x: return str("true") else: return str("false") return str(x) for key, value in my_extra.items(): my_extra[key]['value'] = _tostr(value['value']) self.assertEqual(cat[0].extra, my_extra) self.assertTrue(hasattr(cat[0].picks[0], "extra")) self.assertEqual( cat[0].picks[0].extra, {'weight': {'value': '2', 'namespace': r'http://test.org/xmlns/0.1'}}) self.assertTrue(hasattr(cat, "nsmap")) self.assertTrue(getattr(cat, "nsmap")['ns0'] == nsmap['ns0'])
def __toPick(parser, pick_el, evaluation_mode): """ """ pick = Pick() pick.resource_id = ResourceIdentifier(prefix="/".join([RESOURCE_ROOT, "pick"])) # Raise a warnings if there is a phase delay phase_delay = parser.xpath2obj("phase_delay", pick_el, float) if phase_delay is not None: msg = "The pick has a phase_delay!" raise Exception(msg) waveform = pick_el.xpath("waveform")[0] network = waveform.get("networkCode") station = fix_station_name(waveform.get("stationCode")) # Map some station names. if station in STATION_DICT: station = STATION_DICT[station] if not network: network = NETWORK_DICT[station] location = waveform.get("locationCode") or "" channel = waveform.get("channelCode") or "" pick.waveform_id = WaveformStreamID( network_code=network, station_code=station, channel_code=channel, location_code=location) pick.time, pick.time_errors = __toTimeQuantity(parser, pick_el, "time") # Picks without time are not quakeml conform if pick.time is None: print "Pick has no time and is ignored: %s" % station return None pick.phase_hint = parser.xpath2obj('phaseHint', pick_el, str) onset = parser.xpath2obj('onset', pick_el) # Fixing bad and old typo ... if onset == "implusive": onset = "impulsive" if onset: pick.onset = onset.lower() # Evaluation mode of a pick is global in the SeisHub Event file format. #pick.evaluation_mode = evaluation_mode # The polarity needs to be mapped. polarity = parser.xpath2obj('polarity', pick_el) pol_map_dict = {'up': 'positive', 'positive': 'positive', 'forward': 'positive', 'forwards': 'positive', 'right': 'positive', 'backward': 'negative', 'backwards': 'negative', 'left': 'negative', 'down': 'negative', 'negative': 'negative', 'undecidable': 'undecidable', 'poorup': 'positive', 'poordown': 'negative'} if polarity: if polarity.lower() in pol_map_dict: pick.polarity = pol_map_dict[polarity.lower()] else: pick.polarity = polarity.lower() pick_weight = parser.xpath2obj('weight', pick_el, int) if pick_weight is not None: pick.extra = AttribDict() pick.extra.weight = {'value': pick_weight, 'namespace': NAMESPACE} return pick
def test_write_with_extra_tags_and_read(self): """ Tests that a QuakeML file with additional custom "extra" tags gets written correctly and that when reading it again the extra tags are parsed correctly. """ filename = os.path.join(self.path, "quakeml_1.2_origin.xml") with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") cat = _read_quakeml(filename) self.assertEqual(len(w), 0) # add some custom tags to first event: # - tag with explicit namespace but no explicit ns abbreviation # - tag without explicit namespace (gets obspy default ns) # - tag with explicit namespace and namespace abbreviation my_extra = AttribDict( { "public": { "value": False, "namespace": "http://some-page.de/xmlns/1.0", "attrib": {"some_attrib": "some_value", "another_attrib": "another_value"}, }, "custom": {"value": "True", "namespace": "http://test.org/xmlns/0.1"}, "new_tag": {"value": 1234, "namespace": "http://test.org/xmlns/0.1"}, "tX": {"value": UTCDateTime("2013-01-02T13:12:14.600000Z"), "namespace": "http://test.org/xmlns/0.1"}, "dataid": {"namespace": "http://anss.org/xmlns/catalog/0.1", "type": "attribute", "value": "00999999"}, # some nested tags : "quantity": { "namespace": "http://some-page.de/xmlns/1.0", "attrib": {"attrib1": "attrib_value1", "attrib2": "attrib_value2"}, "value": { "my_nested_tag1": {"namespace": "http://some-page.de/xmlns/1.0", "value": 1.23e10}, "my_nested_tag2": {"namespace": "http://some-page.de/xmlns/1.0", "value": False}, }, }, } ) nsmap = {"ns0": "http://test.org/xmlns/0.1", "catalog": "http://anss.org/xmlns/catalog/0.1"} cat[0].extra = my_extra.copy() # insert a pick with an extra field p = Pick() p.extra = {"weight": {"value": 2, "namespace": "http://test.org/xmlns/0.1"}} cat[0].picks.append(p) with NamedTemporaryFile() as tf: tmpfile = tf.name # write file cat.write(tmpfile, format="QUAKEML", nsmap=nsmap) # check contents with open(tmpfile, "rb") as fh: # enforce reproducible attribute orders through write_c14n obj = etree.fromstring(fh.read()).getroottree() buf = io.BytesIO() obj.write_c14n(buf) buf.seek(0, 0) content = buf.read() # check namespace definitions in root element expected = [ b"<q:quakeml", b'xmlns:catalog="http://anss.org/xmlns/catalog/0.1"', b'xmlns:ns0="http://test.org/xmlns/0.1"', b'xmlns:ns1="http://some-page.de/xmlns/1.0"', b'xmlns:q="http://quakeml.org/xmlns/quakeml/1.2"', b'xmlns="http://quakeml.org/xmlns/bed/1.2"', ] for line in expected: self.assertIn(line, content) # check additional tags expected = [ b"<ns0:custom>True</ns0:custom>", b"<ns0:new_tag>1234</ns0:new_tag>", b"<ns0:tX>2013-01-02T13:12:14.600000Z</ns0:tX>", b"<ns1:public " b'another_attrib="another_value" ' b'some_attrib="some_value">false</ns1:public>', ] for line in expected: self.assertIn(line, content) # now, read again to test if it's parsed correctly.. cat = _read_quakeml(tmpfile) # when reading.. # - namespace abbreviations should be disregarded # - we always end up with a namespace definition, even if it was # omitted when originally setting the custom tag # - custom namespace abbreviations should attached to Catalog self.assertTrue(hasattr(cat[0], "extra")) def _tostr(x): if isinstance(x, bool): if x: return str("true") else: return str("false") elif isinstance(x, AttribDict): for key, value in x.items(): x[key].value = _tostr(value["value"]) return x else: return str(x) for key, value in my_extra.items(): my_extra[key]["value"] = _tostr(value["value"]) self.assertEqual(cat[0].extra, my_extra) self.assertTrue(hasattr(cat[0].picks[0], "extra")) self.assertEqual(cat[0].picks[0].extra, {"weight": {"value": "2", "namespace": "http://test.org/xmlns/0.1"}}) self.assertTrue(hasattr(cat, "nsmap")) self.assertEqual(getattr(cat, "nsmap")["ns0"], nsmap["ns0"])
def __toPick(parser, pick_el, evaluation_mode): """ """ pick = Pick() pick.resource_id = ResourceIdentifier( prefix="/".join([RESOURCE_ROOT, "pick"])) # Raise a warnings if there is a phase delay phase_delay = parser.xpath2obj("phase_delay", pick_el, float) if phase_delay is not None: msg = "The pick has a phase_delay!" raise Exception(msg) waveform = pick_el.xpath("waveform")[0] network = waveform.get("networkCode") station = fix_station_name(waveform.get("stationCode")) # Map some station names. if station in STATION_DICT: station = STATION_DICT[station] if not network: network = NETWORK_DICT[station] location = waveform.get("locationCode") or "" channel = waveform.get("channelCode") or "" pick.waveform_id = WaveformStreamID(network_code=network, station_code=station, channel_code=channel, location_code=location) pick.time, pick.time_errors = __toTimeQuantity(parser, pick_el, "time") # Picks without time are not quakeml conform if pick.time is None: print "Pick has no time and is ignored: %s" % station return None pick.phase_hint = parser.xpath2obj('phaseHint', pick_el, str) onset = parser.xpath2obj('onset', pick_el) # Fixing bad and old typo ... if onset == "implusive": onset = "impulsive" if onset: pick.onset = onset.lower() # Evaluation mode of a pick is global in the SeisHub Event file format. #pick.evaluation_mode = evaluation_mode # The polarity needs to be mapped. polarity = parser.xpath2obj('polarity', pick_el) pol_map_dict = { 'up': 'positive', 'positive': 'positive', 'forward': 'positive', 'forwards': 'positive', 'right': 'positive', 'backward': 'negative', 'backwards': 'negative', 'left': 'negative', 'down': 'negative', 'negative': 'negative', 'undecidable': 'undecidable', 'poorup': 'positive', 'poordown': 'negative' } if polarity: if polarity.lower() in pol_map_dict: pick.polarity = pol_map_dict[polarity.lower()] else: pick.polarity = polarity.lower() pick_weight = parser.xpath2obj('weight', pick_el, int) if pick_weight is not None: pick.extra = AttribDict() pick.extra.weight = {'value': pick_weight, 'namespace': NAMESPACE} return pick
def _read_single_event(event_file, locate_dir, units, local_mag_ph): """ Parse an event file from QuakeMigrate into an obspy Event object. Parameters ---------- event_file : `pathlib.Path` object Path to .event file to read. locate_dir : `pathlib.Path` object Path to locate directory (contains "events", "picks" etc. directories). units : {"km", "m"} Grid projection coordinates for QM LUT (determines units of depths and uncertainties in the .event files). local_mag_ph : {"S", "P"} Amplitude measurement used to calculate local magnitudes. Returns ------- event : `obspy.Event` object Event object populated with all available information output by :class:`~quakemigrate.signal.scan.locate()`, including event locations and uncertainties, picks, and amplitudes and magnitudes if available. """ # Parse information from event file event_info = pd.read_csv(event_file).iloc[0] event_uid = str(event_info["EventID"]) # Set distance conversion factor (from units of QM LUT projection units). if units == "km": factor = 1e3 elif units == "m": factor = 1 else: raise AttributeError(f"units must be 'km' or 'm'; not {units}") # Create event object to store origin and pick information event = Event() event.extra = AttribDict() event.resource_id = str(event_info["EventID"]) event.creation_info = CreationInfo(author="QuakeMigrate", version=quakemigrate.__version__) # Add COA info to extra event.extra.coa = {"value": event_info["COA"], "namespace": ns} event.extra.coa_norm = {"value": event_info["COA_NORM"], "namespace": ns} event.extra.trig_coa = {"value": event_info["TRIG_COA"], "namespace": ns} event.extra.dec_coa = {"value": event_info["DEC_COA"], "namespace": ns} event.extra.dec_coa_norm = { "value": event_info["DEC_COA_NORM"], "namespace": ns } # Determine location of cut waveform data - add to event object as a # custom extra attribute. mseed = locate_dir / "raw_cut_waveforms" / event_uid event.extra.cut_waveforms_file = { "value": str(mseed.with_suffix(".m").resolve()), "namespace": ns } if (locate_dir / "real_cut_waveforms").exists(): mseed = locate_dir / "real_cut_waveforms" / event_uid event.extra.real_cut_waveforms_file = { "value": str(mseed.with_suffix(".m").resolve()), "namespace": ns } if (locate_dir / "wa_cut_waveforms").exists(): mseed = locate_dir / "wa_cut_waveforms" / event_uid event.extra.wa_cut_waveforms_file = { "value": str(mseed.with_suffix(".m").resolve()), "namespace": ns } # Create origin with spline location and set to preferred event origin. origin = Origin() origin.method_id = "spline" origin.longitude = event_info["X"] origin.latitude = event_info["Y"] origin.depth = event_info["Z"] * factor origin.time = UTCDateTime(event_info["DT"]) event.origins = [origin] event.preferred_origin_id = origin.resource_id # Create origin with gaussian location and associate with event origin = Origin() origin.method_id = "gaussian" origin.longitude = event_info["GAU_X"] origin.latitude = event_info["GAU_Y"] origin.depth = event_info["GAU_Z"] * factor origin.time = UTCDateTime(event_info["DT"]) event.origins.append(origin) ouc = OriginUncertainty() ce = ConfidenceEllipsoid() ce.semi_major_axis_length = event_info["COV_ErrY"] * factor ce.semi_intermediate_axis_length = event_info["COV_ErrX"] * factor ce.semi_minor_axis_length = event_info["COV_ErrZ"] * factor ce.major_axis_plunge = 0 ce.major_axis_azimuth = 0 ce.major_axis_rotation = 0 ouc.confidence_ellipsoid = ce ouc.preferred_description = "confidence ellipsoid" # Set uncertainties for both as the gaussian uncertainties for origin in event.origins: origin.longitude_errors.uncertainty = kilometer2degrees( event_info["GAU_ErrX"] * factor / 1e3) origin.latitude_errors.uncertainty = kilometer2degrees( event_info["GAU_ErrY"] * factor / 1e3) origin.depth_errors.uncertainty = event_info["GAU_ErrZ"] * factor origin.origin_uncertainty = ouc # Add OriginQuality info to each origin? for origin in event.origins: origin.origin_type = "hypocenter" origin.evaluation_mode = "automatic" # --- Handle picks file --- pick_file = locate_dir / "picks" / event_uid if pick_file.with_suffix(".picks").is_file(): picks = pd.read_csv(pick_file.with_suffix(".picks")) else: return None for _, pickline in picks.iterrows(): station = str(pickline["Station"]) phase = str(pickline["Phase"]) wid = WaveformStreamID(network_code="", station_code=station) for method in ["modelled", "autopick"]: pick = Pick() pick.extra = AttribDict() pick.waveform_id = wid pick.method_id = method pick.phase_hint = phase if method == "autopick" and str(pickline["PickTime"]) != "-1": pick.time = UTCDateTime(pickline["PickTime"]) pick.time_errors.uncertainty = float(pickline["PickError"]) pick.extra.snr = { "value": float(pickline["SNR"]), "namespace": ns } elif method == "modelled": pick.time = UTCDateTime(pickline["ModelledTime"]) else: continue event.picks.append(pick) # --- Handle amplitudes file --- amps_file = locate_dir / "amplitudes" / event_uid if amps_file.with_suffix(".amps").is_file(): amps = pd.read_csv(amps_file.with_suffix(".amps")) i = 0 for _, ampsline in amps.iterrows(): wid = WaveformStreamID(seed_string=ampsline["id"]) noise_amp = ampsline["Noise_amp"] / 1000 # mm to m for phase in ["P_amp", "S_amp"]: amp = Amplitude() if pd.isna(ampsline[phase]): continue amp.generic_amplitude = ampsline[phase] / 1000 # mm to m amp.generic_amplitude_errors.uncertainty = noise_amp amp.unit = "m" amp.type = "AML" amp.method_id = phase amp.period = 1 / ampsline[f"{phase[0]}_freq"] amp.time_window = TimeWindow( reference=UTCDateTime(ampsline[f"{phase[0]}_time"])) # amp.pick_id = ? amp.waveform_id = wid # amp.filter_id = ? amp.magnitude_hint = "ML" amp.evaluation_mode = "automatic" amp.extra = AttribDict() try: amp.extra.filter_gain = { "value": ampsline[f"{phase[0]}_filter_gain"], "namespace": ns } amp.extra.avg_amp = { "value": ampsline[f"{phase[0]}_avg_amp"] / 1000, # m "namespace": ns } except KeyError: pass if phase[0] == local_mag_ph and not pd.isna(ampsline["ML"]): i += 1 stat_mag = StationMagnitude() stat_mag.extra = AttribDict() # stat_mag.origin_id = ? local_mag_loc stat_mag.mag = ampsline["ML"] stat_mag.mag_errors.uncertainty = ampsline["ML_Err"] stat_mag.station_magnitude_type = "ML" stat_mag.amplitude_id = amp.resource_id stat_mag.extra.picked = { "value": ampsline["is_picked"], "namespace": ns } stat_mag.extra.epi_dist = { "value": ampsline["epi_dist"], "namespace": ns } stat_mag.extra.z_dist = { "value": ampsline["z_dist"], "namespace": ns } event.station_magnitudes.append(stat_mag) event.amplitudes.append(amp) mag = Magnitude() mag.extra = AttribDict() mag.mag = event_info["ML"] mag.mag_errors.uncertainty = event_info["ML_Err"] mag.magnitude_type = "ML" # mag.origin_id = ? mag.station_count = i mag.evaluation_mode = "automatic" mag.extra.r2 = {"value": event_info["ML_r2"], "namespace": ns} event.magnitudes = [mag] event.preferred_magnitude_id = mag.resource_id return event