def add_gpscorrection_into_stationxml(csv_file, input_xml, out_xml=None): """ Read in the correction CSV data from a file, get the station metadata node from input_xml file, then add the CSV data into the station xml node to write into out_xml :param csv_file: input csv file with correction data :param input_xml: input original stationXML file which contains the metadata for the network and station of csv_file :param out_xml: Directory of the output xml file :return: full path of the output xml file """ ns = "https://github.com/GeoscienceAustralia/hiperseis/xmlns/1.0" (net, sta, csv_data) = get_csv_correction_data(csv_file) # path2_myxml = "/home/feizhang/Githubz/hiperseis/tests/testdata/7D_2012_2013.xml" my_inv = read_inventory(input_xml, format='STATIONXML') # https://docs.obspy.org/packages/autogen/obspy.core.inventory.inventory.Inventory.select.html#obspy.core.inventory.inventory.Inventory.select selected_inv = my_inv.select(network=net, station=sta) # print(selected_inv) my_tag = AttribDict() my_tag.namespace = ns my_tag.value = csv_data selected_inv.networks[0].stations[0].extra = AttribDict() selected_inv.networks[0].stations[0].extra.gpsclockcorrection = my_tag stationxml_with_csv = '%s.%s_station_inv_modified.xml' % (net, sta) if out_xml is not None and os.path.isdir(out_xml): stationxml_with_csv = os.path.join(out_xml, stationxml_with_csv) selected_inv.write( stationxml_with_csv, format='STATIONXML', nsmap={ 'GeoscienceAustralia': 'https://github.com/GeoscienceAustralia/hiperseis/xmlns/1.0' }) # my_inv.write('modified_inventory.xml', format='STATIONXML') return stationxml_with_csv
def test_write_with_extra_tags_without_read_extra(self): """ Tests that a Inventory object that was instantiated with custom namespace tags and attributes is written correctly. """ # read the default inventory inv = obspy.read_inventory() # manually add extra to the dictionary network = inv[0] network.extra = {} ns = 'http://test.myns.ns/' # manually add a new custom namespace tag and attribute to the # inventory network.extra['mynsNetworkTag'] = AttribDict({ 'value': 'mynsNetworkTagValue', 'namespace': ns}) network.extra['mynsNetworkAttrib'] = AttribDict({ 'value': 'mynsNetworkAttribValue', 'namespace': ns, 'type': 'attribute'}) station = inv[0][0] station.extra = {} station.extra['mynsStationTag'] = AttribDict({ 'value': 'mynsStationTagValue', 'namespace': ns}) station.extra['mynsStationAttrib'] = AttribDict({ 'value': 'mynsStationAttribValue', 'namespace': ns, 'type': 'attribute'}) channel = inv[0][0][0] # add data availability to inventory channel.data_availability = AttribDict({ 'start': obspy.UTCDateTime('1998-10-26T20:35:58+00:00'), 'end': obspy.UTCDateTime('2014-07-21T12:00:00+00:00')}) channel.extra = {} channel.extra['mynsChannelTag'] = AttribDict({ 'value': 'mynsChannelTagValue', 'namespace': ns}) channel.extra['mynsChannelAttrib'] = AttribDict({ 'value': 'mynsChannelAttribValue', 'namespace': ns, 'type': 'attribute'}) # add nested tags nested_tag = AttribDict() nested_tag.namespace = ns nested_tag.value = AttribDict() # add two nested tags nested_tag.value.my_nested_tag1 = AttribDict() nested_tag.value.my_nested_tag1.namespace = ns nested_tag.value.my_nested_tag1.value = 1.23E+10 nested_tag.value.my_nested_tag2 = AttribDict() nested_tag.value.my_nested_tag2.namespace = ns nested_tag.value.my_nested_tag2.value = True nested_tag.value.my_nested_tag2.attrib = {'{%s}%s' % ( ns, 'nestedAttribute1'): 'nestedAttributeValue1'} channel.extra['nested'] = nested_tag with NamedTemporaryFile() as tf: # manually add custom namespace definition tmpfile = tf.name # set namespace map to include only valid custom namespaces mynsmap = {'myns': ns} # write file with manually defined namespace map inv.write(tmpfile, format="STATIONXML", nsmap=mynsmap) # check contents with open(tmpfile, "rb") as fh: # enforce reproducible attribute orders through write_c14n obj = etree.fromstring(fh.read()).getroottree() buf = io.BytesIO() obj.write_c14n(buf) buf.seek(0, 0) content = buf.read() # check namespace definitions in root element expected = [ b'xmlns="http://www.fdsn.org/xml/station/1"', b'xmlns:myns="http://test.myns.ns/"', b'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'] for line in expected: self.assertIn(line, content) # check additional tags expected = [ b'<myns:mynsNetworkTag>' + b'mynsNetworkTagValue' + b'</myns:mynsNetworkTag>', b'myns:mynsNetworkAttrib="mynsNetworkAttribValue"', b'<myns:mynsStationTag>' + b'mynsStationTagValue' + b'</myns:mynsStationTag>', b'myns:mynsStationAttrib="mynsStationAttribValue"', b'<myns:mynsChannelTag>' + b'mynsChannelTagValue' + b'</myns:mynsChannelTag>', b'myns:mynsChannelAttrib="mynsChannelAttribValue"', b'<myns:nested>', b'<myns:my_nested_tag1>' + b'12300000000.0' + b'</myns:my_nested_tag1>', b'<myns:my_nested_tag2 ' + b'myns:nestedAttribute1="nestedAttributeValue1">' + b'True' + b'</myns:my_nested_tag2>', b'</myns:nested>' ] for line in expected: self.assertIn(line, content)
def modify_invenory(self, gps_clock_corr_csv=None, orient_corr_json=None, equipment_csv=None): """ Modify the existing station XML files to include new metadata: - add equipment sensor digitizer - add extra metadata: GPS correction - add extra metadata: Orientation correction Args: Returns: the final station_xml file modified with new metadata: inv2_xml_file """ # Construct a new inventory object of networks. # This will use new obspy version and new attributes: inv2 = Inventory( # We'll add networks later. networks=[], # The source should be the id whoever create the file. source="Geoscience Australia EFTF AusArray PST") # output dir for modified station inventory xml files out_dir = self.output_dir # "/home/fzhang/tmpdir" net, sta, csv_data = get_csv_correction_data(gps_clock_corr_csv) net_sta, oricorr_json_data = get_orientation_corr(orient_corr_json) my_equip_obj = EquipmentExtractor(csvfile=equipment_csv) big_inv = self.inv_obj for a_net in big_inv.networks: print("The number of station-nodes in the network =", len(a_net.stations)) for a_sta in a_net.stations: # print(a_net.code, a_sta.code) # this contains 328 pairs, but they are NOT unique, station code may repeat. a_inv = big_inv.select( network=a_net.code, station=a_sta.code) # .copy appears to have no effect here # print (a_sta.code, " stations has %s channels"%len(a_sta)) _sensors = my_equip_obj.get_sensors(a_net.code, a_sta.code) if len(_sensors) > 0: sensor_desc = _sensors[0].get("Description") sensor_sernumb = _sensors[0].get("SerNumber") else: print("%s %s No sensors !" % (a_net.code, a_sta.code)) # sensor_desc = "NA Sensor for (%s,%s)" % (a_net.code, a_sta.code) sensor_desc = "Nanometrics Trillium Compact 120s" sensor_sernumb = "N/A" _digitizers = my_equip_obj.get_digitizer( a_net.code, a_sta.code) if len(_digitizers) > 0: dig_desc = _digitizers[0].get("Description") dig_sernumb = _digitizers[0].get("SerNumber") else: print("%s %s No digitizers !" % (a_net.code, a_sta.code)) #dig_desc = "NA Digitizer for (%s,%s)" % (a_net.code, a_sta.code) dig_desc = "Guralp Minimus" dig_sernumb = "N/A" # modify station metadata my_sensor = obspy.core.inventory.util.Equipment( type="Sensor", description=sensor_desc, serial_number=sensor_sernumb) # my_digitizer = obspy.core.inventory.util.Equipment(type="Digitizer", description="Guralp Minimus",serial_number="MIN-A456") my_digitizer = obspy.core.inventory.util.Equipment( type="Digitizer", description=dig_desc, serial_number=dig_sernumb) a_sta.equipments = [my_sensor, my_digitizer] # get station start_ end_date and split csv_data start_dt = a_sta.start_date end_dt = a_sta.end_date ajson = StationMetadataExtra(a_net.code, a_sta.code, start_datetime=start_dt, end_datetime=end_dt) # generate/format extra metadata from inputs mpdf = ajson.add_gps_correction_from_csv(csv_data) # updated the ajson object with more metadata, such as orientation corr ajson.add_orientation_correction(oricorr_json_data) ajson.write_metadata2json( os.path.join( out_dir, "%s.%s_%s_extra_metadata.json" % (a_net.code, a_sta.code, str(start_dt)))) # Now, ready to write the ajson obj into new xml file mformat = "JSON" my_tag = AttribDict() my_tag.namespace = GA_NameSpace my_tag.value = ajson.make_json_string( ) # store all the extra metadata into a json string. a_sta.extra = AttribDict() a_sta.extra.GAMetadata = my_tag # prepare to write out a modified xml file stationxml_with_extra = '%s.%s_station_metadata_%s.xml' % ( a_net.code, a_sta.code, mformat) if out_dir is not None and os.path.isdir(out_dir): stationxml_with_extra = os.path.join( out_dir, stationxml_with_extra) a_inv.write(stationxml_with_extra, format='STATIONXML', nsmap={'GeoscienceAustralia': GA_NameSpace}) # Problem: # sta_file_name2 = "%s_%s_station2.xml"%(a_net.code, a_sta.code) # # OA_CE28 was written 3-times!!!!!! due to multiple (OA,CE28)-station-nodes # There will be 119 xml files written in this loop of 328 items. However, the final results missed 119 equipments!! # outxml2 = os.path.join(OUTPUT_DIR, sta_file_name2) # # inv2.networks = a_inv.networks # # inv2.write(outxml2,format="stationxml", validate=True) # nsmap={'GeoscienceAustralia': GA_NameSpace}) # After the modification of ALL the station objects, # write the big inventory in new object inv2 inv2.networks = [] inv2.networks.append(a_net) inv2_xml_file = os.path.join(out_dir, a_net.code + "_stations2.xml") inv2.write(inv2_xml_file, format="stationxml", nsmap={'GeoscienceAustralia': GA_NameSpace}, validate=True) # every Station got equipment # Add responses: resp_obj = read_response() self.add_response_into_stationxml(inv2, resp_obj) # and the original write out again to check what has been modified? post_orig = os.path.join(out_dir, a_net.code + "_stations_post_orig.xml") big_inv.write(post_orig, format="stationxml", nsmap={'GeoscienceAustralia': GA_NameSpace}, validate=True) # also has the Sensors etc return inv2_xml_file
def test_write_with_extra_tags_without_read_extra(self): """ Tests that a Inventory object that was instantiated with custom namespace tags and attributes is written correctly. """ # read the default inventory inv = obspy.read_inventory() # manually add extra to the dictionary network = inv[0] network.extra = {} ns = 'http://test.myns.ns/' # manually add a new custom namespace tag and attribute to the # inventory network.extra['mynsNetworkTag'] = AttribDict({ 'value': 'mynsNetworkTagValue', 'namespace': ns }) network.extra['mynsNetworkAttrib'] = AttribDict({ 'value': 'mynsNetworkAttribValue', 'namespace': ns, 'type': 'attribute' }) station = inv[0][0] station.extra = {} station.extra['mynsStationTag'] = AttribDict({ 'value': 'mynsStationTagValue', 'namespace': ns }) station.extra['mynsStationAttrib'] = AttribDict({ 'value': 'mynsStationAttribValue', 'namespace': ns, 'type': 'attribute' }) channel = inv[0][0][0] # add data availability to inventory channel.data_availability = AttribDict({ 'start': obspy.UTCDateTime('1998-10-26T20:35:58+00:00'), 'end': obspy.UTCDateTime('2014-07-21T12:00:00+00:00') }) channel.extra = {} channel.extra['mynsChannelTag'] = AttribDict({ 'value': 'mynsChannelTagValue', 'namespace': ns }) channel.extra['mynsChannelAttrib'] = AttribDict({ 'value': 'mynsChannelAttribValue', 'namespace': ns, 'type': 'attribute' }) # add nested tags nested_tag = AttribDict() nested_tag.namespace = ns nested_tag.value = AttribDict() # add two nested tags nested_tag.value.my_nested_tag1 = AttribDict() nested_tag.value.my_nested_tag1.namespace = ns nested_tag.value.my_nested_tag1.value = 1.23E+10 nested_tag.value.my_nested_tag2 = AttribDict() nested_tag.value.my_nested_tag2.namespace = ns nested_tag.value.my_nested_tag2.value = True nested_tag.value.my_nested_tag2.attrib = { '{%s}%s' % (ns, 'nestedAttribute1'): 'nestedAttributeValue1' } channel.extra['nested'] = nested_tag with NamedTemporaryFile() as tf: # manually add custom namespace definition tmpfile = tf.name # set namespace map to include only valid custom namespaces mynsmap = {'myns': ns} # write file with manually defined namespace map inv.write(tmpfile, format="STATIONXML", nsmap=mynsmap) # check contents with open(tmpfile, "rb") as fh: # enforce reproducible attribute orders through write_c14n obj = etree.fromstring(fh.read()).getroottree() buf = io.BytesIO() obj.write_c14n(buf) buf.seek(0, 0) content = buf.read() # check namespace definitions in root element expected = [ b'xmlns="http://www.fdsn.org/xml/station/1"', b'xmlns:myns="http://test.myns.ns/"', b'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' ] for line in expected: self.assertIn(line, content) # check additional tags expected = [ b'<myns:mynsNetworkTag>' + b'mynsNetworkTagValue' + b'</myns:mynsNetworkTag>', b'myns:mynsNetworkAttrib="mynsNetworkAttribValue"', b'<myns:mynsStationTag>' + b'mynsStationTagValue' + b'</myns:mynsStationTag>', b'myns:mynsStationAttrib="mynsStationAttribValue"', b'<myns:mynsChannelTag>' + b'mynsChannelTagValue' + b'</myns:mynsChannelTag>', b'myns:mynsChannelAttrib="mynsChannelAttribValue"', b'<myns:nested>', b'<myns:my_nested_tag1>' + b'12300000000.0' + b'</myns:my_nested_tag1>', b'<myns:my_nested_tag2 ' + b'myns:nestedAttribute1="nestedAttributeValue1">' + b'True' + b'</myns:my_nested_tag2>', b'</myns:nested>' ] for line in expected: self.assertIn(line, content)
# -*- coding: utf-8 -*- """ Created on Mon Jun 10 19:26:01 2019 @author: jpeacock """ from obspy import Inventory from obspy.core.inventory import Network from obspy.core.util import AttribDict ns = "http://some-page.de/xmlns/1.0" Channel = AttribDict() Channel.namespace = ns Channel.value = AttribDict() Channel.value.my_nested_tag1 = AttribDict() Channel.value.my_nested_tag1.namespace = ns Channel.value.my_nested_tag1.value = 1.23e10 Channel.value.my_nested_tag2 = AttribDict() Channel.value.my_nested_tag2.namespace = ns Channel.value.my_nested_tag2.value = True inv = Inventory([Network("XX")], "XX") inv[0].extra = AttribDict() inv[0].extra.Channel = Channel inv.write( "my_inventory.xml", format="STATIONXML",
# generate/format extra metadata from inputs mpdf = ajson.add_gps_correction_from_csv(csv_data) # updated the ajson object with more metadata, such as orientation corr ajson.add_orientation_correction(oricorr_json_data) ajson.write_metadata2json( os.path.join( out_dir, "%s.%s_%s_extra_metadata.json" % (net, sta, str(start_dt)))) # Now, ready to write the ajson obj into new xml file mformat = "JSON" my_tag = AttribDict() my_tag.namespace = GA_NameSpace my_tag.value = ajson.make_json_string( ) # store all the extra metadata into a json string. a_station.extra = AttribDict() a_station.extra.GAMetadata = my_tag # prepare to write out a modified xml file stationxml_with_extra = '%s.%s_station_metadata_%s.xml' % ( net, sta, mformat) if out_dir is not None and os.path.isdir(out_dir): stationxml_with_extra = os.path.join(out_dir, stationxml_with_extra)
def add_gpscorrection_into_stationxml(csv_file, input_xml, out_dir=None, mformat="CSV"): """ Read in the correction CSV data from a file, get the station metadata node from input_xml file, then add the CSV data into the station xml node to write into out_xml :param csv_file: input csv file with correction data :param input_xml: input original stationXML file which contains the metadata for the network and station of csv_file :param out_xml: Directory of the output xml file :return: full path of the output xml file """ GA_NameSpace = "https://github.com/GeoscienceAustralia/hiperseis" (net, sta, csv_data) = get_csv_correction_data(csv_file) # path2_myxml = "/home/feizhang/Githubz/hiperseis/tests/testdata/7D_2012_2013.xml" my_inv = read_inventory(input_xml, format='STATIONXML') # https://docs.obspy.org/packages/autogen/obspy.core.inventory.inventory.Inventory.select.html#obspy.core.inventory.inventory.Inventory.select selected_inv = my_inv.select(network=net, station=sta) # print(selected_inv) station_list = selected_inv.networks[0].stations # redefine the selected_inv for a_station in station_list: # loop over all Stations # get station star end date and split csv_data start_dt = a_station.start_date end_dt = a_station.end_date mpdf = get_metadata_by_date_range(csv_data, net, sta, start_dt, end_dt) #print("Station %s= %s %s" %(a_station, start_dt,end_dt)) my_tag = AttribDict() my_tag.namespace = GA_NameSpace if mformat == "CSV": my_tag.value = mpdf.to_csv(index=False) elif mformat == "JSON": my_tag.value = mpdf.to_json(orient="records", date_format="epoch", force_ascii=True, date_unit="ms", default_handler=None, indent=2) else: print("The format %s is not supported" % mformat) a_station.extra = AttribDict() a_station.extra.GAMetadata = my_tag # prepare to write out a modified xml file mod_stationxml_with_extra = '%s.%s_station_metadata_%s.xml' % (net, sta, mformat) if out_dir is not None and os.path.isdir(out_dir): mod_stationxml_with_extra = os.path.join(out_dir, mod_stationxml_with_extra) selected_inv.write(mod_stationxml_with_extra, format='STATIONXML', nsmap={'GeoscienceAustralia': GA_NameSpace}) # my_inv.write('modified_inventory.xml', format='STATIONXML') return mod_stationxml_with_extra