def add_gpscorrection_into_stationxml(csv_file, input_xml, out_xml=None):
"""
Read in the correction CSV data from a file, get the station metadata node from input_xml file,
then add the CSV data into the station xml node to write into out_xml
:param csv_file: input csv file with correction data
:param input_xml: input original stationXML file which contains the metadata for the network and station of csv_file
:param out_xml: Directory of the output xml file
:return: full path of the output xml file
"""
ns = "https://github.com/GeoscienceAustralia/hiperseis/xmlns/1.0"
(net, sta, csv_data) = get_csv_correction_data(csv_file)
# path2_myxml = "/home/feizhang/Githubz/hiperseis/tests/testdata/7D_2012_2013.xml"
my_inv = read_inventory(input_xml, format='STATIONXML')
# https://docs.obspy.org/packages/autogen/obspy.core.inventory.inventory.Inventory.select.html#obspy.core.inventory.inventory.Inventory.select
selected_inv = my_inv.select(network=net, station=sta)
# print(selected_inv)
my_tag = AttribDict()
my_tag.namespace = ns
my_tag.value = csv_data
selected_inv.networks[0].stations[0].extra = AttribDict()
selected_inv.networks[0].stations[0].extra.gpsclockcorrection = my_tag
stationxml_with_csv = '%s.%s_station_inv_modified.xml' % (net, sta)
if out_xml is not None and os.path.isdir(out_xml):
stationxml_with_csv = os.path.join(out_xml, stationxml_with_csv)
selected_inv.write(
stationxml_with_csv,
format='STATIONXML',
nsmap={
'GeoscienceAustralia':
'https://github.com/GeoscienceAustralia/hiperseis/xmlns/1.0'
})
# my_inv.write('modified_inventory.xml', format='STATIONXML')
return stationxml_with_csv
def test_write_with_extra_tags_without_read_extra(self):
"""
Tests that a Inventory object that was instantiated with
custom namespace tags and attributes is written correctly.
"""
# read the default inventory
inv = obspy.read_inventory()
# manually add extra to the dictionary
network = inv[0]
network.extra = {}
ns = 'http://test.myns.ns/'
# manually add a new custom namespace tag and attribute to the
# inventory
network.extra['mynsNetworkTag'] = AttribDict({
'value': 'mynsNetworkTagValue',
'namespace': ns})
network.extra['mynsNetworkAttrib'] = AttribDict({
'value': 'mynsNetworkAttribValue',
'namespace': ns,
'type': 'attribute'})
station = inv[0][0]
station.extra = {}
station.extra['mynsStationTag'] = AttribDict({
'value': 'mynsStationTagValue',
'namespace': ns})
station.extra['mynsStationAttrib'] = AttribDict({
'value': 'mynsStationAttribValue',
'namespace': ns,
'type': 'attribute'})
channel = inv[0][0][0]
# add data availability to inventory
channel.data_availability = AttribDict({
'start': obspy.UTCDateTime('1998-10-26T20:35:58+00:00'),
'end': obspy.UTCDateTime('2014-07-21T12:00:00+00:00')})
channel.extra = {}
channel.extra['mynsChannelTag'] = AttribDict({
'value': 'mynsChannelTagValue',
'namespace': ns})
channel.extra['mynsChannelAttrib'] = AttribDict({
'value': 'mynsChannelAttribValue',
'namespace': ns,
'type': 'attribute'})
# add nested tags
nested_tag = AttribDict()
nested_tag.namespace = ns
nested_tag.value = AttribDict()
# add two nested tags
nested_tag.value.my_nested_tag1 = AttribDict()
nested_tag.value.my_nested_tag1.namespace = ns
nested_tag.value.my_nested_tag1.value = 1.23E+10
nested_tag.value.my_nested_tag2 = AttribDict()
nested_tag.value.my_nested_tag2.namespace = ns
nested_tag.value.my_nested_tag2.value = True
nested_tag.value.my_nested_tag2.attrib = {'{%s}%s' % (
ns, 'nestedAttribute1'): 'nestedAttributeValue1'}
channel.extra['nested'] = nested_tag
with NamedTemporaryFile() as tf:
# manually add custom namespace definition
tmpfile = tf.name
# set namespace map to include only valid custom namespaces
mynsmap = {'myns': ns}
# write file with manually defined namespace map
inv.write(tmpfile, format="STATIONXML", nsmap=mynsmap)
# check contents
with open(tmpfile, "rb") as fh:
# enforce reproducible attribute orders through write_c14n
obj = etree.fromstring(fh.read()).getroottree()
buf = io.BytesIO()
obj.write_c14n(buf)
buf.seek(0, 0)
content = buf.read()
# check namespace definitions in root element
expected = [
b'xmlns="http://www.fdsn.org/xml/station/1"',
b'xmlns:myns="http://test.myns.ns/"',
b'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"']
for line in expected:
self.assertIn(line, content)
# check additional tags
expected = [
b'<myns:mynsNetworkTag>' +
b'mynsNetworkTagValue' +
b'</myns:mynsNetworkTag>',
b'myns:mynsNetworkAttrib="mynsNetworkAttribValue"',
b'<myns:mynsStationTag>' +
b'mynsStationTagValue' +
b'</myns:mynsStationTag>',
b'myns:mynsStationAttrib="mynsStationAttribValue"',
b'<myns:mynsChannelTag>' +
b'mynsChannelTagValue' +
b'</myns:mynsChannelTag>',
b'myns:mynsChannelAttrib="mynsChannelAttribValue"',
b'<myns:nested>',
b'<myns:my_nested_tag1>' +
b'12300000000.0' +
b'</myns:my_nested_tag1>',
b'<myns:my_nested_tag2 ' +
b'myns:nestedAttribute1="nestedAttributeValue1">' +
b'True' +
b'</myns:my_nested_tag2>',
b'</myns:nested>'
]
for line in expected:
self.assertIn(line, content)
def modify_invenory(self,
gps_clock_corr_csv=None,
orient_corr_json=None,
equipment_csv=None):
"""
Modify the existing station XML files to include new metadata:
- add equipment sensor digitizer
- add extra metadata: GPS correction
- add extra metadata: Orientation correction
Args:
Returns: the final station_xml file modified with new metadata: inv2_xml_file
"""
# Construct a new inventory object of networks.
# This will use new obspy version and new attributes:
inv2 = Inventory(
# We'll add networks later.
networks=[],
# The source should be the id whoever create the file.
source="Geoscience Australia EFTF AusArray PST")
# output dir for modified station inventory xml files
out_dir = self.output_dir # "/home/fzhang/tmpdir"
net, sta, csv_data = get_csv_correction_data(gps_clock_corr_csv)
net_sta, oricorr_json_data = get_orientation_corr(orient_corr_json)
my_equip_obj = EquipmentExtractor(csvfile=equipment_csv)
big_inv = self.inv_obj
for a_net in big_inv.networks:
print("The number of station-nodes in the network =",
len(a_net.stations))
for a_sta in a_net.stations:
# print(a_net.code, a_sta.code) # this contains 328 pairs, but they are NOT unique, station code may repeat.
a_inv = big_inv.select(
network=a_net.code,
station=a_sta.code) # .copy appears to have no effect here
# print (a_sta.code, " stations has %s channels"%len(a_sta))
_sensors = my_equip_obj.get_sensors(a_net.code, a_sta.code)
if len(_sensors) > 0:
sensor_desc = _sensors[0].get("Description")
sensor_sernumb = _sensors[0].get("SerNumber")
else:
print("%s %s No sensors !" % (a_net.code, a_sta.code))
# sensor_desc = "NA Sensor for (%s,%s)" % (a_net.code, a_sta.code)
sensor_desc = "Nanometrics Trillium Compact 120s"
sensor_sernumb = "N/A"
_digitizers = my_equip_obj.get_digitizer(
a_net.code, a_sta.code)
if len(_digitizers) > 0:
dig_desc = _digitizers[0].get("Description")
dig_sernumb = _digitizers[0].get("SerNumber")
else:
print("%s %s No digitizers !" % (a_net.code, a_sta.code))
#dig_desc = "NA Digitizer for (%s,%s)" % (a_net.code, a_sta.code)
dig_desc = "Guralp Minimus"
dig_sernumb = "N/A"
# modify station metadata
my_sensor = obspy.core.inventory.util.Equipment(
type="Sensor",
description=sensor_desc,
serial_number=sensor_sernumb)
# my_digitizer = obspy.core.inventory.util.Equipment(type="Digitizer", description="Guralp Minimus",serial_number="MIN-A456")
my_digitizer = obspy.core.inventory.util.Equipment(
type="Digitizer",
description=dig_desc,
serial_number=dig_sernumb)
a_sta.equipments = [my_sensor, my_digitizer]
# get station start_ end_date and split csv_data
start_dt = a_sta.start_date
end_dt = a_sta.end_date
ajson = StationMetadataExtra(a_net.code,
a_sta.code,
start_datetime=start_dt,
end_datetime=end_dt)
# generate/format extra metadata from inputs
mpdf = ajson.add_gps_correction_from_csv(csv_data)
# updated the ajson object with more metadata, such as orientation corr
ajson.add_orientation_correction(oricorr_json_data)
ajson.write_metadata2json(
os.path.join(
out_dir, "%s.%s_%s_extra_metadata.json" %
(a_net.code, a_sta.code, str(start_dt))))
# Now, ready to write the ajson obj into new xml file
mformat = "JSON"
my_tag = AttribDict()
my_tag.namespace = GA_NameSpace
my_tag.value = ajson.make_json_string(
) # store all the extra metadata into a json string.
a_sta.extra = AttribDict()
a_sta.extra.GAMetadata = my_tag
# prepare to write out a modified xml file
stationxml_with_extra = '%s.%s_station_metadata_%s.xml' % (
a_net.code, a_sta.code, mformat)
if out_dir is not None and os.path.isdir(out_dir):
stationxml_with_extra = os.path.join(
out_dir, stationxml_with_extra)
a_inv.write(stationxml_with_extra,
format='STATIONXML',
nsmap={'GeoscienceAustralia': GA_NameSpace})
# Problem:
# sta_file_name2 = "%s_%s_station2.xml"%(a_net.code, a_sta.code)
# # OA_CE28 was written 3-times!!!!!! due to multiple (OA,CE28)-station-nodes
# There will be 119 xml files written in this loop of 328 items. However, the final results missed 119 equipments!!
# outxml2 = os.path.join(OUTPUT_DIR, sta_file_name2)
#
# inv2.networks = a_inv.networks
#
# inv2.write(outxml2,format="stationxml", validate=True) # nsmap={'GeoscienceAustralia': GA_NameSpace})
# After the modification of ALL the station objects,
# write the big inventory in new object inv2
inv2.networks = []
inv2.networks.append(a_net)
inv2_xml_file = os.path.join(out_dir,
a_net.code + "_stations2.xml")
inv2.write(inv2_xml_file,
format="stationxml",
nsmap={'GeoscienceAustralia': GA_NameSpace},
validate=True) # every Station got equipment
# Add responses:
resp_obj = read_response()
self.add_response_into_stationxml(inv2, resp_obj)
# and the original write out again to check what has been modified?
post_orig = os.path.join(out_dir,
a_net.code + "_stations_post_orig.xml")
big_inv.write(post_orig,
format="stationxml",
nsmap={'GeoscienceAustralia': GA_NameSpace},
validate=True) # also has the Sensors etc
return inv2_xml_file
def test_write_with_extra_tags_without_read_extra(self):
"""
Tests that a Inventory object that was instantiated with
custom namespace tags and attributes is written correctly.
"""
# read the default inventory
inv = obspy.read_inventory()
# manually add extra to the dictionary
network = inv[0]
network.extra = {}
ns = 'http://test.myns.ns/'
# manually add a new custom namespace tag and attribute to the
# inventory
network.extra['mynsNetworkTag'] = AttribDict({
'value': 'mynsNetworkTagValue',
'namespace': ns
})
network.extra['mynsNetworkAttrib'] = AttribDict({
'value': 'mynsNetworkAttribValue',
'namespace': ns,
'type': 'attribute'
})
station = inv[0][0]
station.extra = {}
station.extra['mynsStationTag'] = AttribDict({
'value': 'mynsStationTagValue',
'namespace': ns
})
station.extra['mynsStationAttrib'] = AttribDict({
'value': 'mynsStationAttribValue',
'namespace': ns,
'type': 'attribute'
})
channel = inv[0][0][0]
# add data availability to inventory
channel.data_availability = AttribDict({
'start':
obspy.UTCDateTime('1998-10-26T20:35:58+00:00'),
'end':
obspy.UTCDateTime('2014-07-21T12:00:00+00:00')
})
channel.extra = {}
channel.extra['mynsChannelTag'] = AttribDict({
'value': 'mynsChannelTagValue',
'namespace': ns
})
channel.extra['mynsChannelAttrib'] = AttribDict({
'value': 'mynsChannelAttribValue',
'namespace': ns,
'type': 'attribute'
})
# add nested tags
nested_tag = AttribDict()
nested_tag.namespace = ns
nested_tag.value = AttribDict()
# add two nested tags
nested_tag.value.my_nested_tag1 = AttribDict()
nested_tag.value.my_nested_tag1.namespace = ns
nested_tag.value.my_nested_tag1.value = 1.23E+10
nested_tag.value.my_nested_tag2 = AttribDict()
nested_tag.value.my_nested_tag2.namespace = ns
nested_tag.value.my_nested_tag2.value = True
nested_tag.value.my_nested_tag2.attrib = {
'{%s}%s' % (ns, 'nestedAttribute1'): 'nestedAttributeValue1'
}
channel.extra['nested'] = nested_tag
with NamedTemporaryFile() as tf:
# manually add custom namespace definition
tmpfile = tf.name
# set namespace map to include only valid custom namespaces
mynsmap = {'myns': ns}
# write file with manually defined namespace map
inv.write(tmpfile, format="STATIONXML", nsmap=mynsmap)
# check contents
with open(tmpfile, "rb") as fh:
# enforce reproducible attribute orders through write_c14n
obj = etree.fromstring(fh.read()).getroottree()
buf = io.BytesIO()
obj.write_c14n(buf)
buf.seek(0, 0)
content = buf.read()
# check namespace definitions in root element
expected = [
b'xmlns="http://www.fdsn.org/xml/station/1"',
b'xmlns:myns="http://test.myns.ns/"',
b'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"'
]
for line in expected:
self.assertIn(line, content)
# check additional tags
expected = [
b'<myns:mynsNetworkTag>' + b'mynsNetworkTagValue' +
b'</myns:mynsNetworkTag>',
b'myns:mynsNetworkAttrib="mynsNetworkAttribValue"',
b'<myns:mynsStationTag>' + b'mynsStationTagValue' +
b'</myns:mynsStationTag>',
b'myns:mynsStationAttrib="mynsStationAttribValue"',
b'<myns:mynsChannelTag>' + b'mynsChannelTagValue' +
b'</myns:mynsChannelTag>',
b'myns:mynsChannelAttrib="mynsChannelAttribValue"',
b'<myns:nested>', b'<myns:my_nested_tag1>' + b'12300000000.0' +
b'</myns:my_nested_tag1>', b'<myns:my_nested_tag2 ' +
b'myns:nestedAttribute1="nestedAttributeValue1">' + b'True' +
b'</myns:my_nested_tag2>', b'</myns:nested>'
]
for line in expected:
self.assertIn(line, content)
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 10 19:26:01 2019
@author: jpeacock
"""
from obspy import Inventory
from obspy.core.inventory import Network
from obspy.core.util import AttribDict
ns = "http://some-page.de/xmlns/1.0"
Channel = AttribDict()
Channel.namespace = ns
Channel.value = AttribDict()
Channel.value.my_nested_tag1 = AttribDict()
Channel.value.my_nested_tag1.namespace = ns
Channel.value.my_nested_tag1.value = 1.23e10
Channel.value.my_nested_tag2 = AttribDict()
Channel.value.my_nested_tag2.namespace = ns
Channel.value.my_nested_tag2.value = True
inv = Inventory([Network("XX")], "XX")
inv[0].extra = AttribDict()
inv[0].extra.Channel = Channel
inv.write(
"my_inventory.xml",
format="STATIONXML",
# generate/format extra metadata from inputs
mpdf = ajson.add_gps_correction_from_csv(csv_data)
# updated the ajson object with more metadata, such as orientation corr
ajson.add_orientation_correction(oricorr_json_data)
ajson.write_metadata2json(
os.path.join(
out_dir, "%s.%s_%s_extra_metadata.json" %
(net, sta, str(start_dt))))
# Now, ready to write the ajson obj into new xml file
mformat = "JSON"
my_tag = AttribDict()
my_tag.namespace = GA_NameSpace
my_tag.value = ajson.make_json_string(
) # store all the extra metadata into a json string.
a_station.extra = AttribDict()
a_station.extra.GAMetadata = my_tag
# prepare to write out a modified xml file
stationxml_with_extra = '%s.%s_station_metadata_%s.xml' % (
net, sta, mformat)
if out_dir is not None and os.path.isdir(out_dir):
stationxml_with_extra = os.path.join(out_dir,
stationxml_with_extra)
def add_gpscorrection_into_stationxml(csv_file,
input_xml,
out_dir=None,
mformat="CSV"):
"""
Read in the correction CSV data from a file, get the station metadata node from input_xml file,
then add the CSV data into the station xml node to write into out_xml
:param csv_file: input csv file with correction data
:param input_xml: input original stationXML file which contains the metadata for the network and station of csv_file
:param out_xml: Directory of the output xml file
:return: full path of the output xml file
"""
GA_NameSpace = "https://github.com/GeoscienceAustralia/hiperseis"
(net, sta, csv_data) = get_csv_correction_data(csv_file)
# path2_myxml = "/home/feizhang/Githubz/hiperseis/tests/testdata/7D_2012_2013.xml"
my_inv = read_inventory(input_xml, format='STATIONXML')
# https://docs.obspy.org/packages/autogen/obspy.core.inventory.inventory.Inventory.select.html#obspy.core.inventory.inventory.Inventory.select
selected_inv = my_inv.select(network=net, station=sta)
# print(selected_inv)
station_list = selected_inv.networks[0].stations
# redefine the selected_inv
for a_station in station_list: # loop over all Stations
# get station star end date and split csv_data
start_dt = a_station.start_date
end_dt = a_station.end_date
mpdf = get_metadata_by_date_range(csv_data, net, sta, start_dt, end_dt)
#print("Station %s= %s %s" %(a_station, start_dt,end_dt))
my_tag = AttribDict()
my_tag.namespace = GA_NameSpace
if mformat == "CSV":
my_tag.value = mpdf.to_csv(index=False)
elif mformat == "JSON":
my_tag.value = mpdf.to_json(orient="records",
date_format="epoch",
force_ascii=True,
date_unit="ms",
default_handler=None,
indent=2)
else:
print("The format %s is not supported" % mformat)
a_station.extra = AttribDict()
a_station.extra.GAMetadata = my_tag
# prepare to write out a modified xml file
mod_stationxml_with_extra = '%s.%s_station_metadata_%s.xml' % (net, sta,
mformat)
if out_dir is not None and os.path.isdir(out_dir):
mod_stationxml_with_extra = os.path.join(out_dir,
mod_stationxml_with_extra)
selected_inv.write(mod_stationxml_with_extra,
format='STATIONXML',
nsmap={'GeoscienceAustralia': GA_NameSpace})
# my_inv.write('modified_inventory.xml', format='STATIONXML')
return mod_stationxml_with_extra
