def test_raise_polynomial(self):
with self.assertRaises(NotImplementedError) as e:
read_inventory(self.arclink_xml_poly)
self.assertEqual(
e.exception.args[0], "responsePolynomial not"
"implemented. Contact the ObsPy developers")
def setUp(self):
"""
Read example stationXML/sc3ml format to Inventory
"""
self.data_dir = os.path.join(os.path.dirname(__file__), "data")
stationxml_path = os.path.join(self.data_dir, "EB_response_stationXML")
sc3ml_path = os.path.join(self.data_dir, "EB_response_sc3ml")
self.stationxml_inventory = read_inventory(stationxml_path,
format="STATIONXML")
self.sc3ml_inventory = read_inventory(sc3ml_path, format="SC3ML")
def setUp(self):
"""
Read example stationXML/sc3ml format to Inventory
"""
self.data_dir = os.path.join(os.path.dirname(__file__), "data")
stationxml_path = os.path.join(self.data_dir, "EB_response_stationXML")
sc3ml_path = os.path.join(self.data_dir, "EB_response_sc3ml")
self.stationxml_inventory = read_inventory(stationxml_path,
format="STATIONXML")
self.sc3ml_inventory = read_inventory(sc3ml_path, format="SC3ML")
def setUp(self):
"""
Read example stationXML/sc3ml format to Inventory
"""
self.data_dir = os.path.join(os.path.dirname(os.path.abspath(
inspect.getfile(inspect.currentframe()))), "data")
stationxml_path = os.path.join(self.data_dir, "EB_response_stationXML")
sc3ml_path = os.path.join(self.data_dir, "EB_response_sc3ml")
self.stationxml_inventory = read_inventory(stationxml_path,
format="STATIONXML")
self.sc3ml_inventory = read_inventory(sc3ml_path, format="SC3ML")
def setUp(self):
"""
Read example stationXML/sc3ml format to Inventory
"""
self.data_dir = os.path.join(os.path.dirname(os.path.abspath(
inspect.getfile(inspect.currentframe()))), "data")
stationxml_path = os.path.join(self.data_dir, "EB_response_stationXML")
sc3ml_path = os.path.join(self.data_dir, "EB_response_sc3ml")
self.stationxml_inventory = read_inventory(stationxml_path,
format="STATIONXML")
self.sc3ml_inventory = read_inventory(sc3ml_path, format="SC3ML")
def test_sc3ml_versions(self):
"""
Test multiple schema versions
"""
for version in ['0.5', '0.99']:
filename = os.path.join(self.data_dir, 'version%s' % version)
with self.assertRaises(ValueError) as e:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
read_inventory(filename)
self.assertEqual(e.exception.args[0],
"Schema version not supported.")
def test_readwrite(self):
"""
Test calculating amplitudes
"""
fmin = 0.001
inv = read_inventory(
str(Path(self.testing_path) / "stations_mayobs.xml"), 'STATIONXML')
for net in inv:
for sta in net:
# print(f'{sta.code=}')
ch = sta.select(channel='*[Z3]')[0]
info_str = f'{sta.code}.{ch.code}'
resp = ch.response
# fig = resp.plot(min_freq=fmin, label=info_str, show=False)
paz = PAZ.from_obspy_response(resp)
# paz.plot(min_freq=fmin, label='PAZ', axes=fig.axes, sym='g.',
# show=False)
fname = f'{info_str}.json'
paz.write_json_pz(fname)
paz2 = PAZ.read_json_pz(fname)
paz2.input_units = paz.input_units
self.assertEqual(paz, paz2)
self.assertTextFilesEqual(fname,
Path(self.testing_path) / fname)
os.remove(fname)
def read_STATIONXML(cls, filename, component):
"""
read JSON paz format
"""
inv = read_inventory(filename, 'STATIONXML')
resp = inv.select(channel='*' + component)[0][0][0].response
s_pz = []
s_others = []
stages = resp.response_stages
sens = resp.instrument_sensitivity
for s in stages:
if isinstance(s, PolesZerosResponseStage):
s_pz.append(s)
else:
s_others.append(s)
if len(s_pz) > 1:
paz_base = s_pz[0]
for s in s_pz[1:]:
paz_base.poles.extend(s.poles)
paz_base.zeros.extend(s.zeros)
paz_base.normalization_factor *= s.normalization_factor
resp.response_stages = [paz_base] + s_others
paz = resp.get_paz()
val = cls(poles=paz.poles,
zeros=paz.zeros,
passband_gain=sens.value,
ref_freq=sens.frequency,
input_units=sens.input_units,
output_units=sens.output_units)
return val
def test_empty_depth(self):
"""
Assert depth, latitude, longitude, elevation set to 0.0 if left empty
"""
with warnings.catch_warnings(record=True) as w:
read_inventory(os.path.join(self.data_dir,
"sc3ml_empty_depth_and_id.sc3ml"))
self.assertEqual(str(w[0].message), "Sensor is missing "
"longitude information, "
"using 0.0")
self.assertEqual(str(w[1].message), "Sensor is missing "
"latitude information, "
"using 0.0")
self.assertEqual(str(w[2].message), "Sensor is missing elevation "
"information, using 0.0")
self.assertEqual(str(w[3].message), "Channel is missing depth "
"information, using 0.0")
def test_empty_depth(self):
"""
Assert depth, latitude, longitude, elevation set to 0.0 if left empty
"""
with warnings.catch_warnings(record=True) as w:
read_inventory(os.path.join(self.data_dir,
"sc3ml_empty_depth_and_id.sc3ml"))
self.assertEqual(str(w[0].message), "Sensor is missing "
"longitude information, "
"using 0.0")
self.assertEqual(str(w[1].message), "Sensor is missing "
"latitude information, "
"using 0.0")
self.assertEqual(str(w[2].message), "Sensor is missing elevation "
"information, using 0.0")
self.assertEqual(str(w[3].message), "Channel is missing depth "
"information, using 0.0")
def read_sta_inv(invfiles, sta):
for invv in invfiles:
inv = read_inventory(invv)
nt0 = inv[0].select(station=sta)
if bool(nt0):
lat = nt0[0].latitude
lon = nt0[0].longitude
elev = nt0[0].elevation
return lat, lon, elev
def test_sc3ml_versions(self):
"""
Test multiple schema versions
"""
for version in ['0.5', '0.6', '0.7', '0.8', '0.9']:
filename = os.path.join(self.data_dir, 'version%s' % version)
read_inventory(filename)
for version in ['0.3', '0.10']:
filename = os.path.join(self.data_dir, 'version%s' % version)
with self.assertRaises(ValueError) as e:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
read_inventory(filename)
self.assertEqual(e.exception.args[0],
"Schema version not supported.")
def test_channel_level(self):
"""
Test inventory without repsonse information up to
channel level
"""
inv = read_inventory(os.path.join(self.data_dir,
"channel_level.sc3ml"))
self.assertEqual(inv[0].code, "NL")
self.assertEqual(inv[0][0].code, "HGN")
for cha in inv[0][0].channels:
self.assertTrue(cha.code in ["BHE", "BHN", "BHZ"])
def test_channel_level(self):
"""
Test inventory without repsonse information up to
channel level
"""
inv = read_inventory(os.path.join(self.data_dir,
"channel_level.sc3ml"))
self.assertEqual(inv[0].code, "NL")
self.assertEqual(inv[0][0].code, "HGN")
for cha in inv[0][0].channels:
self.assertTrue(cha.code in ["BHE", "BHN", "BHZ"])
def test_stage_empty_poles_and_zeros(self):
"""
Tests for a case where the poles and zeros are empty see #2633
"""
sc3ml_mbar_path = os.path.join(self.data_dir,
"zero_poles_and_zeros.sc3ml")
sc3ml_inv = read_inventory(sc3ml_mbar_path)
response = sc3ml_inv[0][0][0].response
zeros = response.response_stages[1].zeros
poles = response.response_stages[1].poles
self.assertEqual(zeros, [])
self.assertEqual(poles, [])
def test_sc3ml_versions(self):
"""
Test multiple schema versions
"""
read_inventory(os.path.join(self.data_dir, "version0.7"))
read_inventory(os.path.join(self.data_dir, "version0.8"))
read_inventory(os.path.join(self.data_dir, "version0.9"))
with self.assertRaises(ValueError) as e:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
read_inventory(os.path.join(self.data_dir, "version0.10"))
self.assertEqual(e.exception.args[0], "Schema version not supported.")
def read_sta_inv(invfile, sta):
inv = read_inventory(invfile)
nt0 = inv[0].select(station=sta)
if nt0:
lat = nt0[0].latitude
lon = nt0[0].longitude
elev = nt0[0].elevation
print(sta, lat, lon, elev)
else:
lat = 999
lon = 999
elev = 999
return lat, lon, elev
def test_sc3ml_versions(self):
"""
Test multiple schema versions
"""
read_inventory(os.path.join(self.data_dir, "version0.7"))
read_inventory(os.path.join(self.data_dir, "version0.8"))
read_inventory(os.path.join(self.data_dir, "version0.9"))
with self.assertRaises(ValueError) as e:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
read_inventory(os.path.join(self.data_dir,
"version0.10"))
self.assertEqual(e.exception.args[0], "Schema version not supported.")
def test_analogue_filter_chain(self):
"""
Test analogue filter chain and compare to stationXML equivalent
"""
arclink_inv = read_inventory(self.arclink_afc_path)
station_inv = read_inventory(self.station_afc_path)
arclink_resp = arclink_inv[0][0][0].response.response_stages
station_resp = station_inv[0][0][0].response.response_stages
for arclink_stage, station_stage in zip(arclink_resp, station_resp):
self.assertEqual(arclink_stage.stage_gain,
station_stage.stage_gain)
self.assertEqual(arclink_stage.stage_sequence_number,
station_stage.stage_sequence_number)
for arc_cs, sta_cs in zip(arclink_stage.__dict__.items(),
station_stage.__dict__.items()):
if arc_cs[0] in ['name', 'resource_id2']:
continue
self.assertEqual(arc_cs, sta_cs)
def read_stationxml(cls, filename, channel, station='*'):
"""
read JSON paz format
"""
inv = read_inventory(filename, 'STATIONXML')
inv = inv.select(channel=channel, station=station)
nscs = [{
'net': n.code,
'sta': s.code,
'chan': c
} for n in inv for s in n for c in s]
if len(nscs) > 1:
nsc_strs = [
f'{x["net"]}.{x["sta"]}.{x["chan"].code}' for x in nscs
]
print('Multiple channels were selected, using {nsc_strs[0]}')
print('Other channels were: ' + ','.join(nsc_strs[1:]))
resp = nscs[0]["chan"].response
return cls.from_obspy_response(resp)
def read_inventory(path_or_file_object,
format='STATIONXML',
xy_from_lat_lon=False,
input_projection=4326,
output_projection=None,
*args,
**kwargs) -> Inventory:
"""
Read inventory file
:param path_or_file_object: the path to the inventory file or a file object
:param format: the format
:param xy_from_lat_lon: if True convert populate the XY field by converting
the latitude and longitude to UTM
:param input_projection: The input projection. Applicable if
xy_from_lat_lon is True (default=4326 for for latitude longitude)
:param output_projection: The output projection. Has to be specified if
xy_from_lat_lon is True. Default=None
:param args: see obspy.core.inventory.read_inventory for more information
:param kwargs: see obspy.core.inventory.read_inventory for more information
:return: an inventory object
:rtype: ~uquake.core.inventory.Inventory
"""
if xy_from_lat_lon and (output_projection is None):
raise ValueError('the output projection is needed for conversion'
'from latitude-longitude to UTM')
if type(path_or_file_object) is Path:
path_or_file_object = str(path_or_file_object)
# del kwargs['xy_from_lat_lon']
obspy_inv = inventory.read_inventory(str(path_or_file_object),
format=format,
*args,
**kwargs)
return Inventory.from_obspy_inventory_object(
obspy_inv,
xy_from_lat_lon=xy_from_lat_lon,
output_projection=output_projection,
input_projection=input_projection)
def _data():
"""returns a dict with fields 'mseed', 'mseed_ACC', 'mseed_VEL', 'mseed_DISP' (all Streams.
The latter three after removing the response)
'inventory' (an inventory object) and two strings: 'mseed_path' and 'inventory_path'"""
folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data')
mseed_path = os.path.join(folder, 'trace_GE.APE.mseed')
mseed = obspy_read(mseed_path)
inv_path = os.path.join(folder, 'inventory_GE.APE.xml')
s = StringIO()
with open(inv_path) as _opn:
s.write(_opn.read())
s.seek(0)
inv_obj = read_inventory(s)
ret = {'mseed': mseed, 'inventory': inv_obj, 'mseed_path': mseed_path,
'data_path': folder,
'inventory_path': inv_path}
for inv_output in ['ACC', 'VEL', 'DISP']:
mseed2 = remove_response(mseed, inv_obj, output=inv_output)
ret['mseed_'+inv_output] = mseed2
return ret
def read_stationxml(cls, filename, channel, station='*'):
"""
read StationXML (returns first valid response)
"""
inv = read_inventory(filename, 'STATIONXML')
inv = inv.select(channel=channel, station=station)
nscs = [{
'net': n.code,
'sta': s.code,
'chan': c
} for n in inv for s in n for c in s]
if len(nscs) > 1:
nsc_strs = [
f'{x["net"]}.{x["sta"]}.{x["chan"].code}' for x in nscs
]
print('Multiple channels were selected, using {nsc_strs[0]}')
print('Other channels were: ' + ','.join(nsc_strs[1:]))
elif len(nscs) == 0:
raise NameError(f'Did not find {station}, {channel} in {filename}')
resp = nscs[0]["chan"].response
return cls.from_obspy_response(resp)
def test_auto_read_arclink_xml(self):
arclink_inv = read_inventory(self.arclink_xml_path)
self.assertIsNotNone(arclink_inv)
station_inv = read_inventory(self.station_xml_path)
arclink_inv = arclink_inv.select(station="GOLS")
channels = ['HH1', 'HH2', 'HHZ', 'BHZ', 'BH1', 'BH2', 'LH1', 'LH2',
'LHZ']
for channel in channels:
arclink_cha = arclink_inv.select(channel=channel)
station_cha = station_inv.select(channel=channel)
for arc, st_xml in zip(arclink_cha[0][0], station_cha[0][0]):
self.assertEqual(arc.code, st_xml.code)
self.assertEqual(arc.latitude, st_xml.latitude)
self.assertEqual(arc.longitude, st_xml.longitude)
self.assertEqual(arc.depth, st_xml.depth)
self.assertEqual(arc.azimuth, st_xml.azimuth)
self.assertEqual(arc.dip, st_xml.dip)
self.assertEqual(arc.sample_rate, st_xml.sample_rate)
self.assertEqual(arc.start_date, st_xml.start_date)
self.assertEqual(arc.end_date, st_xml.end_date)
# reading stationxml will ignore old StationXML 1.0 defined
# StorageFormat, Arclink Inventory XML and SC3ML get it stored
# in extra now
with warnings.catch_warnings(record=True):
self.assertEqual(st_xml.storage_format, None)
self.assertEqual(arc.storage_format, None)
self.assertEqual(
arc.extra['format'],
{'namespace': SCHEMA_NAMESPACE, 'value': None})
cdisps = "clock_drift_in_seconds_per_sample"
self.assertEqual(getattr(arc, cdisps), getattr(st_xml, cdisps))
# Compare datalogger element
for arc_el, st_xml_el in zip(arc.data_logger.__dict__.items(),
st_xml.data_logger.__dict__
.items()):
self.assertEqual(arc_el, st_xml_el)
# Compare sensor element
for arc_sen, st_xml_sen in zip(arc.sensor.__dict__.items(),
st_xml.sensor.__dict__.items()):
# Skip the type; set for stationXML not for ArclinkXML
if arc_sen[0] == 'type':
continue
self.assertEqual(arc_sen, st_xml_sen)
# Same number of response stages
self.assertEqual(len(arc.response.response_stages),
len(st_xml.response.response_stages))
# Check the response stages
for arc_resp, sta_resp in zip(arc.response.response_stages,
st_xml.response.response_stages):
self.assertEqual(arc_resp.stage_gain, sta_resp.stage_gain)
self.assertEqual(arc_resp.stage_sequence_number,
sta_resp.stage_sequence_number)
for arc_cs, sta_cs in zip(arc_resp.__dict__.items(),
sta_resp.__dict__.items()):
# Name can be different and resource_id is not given
# in stationXML
if arc_cs[0] in ['name', 'resource_id2']:
continue
# Floating point precision troubles..
# 0.021099999999999997 != 0.0211
if isinstance(arc_cs[1], float):
self.assertAlmostEqual(arc_cs[1], sta_cs[1])
else:
self.assertEqual(arc_cs, sta_cs)
def test_publicid_slash(self):
v = read_inventory(os.path.join(self.data_dir, "public-id-slash.xml"))
self.assertEqual(len(v[0][0][0].response.response_stages), 11)
def process(input_folder, inventory, output_file_name, min_length_sec,
merge_threshold, ntraces_per_file):
"""
INPUT_FOLDER: Path to input folder containing miniseed files \n
INVENTORY: Path to FDSNStationXML inventory containing channel-level metadata for all stations \n
OUTPUT_FILE_NAME: Name of output ASDF file \n
"""
comm = MPI.COMM_WORLD
nproc = comm.Get_size()
rank = comm.Get_rank()
# Read inventory
inv = None
try:
inv = read_inventory(inventory)
except Exception as e:
print(e)
# end try
files = np.array(glob.glob(input_folder + '/*.mseed'))
random.Random(nproc).shuffle(files)
# files = files[:100]
ustations = set()
ustationInv = defaultdict(list)
networklist = []
stationlist = []
for file in files:
_, _, net, sta, _ = file.split('.')
ustations.add('%s.%s' % (net, sta))
networklist.append(net)
stationlist.append(sta)
# end for
networklist = np.array(networklist)
stationlist = np.array(stationlist)
idx = np.lexsort((networklist, stationlist))
files = files[idx]
myfiles = split_list(files, nproc)[rank]
if (rank == 0):
for i, ustation in enumerate(ustations):
net, sta = ustation.split('.')
sinv = inv.select(network=net, station=sta)
if (not len(sinv.networks)):
print(('Missing station: %s.%s' % (net, sta)))
ustationInv[ustation] = None
else:
ustationInv[ustation] = sinv
# end if
# end for
# end if
ustationInv = comm.bcast(ustationInv, root=0)
# Extract trace-count-lists in parallel
mytrccountlist = np.zeros(len(myfiles))
for ifile, file in enumerate(tqdm(myfiles)):
try:
st = read(file, headonly=True)
mytrccountlist[ifile] = len(st)
# end if
except Exception as e:
print(e)
# end try
# end for
trccountlist = comm.gather(mytrccountlist, root=0)
if (rank == 0):
trccountlist = np.array(
[item for sublist in trccountlist for item in sublist])
# Some mseed files can be problematic in terms of having way too many traces --
# e.g. 250k+ traces, each a couple of samples long, for a day mseed file. We
# need to blacklist them and exclude them from the ASDF file.
print(('Blacklisted %d files out of %d files; ' \
'average trace-count %f, std: %f' % (np.sum(trccountlist > ntraces_per_file),
len(trccountlist),
np.mean(trccountlist),
np.std(trccountlist))))
f = open(str(os.path.splitext(output_file_name)[0] + '.trccount.txt'),
'w+')
for i in range(len(files)):
f.write('%s\t%d\n' % (files[i], trccountlist[i]))
# end for
f.close()
if (os.path.exists(output_file_name)): os.remove(output_file_name)
ds = pyasdf.ASDFDataSet(output_file_name,
compression='gzip-3',
mpi=False)
for ifile, file in enumerate(tqdm(files)):
st = []
if (trccountlist[ifile] > ntraces_per_file):
continue
else:
try:
st = read(file)
except Exception as e:
print(e)
continue
# end try
# end if
if (len(st)):
netsta = st[0].stats.network + '.' + st[0].stats.station
if (ustationInv[netsta]):
if (merge_threshold):
ntraces = len(st)
if (ntraces > merge_threshold):
try:
st = st.merge(method=1,
fill_value='interpolate')
except:
print('Failed to merge traces. Moving along..')
continue
# end try
print(
('Merging stream with %d traces' % (ntraces)))
# end if
# end if
# end if
for tr in st:
if (tr.stats.npts == 0): continue
if (min_length_sec):
if (tr.stats.npts * tr.stats.delta < min_length_sec):
continue
# end if
asdfTag = make_ASDF_tag(tr,
"raw_recording").encode('ascii')
try:
ds.add_waveforms(tr, tag='raw_recording')
except Exception as e:
print(e)
print('Failed to append trace:')
print(tr)
# end try
# end for
try:
ds.add_stationxml(ustationInv[netsta])
except Exception as e:
print(e)
print('Failed to append inventory:')
print((ustationInv[netsta]))
# end try
# end if
# end for
print('Closing asdf file..')
del ds
f.write('Orient 1 2, ' + str(st[0].stats.starttime.julday).zfill(3) +
', ' + str(st[0].stats.starttime.hour).zfill(2) + ', ' +
str(theta1) + ', ' + str(rot1(theta1)) + '\n')
f.write('Orient 2 3, ' + str(st[0].stats.starttime.julday).zfill(3) +
', ' + str(st[0].stats.starttime.hour).zfill(2) + ', ' +
str(theta2) + ', ' + str(rot2(theta2)) + '\n')
f.write('Orient 1 3, ' + str(st[0].stats.starttime.julday).zfill(3) +
', ' + str(st[0].stats.starttime.hour).zfill(2) + ', ' +
str(theta3) + ', ' + str(rot3(theta3)) + '\n')
return
resps = glob.glob('metadata/RESP*')
for resp in resps:
if 'inv' not in vars():
inv = read_inventory(resp)
else:
inv += read_inventory(resp)
for stime, etime in zip(stimes, etimes):
# This loop represents a single test (we will parse data in windows for each test)
ctime = stime
st = Stream()
while ctime <= etime:
st += read('/msd/XX_TST4/2020/' + str(ctime.julday).zfill(3) + '/*BH*')
st += read('/msd/GS_ALQ2/2020/' + str(ctime.julday).zfill(3) +
'/00_BH*')
ctime += 24 * 60 * 60
st.merge(fill_value=0)
st.trim(stime, etime)
st.sort()
