def _get_ppsd():
"""
Returns ready computed ppsd for testing purposes.
"""
tr, paz = _get_sample_data()
st = Stream([tr])
ppsd = PPSD(tr.stats, paz, db_bins=(-200, -50, 0.5))
ppsd.add(st)
return ppsd
def test_PPSD_w_IRIS_against_obspy_results(self):
"""
Test against results obtained after merging of #1108.
"""
# Read in ANMO data for one day
st = read(os.path.join(self.path, 'IUANMO.seed'))
# Read in metadata in various different formats
paz = {
'gain':
86298.5,
'zeros': [0, 0],
'poles': [
-59.4313, -22.7121 + 27.1065j, -22.7121 + 27.1065j, -0.0048004,
-0.073199
],
'sensitivity':
3.3554 * 10**9
}
resp = os.path.join(self.path, 'IUANMO.resp')
parser = Parser(os.path.join(self.path, 'IUANMO.dataless'))
inv = read_inventory(os.path.join(self.path, 'IUANMO.xml'))
# load expected results, for both only PAZ and full response
results_paz = np.load(os.path.join(self.path, 'IUANMO_ppsd_paz.npz'))
results_full = np.load(
os.path.join(self.path, 'IUANMO_ppsd_fullresponse.npz'))
arrays_to_check = ['hist_stack', 'spec_bins', 'period_bins']
# Calculate the PPSDs and test against expected results
# first: only PAZ
ppsd = PPSD(st[0].stats, paz)
ppsd.add(st)
for key in arrays_to_check:
self.assertTrue(
np.allclose(getattr(ppsd, key), results_paz[key], rtol=1e-5))
# second: various methods for full response
# (also test various means of initialization, basically testing the
# decorator that maps the deprecated keywords)
for metadata in [parser, inv, resp]:
ppsd = PPSD(st[0].stats, paz=metadata)
ppsd = PPSD(st[0].stats, parser=metadata)
ppsd = PPSD(st[0].stats, metadata)
ppsd.add(st)
for key in arrays_to_check:
self.assertTrue(
np.allclose(getattr(ppsd, key),
results_full[key],
rtol=1e-5))
def test_issue1216(self):
tr, paz = _get_sample_data()
st = Stream([tr])
ppsd = PPSD(tr.stats, paz, db_bins=(-200, -50, 0.5))
ppsd.add(st)
# After adding data internal representation of hist stack is None
assert ppsd._current_hist_stack is None
# Accessing the current_histogram property calculates the default stack
assert ppsd.current_histogram is not None
assert ppsd._current_hist_stack is not None
# Adding the same data again does not invalidate the internal stack
# but raises "UserWarning: Already covered time spans detected"
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always', UserWarning)
ppsd.add(st)
msg = 'Already covered time spans detected'
w2 = [w_ for w_ in w if str(w_.message).startswith(msg)]
assert len(w2) == 4
assert ppsd._current_hist_stack is not None
# Adding new data invalidates the internal stack
tr.stats.starttime += 3600
st2 = Stream([tr])
# raises "UserWarning: Already covered time spans detected"
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always', UserWarning)
ppsd.add(st2)
msg = 'Already covered time spans detected'
w2 = [w_ for w_ in w if str(w_.message).startswith(msg)]
assert len(w2) == 2
assert ppsd._current_hist_stack is None
# Accessing current_histogram again calculates the stack
assert ppsd.current_histogram is not None
assert ppsd._current_hist_stack is not None
def test_nice_ringlaser_metadata_error_msg(self):
expected = ("When using `special_handling='ringlaser'`, `metadata` "
"must be a plain dictionary with key 'sensitivity' "
"stating the overall sensitivity`.")
with pytest.raises(TypeError, match=re.escape(expected)):
PPSD(stats=Stats(),
metadata=Inventory(networks=[], source=""),
special_handling='ringlaser')
def test_nice_ringlaser_metadata_error_msg(self):
with self.assertRaises(TypeError) as e:
PPSD(stats=Stats(), metadata=Inventory(networks=[], source=""),
special_handling='ringlaser')
expected = ("When using `special_handling='ringlaser'`, `metadata` "
"must be a plain dictionary with key 'sensitivity' "
"stating the overall sensitivity`.")
self.assertEqual(str(e.exception), expected)
def psd(segment, raw_trace, inventory=None):
if inventory is None:
inventory = segment.inventory()
# tr = segment.stream(True)[0]
dt = (segment.arrival_time - raw_trace.stats.starttime.datetime).total_seconds()
raw_trace = raw_trace.slice(raw_trace.stats.starttime,
UTCDateTime(segment.arrival_time))
ppsd = PPSD(raw_trace.stats, metadata=inventory, ppsd_length=int(dt))
ppsd.add(raw_trace)
return ppsd
def test_ppsd_time_checks(self):
"""
Some tests that make sure checking if a new PSD slice to be addded to
existing PPSD has an invalid overlap or not works as expected.
"""
ppsd = PPSD(Stats(), Response())
one_second = 1000000000
t0 = 946684800000000000 # 2000-01-01T00:00:00
time_diffs = [
0, one_second, one_second * 2, one_second * 3, one_second * 8,
one_second * 9, one_second * 10
]
ppsd._times_processed = [t0 + td for td in time_diffs]
ppsd.ppsd_length = 2
ppsd.overlap = 0.5
# valid time stamps to insert data for (i.e. data that overlaps with
# existing data at most "overlap" times "ppsd_length")
ns_ok = [
t0 - 3 * one_second,
t0 - 1.01 * one_second,
t0 - one_second,
t0 + 4 * one_second,
t0 + 4.01 * one_second,
t0 + 6 * one_second,
t0 + 7 * one_second,
t0 + 6.99 * one_second,
t0 + 11 * one_second,
t0 + 11.01 * one_second,
t0 + 15 * one_second,
]
for ns in ns_ok:
t = UTCDateTime(ns=int(ns))
# getting False means time is not present yet and a PSD slice would
# be added to the PPSD data
self.assertFalse(ppsd._PPSD__check_time_present(t))
# invalid time stamps to insert data for (i.e. data that overlaps with
# existing data more than "overlap" times "ppsd_length")
ns_bad = [
t0 - 0.99 * one_second,
t0 - 0.5 * one_second,
t0,
t0 + 1.1 * one_second,
t0 + 3.99 * one_second,
t0 + 7.01 * one_second,
t0 + 7.5 * one_second,
t0 + 8 * one_second,
t0 + 8.8 * one_second,
t0 + 10 * one_second,
t0 + 10.99 * one_second,
]
for ns in ns_bad:
t = UTCDateTime(ns=int(ns))
# getting False means time is not present yet and a PSD slice would
# be added to the PPSD data
self.assertTrue(ppsd._PPSD__check_time_present(t))
def psd(trace, inventory):
'''Returns the tuple (psd_x, psd_y) values where the first argument is
a numopy array of periods and the second argument is a numpy array of
power spectrum values in Decibel
'''
tr = trace
dt = (tr.stats.endtime.datetime -
tr.stats.starttime.datetime).total_seconds()
ppsd = PPSD(tr.stats, metadata=inventory, ppsd_length=int(dt))
ppsd.add(tr)
return ppsd.period_bin_centers, ppsd.psd_values[0]
def test_ppsd_add_npz(self):
"""
Test PPSD.add_npz().
"""
# set up a bogus PPSD, with fixed random psds but with real start times
# of psd pieces, to facilitate testing the stack selection.
ppsd = PPSD(stats=Stats(dict(sampling_rate=150)),
metadata=None,
db_bins=(-200, -50, 20.),
period_step_octaves=1.4)
_times_processed = np.load(
os.path.join(self.path, "ppsd_times_processed.npy")).tolist()
# change data to nowadays used nanoseconds POSIX timestamp
_times_processed = [UTCDateTime(t)._ns for t in _times_processed]
np.random.seed(1234)
_binned_psds = [
arr for arr in np.random.uniform(-200, -50, (
len(_times_processed), len(ppsd.period_bin_centers)))
]
with NamedTemporaryFile(suffix=".npz") as tf1, \
NamedTemporaryFile(suffix=".npz") as tf2, \
NamedTemporaryFile(suffix=".npz") as tf3:
# save data split up over three separate temporary files
ppsd._times_processed = _times_processed[:200]
ppsd._binned_psds = _binned_psds[:200]
ppsd.save_npz(tf1.name)
ppsd._times_processed = _times_processed[200:400]
ppsd._binned_psds = _binned_psds[200:400]
ppsd.save_npz(tf2.name)
ppsd._times_processed = _times_processed[400:]
ppsd._binned_psds = _binned_psds[400:]
ppsd.matplotlib_version = "X.X.X"
ppsd.save_npz(tf3.name)
# now load these saved npz files and check if all data is present
ppsd = PPSD.load_npz(tf1.name, metadata=None)
ppsd.add_npz(tf2.name)
# we changed a version number so this should emit a warning
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
ppsd.add_npz(tf3.name)
self.assertEqual(len(w), 1)
np.testing.assert_array_equal(_binned_psds, ppsd._binned_psds)
np.testing.assert_array_equal(_times_processed,
ppsd._times_processed)
# adding data already present should also emit a warning and the
# PPSD should not be changed
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
ppsd.add_npz(tf2.name)
self.assertEqual(len(w), 1)
np.testing.assert_array_equal(_binned_psds, ppsd._binned_psds)
np.testing.assert_array_equal(_times_processed,
ppsd._times_processed)
def test_short_trace_warning(self):
"""
Makes sure a warning is shown if a trace shorter than ppsd length is
added which is doing nothing. see #2386
"""
tr, paz = _get_sample_data()
ppsd = PPSD(stats=tr.stats, metadata=paz)
tr.data = tr.data[:4] # shorten trace
msg = (f"Trace is shorter than this PPSD's 'ppsd_length' "
f"({str(ppsd.ppsd_length)} seconds). Skipping trace: "
f"{str(tr)}")
with CatchAndAssertWarnings(expected=[(UserWarning, msg)]):
success = ppsd.add(tr)
assert not success # add returns False on noop
assert not len(ppsd.times_processed) # should be empty, nothing added
# contains start/end times of traces added in, even if not processed
assert len(ppsd.times_data) == 1
def comp_ppsd(sta, debug=True):
net = "IU"
for chan in chans:
print('On:' + sta + ' ' + chan)
stime = UTCDateTime('2016-280T00:00:00')
etime = UTCDateTime('2019-280T00:00:00')
ctime = stime
if sta == 'QSPA':
sen = 3.43 * 10**9
else:
sen = 41.943
paz = {'gain': 1., 'poles': [], 'zeros': [], 'sensitivity': sen}
while ctime < etime:
if debug:
print(ctime)
try:
#if True:
st = Stream()
nctime = ctime
for addday in range(5):
nctime = ctime + addday * 24 * 60 * 60
st += read('/msd/' + net + '_' + sta + '/' +
str(nctime.year) + '/' +
str(nctime.julday).zfill(3) + '/*' + chan + '*')
if debug:
print(st)
except:
ctime += 5 * 24 * 60 * 60
continue
st.merge(fill_value=0)
if 'ppsd' not in vars():
ppsd = PPSD(st[0].stats,
paz,
db_bins=(-100, 100, 1.),
ppsd_length=2**14,
period_smoothing_width_octaves=0.50,
special_handling="ringlaser")
ppsd.add(st)
ctime += 5 * 24 * 60 * 60
ppsd.save_npz('PDF_DATA_' + net + '_' + sta + '_' + chan + ".npz")
ppsd.plot(filename='PPSD_' + net + '_' + sta + '_' + chan + '.PNG',
show_noise_models=False,
period_lim=(2., 1000.))
del ppsd
return
def test_ppsd_infrasound(self):
"""
Test plotting psds on infrasound data
"""
wf = os.path.join(PATH, 'IM.I59H1..BDF_2020_10_31.mseed')
md = os.path.join(PATH, 'IM.I59H1..BDF_2020_10_31.xml')
st = read(wf)
inv = read_inventory(md)
tr = st[0]
ppsd = PPSD(tr.stats,
metadata=inv,
special_handling='infrasound',
db_bins=(-100, 40, 1.),
ppsd_length=300,
overlap=0.5)
ppsd.add(st)
fig = ppsd.plot(xaxis_frequency=True,
period_lim=(0.01, 10),
show=False)
models = (get_idc_infra_hi_noise(), get_idc_infra_low_noise())
lines = fig.axes[0].lines
freq1 = lines[0].get_xdata()
per1 = 1 / freq1
hn = lines[0].get_ydata()
freq2 = lines[1].get_xdata()
per2 = 1 / freq2
ln = lines[1].get_ydata()
per1_m, hn_m = models[0]
per2_m, ln_m = models[1]
np.testing.assert_array_equal(hn, hn_m)
np.testing.assert_array_equal(ln, ln_m)
np.testing.assert_array_equal(per1, per1_m)
np.testing.assert_array_equal(per2, per2_m)
# test calculated psd values
psd = ppsd.psd_values[0]
assert len(psd) == 73
psd = psd[:20]
expected = np.array([
-63.424206, -64.07918, -64.47593, -64.77374, -65.09937, -67.17343,
-66.36576, -65.75002, -65.34155, -64.58012, -63.72327, -62.615784,
-61.612656, -61.085754, -60.09534, -58.949272, -57.600315,
-56.43776, -55.448067, -54.242218
],
dtype=np.float32)
np.testing.assert_array_almost_equal(psd, expected, decimal=2)
def test_exclude_last_sample(self):
start = UTCDateTime("2017-01-01T00:00:00")
header = {
"starttime": start,
"network": "GR",
"station": "FUR",
"channel": "BHZ"
}
# 49 segments of 30 minutes to allow 30 minutes overlap in next day
tr = Trace(data=np.arange(30 * 60 * 4, dtype=np.int32), header=header)
ppsd = PPSD(tr.stats, read_inventory())
ppsd.add(tr)
self.assertEqual(3, len(ppsd._times_processed))
self.assertEqual(3600, ppsd.len)
for i, time in enumerate(ppsd._times_processed):
current = start.ns + (i * 30 * 60) * 1e9
self.assertTrue(time == current)
def test_issue1216(self):
tr, paz = _get_sample_data()
st = Stream([tr])
ppsd = PPSD(tr.stats, paz, db_bins=(-200, -50, 0.5))
ppsd.add(st)
# After adding data internal representation of hist stack is None
self.assertIsNone(ppsd._current_hist_stack)
# Accessing the current_histogram property calculates the default stack
self.assertIsNotNone(ppsd.current_histogram)
self.assertIsNotNone(ppsd._current_hist_stack)
# Adding the same data again does not invalidate the internal stack
ppsd.add(st)
self.assertIsNotNone(ppsd._current_hist_stack)
# Adding new data invalidates the internal stack
tr.stats.starttime += 3600
st2 = Stream([tr])
ppsd.add(st2)
self.assertIsNone(ppsd._current_hist_stack)
# Accessing current_histogram again calculates the stack
self.assertIsNotNone(ppsd.current_histogram)
self.assertIsNotNone(ppsd._current_hist_stack)
def test_exception_reading_newer_npz(self):
"""
Checks that an exception is properly raised when trying to read a npz
that was written on a more recent ObsPy version (specifically that has
a higher 'ppsd_version' number which is used to keep track of changes
in PPSD and the npz file used for serialization).
"""
msg = ("Trying to read/add a PPSD npz with 'ppsd_version=100'. This "
"file was written on a more recent ObsPy version that very "
"likely has incompatible changes in PPSD internal structure "
"and npz serialization. It can not safely be read with this "
"ObsPy version (current 'ppsd_version' is {!s}). Please "
"consider updating your ObsPy installation.".format(
PPSD(stats=Stats(), metadata=None).ppsd_version))
# 1 - loading a npz
data = np.load(self.example_ppsd_npz)
# we have to load, modify 'ppsd_version' and save the npz file for the
# test..
items = {key: data[key] for key in data.files}
# deliberately set a higher ppsd_version number
items['ppsd_version'] = items['ppsd_version'].copy()
items['ppsd_version'].fill(100)
with NamedTemporaryFile() as tf:
filename = tf.name
with open(filename, 'wb') as fh:
np.savez(fh, **items)
with self.assertRaises(ObsPyException) as e:
PPSD.load_npz(filename)
self.assertEqual(str(e.exception), msg)
# 2 - adding a npz
ppsd = PPSD.load_npz(self.example_ppsd_npz)
for method in (ppsd.add_npz, ppsd._add_npz):
with NamedTemporaryFile() as tf:
filename = tf.name
with open(filename, 'wb') as fh:
np.savez(fh, **items)
with self.assertRaises(ObsPyException) as e:
method(filename)
self.assertEqual(str(e.exception), msg)
def test_wrong_trace_id_message(self, state):
"""
Test that we get the expected warning message on waveform/metadata
mismatch.
"""
tr, _paz = _get_sample_data()
inv = read_inventory(os.path.join(state.path, 'IUANMO.xml'))
st = Stream([tr])
ppsd = PPSD(tr.stats, inv)
# metadata doesn't fit the trace ID specified via stats
# should show a warning..
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
ret = ppsd.add(st)
# the trace is sliced into four segments, so we get the warning
# message four times..
assert len(w) == 4
for w_ in w:
assert str(w_.message).startswith(
"Error getting response from provided metadata")
# should not add the data to the ppsd
assert not ret
def psd(tr, inventory, psd_periods, obspy=False):
"""Returns the tuple (psd_x, psd_y) values where the first argument is
a numopy array of periods and the second argument is a numpy array of
power spectrum values in Decibel
"""
from obspy.signal.spectral_estimation import PPSD
# tr = trace.trim(endtime=trace.stats.endtime + 60)
if obspy:
dt = (tr.stats.endtime.datetime -
tr.stats.starttime.datetime).total_seconds()
ppsd = PPSD(tr.stats, metadata=inventory, ppsd_length=int(dt))
ppsd.add(tr)
try:
ppsd.psd_values[0] # just a check (do we have 1 element?)
val = np.interp(psd_periods, ppsd.period_bin_centers,
ppsd.psd_values[0])
val[psd_periods < ppsd.period_bin_centers[0]] = np.nan
val[psd_periods > ppsd.period_bin_centers[-1]] = np.nan
return val
except IndexError:
return []
# return ppsd.period_bin_centers, ppsd.psd_values[0]
else:
return trace_psd(tr, inventory, psd_periods)[0]
def test_PPSD(self):
"""
Test PPSD routine with some real data. Data was downsampled to 100Hz
so the ppsd is a bit distorted which does not matter for the purpose
of testing.
"""
# load test file
file_data = os.path.join(self.path,
'BW.KW1._.EHZ.D.2011.090_downsampled.asc.gz')
file_histogram = os.path.join(
self.path,
'BW.KW1._.EHZ.D.2011.090_downsampled__ppsd_hist_stack.npy')
file_binning = os.path.join(
self.path, 'BW.KW1._.EHZ.D.2011.090_downsampled__ppsd_mixed.npz')
# parameters for the test
data = np.loadtxt(file_data)
stats = {
'_format': 'MSEED',
'calib': 1.0,
'channel': 'EHZ',
'delta': 0.01,
'endtime': UTCDateTime(2011, 3, 31, 2, 36, 0, 180000),
'location': '',
'mseed': {
'dataquality': 'D',
'record_length': 512,
'encoding': 'STEIM2',
'byteorder': '>'
},
'network': 'BW',
'npts': 936001,
'sampling_rate': 100.0,
'starttime': UTCDateTime(2011, 3, 31, 0, 0, 0, 180000),
'station': 'KW1'
}
tr = Trace(data, stats)
st = Stream([tr])
paz = {
'gain':
60077000.0,
'poles': [(-0.037004 + 0.037016j), (-0.037004 - 0.037016j),
(-251.33 + 0j), (-131.04 - 467.29j),
(-131.04 + 467.29j)],
'sensitivity':
2516778400.0,
'zeros': [0j, 0j]
}
ppsd = PPSD(tr.stats, paz)
ppsd.add(st)
# read results and compare
result_hist = np.load(file_histogram)
self.assertEqual(len(ppsd.times), 4)
self.assertEqual(ppsd.nfft, 65536)
self.assertEqual(ppsd.nlap, 49152)
np.testing.assert_array_equal(ppsd.hist_stack, result_hist)
# add the same data a second time (which should do nothing at all) and
# test again - but it will raise UserWarnings, which we omit for now
with warnings.catch_warnings(record=True):
warnings.simplefilter('ignore', UserWarning)
ppsd.add(st)
np.testing.assert_array_equal(ppsd.hist_stack, result_hist)
# test the binning arrays
binning = np.load(file_binning)
np.testing.assert_array_equal(ppsd.spec_bins, binning['spec_bins'])
np.testing.assert_array_equal(ppsd.period_bins, binning['period_bins'])
#st= read(curfile)
#if idx == 0:
#ppsd = PPSD(st[0].stats, paz)
#ppsd.add(st)
#ppsd.save_npz(st[0].id + 'PDF.npz')
debug = True
string = 'newdata/output_raw/2017/XX/FF'
pazNEW = corn_freq_2_paz(4.9, .96)
pazNEW['sensitivity'] = 75900.
chans = ['HHZ', 'HHE', 'HHN']
for idx, chan in enumerate(chans):
st = Stream()
for s in range(1, 9):
if debug:
print('On ' + str(s) + ' ' + chan)
st += read(string + str(s) + '/' + chan + '.D/XX*')
stime = UTCDateTime('2017-05-03T01:10:00.0')
etime = UTCDateTime('2017-05-04T01:10:00.0')
st.trim(starttime=stime, endtime=etime)
st.decimate(4)
st.decimate(2)
print(st)
ppsd = PPSD(st[0].stats, pazNEW)
ppsd.add(st)
ppsd.save_npz(st[0].id + 'PDF.npz')
def test_ppsd_restricted_stacks(self):
"""
Test PPSD.calculate_histogram() with restrictions to what data should
be stacked. Also includes image tests.
"""
# set up a bogus PPSD, with fixed random psds but with real start times
# of psd pieces, to facilitate testing the stack selection.
ppsd = PPSD(stats=Stats(dict(sampling_rate=150)),
metadata=None,
db_bins=(-200, -50, 20.),
period_step_octaves=1.4)
# change data to nowadays used nanoseconds POSIX timestamp
ppsd._times_processed = [
UTCDateTime(t)._ns for t in np.load(
os.path.join(self.path, "ppsd_times_processed.npy")).tolist()
]
np.random.seed(1234)
ppsd._binned_psds = [
arr for arr in np.random.uniform(-200, -50, (
len(ppsd._times_processed), len(ppsd.period_bin_centers)))
]
# Test callback function that selects a fixed random set of the
# timestamps. Also checks that we get passed the type we expect,
# which is 1D numpy ndarray of int type.
def callback(t_array):
self.assertIsInstance(t_array, np.ndarray)
self.assertEqual(t_array.shape, (len(ppsd._times_processed), ))
self.assertTrue(np.issubdtype(t_array.dtype, np.integer))
np.random.seed(1234)
res = np.random.randint(0, 2, len(t_array)).astype(np.bool)
return res
# test several different sets of stack criteria, should cover
# everything, even with lots of combined criteria
stack_criteria_list = [
dict(starttime=UTCDateTime(2015, 3, 8), month=[2, 3, 5, 7, 8]),
dict(endtime=UTCDateTime(2015, 6, 7),
year=[2015],
time_of_weekday=[(1, 0, 24), (2, 0, 24), (-1, 0, 11)]),
dict(year=[2013, 2014, 2016, 2017], month=[2, 3, 4]),
dict(month=[1, 2, 5, 6, 8], year=2015),
dict(isoweek=[4, 5, 6, 13, 22, 23, 24, 44, 45]),
dict(time_of_weekday=[(5, 22, 24), (6, 0, 2), (6, 22, 24)]),
dict(callback=callback, month=[1, 3, 5, 7]),
dict(callback=callback)
]
expected_selections = np.load(
os.path.join(self.path, "ppsd_stack_selections.npy"))
# test every set of criteria
for stack_criteria, expected_selection in zip(stack_criteria_list,
expected_selections):
selection_got = ppsd._stack_selection(**stack_criteria)
np.testing.assert_array_equal(selection_got, expected_selection)
# test one particular selection as an image test
plot_kwargs = dict(max_percentage=15,
xaxis_frequency=True,
period_lim=(0.01, 50))
ppsd.calculate_histogram(**stack_criteria_list[1])
with ImageComparison(self.path_images,
'ppsd_restricted_stack.png',
reltol=1.5) as ic:
fig = ppsd.plot(show=False, **plot_kwargs)
# some matplotlib/Python version combinations lack the left-most
# tick/label "Jan 2015". Try to circumvent and get the (otherwise
# OK) test by changing the left x limit a bit further out (by two
# days, axis is in mpl days). See e.g.
# https://tests.obspy.org/30657/#1
fig.axes[1].set_xlim(left=fig.axes[1].get_xlim()[0] - 2)
with np.errstate(under='ignore'):
fig.savefig(ic.name)
# test it again, checking that updating an existing plot with different
# stack selection works..
# a) we start with the stack for the expected image and test that it
# matches (like above):
ppsd.calculate_histogram(**stack_criteria_list[1])
with ImageComparison(self.path_images,
'ppsd_restricted_stack.png',
reltol=1.5,
plt_close_all_exit=False) as ic:
fig = ppsd.plot(show=False, **plot_kwargs)
# some matplotlib/Python version combinations lack the left-most
# tick/label "Jan 2015". Try to circumvent and get the (otherwise
# OK) test by changing the left x limit a bit further out (by two
# days, axis is in mpl days). See e.g.
# https://tests.obspy.org/30657/#1
fig.axes[1].set_xlim(left=fig.axes[1].get_xlim()[0] - 2)
with np.errstate(under='ignore'):
fig.savefig(ic.name)
# b) now reuse figure and set the histogram with a different stack,
# image test should fail:
ppsd.calculate_histogram(**stack_criteria_list[3])
try:
with ImageComparison(self.path_images,
'ppsd_restricted_stack.png',
adjust_tolerance=False,
plt_close_all_enter=False,
plt_close_all_exit=False) as ic:
# rms of the valid comparison above is ~31,
# rms of the invalid comparison we test here is ~36
if MATPLOTLIB_VERSION == [1, 1, 1]:
ic.tol = 33
ppsd._plot_histogram(fig=fig, draw=True)
with np.errstate(under='ignore'):
fig.savefig(ic.name)
except ImageComparisonException:
pass
else:
msg = "Expected ImageComparisonException was not raised."
self.fail(msg)
# c) now reuse figure and set the original histogram stack again,
# image test should pass agin:
ppsd.calculate_histogram(**stack_criteria_list[1])
with ImageComparison(self.path_images,
'ppsd_restricted_stack.png',
reltol=1.5,
plt_close_all_enter=False) as ic:
ppsd._plot_histogram(fig=fig, draw=True)
with np.errstate(under='ignore'):
fig.savefig(ic.name)
def test_ppsd_w_iris_against_obspy_results(self):
"""
Test against results obtained after merging of #1108.
"""
# Read in ANMO data for one day
st = read(os.path.join(self.path, 'IUANMO.seed'))
# Read in metadata in various different formats
paz = {
'gain':
86298.5,
'zeros': [0, 0],
'poles': [
-59.4313, -22.7121 + 27.1065j, -22.7121 + 27.1065j, -0.0048004,
-0.073199
],
'sensitivity':
3.3554 * 10**9
}
resp = os.path.join(self.path, 'IUANMO.resp')
parser = Parser(os.path.join(self.path, 'IUANMO.dataless'))
inv = read_inventory(os.path.join(self.path, 'IUANMO.xml'))
# load expected results, for both only PAZ and full response
filename_paz = os.path.join(self.path, 'IUANMO_ppsd_paz.npz')
results_paz = PPSD.load_npz(filename_paz, metadata=None)
filename_full = os.path.join(self.path, 'IUANMO_ppsd_fullresponse.npz')
results_full = PPSD.load_npz(filename_full, metadata=None)
# Calculate the PPSDs and test against expected results
# first: only PAZ
ppsd = PPSD(st[0].stats, paz)
ppsd.add(st)
# commented code to generate the test data:
# ## np.savez(filename_paz,
# ## **dict([(k, getattr(ppsd, k))
# ## for k in PPSD.NPZ_STORE_KEYS]))
for key in PPSD.NPZ_STORE_KEYS_ARRAY_TYPES:
np.testing.assert_allclose(getattr(ppsd, key),
getattr(results_paz, key),
rtol=1e-5)
for key in PPSD.NPZ_STORE_KEYS_LIST_TYPES:
for got, expected in zip(getattr(ppsd, key),
getattr(results_paz, key)):
np.testing.assert_allclose(got, expected, rtol=1e-5)
for key in PPSD.NPZ_STORE_KEYS_SIMPLE_TYPES:
if key in ["obspy_version", "numpy_version", "matplotlib_version"]:
continue
self.assertEqual(getattr(ppsd, key), getattr(results_paz, key))
# second: various methods for full response
for metadata in [parser, inv, resp]:
ppsd = PPSD(st[0].stats, metadata)
ppsd.add(st)
# commented code to generate the test data:
# ## np.savez(filename_full,
# ## **dict([(k, getattr(ppsd, k))
# ## for k in PPSD.NPZ_STORE_KEYS]))
for key in PPSD.NPZ_STORE_KEYS_ARRAY_TYPES:
np.testing.assert_allclose(getattr(ppsd, key),
getattr(results_full, key),
rtol=1e-5)
for key in PPSD.NPZ_STORE_KEYS_LIST_TYPES:
for got, expected in zip(getattr(ppsd, key),
getattr(results_full, key)):
np.testing.assert_allclose(got, expected, rtol=1e-5)
for key in PPSD.NPZ_STORE_KEYS_SIMPLE_TYPES:
if key in [
"obspy_version", "numpy_version", "matplotlib_version"
]:
continue
self.assertEqual(getattr(ppsd, key),
getattr(results_full, key))
def test_ppsd_w_iris(self):
# Bands to be used this is the upper and lower frequency band pairs
fres = zip([0.1, 0.05], [0.2, 0.1])
file_data_anmo = os.path.join(self.path, 'IUANMO.seed')
# Read in ANMO data for one day
st = read(file_data_anmo)
# Use a canned ANMO response which will stay static
paz = {
'gain':
86298.5,
'zeros': [0, 0],
'poles': [
-59.4313, -22.7121 + 27.1065j, -22.7121 + 27.1065j, -0.0048004,
-0.073199
],
'sensitivity':
3.3554 * 10**9
}
# Make an empty PPSD and add the data
# use highest frequency given by IRIS Mustang noise-pdf web service
# (0.475683 Hz == 2.10224036 s) as center of first bin, so that we
# end up with the same bins.
ppsd = PPSD(st[0].stats, paz, period_limits=(2.10224036, 1400))
ppsd.add(st)
ppsd.calculate_histogram()
# Get the 50th percentile from the PPSD
(per, perval) = ppsd.get_percentile(percentile=50)
perinv = 1 / per
# Read in the results obtained from a Mustang flat file
file_data_iris = os.path.join(self.path, 'IRISpdfExample')
data = np.genfromtxt(file_data_iris,
comments='#',
delimiter=',',
dtype=[(native_str("freq"), np.float64),
(native_str("power"), np.int32),
(native_str("hits"), np.int32)])
freq = data["freq"]
power = data["power"]
hits = data["hits"]
# cut data to same period range as in the ppsd we computed
# (Mustang returns more long periods, probably due to some zero padding
# or longer nfft in psd)
num_periods = len(ppsd.period_bin_centers)
freqdistinct = np.array(sorted(set(freq), reverse=True)[:num_periods])
# just make sure that we compare the same periods in the following
# (as we access both frequency arrays by indices from now on)
np.testing.assert_allclose(freqdistinct,
1 / ppsd.period_bin_centers,
rtol=1e-4)
# For each frequency pair we want to compare the mean of the bands
for fre in fres:
# determine which bins we want to compare
mask = (fre[0] < perinv) & (perinv < fre[1])
# Get the values for the bands from the PPSD
per_val_good_obspy = perval[mask]
percenlist = []
# Now we sort out all of the data from the IRIS flat file
# We will loop through the frequency values and compute a
# 50th percentile
for curfreq in freqdistinct[mask]:
mask_ = curfreq == freq
tempvalslist = np.repeat(power[mask_], hits[mask_])
percenlist.append(np.percentile(tempvalslist, 50))
# Here is the actual test
np.testing.assert_allclose(np.mean(per_val_good_obspy),
np.mean(percenlist),
rtol=0.0,
atol=1.2)
def test_ppsd_restricted_stacks(self, state, image_path):
"""
Test PPSD.calculate_histogram() with restrictions to what data should
be stacked. Also includes image tests.
"""
# set up a bogus PPSD, with fixed random psds but with real start times
# of psd pieces, to facilitate testing the stack selection.
ppsd = PPSD(stats=Stats(dict(sampling_rate=150)),
metadata=None,
db_bins=(-200, -50, 20.),
period_step_octaves=1.4)
# change data to nowadays used nanoseconds POSIX timestamp
ppsd._times_processed = [
UTCDateTime(t)._ns for t in np.load(
os.path.join(state.path, "ppsd_times_processed.npy")).tolist()
]
np.random.seed(1234)
ppsd._binned_psds = [
arr for arr in np.random.uniform(-200, -50, (
len(ppsd._times_processed), len(ppsd.period_bin_centers)))
]
# Test callback function that selects a fixed random set of the
# timestamps. Also checks that we get passed the type we expect,
# which is 1D numpy ndarray of int type.
def callback(t_array):
assert isinstance(t_array, np.ndarray)
assert t_array.shape == (len(ppsd._times_processed), )
assert np.issubdtype(t_array.dtype, np.integer)
np.random.seed(1234)
res = np.random.randint(0, 2, len(t_array)).astype(bool)
return res
# test several different sets of stack criteria, should cover
# everything, even with lots of combined criteria
stack_criteria_list = [
dict(starttime=UTCDateTime(2015, 3, 8), month=[2, 3, 5, 7, 8]),
dict(endtime=UTCDateTime(2015, 6, 7),
year=[2015],
time_of_weekday=[(1, 0, 24), (2, 0, 24), (-1, 0, 11)]),
dict(year=[2013, 2014, 2016, 2017], month=[2, 3, 4]),
dict(month=[1, 2, 5, 6, 8], year=2015),
dict(isoweek=[4, 5, 6, 13, 22, 23, 24, 44, 45]),
dict(time_of_weekday=[(5, 22, 24), (6, 0, 2), (6, 22, 24)]),
dict(callback=callback, month=[1, 3, 5, 7]),
dict(callback=callback)
]
expected_selections = np.load(
os.path.join(state.path, "ppsd_stack_selections.npy"))
# test every set of criteria
for stack_criteria, expected_selection in zip(stack_criteria_list,
expected_selections):
selection_got = ppsd._stack_selection(**stack_criteria)
np.testing.assert_array_equal(selection_got, expected_selection)
plot_kwargs = dict(max_percentage=15,
xaxis_frequency=True,
period_lim=(0.01, 50))
ppsd.calculate_histogram(**stack_criteria_list[1])
fig = ppsd.plot(show=False, **plot_kwargs)
fig.axes[1].set_xlim(left=fig.axes[1].get_xlim()[0] - 2)
image_path_1 = image_path.parent / 'test_ppsd_restricted_stacks_1.png'
with np.errstate(under='ignore'):
fig.savefig(image_path_1)
# test it again, checking that updating an existing plot with different
# stack selection works..
# a) we start with the stack for the expected image and test that it
# matches (like above):
ppsd.calculate_histogram(**stack_criteria_list[1])
image_path_2 = image_path.parent / 'test_ppsd_restricted_stacks_2.png'
with np.errstate(under='ignore'):
fig.savefig(image_path_2)
ppsd.calculate_histogram(**stack_criteria_list[1])
image_path_3 = image_path.parent / 'test_ppsd_restricted_stacks_3.png'
ppsd._plot_histogram(fig=fig, draw=True)
with np.errstate(under='ignore'):
fig.savefig(image_path_3)
def psd(raw_trace, inventory):
# tr = segment.stream(True)[0]
dt = raw_trace.stats.endtime - raw_trace.stats.starttime # total_seconds
ppsd = PPSD(raw_trace.stats, metadata=inventory, ppsd_length=int(dt))
ppsd.add(raw_trace)
return ppsd
def test_PPSD_w_IRIS(self):
# Bands to be used this is the upper and lower frequency band pairs
fres = zip([0.1, 0.05], [0.2, 0.1])
file_dataANMO = os.path.join(self.path, 'IUANMO.seed')
# Read in ANMO data for one day
st = read(file_dataANMO)
# Use a canned ANMO response which will stay static
paz = {
'gain':
86298.5,
'zeros': [0, 0],
'poles': [
-59.4313, -22.7121 + 27.1065j, -22.7121 + 27.1065j, -0.0048004,
-0.073199
],
'sensitivity':
3.3554 * 10**9
}
# Make an empty PPSD and add the data
ppsd = PPSD(st[0].stats, paz)
ppsd.add(st)
# Get the 50th percentile from the PPSD
(per, perval) = ppsd.get_percentile(percentile=50)
# Read in the results obtained from a Mustang flat file
file_dataIRIS = os.path.join(self.path, 'IRISpdfExample')
freq, power, hits = np.genfromtxt(file_dataIRIS,
comments='#',
delimiter=',',
unpack=True)
# For each frequency pair we want to compare the mean of the bands
for fre in fres:
pervalGoodOBSPY = []
# Get the values for the bands from the PPSD
perinv = 1 / per
mask = (fre[0] < perinv) & (perinv < fre[1])
pervalGoodOBSPY = perval[mask]
# Now we sort out all of the data from the IRIS flat file
mask = (fre[0] < freq) & (freq < fre[1])
triples = list(zip(freq[mask], hits[mask], power[mask]))
# We now have all of the frequency values of interest
# We will get the distinct frequency values
freqdistinct = sorted(list(set(freq[mask])), reverse=True)
percenlist = []
# We will loop through the frequency values and compute a
# 50th percentile
for curfreq in freqdistinct:
tempvalslist = []
for triple in triples:
if np.isclose(curfreq, triple[0], atol=1e-3, rtol=0.0):
tempvalslist += [int(triple[2])] * int(triple[1])
percenlist.append(np.percentile(tempvalslist, 50))
# Here is the actual test
np.testing.assert_allclose(np.mean(pervalGoodOBSPY),
np.mean(percenlist),
rtol=0.0,
atol=1.0)
